big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13983. TestOfflineImageViewer crashes in windows. Contributed by Vinayakumar B.

Browse Source
This commit is contained in:
Inigo Goiri 2018-10-19 17:16:04 -07:00
parent e2a9fa8448
commit f069d38c8d
2 changed files with 63 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
/**
@ -135,7 +136,7 @@ private static Options buildOptions() {
*/
public static void main(String[] args) throws Exception {
int status = run(args);
System.exit(status);
ExitUtil.terminate(status);
}
public static int run(String[] args) throws Exception {
@ -175,19 +176,24 @@ public static int run(String[] args) throws Exception {
String tempPath = cmd.getOptionValue("t", "");
Configuration conf = new Configuration();
try (PrintStream out = outputFile.equals("-") ?
System.out : new PrintStream(outputFile, "UTF-8")) {
PrintStream out = null;
try {
out = outputFile.equals("-") || "REVERSEXML".equalsIgnoreCase(processor) ?
System.out : new PrintStream(outputFile, "UTF-8");
switch (StringUtils.toUpperCase(processor)) {
case "FILEDISTRIBUTION":
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
boolean formatOutput = cmd.hasOption("format");
new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
.visit(new RandomAccessFile(inputFile, "r"));
try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
.visit(r);
}
break;
case "XML":
new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
"r"));
try (RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
new PBImageXmlWriter(conf, out).visit(r);
}
break;
case "REVERSEXML":
try {
@ -196,7 +202,7 @@ public static int run(String[] args) throws Exception {
System.err.println("OfflineImageReconstructor failed: "
+ e.getMessage());
e.printStackTrace(System.err);
System.exit(1);
ExitUtil.terminate(1);
}
break;
case "WEB":
@ -208,8 +214,9 @@ public static int run(String[] args) throws Exception {
break;
case "DELIMITED":
try (PBImageDelimitedTextWriter writer =
new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
writer.visit(new RandomAccessFile(inputFile, "r"));
new PBImageDelimitedTextWriter(out, delimiter, tempPath);
RandomAccessFile r = new RandomAccessFile(inputFile, "r")) {
writer.visit(r);
}
break;
default:
@ -223,6 +230,10 @@ public static int run(String[] args) throws Exception {
} catch (IOException e) {
System.err.println("Encountered exception. Exiting: " + e.getMessage());
e.printStackTrace(System.err);
} finally {
if (out != null && out != System.out) {
out.close();
}
}
return -1;
}

76
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -355,8 +355,10 @@ public void testTruncatedFSImage() throws IOException {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(new RandomAccessFile(truncatedFile, "r"));
try (RandomAccessFile r = new RandomAccessFile(truncatedFile, "r")) {
new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(r);
}
}
private void copyPartOfFile(File src, File dest) throws IOException {
@ -377,38 +379,41 @@ private void copyPartOfFile(File src, File dest) throws IOException {
@Test
public void testFileDistributionCalculator() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
.visit(new RandomAccessFile(originalFsimage, "r"));
o.close();
try (ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
.visit(r);
o.close();
String outputString = output.toString();
Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
Matcher matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1));
assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
String outputString = output.toString();
Pattern p = Pattern.compile("totalFiles = (\\d+)\n");
Matcher matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalFiles = Integer.parseInt(matcher.group(1));
assertEquals(NUM_DIRS * FILES_PER_DIR + filesECCount + 1, totalFiles);
p = Pattern.compile("totalDirectories = (\\d+)\n");
matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalDirs = Integer.parseInt(matcher.group(1));
// totalDirs includes root directory
assertEquals(dirCount + 1, totalDirs);
p = Pattern.compile("totalDirectories = (\\d+)\n");
matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalDirs = Integer.parseInt(matcher.group(1));
// totalDirs includes root directory
assertEquals(dirCount + 1, totalDirs);
FileStatus maxFile = Collections.max(writtenFiles.values(),
new Comparator<FileStatus>() {
@Override
public int compare(FileStatus first, FileStatus second) {
return first.getLen() < second.getLen() ? -1 :
((first.getLen() == second.getLen()) ? 0 : 1);
}
});
p = Pattern.compile("maxFileSize = (\\d+)\n");
matcher = p.matcher(output.toString("UTF-8"));
assertTrue(matcher.find() && matcher.groupCount() == 1);
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
FileStatus maxFile = Collections.max(writtenFiles.values(),
new Comparator<FileStatus>() {
@Override
public int compare(FileStatus first, FileStatus second) {
return first.getLen() < second.getLen() ?
-1 :
((first.getLen() == second.getLen()) ? 0 : 1);
}
});
p = Pattern.compile("maxFileSize = (\\d+)\n");
matcher = p.matcher(output.toString("UTF-8"));
assertTrue(matcher.find() && matcher.groupCount() == 1);
assertEquals(maxFile.getLen(), Long.parseLong(matcher.group(1)));
}
}
@Test
@ -514,7 +519,9 @@ public void testPBImageXmlWriter() throws IOException, SAXException,
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
v.visit(new RandomAccessFile(originalFsimage, "r"));
try (RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
v.visit(r);
}
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser();
final String xml = output.toString();
@ -667,10 +674,11 @@ private void testPBDelimitedWriter(String db)
final String DELIMITER = "\t";
ByteArrayOutputStream output = new ByteArrayOutputStream();
try (PrintStream o = new PrintStream(output)) {
try (PrintStream o = new PrintStream(output);
RandomAccessFile r = new RandomAccessFile(originalFsimage, "r")) {
PBImageDelimitedTextWriter v =
new PBImageDelimitedTextWriter(o, DELIMITER, db);
v.visit(new RandomAccessFile(originalFsimage, "r"));
v.visit(r);
}
Set<String> fileNames = new HashSet<>();