big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-11012. hadoop fs -text of zero-length file causes EOFException. Contributed by Eric Payne

Browse Source
This commit is contained in:
Jason Lowe 2014-09-02 18:15:38 +00:00
parent e1109fb656
commit 01e8f056d9
3 changed files with 72 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -729,6 +729,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11036. Add build directory to .gitignore (Tsuyoshi OZAWA via aw)
HADOOP-11012. hadoop fs -text of zero-length file causes EOFException
(Eric Payne via jlowe)
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES

12
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs.shell;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.util.LinkedList;
@ -126,8 +127,17 @@ public static class Text extends Cat {
protected InputStream getInputStream(PathData item) throws IOException {
FSDataInputStream i = (FSDataInputStream)super.getInputStream(item);
// Handle 0 and 1-byte files
short leadBytes;
try {
leadBytes = i.readShort();
} catch (EOFException e) {
i.seek(0);
return i;
}
// Check type of stream first
switch(i.readShort()) {
switch(leadBytes) {
case 0x1f8b: { // RFC 1952
// Must be gzip
i.seek(0);

77
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
View File

@ -42,29 +42,14 @@ public class TestTextCommand {
System.getProperty("test.build.data", "build/test/data/") + "/testText";
private static final String AVRO_FILENAME =
new Path(TEST_ROOT_DIR, "weather.avro").toUri().getPath();
private static final String TEXT_FILENAME =
new Path(TEST_ROOT_DIR, "testtextfile.txt").toUri().getPath();
/**
* Tests whether binary Avro data files are displayed correctly.
*/
@Test (timeout = 30000)
public void testDisplayForAvroFiles() throws Exception {
// Create a small Avro data file on the local file system.
createAvroFile(generateWeatherAvroBinaryData());
// Prepare and call the Text command's protected getInputStream method
// using reflection.
Configuration conf = new Configuration();
URI localPath = new URI(AVRO_FILENAME);
PathData pathData = new PathData(localPath, conf);
Display.Text text = new Display.Text();
text.setConf(conf);
Method method = text.getClass().getDeclaredMethod(
"getInputStream", PathData.class);
method.setAccessible(true);
InputStream stream = (InputStream) method.invoke(text, pathData);
String output = inputStreamToString(stream);
// Check the output.
String expectedOutput =
"{\"station\":\"011990-99999\",\"time\":-619524000000,\"temp\":0}" +
System.getProperty("line.separator") +
@ -77,18 +62,72 @@ public void testDisplayForAvroFiles() throws Exception {
"{\"station\":\"012650-99999\",\"time\":-655509600000,\"temp\":78}" +
System.getProperty("line.separator");
String output = readUsingTextCommand(AVRO_FILENAME,
generateWeatherAvroBinaryData());
assertEquals(expectedOutput, output);
}
/**
* Tests that a zero-length file is displayed correctly.
*/
@Test (timeout = 30000)
public void testEmptyTextFil() throws Exception {
byte[] emptyContents = { };
String output = readUsingTextCommand(TEXT_FILENAME, emptyContents);
assertTrue("".equals(output));
}
/**
* Tests that a one-byte file is displayed correctly.
*/
@Test (timeout = 30000)
public void testOneByteTextFil() throws Exception {
byte[] oneByteContents = { 'x' };
String output = readUsingTextCommand(TEXT_FILENAME, oneByteContents);
assertTrue(new String(oneByteContents).equals(output));
}
/**
* Tests that a one-byte file is displayed correctly.
*/
@Test (timeout = 30000)
public void testTwoByteTextFil() throws Exception {
byte[] twoByteContents = { 'x', 'y' };
String output = readUsingTextCommand(TEXT_FILENAME, twoByteContents);
assertTrue(new String(twoByteContents).equals(output));
}
// Create a file on the local file system and read it using
// the Display.Text class.
private String readUsingTextCommand(String fileName, byte[] fileContents)
throws Exception {
createFile(fileName, fileContents);
// Prepare and call the Text command's protected getInputStream method
// using reflection.
Configuration conf = new Configuration();
URI localPath = new URI(fileName);
PathData pathData = new PathData(localPath, conf);
Display.Text text = new Display.Text() {
@Override
public InputStream getInputStream(PathData item) throws IOException {
return super.getInputStream(item);
}
};
text.setConf(conf);
InputStream stream = (InputStream) text.getInputStream(pathData);
return inputStreamToString(stream);
}
private String inputStreamToString(InputStream stream) throws IOException {
StringWriter writer = new StringWriter();
IOUtils.copy(stream, writer);
return writer.toString();
}
private void createAvroFile(byte[] contents) throws IOException {
private void createFile(String fileName, byte[] contents) throws IOException {
(new File(TEST_ROOT_DIR)).mkdir();
File file = new File(AVRO_FILENAME);
File file = new File(fileName);
file.createNewFile();
FileOutputStream stream = new FileOutputStream(file);
stream.write(contents);