big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13960. hdfs dfs -checksum command should optionally show block size in output. Contributed by Lokesh Jain.

Browse Source 
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
Lokesh Jain 2019-04-02 12:23:18 -07:00 committed by Wei-Chiu Chuang
parent bfc90bdc64
commit cf268114c9
4 changed files with 51 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
View File

@ -175,7 +175,7 @@ protected InputStream getInputStream(PathData item) throws IOException {
public static class Checksum extends Display {
public static final String NAME = "checksum";
public static final String USAGE = "<src> ...";
public static final String USAGE = "[-v] <src> ...";
public static final String DESCRIPTION =
"Dump checksum information for files that match the file " +
"pattern <src> to stdout. Note that this requires a round-trip " +
@ -184,6 +184,16 @@ public static class Checksum extends Display {
"file depends on its content, block size and the checksum " +
"algorithm and parameters used for creating the file.";
private boolean displayBlockSize;
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "v");
cf.parse(args);
displayBlockSize = cf.getOpt("v");
}
@Override
protected void processPath(PathData item) throws IOException {
if (item.stat.isDirectory()) {
@ -191,14 +201,15 @@ protected void processPath(PathData item) throws IOException {
}
FileChecksum checksum = item.fs.getFileChecksum(item.path);
if (checksum == null) {
out.printf("%s\tNONE\t%n", item.toString());
String outputChecksum = checksum == null ? "NONE" :
String.format("%s\t%s", checksum.getAlgorithmName(), StringUtils
.byteToHexString(checksum.getBytes(), 0, checksum.getLength()));
if (displayBlockSize) {
FileStatus fileStatus = item.fs.getFileStatus(item.path);
out.printf("%s\t%s\tBlockSize=%s%n", item.toString(), outputChecksum,
fileStatus != null ? fileStatus.getBlockSize() : "NONE");
} else {
String checksumString = StringUtils.byteToHexString(
checksum.getBytes(), 0, checksum.getLength());
out.printf("%s\t%s\t%s%n",
item.toString(), checksum.getAlgorithmName(),
checksumString);
out.printf("%s\t%s%n", item.toString(), outputChecksum);
}
}
}

6
hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
View File

@ -73,10 +73,14 @@ Returns 0 on success and -1 on error.
checksum
--------
Usage: `hadoop fs -checksum URI`
Usage: `hadoop fs -checksum [-v] URI`
Returns the checksum information of a file.
Options
* The `-v` option displays blocks size for the file.
Example:
* `hadoop fs -checksum hdfs://nn1.example.com/file1`

2
hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
View File

@ -714,7 +714,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^-checksum &lt;src&gt; \.\.\. :\s*</expected-output>
<expected-output>^-checksum \[-v\] &lt;src&gt; \.\.\. :\s*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
View File

@ -71,6 +71,7 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Assert;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
@ -1121,6 +1122,31 @@ private void textTest(Path root, Configuration conf) throws Exception {
}
}
@Test (timeout = 30000)
public void testChecksum() throws Exception {
PrintStream printStream = System.out;
try {
ByteArrayOutputStream out = new ByteArrayOutputStream();
System.setOut(new PrintStream(out));
FsShell shell = new FsShell(dfs.getConf());
final Path filePath = new Path("/testChecksum/file1");
writeFile(dfs, filePath);
FileStatus fileStatus = dfs.getFileStatus(filePath);
FileChecksum checksum = dfs.getFileChecksum(filePath);
String[] args = {"-checksum", "-v", filePath.toString()};
assertEquals(0, shell.run(args));
// verify block size is printed in the output
assertTrue(out.toString()
.contains(String.format("BlockSize=%s", fileStatus.getBlockSize())));
// verify checksum is printed in the output
assertTrue(out.toString().contains(StringUtils
.byteToHexString(checksum.getBytes(), 0, checksum.getLength())));
} finally {
Assert.assertNotNull(printStream);
System.setOut(printStream);
}
}
@Test (timeout = 30000)
public void testCopyToLocal() throws IOException {
FsShell shell = new FsShell(dfs.getConf());