HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing. (Arpit Agarwal)

This commit is contained in:
arp 2014-08-28 15:53:12 -07:00
parent 722cc92fa5
commit bbaa7dc28d
7 changed files with 115 additions and 19 deletions

View File

@ -179,8 +179,19 @@ public FSDataOutputStream create(Path f, FsPermission permission,
return fs.create(f, permission, return fs.create(f, permission,
overwrite, bufferSize, replication, blockSize, progress); overwrite, bufferSize, replication, blockSize, progress);
} }
@Override
public FSDataOutputStream create(Path f,
FsPermission permission,
EnumSet<CreateFlag> flags,
int bufferSize,
short replication,
long blockSize,
Progressable progress,
ChecksumOpt checksumOpt) throws IOException {
return fs.create(f, permission,
flags, bufferSize, replication, blockSize, progress);
}
@Override @Override
@Deprecated @Deprecated

View File

@ -30,6 +30,7 @@
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.FilterFileSystem;
@ -45,6 +46,9 @@
import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IOUtils;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.LAZY_PERSIST;
/** /**
* Provides: argument processing to ensure the destination is valid * Provides: argument processing to ensure the destination is valid
* for the number of source arguments. A processPaths that accepts both * for the number of source arguments. A processPaths that accepts both
@ -56,6 +60,7 @@ abstract class CommandWithDestination extends FsCommand {
private boolean overwrite = false; private boolean overwrite = false;
private boolean verifyChecksum = true; private boolean verifyChecksum = true;
private boolean writeChecksum = true; private boolean writeChecksum = true;
private boolean lazyPersist = false;
/** /**
* The name of the raw xattr namespace. It would be nice to use * The name of the raw xattr namespace. It would be nice to use
@ -78,6 +83,10 @@ protected void setOverwrite(boolean flag) {
overwrite = flag; overwrite = flag;
} }
protected void setLazyPersist(boolean flag) {
lazyPersist = flag;
}
protected void setVerifyChecksum(boolean flag) { protected void setVerifyChecksum(boolean flag) {
verifyChecksum = flag; verifyChecksum = flag;
} }
@ -379,7 +388,7 @@ protected void copyStreamToTarget(InputStream in, PathData target)
try { try {
PathData tempTarget = target.suffix("._COPYING_"); PathData tempTarget = target.suffix("._COPYING_");
targetFs.setWriteChecksum(writeChecksum); targetFs.setWriteChecksum(writeChecksum);
targetFs.writeStreamToFile(in, tempTarget); targetFs.writeStreamToFile(in, tempTarget, lazyPersist);
targetFs.rename(tempTarget, target); targetFs.rename(tempTarget, target);
} finally { } finally {
targetFs.close(); // last ditch effort to ensure temp file is removed targetFs.close(); // last ditch effort to ensure temp file is removed
@ -449,10 +458,11 @@ private static class TargetFileSystem extends FilterFileSystem {
super(fs); super(fs);
} }
void writeStreamToFile(InputStream in, PathData target) throws IOException { void writeStreamToFile(InputStream in, PathData target,
boolean lazyPersist) throws IOException {
FSDataOutputStream out = null; FSDataOutputStream out = null;
try { try {
out = create(target); out = create(target, lazyPersist);
IOUtils.copyBytes(in, out, getConf(), true); IOUtils.copyBytes(in, out, getConf(), true);
} finally { } finally {
IOUtils.closeStream(out); // just in case copyBytes didn't IOUtils.closeStream(out); // just in case copyBytes didn't
@ -460,9 +470,21 @@ void writeStreamToFile(InputStream in, PathData target) throws IOException {
} }
// tag created files as temp files // tag created files as temp files
FSDataOutputStream create(PathData item) throws IOException { FSDataOutputStream create(PathData item, boolean lazyPersist)
throws IOException {
try { try {
return create(item.path, true); EnumSet<CreateFlag> createFlags = EnumSet.of(CREATE);
if (lazyPersist) {
createFlags.add(LAZY_PERSIST);
}
return create(item.path,
null,
createFlags,
getConf().getInt("io.file.buffer.size", 4096),
lazyPersist ? 1 : getDefaultReplication(item.path),
getDefaultBlockSize(),
null,
null);
} finally { // might have been created but stream was interrupted } finally { // might have been created but stream was interrupted
deleteOnExit(item.path); deleteOnExit(item.path);
} }

View File

@ -215,21 +215,25 @@ protected void processOptions(LinkedList<String> args)
*/ */
public static class Put extends CommandWithDestination { public static class Put extends CommandWithDestination {
public static final String NAME = "put"; public static final String NAME = "put";
public static final String USAGE = "[-f] [-p] <localsrc> ... <dst>"; public static final String USAGE = "[-f] [-p] [-l] <localsrc> ... <dst>";
public static final String DESCRIPTION = public static final String DESCRIPTION =
"Copy files from the local file system " + "Copy files from the local file system " +
"into fs. Copying fails if the file already " + "into fs. Copying fails if the file already " +
"exists, unless the -f flag is given. Passing " + "exists, unless the -f flag is given.\n" +
"-p preserves access and modification times, " + "Flags:\n" +
"ownership and the mode. Passing -f overwrites " + " -p : Preserves access and modification times, ownership and the mode.\n" +
"the destination if it already exists.\n"; " -f : Overwrites the destination if it already exists.\n" +
" -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
" replication factor of 1. This flag will result in reduced\n" +
" durability. Use with care.\n";
@Override @Override
protected void processOptions(LinkedList<String> args) throws IOException { protected void processOptions(LinkedList<String> args) throws IOException {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p"); CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l");
cf.parse(args); cf.parse(args);
setOverwrite(cf.getOpt("f")); setOverwrite(cf.getOpt("f"));
setPreserve(cf.getOpt("p")); setPreserve(cf.getOpt("p"));
setLazyPersist(cf.getOpt("l"));
getRemoteDestination(args); getRemoteDestination(args);
// should have a -r option // should have a -r option
setRecursive(true); setRecursive(true);

View File

@ -39,6 +39,7 @@
* %u: User name of owner * %u: User name of owner
* %y: UTC date as &quot;yyyy-MM-dd HH:mm:ss&quot; * %y: UTC date as &quot;yyyy-MM-dd HH:mm:ss&quot;
* %Y: Milliseconds since January 1, 1970 UTC * %Y: Milliseconds since January 1, 1970 UTC
* %l: Whether lazyPersist flag is set on the file.
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Unstable @InterfaceStability.Unstable
@ -53,7 +54,8 @@ public static void registerCommands(CommandFactory factory) {
public static final String DESCRIPTION = public static final String DESCRIPTION =
"Print statistics about the file/directory at <path> " + "Print statistics about the file/directory at <path> " +
"in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " + "in the specified format. Format accepts filesize in blocks (%b), group name of owner(%g), " +
"filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y)\n"; "filename (%n), block size (%o), replication (%r), user name of owner(%u), modification date (%y, %Y), " +
"lazyPersist flag (%l)\n";
protected static final SimpleDateFormat timeFmt; protected static final SimpleDateFormat timeFmt;
static { static {
@ -115,6 +117,9 @@ protected void processPath(PathData item) throws IOException {
case 'Y': case 'Y':
buf.append(stat.getModificationTime()); buf.append(stat.getModificationTime());
break; break;
case 'l':
buf.append(stat.isLazyPersist());
break;
default: default:
// this leaves %<unknown> alone, which causes the potential for // this leaves %<unknown> alone, which causes the potential for
// future format options to break strings; should use %% to // future format options to break strings; should use %% to

View File

@ -436,7 +436,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^-put \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output> <expected-output>^-put \[-f\] \[-p\] \[-l\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
@ -444,15 +444,31 @@
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^\s*exists, unless the -f flag is given.( )*Passing -p preserves access and( )*</expected-output> <expected-output>^\s*exists, unless the -f flag is given.( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^\s*modification times, ownership and the mode. Passing -f overwrites the( )*</expected-output> <expected-output>^\s*Flags:( )*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*destination if it already exists.( )*</expected-output> <expected-output>^\s*-p Preserves access and modification times, ownership and the mode.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-f Overwrites the destination if it already exists.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-l Allow DataNode to lazily persist the file to disk. Forces( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*replication factor of 1. This flag will result in reduced( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*durability. Use with care.( )*</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -467,7 +483,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^-copyFromLocal \[-f\] \[-p\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output> <expected-output>^-copyFromLocal \[-f\] \[-p\] \[-l\] &lt;localsrc&gt; \.\.\. &lt;dst&gt; :\s*</expected-output>
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
@ -782,7 +798,7 @@
</comparator> </comparator>
<comparator> <comparator>
<type>RegexpComparator</type> <type>RegexpComparator</type>
<expected-output>^( |\t)*\(%y, %Y\)( )*</expected-output> <expected-output>^( |\t)*\(%y, %Y\), lazyPersist flag \(\%l\)( )*</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>

View File

@ -21,4 +21,6 @@
HDFS-6929. NN periodically unlinks lazy persist files with missing HDFS-6929. NN periodically unlinks lazy persist files with missing
replicas from namespace. (Arpit Agarwal) replicas from namespace. (Arpit Agarwal)
HDFS-6928. 'hdfs put' command should accept lazyPersist flag for testing.
(Arpit Agarwal)

View File

@ -4554,6 +4554,42 @@
</comparators> </comparators>
</test> </test>
<test> <!-- TESTED -->
<description>put: The LazyPersist flag is set when requested</description>
<test-commands>
<command>-fs NAMENODE -mkdir /dirLp</command>
<command>-fs NAMENODE -put -l CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
<command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm -r /dirLp/</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^true</expected-output>
</comparator>
</comparators>
</test>
<test> <!-- TESTED -->
<description>put: The LazyPersist flag is not set by default</description>
<test-commands>
<command>-fs NAMENODE -mkdir /dirLp</command>
<command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dirLp/data15bytes</command>
<command>-fs NAMENODE -stat %l /dirLp/data15bytes</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm -r /dirLp/</command>
</cleanup-commands>
<comparators>
<comparator>
<type>RegexpComparator</type>
<expected-output>^false</expected-output>
</comparator>
</comparators>
</test>
<!-- Tests for copyFromLocal --> <!-- Tests for copyFromLocal -->
<test> <!-- TESTED --> <test> <!-- TESTED -->
<description>copyFromLocal: copying file into a file (absolute path)</description> <description>copyFromLocal: copying file into a file (absolute path)</description>