diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bc333e6e9d..7586f950a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -192,6 +192,8 @@ Trunk (Unreleased) HDFS-5450. Better API for getting the cached blocks locations. (wang) + HDFS-5485. Add command-line support for modifyDirective. (cmccabe) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java index 6ae902208d..5352d669e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java @@ -248,6 +248,84 @@ public int run(Configuration conf, List args) throws IOException { } } + private static class ModifyPathBasedCacheDirectiveCommand implements Command { + @Override + public String getName() { + return "-modifyDirective"; + } + + @Override + public String getShortUsage() { + return "[" + getName() + + " -id [-path ] [-replication ] " + + "[-pool ] ]\n"; + } + + @Override + public String getLongUsage() { + TableListing listing = getOptionDescriptionListing(); + listing.addRow("", "The ID of the directive to modify (required)"); + listing.addRow("", "A path to cache. The path can be " + + "a directory or a file. (optional)"); + listing.addRow("", "The cache replication factor to use. " + + "(optional)"); + listing.addRow("", "The pool to which the directive will be " + + "added. You must have write permission on the cache pool " + + "in order to move a directive into it. (optional)"); + return getShortUsage() + "\n" + + "Modify a PathBasedCache directive.\n\n" + + listing.toString(); + } + + @Override + public int run(Configuration conf, List args) throws IOException { + PathBasedCacheDirective.Builder builder = + new PathBasedCacheDirective.Builder(); + boolean modified = false; + String idString = StringUtils.popOptionWithArgument("-id", args); + if (idString == null) { + System.err.println("You must specify a directive ID with -id."); + return 1; + } + builder.setId(Long.parseLong(idString)); + String path = StringUtils.popOptionWithArgument("-path", args); + if (path != null) { + builder.setPath(new Path(path)); + modified = true; + } + String replicationString = + StringUtils.popOptionWithArgument("-replication", args); + if (replicationString != null) { + builder.setReplication(Short.parseShort(replicationString)); + modified = true; + } + String poolName = + StringUtils.popOptionWithArgument("-pool", args); + if (poolName != null) { + builder.setPool(poolName); + modified = true; + } + if (!args.isEmpty()) { + System.err.println("Can't understand argument: " + args.get(0)); + System.err.println("Usage is " + getShortUsage()); + return 1; + } + if (!modified) { + System.err.println("No modifications were specified."); + return 1; + } + DistributedFileSystem dfs = getDFS(conf); + try { + dfs.modifyPathBasedCacheDirective(builder.build()); + System.out.println("Modified PathBasedCache entry " + idString); + } catch (IOException e) { + System.err.println(prettifyException(e)); + return 2; + } + return 0; + } + } + private static class RemovePathBasedCacheDirectivesCommand implements Command { @Override public String getName() { @@ -352,6 +430,7 @@ public int run(Configuration conf, List args) throws IOException { TableListing tableListing = new TableListing.Builder(). addField("ID", Justification.LEFT). addField("POOL", Justification.LEFT). + addField("REPLICATION", Justification.LEFT). addField("PATH", Justification.LEFT). build(); DistributedFileSystem dfs = getDFS(conf); @@ -362,6 +441,7 @@ public int run(Configuration conf, List args) throws IOException { PathBasedCacheDirective directive = iter.next(); String row[] = new String[] { "" + directive.getId(), directive.getPool(), + "" + directive.getReplication(), directive.getPath().toUri().getPath(), }; tableListing.addRow(row); @@ -744,9 +824,10 @@ public int run(Configuration conf, List args) throws IOException { private static Command[] COMMANDS = { new AddPathBasedCacheDirectiveCommand(), + new ModifyPathBasedCacheDirectiveCommand(), + new ListPathBasedCacheDirectiveCommand(), new RemovePathBasedCacheDirectiveCommand(), new RemovePathBasedCacheDirectivesCommand(), - new ListPathBasedCacheDirectiveCommand(), new AddCachePoolCommand(), new ModifyCachePoolCommand(), new RemoveCachePoolCommand(), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml index 54dd7a97b7..eba62ba1fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml @@ -180,15 +180,15 @@ SubstringComparator - 1 pool1 /foo + 1 pool1 1 /foo SubstringComparator - 2 pool1 /bar + 2 pool1 1 /bar SubstringComparator - 3 pool1 /baz + 3 pool1 2 /baz @@ -234,11 +234,11 @@ SubstringComparator - 8 pool2 /baz + 8 pool2 1 /baz SubstringComparator - 9 pool2 /buz + 9 pool2 1 /buz @@ -265,11 +265,11 @@ SubstringComparator - 10 pool1 /foo + 10 pool1 1 /foo SubstringComparator - 12 pool2 /foo + 12 pool2 1 /foo @@ -296,7 +296,7 @@ SubstringComparator - 16 pool2 /foo + 16 pool2 1 /foo @@ -320,7 +320,7 @@ SubstringComparator - 19 pool1 /bar + 19 pool1 1 /bar @@ -349,11 +349,37 @@ SubstringComparator - 22 pool1 /bar + 22 pool1 1 /bar SubstringComparator - 24 pool2 /bar + 24 pool2 1 /bar + + + + + + Testing modifying directives + + -addPool pool1 + -addPool pool2 + -addDirective -path /foo -pool pool2 + -modifyDirective -id 25 -path /bar2 + -modifyDirective -id 25 -pool pool1 -path /bar3 + -listDirectives -path /bar3 + + + -removePool pool1 + -removePool pool2 + + + + SubstringComparator + Found 1 entry + + + SubstringComparator + 25 pool1 1 /bar3