big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-5485. add command-line support for modifyDirective (cmccabe)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1541377 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Colin McCabe 2013-11-13 03:29:02 +00:00
parent 656e859f5c
commit 39def49661
3 changed files with 121 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -192,6 +192,8 @@ Trunk (Unreleased)
HDFS-5450. Better API for getting the cached blocks locations. (wang)
HDFS-5485. Add command-line support for modifyDirective. (cmccabe)
OPTIMIZATIONS
HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe)

83
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
View File

@ -248,6 +248,84 @@ public int run(Configuration conf, List<String> args) throws IOException {
}
}
private static class ModifyPathBasedCacheDirectiveCommand implements Command {
@Override
public String getName() {
return "-modifyDirective";
}
@Override
public String getShortUsage() {
return "[" + getName() +
" -id <id> [-path <path>] [-replication <replication>] " +
"[-pool <pool-name>] ]\n";
}
@Override
public String getLongUsage() {
TableListing listing = getOptionDescriptionListing();
listing.addRow("<id>", "The ID of the directive to modify (required)");
listing.addRow("<path>", "A path to cache. The path can be " +
"a directory or a file. (optional)");
listing.addRow("<replication>", "The cache replication factor to use. " +
"(optional)");
listing.addRow("<pool-name>", "The pool to which the directive will be " +
"added. You must have write permission on the cache pool "
+ "in order to move a directive into it. (optional)");
return getShortUsage() + "\n" +
"Modify a PathBasedCache directive.\n\n" +
listing.toString();
}
@Override
public int run(Configuration conf, List<String> args) throws IOException {
PathBasedCacheDirective.Builder builder =
new PathBasedCacheDirective.Builder();
boolean modified = false;
String idString = StringUtils.popOptionWithArgument("-id", args);
if (idString == null) {
System.err.println("You must specify a directive ID with -id.");
return 1;
}
builder.setId(Long.parseLong(idString));
String path = StringUtils.popOptionWithArgument("-path", args);
if (path != null) {
builder.setPath(new Path(path));
modified = true;
}
String replicationString =
StringUtils.popOptionWithArgument("-replication", args);
if (replicationString != null) {
builder.setReplication(Short.parseShort(replicationString));
modified = true;
}
String poolName =
StringUtils.popOptionWithArgument("-pool", args);
if (poolName != null) {
builder.setPool(poolName);
modified = true;
}
if (!args.isEmpty()) {
System.err.println("Can't understand argument: " + args.get(0));
System.err.println("Usage is " + getShortUsage());
return 1;
}
if (!modified) {
System.err.println("No modifications were specified.");
return 1;
}
DistributedFileSystem dfs = getDFS(conf);
try {
dfs.modifyPathBasedCacheDirective(builder.build());
System.out.println("Modified PathBasedCache entry " + idString);
} catch (IOException e) {
System.err.println(prettifyException(e));
return 2;
}
return 0;
}
}
private static class RemovePathBasedCacheDirectivesCommand implements Command {
@Override
public String getName() {
@ -352,6 +430,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
TableListing tableListing = new TableListing.Builder().
addField("ID", Justification.LEFT).
addField("POOL", Justification.LEFT).
addField("REPLICATION", Justification.LEFT).
addField("PATH", Justification.LEFT).
build();
DistributedFileSystem dfs = getDFS(conf);
@ -362,6 +441,7 @@ public int run(Configuration conf, List<String> args) throws IOException {
PathBasedCacheDirective directive = iter.next();
String row[] = new String[] {
"" + directive.getId(), directive.getPool(),
"" + directive.getReplication(),
directive.getPath().toUri().getPath(),
};
tableListing.addRow(row);
@ -744,9 +824,10 @@ public int run(Configuration conf, List<String> args) throws IOException {
private static Command[] COMMANDS = {
new AddPathBasedCacheDirectiveCommand(),
new ModifyPathBasedCacheDirectiveCommand(),
new ListPathBasedCacheDirectiveCommand(),
new RemovePathBasedCacheDirectiveCommand(),
new RemovePathBasedCacheDirectivesCommand(),
new ListPathBasedCacheDirectiveCommand(),
new AddCachePoolCommand(),
new ModifyCachePoolCommand(),
new RemoveCachePoolCommand(),

48
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
View File

@ -180,15 +180,15 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>1 pool1 /foo</expected-output>
<expected-output>1 pool1 1 /foo</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>2 pool1 /bar</expected-output>
<expected-output>2 pool1 1 /bar</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>3 pool1 /baz</expected-output>
<expected-output>3 pool1 2 /baz</expected-output>
</comparator>
</comparators>
</test>
@ -234,11 +234,11 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>8 pool2 /baz</expected-output>
<expected-output>8 pool2 1 /baz</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>9 pool2 /buz</expected-output>
<expected-output>9 pool2 1 /buz</expected-output>
</comparator>
</comparators>
</test>
@ -265,11 +265,11 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>10 pool1 /foo</expected-output>
<expected-output>10 pool1 1 /foo</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>12 pool2 /foo</expected-output>
<expected-output>12 pool2 1 /foo</expected-output>
</comparator>
</comparators>
</test>
@ -296,7 +296,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>16 pool2 /foo</expected-output>
<expected-output>16 pool2 1 /foo</expected-output>
</comparator>
</comparators>
</test>
@ -320,7 +320,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>19 pool1 /bar</expected-output>
<expected-output>19 pool1 1 /bar</expected-output>
</comparator>
</comparators>
</test>
@ -349,11 +349,37 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>22 pool1 /bar</expected-output>
<expected-output>22 pool1 1 /bar</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>24 pool2 /bar</expected-output>
<expected-output>24 pool2 1 /bar</expected-output>
</comparator>
</comparators>
</test>
<test> <!--Tested -->
<description>Testing modifying directives</description>
<test-commands>
<cache-admin-command>-addPool pool1</cache-admin-command>
<cache-admin-command>-addPool pool2</cache-admin-command>
<cache-admin-command>-addDirective -path /foo -pool pool2</cache-admin-command>
<cache-admin-command>-modifyDirective -id 25 -path /bar2</cache-admin-command>
<cache-admin-command>-modifyDirective -id 25 -pool pool1 -path /bar3</cache-admin-command>
<cache-admin-command>-listDirectives -path /bar3</cache-admin-command>
</test-commands>
<cleanup-commands>
<cache-admin-command>-removePool pool1</cache-admin-command>
<cache-admin-command>-removePool pool2</cache-admin-command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Found 1 entry</expected-output>
</comparator>
<comparator>
<type>SubstringComparator</type>
<expected-output>25 pool1 1 /bar3</expected-output>
</comparator>
</comparators>
</test>