big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-10583. bin/hadoop key throws NPE with no args and assorted other fixups. (clamb via tucu)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1594320 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Alejandro Abdelnur 2014-05-13 18:29:40 +00:00
parent 4bc3371824
commit c54a4bb666
5 changed files with 137 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -326,6 +326,8 @@ Trunk (Unreleased)
HADOOP-10431. Change visibility of KeyStore.Options getter methods to public. (tucu) HADOOP-10431. Change visibility of KeyStore.Options getter methods to public. (tucu)
HADOOP-10583. bin/hadoop key throws NPE with no args and assorted other fixups. (clamb via tucu)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd) HADOOP-7761. Improve the performance of raw comparisons. (todd)

71
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
View File

@ -27,9 +27,7 @@
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat; import java.text.MessageFormat;
import java.util.Date; import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter; import com.google.gson.stream.JsonWriter;
@ -176,22 +174,26 @@ protected int addVersion() {
protected byte[] serialize() throws IOException { protected byte[] serialize() throws IOException {
ByteArrayOutputStream buffer = new ByteArrayOutputStream(); ByteArrayOutputStream buffer = new ByteArrayOutputStream();
JsonWriter writer = new JsonWriter(new OutputStreamWriter(buffer)); JsonWriter writer = new JsonWriter(new OutputStreamWriter(buffer));
writer.beginObject(); try {
if (cipher != null) { writer.beginObject();
writer.name(CIPHER_FIELD).value(cipher); if (cipher != null) {
writer.name(CIPHER_FIELD).value(cipher);
}
if (bitLength != 0) {
writer.name(BIT_LENGTH_FIELD).value(bitLength);
}
if (created != null) {
writer.name(CREATED_FIELD).value(created.getTime());
}
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
writer.name(VERSIONS_FIELD).value(versions);
writer.endObject();
writer.flush();
} finally {
writer.close();
} }
if (bitLength != 0) {
writer.name(BIT_LENGTH_FIELD).value(bitLength);
}
if (created != null) {
writer.name(CREATED_FIELD).value(created.getTime());
}
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
writer.name(VERSIONS_FIELD).value(versions);
writer.endObject();
writer.flush();
return buffer.toByteArray(); return buffer.toByteArray();
} }
@ -207,23 +209,27 @@ protected Metadata(byte[] bytes) throws IOException {
int versions = 0; int versions = 0;
String description = null; String description = null;
JsonReader reader = new JsonReader(new InputStreamReader JsonReader reader = new JsonReader(new InputStreamReader
(new ByteArrayInputStream(bytes))); (new ByteArrayInputStream(bytes)));
reader.beginObject(); try {
while (reader.hasNext()) { reader.beginObject();
String field = reader.nextName(); while (reader.hasNext()) {
if (CIPHER_FIELD.equals(field)) { String field = reader.nextName();
cipher = reader.nextString(); if (CIPHER_FIELD.equals(field)) {
} else if (BIT_LENGTH_FIELD.equals(field)) { cipher = reader.nextString();
bitLength = reader.nextInt(); } else if (BIT_LENGTH_FIELD.equals(field)) {
} else if (CREATED_FIELD.equals(field)) { bitLength = reader.nextInt();
created = new Date(reader.nextLong()); } else if (CREATED_FIELD.equals(field)) {
} else if (VERSIONS_FIELD.equals(field)) { created = new Date(reader.nextLong());
versions = reader.nextInt(); } else if (VERSIONS_FIELD.equals(field)) {
} else if (DESCRIPTION_FIELD.equals(field)) { versions = reader.nextInt();
description = reader.nextString(); } else if (DESCRIPTION_FIELD.equals(field)) {
description = reader.nextString();
}
} }
reader.endObject();
} finally {
reader.close();
} }
reader.endObject();
this.cipher = cipher; this.cipher = cipher;
this.bitLength = bitLength; this.bitLength = bitLength;
this.created = created; this.created = created;
@ -310,7 +316,6 @@ public abstract KeyVersion getKeyVersion(String versionName
*/ */
public abstract List<String> getKeys() throws IOException; public abstract List<String> getKeys() throws IOException;
/** /**
* Get key metadata in bulk. * Get key metadata in bulk.
* @param names the names of the keys to get * @param names the names of the keys to get

168
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
View File

@ -23,9 +23,6 @@
import java.security.InvalidParameterException; import java.security.InvalidParameterException;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.List; import java.util.List;
import java.util.Map;
import javax.crypto.KeyGenerator;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
@ -93,41 +90,54 @@ public int run(String[] args) throws Exception {
*/ */
private int init(String[] args) throws IOException { private int init(String[] args) throws IOException {
for (int i = 0; i < args.length; i++) { // parse command line for (int i = 0; i < args.length; i++) { // parse command line
boolean moreTokens = (i < args.length - 1);
if (args[i].equals("create")) { if (args[i].equals("create")) {
String keyName = args[++i]; String keyName = "--help";
if (moreTokens) {
keyName = args[++i];
}
command = new CreateCommand(keyName); command = new CreateCommand(keyName);
if (keyName.equals("--help")) { if ("--help".equals(keyName)) {
printKeyShellUsage(); printKeyShellUsage();
return -1; return -1;
} }
} else if (args[i].equals("delete")) { } else if (args[i].equals("delete")) {
String keyName = args[++i]; String keyName = "--help";
if (moreTokens) {
keyName = args[++i];
}
command = new DeleteCommand(keyName); command = new DeleteCommand(keyName);
if (keyName.equals("--help")) { if ("--help".equals(keyName)) {
printKeyShellUsage(); printKeyShellUsage();
return -1; return -1;
} }
} else if (args[i].equals("roll")) { } else if (args[i].equals("roll")) {
String keyName = args[++i]; String keyName = "--help";
if (moreTokens) {
keyName = args[++i];
}
command = new RollCommand(keyName); command = new RollCommand(keyName);
if (keyName.equals("--help")) { if ("--help".equals(keyName)) {
printKeyShellUsage(); printKeyShellUsage();
return -1; return -1;
} }
} else if (args[i].equals("list")) { } else if ("list".equals(args[i])) {
command = new ListCommand(); command = new ListCommand();
} else if (args[i].equals("--size")) { } else if ("--size".equals(args[i]) && moreTokens) {
getConf().set(KeyProvider.DEFAULT_BITLENGTH_NAME, args[++i]); getConf().set(KeyProvider.DEFAULT_BITLENGTH_NAME, args[++i]);
} else if (args[i].equals("--cipher")) { } else if ("--cipher".equals(args[i]) && moreTokens) {
getConf().set(KeyProvider.DEFAULT_CIPHER_NAME, args[++i]); getConf().set(KeyProvider.DEFAULT_CIPHER_NAME, args[++i]);
} else if (args[i].equals("--provider")) { } else if ("--provider".equals(args[i]) && moreTokens) {
userSuppliedProvider = true; userSuppliedProvider = true;
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]); getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
} else if (args[i].equals("--metadata")) { } else if ("--metadata".equals(args[i])) {
getConf().setBoolean(LIST_METADATA, true); getConf().setBoolean(LIST_METADATA, true);
} else if (args[i].equals("-i") || (args[i].equals("--interactive"))) { } else if ("-i".equals(args[i]) || ("--interactive".equals(args[i]))) {
interactive = true; interactive = true;
} else if (args[i].equals("--help")) { } else if ("--help".equals(args[i])) {
printKeyShellUsage(); printKeyShellUsage();
return -1; return -1;
} else { } else {
@ -136,6 +146,12 @@ private int init(String[] args) throws IOException {
return -1; return -1;
} }
} }
if (command == null) {
printKeyShellUsage();
return -1;
}
return 0; return 0;
} }
@ -143,8 +159,7 @@ private void printKeyShellUsage() {
out.println(USAGE_PREFIX + COMMANDS); out.println(USAGE_PREFIX + COMMANDS);
if (command != null) { if (command != null) {
out.println(command.getUsage()); out.println(command.getUsage());
} } else {
else {
out.println("=========================================================" + out.println("=========================================================" +
"======"); "======");
out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC); out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
@ -174,8 +189,7 @@ protected KeyProvider getKeyProvider() {
providers = KeyProviderFactory.getProviders(getConf()); providers = KeyProviderFactory.getProviders(getConf());
if (userSuppliedProvider) { if (userSuppliedProvider) {
provider = providers.get(0); provider = providers.get(0);
} } else {
else {
for (KeyProvider p : providers) { for (KeyProvider p : providers) {
if (!p.isTransient()) { if (!p.isTransient()) {
provider = p; provider = p;
@ -190,7 +204,7 @@ protected KeyProvider getKeyProvider() {
} }
protected void printProviderWritten() { protected void printProviderWritten() {
out.println(provider.getClass().getName() + " has been updated."); out.println(provider + " has been updated.");
} }
protected void warnIfTransientProvider() { protected void warnIfTransientProvider() {
@ -206,12 +220,12 @@ protected void warnIfTransientProvider() {
private class ListCommand extends Command { private class ListCommand extends Command {
public static final String USAGE = public static final String USAGE =
"list [--provider] [--metadata] [--help]"; "list [--provider <provider>] [--metadata] [--help]";
public static final String DESC = public static final String DESC =
"The list subcommand displays the keynames contained within \n" + "The list subcommand displays the keynames contained within\n" +
"a particular provider - as configured in core-site.xml or " + "a particular provider as configured in core-site.xml or\n" +
"indicated\nthrough the --provider argument.\n" + "specified with the --provider argument. --metadata displays\n" +
"If the --metadata option is used, the keys metadata will be printed"; "the metadata.";
private boolean metadata = false; private boolean metadata = false;
@ -220,9 +234,9 @@ public boolean validate() {
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no non-transient KeyProviders configured.\n" out.println("There are no non-transient KeyProviders configured.\n"
+ "Consider using the --provider option to indicate the provider\n" + "Use the --provider option to specify a provider. If you\n"
+ "to use. If you want to list a transient provider then you\n" + "want to list a transient provider then you must use the\n"
+ "you MUST use the --provider argument."); + "--provider argument.");
rc = false; rc = false;
} }
metadata = getConf().getBoolean(LIST_METADATA, false); metadata = getConf().getBoolean(LIST_METADATA, false);
@ -231,12 +245,12 @@ public boolean validate() {
public void execute() throws IOException { public void execute() throws IOException {
try { try {
List<String> keys = provider.getKeys(); final List<String> keys = provider.getKeys();
out.println("Listing keys for KeyProvider: " + provider.toString()); out.println("Listing keys for KeyProvider: " + provider);
if (metadata) { if (metadata) {
Metadata[] meta = final Metadata[] meta =
provider.getKeysMetadata(keys.toArray(new String[keys.size()])); provider.getKeysMetadata(keys.toArray(new String[keys.size()]));
for(int i=0; i < meta.length; ++i) { for (int i = 0; i < meta.length; ++i) {
out.println(keys.get(i) + " : " + meta[i]); out.println(keys.get(i) + " : " + meta[i]);
} }
} else { } else {
@ -245,7 +259,7 @@ public void execute() throws IOException {
} }
} }
} catch (IOException e) { } catch (IOException e) {
out.println("Cannot list keys for KeyProvider: " + provider.toString() out.println("Cannot list keys for KeyProvider: " + provider
+ ": " + e.getMessage()); + ": " + e.getMessage());
throw e; throw e;
} }
@ -258,11 +272,10 @@ public String getUsage() {
} }
private class RollCommand extends Command { private class RollCommand extends Command {
public static final String USAGE = "roll <keyname> [--provider] [--help]"; public static final String USAGE = "roll <keyname> [--provider <provider>] [--help]";
public static final String DESC = public static final String DESC =
"The roll subcommand creates a new version of the key specified\n" + "The roll subcommand creates a new version for the specified key\n" +
"through the <keyname> argument within the provider indicated using\n" + "within the provider indicated using the --provider argument\n";
"the --provider argument";
String keyName = null; String keyName = null;
@ -274,15 +287,14 @@ public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid KeyProviders configured.\n" out.println("There are no valid KeyProviders configured. The key\n" +
+ "Key will not be rolled.\n" "has not been rolled. Use the --provider option to specify\n" +
+ "Consider using the --provider option to indicate the provider" "a provider.");
+ " to use.");
rc = false; rc = false;
} }
if (keyName == null) { if (keyName == null) {
out.println("There is no keyName specified. Please provide the" + out.println("Please provide a <keyname>.\n" +
"mandatory <keyname>. See the usage description with --help."); "See the usage description by using --help.");
rc = false; rc = false;
} }
return rc; return rc;
@ -290,10 +302,9 @@ public boolean validate() {
public void execute() throws NoSuchAlgorithmException, IOException { public void execute() throws NoSuchAlgorithmException, IOException {
try { try {
Metadata md = provider.getMetadata(keyName);
warnIfTransientProvider(); warnIfTransientProvider();
out.println("Rolling key version from KeyProvider: " out.println("Rolling key version from KeyProvider: "
+ provider.toString() + " for key name: " + keyName); + provider + "\n for key name: " + keyName);
try { try {
provider.rollNewVersion(keyName); provider.rollNewVersion(keyName);
out.println(keyName + " has been successfully rolled."); out.println(keyName + " has been successfully rolled.");
@ -301,12 +312,12 @@ public void execute() throws NoSuchAlgorithmException, IOException {
printProviderWritten(); printProviderWritten();
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
out.println("Cannot roll key: " + keyName + " within KeyProvider: " out.println("Cannot roll key: " + keyName + " within KeyProvider: "
+ provider.toString()); + provider);
throw e; throw e;
} }
} catch (IOException e1) { } catch (IOException e1) {
out.println("Cannot roll key: " + keyName + " within KeyProvider: " out.println("Cannot roll key: " + keyName + " within KeyProvider: "
+ provider.toString()); + provider);
throw e1; throw e1;
} }
} }
@ -318,11 +329,11 @@ public String getUsage() {
} }
private class DeleteCommand extends Command { private class DeleteCommand extends Command {
public static final String USAGE = "delete <keyname> [--provider] [--help]"; public static final String USAGE = "delete <keyname> [--provider <provider>] [--help]";
public static final String DESC = public static final String DESC =
"The delete subcommand deletes all of the versions of the key\n" + "The delete subcommand deletes all versions of the key\n" +
"specified as the <keyname> argument from within the provider\n" + "specified by the <keyname> argument from within the\n" +
"indicated through the --provider argument"; "provider specified --provider.";
String keyName = null; String keyName = null;
boolean cont = true; boolean cont = true;
@ -335,23 +346,21 @@ public DeleteCommand(String keyName) {
public boolean validate() { public boolean validate() {
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid KeyProviders configured.\n" out.println("There are no valid KeyProviders configured. Nothing\n"
+ "Nothing will be deleted.\n" + "was deleted. Use the --provider option to specify a provider.");
+ "Consider using the --provider option to indicate the provider"
+ " to use.");
return false; return false;
} }
if (keyName == null) { if (keyName == null) {
out.println("There is no keyName specified. Please provide the" + out.println("There is no keyName specified. Please specify a " +
"mandatory <keyname>. See the usage description with --help."); "<keyname>. See the usage description with --help.");
return false; return false;
} }
if (interactive) { if (interactive) {
try { try {
cont = ToolRunner cont = ToolRunner
.confirmPrompt("You are about to DELETE all versions of " .confirmPrompt("You are about to DELETE all versions of "
+ "the key: " + keyName + " from KeyProvider " + " key: " + keyName + " from KeyProvider "
+ provider.toString() + ". Continue?:"); + provider + ". Continue?:");
if (!cont) { if (!cont) {
out.println("Nothing has been be deleted."); out.println("Nothing has been be deleted.");
} }
@ -367,7 +376,7 @@ public boolean validate() {
public void execute() throws IOException { public void execute() throws IOException {
warnIfTransientProvider(); warnIfTransientProvider();
out.println("Deleting key: " + keyName + " from KeyProvider: " out.println("Deleting key: " + keyName + " from KeyProvider: "
+ provider.toString()); + provider);
if (cont) { if (cont) {
try { try {
provider.deleteKey(keyName); provider.deleteKey(keyName);
@ -375,7 +384,7 @@ public void execute() throws IOException {
provider.flush(); provider.flush();
printProviderWritten(); printProviderWritten();
} catch (IOException e) { } catch (IOException e) {
out.println(keyName + "has NOT been deleted."); out.println(keyName + " has not been deleted.");
throw e; throw e;
} }
} }
@ -388,16 +397,16 @@ public String getUsage() {
} }
private class CreateCommand extends Command { private class CreateCommand extends Command {
public static final String USAGE = "create <keyname> [--cipher] " + public static final String USAGE =
"[--size] [--provider] [--help]"; "create <keyname> [--cipher <cipher>] [--size <size>]\n" +
" [--provider <provider>] [--help]";
public static final String DESC = public static final String DESC =
"The create subcommand creates a new key for the name specified\n" + "The create subcommand creates a new key for the name specified\n" +
"as the <keyname> argument within the provider indicated through\n" + "by the <keyname> argument within the provider specified by the\n" +
"the --provider argument. You may also indicate the specific\n" + "--provider argument. You may specify a cipher with the --cipher\n" +
"cipher through the --cipher argument. The default for cipher is\n" + "argument. The default cipher is currently \"AES/CTR/NoPadding\".\n" +
"currently \"AES/CTR/NoPadding\". The default keysize is \"256\".\n" + "The default keysize is 256. You may specify the requested key\n" +
"You may also indicate the requested key length through the --size\n" + "length using the --size argument.\n";
"argument.";
String keyName = null; String keyName = null;
@ -409,15 +418,14 @@ public boolean validate() {
boolean rc = true; boolean rc = true;
provider = getKeyProvider(); provider = getKeyProvider();
if (provider == null) { if (provider == null) {
out.println("There are no valid KeyProviders configured.\nKey" + out.println("There are no valid KeyProviders configured. No key\n" +
" will not be created.\n" " was created. You can use the --provider option to specify\n" +
+ "Consider using the --provider option to indicate the provider" + " a provider to use.");
" to use.");
rc = false; rc = false;
} }
if (keyName == null) { if (keyName == null) {
out.println("There is no keyName specified. Please provide the" + out.println("Please provide a <keyname>. See the usage description" +
"mandatory <keyname>. See the usage description with --help."); " with --help.");
rc = false; rc = false;
} }
return rc; return rc;
@ -432,13 +440,13 @@ public void execute() throws IOException, NoSuchAlgorithmException {
provider.flush(); provider.flush();
printProviderWritten(); printProviderWritten();
} catch (InvalidParameterException e) { } catch (InvalidParameterException e) {
out.println(keyName + " has NOT been created. " + e.getMessage()); out.println(keyName + " has not been created. " + e.getMessage());
throw e; throw e;
} catch (IOException e) { } catch (IOException e) {
out.println(keyName + " has NOT been created. " + e.getMessage()); out.println(keyName + " has not been created. " + e.getMessage());
throw e; throw e;
} catch (NoSuchAlgorithmException e) { } catch (NoSuchAlgorithmException e) {
out.println(keyName + " has NOT been created. " + e.getMessage()); out.println(keyName + " has not been created. " + e.getMessage());
throw e; throw e;
} }
} }

9
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
View File

@ -126,7 +126,6 @@ public static <T> T checkNotNull(T o, String name)
return o; return o;
} }
public static String checkNotEmpty(String s, String name) public static String checkNotEmpty(String s, String name)
throws IllegalArgumentException { throws IllegalArgumentException {
checkNotNull(s, name); checkNotNull(s, name);
@ -140,6 +139,13 @@ public static String checkNotEmpty(String s, String name)
private String kmsUrl; private String kmsUrl;
private SSLFactory sslFactory; private SSLFactory sslFactory;
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("KMSClientProvider[");
sb.append(kmsUrl).append("]");
return sb.toString();
}
public KMSClientProvider(URI uri, Configuration conf) throws IOException { public KMSClientProvider(URI uri, Configuration conf) throws IOException {
Path path = unnestUri(uri); Path path = unnestUri(uri);
URL url = path.toUri().toURL(); URL url = path.toUri().toURL();
@ -515,5 +521,4 @@ public void flush() throws IOException {
public static String buildVersionName(String name, int version) { public static String buildVersionName(String name, int version) {
return KeyProvider.buildVersionName(name, version); return KeyProvider.buildVersionName(name, version);
} }
} }

4
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java
View File

@ -121,7 +121,7 @@ public void testInvalidKeySize() throws Exception {
ks.setConf(new Configuration()); ks.setConf(new Configuration());
rc = ks.run(args1); rc = ks.run(args1);
assertEquals(-1, rc); assertEquals(-1, rc);
assertTrue(outContent.toString().contains("key1 has NOT been created.")); assertTrue(outContent.toString().contains("key1 has not been created."));
} }
@Test @Test
@ -134,7 +134,7 @@ public void testInvalidCipher() throws Exception {
ks.setConf(new Configuration()); ks.setConf(new Configuration());
rc = ks.run(args1); rc = ks.run(args1);
assertEquals(-1, rc); assertEquals(-1, rc);
assertTrue(outContent.toString().contains("key1 has NOT been created.")); assertTrue(outContent.toString().contains("key1 has not been created."));
} }
@Test @Test