big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15885. Add base64 (urlString) support to DTUtil. Contributed by Inigo Goiri.

Browse Source
This commit is contained in:
Giovanni Matteo Fumarola 2018-11-02 10:54:12 -07:00
parent 6d9c18cfa9
commit 44e37b4fd9
4 changed files with 107 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
View File

@ -89,7 +89,7 @@ private static String formatDate(long date) {
/** Add the service prefix for a local filesystem. */
private static Path fileToPath(File f) {
return new Path("file:" + f.getAbsolutePath());
return new Path(f.toURI().toString());
}
/** Write out a Credentials object as a local file.
@ -294,4 +294,30 @@ public static void renewTokenFile(
}
doFormattedWrite(tokenFile, fileFormat, creds, conf);
}
/** Import a token from a base64 encoding into the local filesystem.
* @param tokenFile A local File object.
* @param fileFormat A string equal to FORMAT_PB or FORMAT_JAVA, for output.
* @param alias overwrite Service field of fetched token with this text.
* @param base64 urlString Encoding of the token to import.
* @param conf Configuration object passed along.
* @throws IOException Error to import the token into the file.
*/
public static void importTokenFile(File tokenFile, String fileFormat,
Text alias, String base64, Configuration conf)
throws IOException {
Credentials creds = tokenFile.exists() ?
Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials();
Token<TokenIdentifier> token = new Token<>();
token.decodeFromUrlString(base64);
if (alias != null) {
token.setService(alias);
}
creds.addToken(token.getService(), token);
LOG.info("Add token with service {}", token.getService());
doFormattedWrite(tokenFile, fileFormat, creds, conf);
}
}

37
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
View File

@ -55,6 +55,7 @@ public class DtUtilShell extends CommandShell {
private static final String CANCEL = "cancel";
private static final String REMOVE = "remove";
private static final String RENEW = "renew";
private static final String IMPORT = "import";
private static final String RENEWER = "-renewer";
private static final String SERVICE = "-service";
private static final String ALIAS = "-alias";
@ -138,6 +139,8 @@ protected int init(String[] args) throws Exception {
setSubCommand(new Remove(false));
} else if (command.equals(RENEW)) {
setSubCommand(new Renew());
} else if (command.equals(IMPORT)) {
setSubCommand(new Import(args[++i]));
}
} else if (args[i].equals(ALIAS)) {
alias = new Text(args[++i]);
@ -176,11 +179,11 @@ protected int init(String[] args) throws Exception {
@Override
public String getCommandUsage() {
return String.format(
"%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
"%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(),
(new Edit()).getUsage(), (new Append()).getUsage(),
(new Remove(true)).getUsage(), (new Remove(false)).getUsage(),
(new Renew()).getUsage());
(new Renew()).getUsage(), (new Import()).getUsage());
}
private class Print extends SubCommand {
@ -357,6 +360,36 @@ public String getUsage() {
}
}
private class Import extends SubCommand {
public static final String IMPORT_USAGE =
"dtutil import <base64> [-alias <alias>] " +
FORMAT_SUBSTRING + " filename";
private String base64 = null;
Import() { }
Import(String arg) {
base64 = arg;
}
@Override
public boolean validate() {
return true;
}
@Override
public void execute() throws Exception {
DtFileOperations.importTokenFile(
firstFile, format, alias, base64, getConf());
}
@Override
public String getUsage() {
return IMPORT_USAGE;
}
}
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args));
}

1
hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
View File

@ -172,6 +172,7 @@ For every subcommand that connects to a service, convenience flags are provided
| `remove -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp; *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
| `cancel -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp; *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. |
| `renew -alias` *alias* <br/>&nbsp;&nbsp; `[-format (java|protobuf)]` <br/>&nbsp;&nbsp; *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
| `import` *base64* <br/>&nbsp;&nbsp; `[-alias` *alias* `]` <br/>&nbsp;&nbsp; *filename* | Import a token from a base64 token. <br/> *alias* will overwrite the service field in the token. |
### `fs`

44
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
View File

@ -57,6 +57,13 @@ public class TestDtUtilShell {
public static Text KIND_GET = new Text("testTokenKindGet");
public static Token<?> MOCK_TOKEN =
new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET);
private static final Text SERVICE_IMPORT =
new Text("testTokenServiceImport");
private static final Text KIND_IMPORT = new Text("testTokenKindImport");
private static final Token<?> IMPORT_TOKEN =
new Token(IDENTIFIER, PASSWORD, KIND_IMPORT, SERVICE_IMPORT);
static {
try {
defaultConf.set("fs.defaultFS", "file:///");
@ -73,9 +80,11 @@ public class TestDtUtilShell {
private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2");
private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3");
private final Path tokenFileGet = new Path(workDir, "testGetTokenFile");
private final Path tokenFileImport = new Path(workDir, "testImportTokenFile");
private final String tokenFilename = tokenFile.toString();
private final String tokenFilename2 = tokenFile2.toString();
private final String tokenFilenameGet = tokenFileGet.toString();
private final String tokenFilenameImport = tokenFileImport.toString();
private String[] args = null;
private DtUtilShell dt = null;
private int rc = 0;
@ -283,4 +292,39 @@ public void testFormatProtoFlag() throws Exception {
spyCreds.readTokenStorageStream(in);
Mockito.verify(spyCreds, Mockito.never()).readFields(in);
}
@Test
public void testImport() throws Exception {
String base64 = IMPORT_TOKEN.encodeToUrlString();
args = new String[] {"import", base64, tokenFilenameImport};
rc = dt.run(args);
assertEquals("test simple import print old exit code", 0, rc);
args = new String[] {"print", tokenFilenameImport};
rc = dt.run(args);
assertEquals("test simple import print old exit code", 0, rc);
assertTrue("test print after import output:\n" + outContent,
outContent.toString().contains(KIND_IMPORT.toString()));
assertTrue("test print after import output:\n" + outContent,
outContent.toString().contains(SERVICE_IMPORT.toString()));
assertTrue("test print after simple import output:\n" + outContent,
outContent.toString().contains(base64));
}
@Test
public void testImportWithAliasFlag() throws Exception {
String base64 = IMPORT_TOKEN.encodeToUrlString();
args = new String[] {"import", base64, "-alias", alias,
tokenFilenameImport};
rc = dt.run(args);
assertEquals("test import with alias print old exit code", 0, rc);
args = new String[] {"print", tokenFilenameImport};
rc = dt.run(args);
assertEquals("test simple import print old exit code", 0, rc);
assertTrue("test print after import output:\n" + outContent,
outContent.toString().contains(KIND_IMPORT.toString()));
assertTrue("test print after import with alias output:\n" + outContent,
outContent.toString().contains(alias));
}
}