big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-15465. Support WebHDFS accesses to the data stored in secure Datanode through insecure Namenode. (#2135)

Browse Source
This commit is contained in:
touchida 2020-07-28 01:55:11 +09:00 committed by GitHub
parent 60a254621a
commit 026dce5334
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 53 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/DataNodeUGIProvider.java
View File

@ -72,9 +72,12 @@ UserGroupInformation ugi() throws IOException {
UserGroupInformation ugi;
try {
if (UserGroupInformation.isSecurityEnabled()) {
final Token<DelegationTokenIdentifier> token = params.delegationToken();
final Token<DelegationTokenIdentifier> token = params.delegationToken();
// Create nonTokenUGI when token is null regardless of security.
// This makes it possible to access the data stored in secure DataNode
// through insecure Namenode.
if (UserGroupInformation.isSecurityEnabled() && token != null) {
ugi = ugiCache.get(buildTokenCacheKey(token),
new Callable<UserGroupInformation>() {
@Override
@ -134,7 +137,8 @@ private String buildNonTokenCacheKey(String doAsUserFromQuery,
return key;
}
private UserGroupInformation nonTokenUGI(String usernameFromQuery,
@VisibleForTesting
UserGroupInformation nonTokenUGI(String usernameFromQuery,
String doAsUserFromQuery, String remoteUser) throws IOException {
UserGroupInformation ugi = UserGroupInformation

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
View File

@ -123,6 +123,9 @@ boolean noredirect() {
Token<DelegationTokenIdentifier> delegationToken() throws IOException {
String delegation = param(DelegationParam.NAME);
if (delegation == null) {
return null;
}
final Token<DelegationTokenIdentifier> token = new
Token<DelegationTokenIdentifier>();
token.decodeFromUrlString(delegation);

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
View File

@ -335,8 +335,8 @@ private static void writeContinueHeader(ChannelHandlerContext ctx) {
}
private void injectToken() throws IOException {
if (UserGroupInformation.isSecurityEnabled()) {
Token<DelegationTokenIdentifier> token = params.delegationToken();
Token<DelegationTokenIdentifier> token = params.delegationToken();
if (UserGroupInformation.isSecurityEnabled() && token != null) {
token.setKind(HDFS_DELEGATION_KIND);
ugi.addToken(token);
}

32
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestDataNodeUGIProvider.java
View File

@ -20,6 +20,8 @@
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import io.netty.handler.codec.http.QueryStringDecoder;
import java.io.IOException;
@ -31,6 +33,7 @@
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
@ -186,6 +189,35 @@ public void testUGICacheInSecure() throws Exception {
ugi11, url22);
}
@Test
public void testUGINullTokenSecure() throws IOException {
SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
String uri1 = WebHdfsFileSystem.PATH_PREFIX
+ PATH
+ "?op=OPEN"
+ Param.toSortedString("&", new OffsetParam((long) OFFSET),
new LengthParam((long) LENGTH), new UserParam("root"));
ParameterParser params = new ParameterParser(
new QueryStringDecoder(URI.create(uri1)), conf);
DataNodeUGIProvider ugiProvider = new DataNodeUGIProvider(params);
String usernameFromQuery = params.userName();
String doAsUserFromQuery = params.doAsUser();
String remoteUser = usernameFromQuery == null ? JspHelper
.getDefaultWebUserName(params.conf())
: usernameFromQuery;
DataNodeUGIProvider spiedUGIProvider = spy(ugiProvider);
spiedUGIProvider.ugi();
verify(spiedUGIProvider).nonTokenUGI(usernameFromQuery, doAsUserFromQuery,
remoteUser);
}
/**
* Wait for expiration of entries from the UGI cache. We need to be careful
* not to touch the entries in the cache while we're waiting for expiration.

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
View File

@ -55,6 +55,15 @@ public void testDeserializeHAToken() throws IOException {
Assert.assertTrue(HAUtilClient.isTokenForLogicalUri(tok2));
}
@Test
public void testNullToken() throws IOException {
Configuration conf = new Configuration();
QueryStringDecoder decoder = new QueryStringDecoder(
WebHdfsHandler.WEBHDFS_PREFIX + "/test");
ParameterParser testParser = new ParameterParser(decoder, conf);
Assert.assertNull(testParser.delegationToken());
}
@Test
public void testDecodePath() {
final String ESCAPED_PATH = "/test%25+1%26%3Dtest?op=OPEN&foo=bar";