big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10339. Fix TimelineClient in NodeManager failing when Simple Http Auth used in Secure Cluster

Browse Source 
Contributed by Tarun Parimi.
This commit is contained in:
Prabhu Joseph 2020-07-16 23:36:38 +05:30 committed by Prabhu Joseph
parent b21cb91c7f
commit cc71d50b21
11 changed files with 59 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -38,6 +38,7 @@
import org.apache.hadoop.ha.ActiveStandbyElector;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.util.BasicDiskValidator;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@ -3266,6 +3267,18 @@ public static boolean isAclEnabled(Configuration conf) {
public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS
= 2000L;
/**
* The configuration prefix of timeline HTTP authentication.
*/
public static final String TIMELINE_HTTP_AUTH_PREFIX =
TIMELINE_SERVICE_PREFIX + "http-authentication.";
/**
* The authentication type for timeline HTTP authentication.
*/
public static final String TIMELINE_HTTP_AUTH_TYPE =
TIMELINE_HTTP_AUTH_PREFIX + AuthenticationFilter.AUTH_TYPE;
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest;
@ -315,7 +316,9 @@ public YarnClientApplication createApplication()
// Automatically add the timeline DT into the CLC
// Only when the security and the timeline service are both enabled
if (isSecurityEnabled() && timelineV1ServiceEnabled) {
if (isSecurityEnabled() && timelineV1ServiceEnabled &&
getConfig().get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE)
.equals(KerberosAuthenticationHandler.TYPE)) {
addTimelineDelegationToken(appContext.getAMContainerSpec());
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
@ -1228,6 +1229,9 @@ private void testCreateTimelineClientWithError(
timelineClientBestEffort);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION,
timelineVersion);
// Timeline Delegation token and client is only used for kerberos
conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
KerberosAuthenticationHandler.TYPE);
MockYarnClient client = new MockYarnClient();
MockYarnClient spyClient = spy(client);
when(spyClient.createTimelineClient()).thenThrow(mockErr);

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientImpl.java
View File

@ -33,6 +33,7 @@
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@ -128,7 +129,8 @@ public void testBestEffortTimelineDelegationToken()
Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
KerberosAuthenticationHandler.TYPE);
YarnClientImpl client = spy(new YarnClientImpl() {
@Override
@ -278,6 +280,8 @@ public void testAutomaticTimelineDelegationTokenLoading()
Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
KerberosAuthenticationHandler.TYPE);
TimelineDelegationTokenIdentifier timelineDT =
new TimelineDelegationTokenIdentifier();
final Token<TimelineDelegationTokenIdentifier> dToken =

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
View File

@ -28,6 +28,7 @@
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -349,7 +350,9 @@ private static void putTimelineDataInJSONFile(String path, String type) {
client.start();
try {
if (UserGroupInformation.isSecurityEnabled()
&& conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
&& conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)
&& conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE)
.equals(KerberosAuthenticationHandler.TYPE)) {
Token<TimelineDelegationTokenIdentifier> token =
client.getDelegationToken(
UserGroupInformation.getCurrentUser().getUserName());

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
View File

@ -34,6 +34,8 @@
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@ -111,8 +113,12 @@ protected void serviceInit(Configuration conf) throws Exception {
} else {
connConfigurator = DEFAULT_TIMEOUT_CONN_CONFIGURATOR;
}
if (UserGroupInformation.isSecurityEnabled()) {
String defaultAuth = UserGroupInformation.isSecurityEnabled() ?
KerberosAuthenticationHandler.TYPE :
PseudoAuthenticationHandler.TYPE;
String authType = conf.get(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE,
defaultAuth);
if (authType.equals(KerberosAuthenticationHandler.TYPE)) {
authenticator = new KerberosDelegationTokenAuthenticator();
} else {
authenticator = new PseudoDelegationTokenAuthenticator();

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
View File

@ -243,6 +243,7 @@ public void testDelegationTokenOperationsRetry() throws Exception {
// use kerberos to bypass the issue in HADOOP-11215
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
conf.set(YarnConfiguration.TIMELINE_HTTP_AUTH_TYPE, "kerberos");
UserGroupInformation.setConfiguration(conf);
TimelineClientImpl client = createTimelineClient(conf);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
View File

@ -48,6 +48,7 @@
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -107,11 +108,11 @@ public static void setup() {
try {
testTimelineServer = new ApplicationHistoryServer();
conf = new Configuration(false);
conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
"kerberos");
conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
KerberosAuthenticationHandler.KEYTAB,
httpSpnegoKeytabFile.getAbsolutePath());
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler;
import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import java.util.HashMap;
import java.util.Map;
@ -48,12 +49,6 @@
*/
public class TimelineAuthenticationFilterInitializer extends FilterInitializer {
/**
* The configuration prefix of timeline HTTP authentication.
*/
public static final String PREFIX =
"yarn.timeline-service.http-authentication.";
@VisibleForTesting
Map<String, String> filterConfig;
@ -68,7 +63,8 @@ protected void setAuthFilterConfig(Configuration conf) {
// yarn.timeline-service.http-authentication.proxyuser will override
// hadoop.proxyuser
Map<String, String> timelineAuthProps =
AuthenticationFilterInitializer.getFilterConfigMap(conf, PREFIX);
AuthenticationFilterInitializer.getFilterConfigMap(conf,
TIMELINE_HTTP_AUTH_PREFIX);
filterConfig.putAll(timelineAuthProps);
}
@ -81,7 +77,8 @@ protected Map<String, String> getFilterConfig() {
* Initializes {@link TimelineAuthenticationFilter}.
* <p>
* Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
* configuration properties prefixed with {@value #PREFIX}.
* configuration properties prefixed with
* {@value org.apache.hadoop.yarn.conf.YarnConfiguration#TIMELINE_HTTP_AUTH_PREFIX}.
*
* @param container
* The filter container.

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java
View File

@ -23,7 +23,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import org.junit.Test;
import org.mockito.Mockito;
@ -46,9 +46,9 @@ public void testProxyUserConfiguration() {
break;
case 1:
// yarn.timeline-service.http-authentication.proxyuser prefix
conf.set(PREFIX + "proxyuser.foo.hosts", "*");
conf.set(PREFIX + "proxyuser.foo.users", "*");
conf.set(PREFIX + "proxyuser.foo.groups", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*");
break;
case 2:
// hadoop.proxyuser prefix has been overwritten by
@ -56,9 +56,9 @@ public void testProxyUserConfiguration() {
conf.set("hadoop.proxyuser.foo.hosts", "bar");
conf.set("hadoop.proxyuser.foo.users", "bar");
conf.set("hadoop.proxyuser.foo.groups", "bar");
conf.set(PREFIX + "proxyuser.foo.hosts", "*");
conf.set(PREFIX + "proxyuser.foo.users", "*");
conf.set(PREFIX + "proxyuser.foo.groups", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.hosts", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.users", "*");
conf.set(TIMELINE_HTTP_AUTH_PREFIX + "proxyuser.foo.groups", "*");
break;
default:
break;

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
View File

@ -68,13 +68,13 @@
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@ -151,11 +151,11 @@ public static void setup() {
conf = new Configuration(false);
conf.setClass("fs.file.impl", RawLocalFileSystem.class,
FileSystem.class);
conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type",
conf.setStrings(TIMELINE_HTTP_AUTH_PREFIX + "type",
"kerberos");
conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal);
conf.set(TimelineAuthenticationFilterInitializer.PREFIX +
conf.set(TIMELINE_HTTP_AUTH_PREFIX +
KerberosAuthenticationHandler.KEYTAB,
httpSpnegoKeytabFile.getAbsolutePath());
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,