big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9668. UGI conf doesn't read user overridden configurations on RM and NM startup. (Contributed by Jonathan Hung)

Browse Source
This commit is contained in:
Haibo Chen 2019-07-11 13:56:26 -07:00
parent 8fb5ca3f40
commit 9b54dd7186
5 changed files with 62 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
View File

@ -381,6 +381,7 @@ protected ContainerExecutor createContainerExecutor(Configuration conf) {
@Override
protected void serviceInit(Configuration conf) throws Exception {
UserGroupInformation.setConfiguration(conf);
rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration
.RM_WORK_PRESERVING_RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
View File

@ -23,6 +23,7 @@
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
@ -30,7 +31,9 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class TestNodeManager {
@ -42,6 +45,9 @@ public void init(Context nmContext) throws IOException {
}
}
@Rule
public ExpectedException thrown = ExpectedException.none();
@Test
public void testContainerExecutorInitCall() {
NodeManager nm = new NodeManager();
@ -170,4 +176,26 @@ public void testCreationOfNodeLabelsProviderService()
e.printStackTrace();
}
}
/**
* Test whether NodeManager passes user-provided conf to
* UserGroupInformation class. If it reads this (incorrect)
* AuthenticationMethod enum an exception is thrown.
*/
@Test
public void testUserProvidedUGIConf() throws Exception {
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Invalid attribute value for "
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
+ " of DUMMYAUTH");
Configuration dummyConf = new YarnConfiguration();
dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"DUMMYAUTH");
NodeManager dummyNodeManager = new NodeManager();
try {
dummyNodeManager.init(dummyConf);
} finally {
dummyNodeManager.stop();
}
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -263,6 +263,7 @@ protected ResourceProfilesManager createResourceProfileManager() {
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
UserGroupInformation.setConfiguration(conf);
this.rmContext = new RMContextImpl();
rmContext.setResourceManager(this);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
View File

@ -2099,9 +2099,10 @@ public void testSynchronouslyRenewDTOnRecovery() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
// start RM
MockRM rm1 = createMockRM(conf);
MockRM rm1 = new TestSecurityMockRM(conf);
rm1.start();
final MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@ -2109,7 +2110,7 @@ public void testSynchronouslyRenewDTOnRecovery() throws Exception {
RMApp app0 = rm1.submitApp(200);
final MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
MockRM rm2 = new MockRM(conf, rm1.getRMStateStore()) {
MockRM rm2 = new TestSecurityMockRM(conf, rm1.getRMStateStore()) {
@Override
protected ResourceTrackerService createResourceTrackerService() {
return new ResourceTrackerService(this.rmContext,

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
View File

@ -25,6 +25,7 @@
import java.util.Collection;
import java.util.concurrent.TimeoutException;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@ -51,14 +52,19 @@
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class TestResourceManager {
private static final Logger LOG =
LoggerFactory.getLogger(TestResourceManager.class);
private ResourceManager resourceManager = null;
@Rule
public ExpectedException thrown = ExpectedException.none();
@Before
public void setUp() throws Exception {
Configuration conf = new YarnConfiguration();
@ -329,4 +335,26 @@ protected void doSecureLogin() throws IOException {
}
}
/**
* Test whether ResourceManager passes user-provided conf to
* UserGroupInformation class. If it reads this (incorrect)
* AuthenticationMethod enum an exception is thrown.
*/
@Test
public void testUserProvidedUGIConf() throws Exception {
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage("Invalid attribute value for "
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION
+ " of DUMMYAUTH");
Configuration dummyConf = new YarnConfiguration();
dummyConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"DUMMYAUTH");
ResourceManager dummyResourceManager = new ResourceManager();
try {
dummyResourceManager.init(dummyConf);
} finally {
dummyResourceManager.stop();
}
}
}