From 0bee3849e323bf71925024992f9e655aee2d75f9 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Wed, 31 Jan 2018 20:51:40 -0500 Subject: [PATCH] YARN-7816. Allow same application name submitted by multiple users. (Contributed by Gour Saha) --- .../org/apache/hadoop/http/HttpServer2.java | 2 +- .../yarn/service/client/ServiceClient.java | 4 + .../hadoop/yarn/service/ServiceTestUtils.java | 5 + .../yarn/service/TestYarnNativeServices.java | 128 ++++++++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java index 65aadf3c05..7e1264059f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java @@ -136,7 +136,7 @@ public final class HttpServer2 implements FilterContainer { public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads"; public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir"; - static final String FILTER_INITIALIZER_PROPERTY + public static final String FILTER_INITIALIZER_PROPERTY = "hadoop.http.filter.initializers"; // The ServletContext attribute where the daemon Configuration diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index ee6e681bf9..612b8b4c3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -540,6 +540,10 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, request.setApplicationTypes(types); request.setApplicationTags(tags); request.setApplicationStates(liveStates); + String user = UserGroupInformation.getCurrentUser().getUserName(); + if (user != null) { + request.setUsers(Collections.singleton(user)); + } List reports = yarnClient.getApplications(request); if (!reports.isEmpty()) { String message = ""; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java index a70a0c280d..1117b76690 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -165,6 +166,10 @@ public class ServiceTestUtils { // Disable vmem check to disallow NM killing the container conf.setBoolean(NM_VMEM_CHECK_ENABLED, false); conf.setBoolean(NM_PMEM_CHECK_ENABLED, false); + // set auth filters + conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY, + "org.apache.hadoop.security.AuthenticationFilterInitializer," + + "org.apache.hadoop.security.HttpCrossOriginFilterInitializer"); // setup zk cluster zkCluster = new TestingCluster(1); zkCluster.start(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 5067ffc917..b6126bfe02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -22,6 +22,7 @@ import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.exceptions.SliderException; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.hamcrest.CoreMatchers; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -49,6 +51,7 @@ import java.util.*; import java.util.concurrent.TimeoutException; import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; /** * End to end tests to test deploying services with MiniYarnCluster and a in-JVM @@ -158,6 +161,109 @@ public class TestYarnNativeServices extends ServiceTestUtils { client.actionDestroy(exampleApp.getName()); } + @Test(timeout = 200000) + public void testCreateServiceSameNameDifferentUser() throws Exception { + String sameAppName = "same-name"; + String userA = "usera"; + String userB = "userb"; + + setupInternal(NUM_NMS); + ServiceClient client = createClient(); + String origBasePath = getConf().get(YARN_SERVICE_BASE_PATH); + + Service userAApp = new Service(); + userAApp.setName(sameAppName); + userAApp.addComponent(createComponent("comp", 1, "sleep 1000")); + Service userBApp = new Service(); + userBApp.setName(sameAppName); + userBApp.addComponent(createComponent("comp", 1, "sleep 1000")); + + File userABasePath = null, userBBasePath = null; + try { + userABasePath = new File(origBasePath, userA); + userABasePath.mkdirs(); + getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath()); + client.actionCreate(userAApp); + waitForServiceToBeStarted(client, userAApp); + + userBBasePath = new File(origBasePath, userB); + userBBasePath.mkdirs(); + getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath()); + client.actionBuild(userBApp); + } catch (Exception e) { + Assert + .fail("Exception should not be thrown - " + e.getLocalizedMessage()); + } finally { + if (userABasePath != null) { + getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath()); + client.actionStop(sameAppName, true); + client.actionDestroy(sameAppName); + } + if (userBBasePath != null) { + getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath()); + client.actionDestroy(sameAppName); + } + } + + // Need to extend this test to validate that different users can create + // apps of exact same name. So far only create followed by build is tested. + // Need to test create followed by create. + } + + @Test(timeout = 200000) + public void testCreateServiceSameNameSameUser() throws Exception { + String sameAppName = "same-name"; + String user = UserGroupInformation.getCurrentUser().getUserName(); + System.setProperty("user.name", user); + + setupInternal(NUM_NMS); + ServiceClient client = createClient(); + + Service appA = new Service(); + appA.setName(sameAppName); + appA.addComponent(createComponent("comp", 1, "sleep 1000")); + Service appB = new Service(); + appB.setName(sameAppName); + appB.addComponent(createComponent("comp", 1, "sleep 1000")); + + try { + client.actionBuild(appA); + client.actionBuild(appB); + } catch (Exception e) { + String expectedMsg = "Service Instance dir already exists:"; + if (e.getLocalizedMessage() != null) { + Assert.assertThat(e.getLocalizedMessage(), + CoreMatchers.containsString(expectedMsg)); + } else { + Assert.fail("Message cannot be null. It has to say - " + expectedMsg); + } + } finally { + // cleanup + client.actionDestroy(sameAppName); + } + + try { + client.actionCreate(appA); + waitForServiceToBeStarted(client, appA); + + client.actionCreate(appB); + waitForServiceToBeStarted(client, appB); + } catch (Exception e) { + String expectedMsg = "Failed to create service " + sameAppName + + ", because it already exists."; + if (e.getLocalizedMessage() != null) { + Assert.assertThat(e.getLocalizedMessage(), + CoreMatchers.containsString(expectedMsg)); + } else { + Assert.fail("Message cannot be null. It has to say - " + expectedMsg); + } + } finally { + // cleanup + client.actionStop(sameAppName, true); + client.actionDestroy(sameAppName); + } + } + // Test to verify recovery of SeviceMaster after RM is restarted. // 1. Create an example service. // 2. Restart RM. @@ -369,6 +475,28 @@ public class TestYarnNativeServices extends ServiceTestUtils { }, 2000, 200000); } + /** + * Wait until service is started. It does not have to reach a stable state. + * + * @param client + * @param exampleApp + * @throws TimeoutException + * @throws InterruptedException + */ + private void waitForServiceToBeStarted(ServiceClient client, + Service exampleApp) throws TimeoutException, InterruptedException { + GenericTestUtils.waitFor(() -> { + try { + Service retrievedApp = client.getStatus(exampleApp.getName()); + System.out.println(retrievedApp); + return retrievedApp.getState() == ServiceState.STARTED; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + }, 2000, 200000); + } + private ServiceClient createClient() throws Exception { ServiceClient client = new ServiceClient() { @Override protected Path addJarResource(String appName,