big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7816. Allow same application name submitted by multiple users. (Contributed by Gour Saha)

Browse Source
This commit is contained in:
Eric Yang 2018-01-31 20:51:40 -05:00
parent 5a725bb886
commit 0bee3849e3
4 changed files with 138 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
View File

@ -136,7 +136,7 @@ public final class HttpServer2 implements FilterContainer {
public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads";
public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir";
static final String FILTER_INITIALIZER_PROPERTY
public static final String FILTER_INITIALIZER_PROPERTY
= "hadoop.http.filter.initializers";
// The ServletContext attribute where the daemon Configuration

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -540,6 +540,10 @@ private void verifyNoLiveAppInRM(String serviceName, String action)
request.setApplicationTypes(types);
request.setApplicationTags(tags);
request.setApplicationStates(liveStates);
String user = UserGroupInformation.getCurrentUser().getUserName();
if (user != null) {
request.setUsers(Collections.singleton(user));
}
List<ApplicationReport> reports = yarnClient.getApplications(request);
if (!reports.isEmpty()) {
String message = "";

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -165,6 +166,10 @@ protected void setupInternal(int numNodeManager)
// Disable vmem check to disallow NM killing the container
conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
// set auth filters
conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
"org.apache.hadoop.security.AuthenticationFilterInitializer,"
+ "org.apache.hadoop.security.HttpCrossOriginFilterInitializer");
// setup zk cluster
zkCluster = new TestingCluster(1);
zkCluster.start();

128
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
View File

@ -22,6 +22,7 @@
import com.google.common.collect.Multimap;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -34,6 +35,7 @@
import org.apache.hadoop.yarn.service.client.ServiceClient;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -49,6 +51,7 @@
import java.util.concurrent.TimeoutException;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.FINISHED;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
/**
* End to end tests to test deploying services with MiniYarnCluster and a in-JVM
@ -158,6 +161,109 @@ public void testComponentStartOrder() throws Exception {
client.actionDestroy(exampleApp.getName());
}
@Test(timeout = 200000)
public void testCreateServiceSameNameDifferentUser() throws Exception {
String sameAppName = "same-name";
String userA = "usera";
String userB = "userb";
setupInternal(NUM_NMS);
ServiceClient client = createClient();
String origBasePath = getConf().get(YARN_SERVICE_BASE_PATH);
Service userAApp = new Service();
userAApp.setName(sameAppName);
userAApp.addComponent(createComponent("comp", 1, "sleep 1000"));
Service userBApp = new Service();
userBApp.setName(sameAppName);
userBApp.addComponent(createComponent("comp", 1, "sleep 1000"));
File userABasePath = null, userBBasePath = null;
try {
userABasePath = new File(origBasePath, userA);
userABasePath.mkdirs();
getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath());
client.actionCreate(userAApp);
waitForServiceToBeStarted(client, userAApp);
userBBasePath = new File(origBasePath, userB);
userBBasePath.mkdirs();
getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath());
client.actionBuild(userBApp);
} catch (Exception e) {
Assert
.fail("Exception should not be thrown - " + e.getLocalizedMessage());
} finally {
if (userABasePath != null) {
getConf().set(YARN_SERVICE_BASE_PATH, userABasePath.getAbsolutePath());
client.actionStop(sameAppName, true);
client.actionDestroy(sameAppName);
}
if (userBBasePath != null) {
getConf().set(YARN_SERVICE_BASE_PATH, userBBasePath.getAbsolutePath());
client.actionDestroy(sameAppName);
}
}
// Need to extend this test to validate that different users can create
// apps of exact same name. So far only create followed by build is tested.
// Need to test create followed by create.
}
@Test(timeout = 200000)
public void testCreateServiceSameNameSameUser() throws Exception {
String sameAppName = "same-name";
String user = UserGroupInformation.getCurrentUser().getUserName();
System.setProperty("user.name", user);
setupInternal(NUM_NMS);
ServiceClient client = createClient();
Service appA = new Service();
appA.setName(sameAppName);
appA.addComponent(createComponent("comp", 1, "sleep 1000"));
Service appB = new Service();
appB.setName(sameAppName);
appB.addComponent(createComponent("comp", 1, "sleep 1000"));
try {
client.actionBuild(appA);
client.actionBuild(appB);
} catch (Exception e) {
String expectedMsg = "Service Instance dir already exists:";
if (e.getLocalizedMessage() != null) {
Assert.assertThat(e.getLocalizedMessage(),
CoreMatchers.containsString(expectedMsg));
} else {
Assert.fail("Message cannot be null. It has to say - " + expectedMsg);
}
} finally {
// cleanup
client.actionDestroy(sameAppName);
}
try {
client.actionCreate(appA);
waitForServiceToBeStarted(client, appA);
client.actionCreate(appB);
waitForServiceToBeStarted(client, appB);
} catch (Exception e) {
String expectedMsg = "Failed to create service " + sameAppName
+ ", because it already exists.";
if (e.getLocalizedMessage() != null) {
Assert.assertThat(e.getLocalizedMessage(),
CoreMatchers.containsString(expectedMsg));
} else {
Assert.fail("Message cannot be null. It has to say - " + expectedMsg);
}
} finally {
// cleanup
client.actionStop(sameAppName, true);
client.actionDestroy(sameAppName);
}
}
// Test to verify recovery of SeviceMaster after RM is restarted.
// 1. Create an example service.
// 2. Restart RM.
@ -369,6 +475,28 @@ private void waitForServiceToBeStable(ServiceClient client,
}, 2000, 200000);
}
/**
* Wait until service is started. It does not have to reach a stable state.
*
* @param client
* @param exampleApp
* @throws TimeoutException
* @throws InterruptedException
*/
private void waitForServiceToBeStarted(ServiceClient client,
Service exampleApp) throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> {
try {
Service retrievedApp = client.getStatus(exampleApp.getName());
System.out.println(retrievedApp);
return retrievedApp.getState() == ServiceState.STARTED;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}, 2000, 200000);
}
private ServiceClient createClient() throws Exception {
ServiceClient client = new ServiceClient() {
@Override protected Path addJarResource(String appName,