big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7799. Improved YARN service jar file handling.

Browse Source 
Contributed by Billie Rinaldi
This commit is contained in:
Eric Yang 2018-05-01 16:46:34 -04:00
parent 9e2cfb2d3f
commit 24eeea8b18
5 changed files with 194 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
View File

@ -172,6 +172,11 @@
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>

93
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -28,7 +28,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
@ -37,8 +39,8 @@
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
@ -896,13 +898,13 @@ private Map<String, String> addAMEnv() throws IOException {
protected Path addJarResource(String serviceName,
Map<String, LocalResource> localResources)
throws IOException, SliderException {
throws IOException, YarnException {
Path libPath = fs.buildClusterDirPath(serviceName);
ProviderUtils
.addProviderJar(localResources, ServiceMaster.class, SERVICE_CORE_JAR, fs,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
if (actionDependency(null, false) == EXIT_SUCCESS) {
LOG.info("Loading lib tar from " + dependencyLibTarGzip);
fs.submitTarGzipAndUpdate(localResources);
} else {
@ -1223,18 +1225,18 @@ public int enableFastLaunch(String destinationFolder)
return actionDependency(destinationFolder, true);
}
public int actionDependency(String destinationFolder, boolean overwrite)
throws IOException, YarnException {
public int actionDependency(String destinationFolder, boolean overwrite) {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
Path dependencyLibTarGzip;
if (destinationFolder == null) {
destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR,
VersionInfo.getVersion());
dependencyLibTarGzip = fs.getDependencyTarGzip();
} else {
dependencyLibTarGzip = new Path(destinationFolder,
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
}
Path dependencyLibTarGzip = new Path(destinationFolder,
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
@ -1247,24 +1249,71 @@ public int actionDependency(String destinationFolder, boolean overwrite)
String[] libDirs = ServiceUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars
tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
File tempLibTarGzipFile = null;
try {
if (!checkPermissions(dependencyLibTarGzip)) {
return EXIT_UNAUTHORIZED;
}
LOG.info("Version Info: " + VersionInfo.getBuildVersion());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
LOG.info("To let apps use this tarball, in yarn-site set config property "
+ "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
dependencyLibTarGzip);
return EXIT_SUCCESS;
tempLibTarGzipFile = File.createTempFile(
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars
tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
LOG.info("To let apps use this tarball, in yarn-site set config " +
"property {} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
dependencyLibTarGzip);
return EXIT_SUCCESS;
} catch (IOException e) {
LOG.error("Got exception creating tarball and uploading to HDFS", e);
return EXIT_EXCEPTION_THROWN;
} finally {
if (tempLibTarGzipFile != null) {
if (!tempLibTarGzipFile.delete()) {
LOG.warn("Failed to delete tmp file {}", tempLibTarGzipFile);
}
}
}
} else {
return EXIT_FALSE;
}
}
private boolean checkPermissions(Path dependencyLibTarGzip) throws
IOException {
AccessControlList yarnAdminAcl = new AccessControlList(getConfig().get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
AccessControlList dfsAdminAcl = new AccessControlList(
getConfig().get(DFSConfigKeys.DFS_ADMIN, " "));
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
if (!yarnAdminAcl.isUserAllowed(ugi) && !dfsAdminAcl.isUserAllowed(ugi)) {
LOG.error("User must be on the {} or {} list to have permission to " +
"upload AM dependency tarball", YarnConfiguration.YARN_ADMIN_ACL,
DFSConfigKeys.DFS_ADMIN);
return false;
}
Path parent = dependencyLibTarGzip.getParent();
while (parent != null) {
if (fs.getFileSystem().exists(parent)) {
FsPermission perm = fs.getFileSystem().getFileStatus(parent)
.getPermission();
if (!perm.getOtherAction().implies(FsAction.READ_EXECUTE)) {
LOG.error("Parent directory {} of {} tarball location {} does not " +
"have world read/execute permission", parent, YarnServiceConf
.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
return false;
}
}
parent = parent.getParent();
}
return true;
}
protected ClientAMProtocol createAMProxy(String serviceName,
ApplicationReport appReport) throws IOException, YarnException {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -363,6 +364,12 @@ public Path getDependencyTarGzip() {
if (configuredDependencyTarballPath != null) {
dependencyLibTarGzip = new Path(configuredDependencyTarballPath);
}
if (dependencyLibTarGzip == null) {
dependencyLibTarGzip = new Path(String.format(YarnServiceConstants
.DEPENDENCY_DIR, VersionInfo.getVersion()),
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
}
return dependencyLibTarGzip;
}

118
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
View File

@ -20,6 +20,10 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -27,12 +31,15 @@
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -40,21 +47,33 @@
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.List;
import static org.apache.hadoop.yarn.client.api.AppAdminClient.YARN_APP_ADMIN_CLIENT_PREFIX;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEPENDENCY_TARBALL_PATH;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_SUCCESS;
import static org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes.EXIT_UNAUTHORIZED;
import static org.mockito.Mockito.spy;
public class TestServiceCLI {
private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
.class);
@Rule
public TemporaryFolder tmpFolder = new TemporaryFolder();
private Configuration conf = new YarnConfiguration();
private File basedir;
private SliderFileSystem fs;
private String basedirProp;
private ApplicationCLI cli;
private File basedir;
private String basedirProp;
private File dependencyTarGzBaseDir;
private Path dependencyTarGz;
private String dependencyTarGzProp;
private String yarnAdminNoneAclProp;
private String dfsAdminAclProp;
private void createCLI() {
cli = new ApplicationCLI();
@ -67,12 +86,17 @@ private void createCLI() {
cli.setConf(conf);
}
private int runCLI(String[] args) throws Exception {
LOG.info("running CLI: yarn {}", Arrays.asList(args));
return ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
}
private void buildApp(String serviceName, String appDef) throws Throwable {
String[] args = {"app",
"-D", basedirProp, "-save", serviceName,
ExampleAppJson.resourceName(appDef),
"-appTypes", DUMMY_APP_TYPE};
ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
}
private void buildApp(String serviceName, String appDef,
@ -83,7 +107,13 @@ private void buildApp(String serviceName, String appDef,
"-appTypes", DUMMY_APP_TYPE,
"-updateLifetime", lifetime,
"-changeQueue", queue};
ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
}
private static Path getDependencyTarGz(File dir) {
return new Path(new File(dir, YarnServiceConstants
.DEPENDENCY_TAR_GZ_FILE_NAME + YarnServiceConstants
.DEPENDENCY_TAR_GZ_FILE_EXT).getAbsolutePath());
}
@Before
@ -91,12 +121,22 @@ public void setup() throws Throwable {
basedir = new File("target", "apps");
basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
dependencyTarGzBaseDir = tmpFolder.getRoot();
dependencyTarGz = getDependencyTarGz(dependencyTarGzBaseDir);
dependencyTarGzProp = DEPENDENCY_TARBALL_PATH + "=" + dependencyTarGz
.toString();
conf.set(DEPENDENCY_TARBALL_PATH, dependencyTarGz.toString());
fs = new SliderFileSystem(conf);
if (basedir.exists()) {
FileUtils.deleteDirectory(basedir);
} else {
basedir.mkdirs();
}
yarnAdminNoneAclProp = YarnConfiguration.YARN_ADMIN_ACL + "=none";
dfsAdminAclProp = DFSConfigKeys.DFS_ADMIN + "=" +
UserGroupInformation.getCurrentUser();
System.setProperty(YarnServiceConstants.PROPERTY_LIB_DIR, basedir
.getAbsolutePath());
createCLI();
}
@ -108,7 +148,7 @@ public void tearDown() throws IOException {
cli.stop();
}
@Test
@Test (timeout = 180000)
public void testFlexComponents() throws Throwable {
// currently can only test building apps, since that is the only
// operation that doesn't require an RM
@ -122,7 +162,7 @@ public void testFlexComponents() throws Throwable {
checkApp(serviceName, "master", 1L, 1000L, "qname");
}
@Test
@Test (timeout = 180000)
public void testInitiateServiceUpgrade() throws Exception {
String[] args = {"app", "-upgrade", "app-1",
"-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@ -131,7 +171,7 @@ public void testInitiateServiceUpgrade() throws Exception {
Assert.assertEquals(result, 0);
}
@Test
@Test (timeout = 180000)
public void testInitiateAutoFinalizeServiceUpgrade() throws Exception {
String[] args = {"app", "-upgrade", "app-1",
"-initiate", ExampleAppJson.resourceName(ExampleAppJson.APP_JSON),
@ -141,7 +181,7 @@ public void testInitiateAutoFinalizeServiceUpgrade() throws Exception {
Assert.assertEquals(result, 0);
}
@Test
@Test (timeout = 180000)
public void testUpgradeInstances() throws Exception {
conf.set(YARN_APP_ADMIN_CLIENT_PREFIX + DUMMY_APP_TYPE,
DummyServiceClient.class.getName());
@ -153,6 +193,68 @@ public void testUpgradeInstances() throws Exception {
Assert.assertEquals(result, 0);
}
@Test (timeout = 180000)
public void testEnableFastLaunch() throws Exception {
fs.getFileSystem().create(new Path(basedir.getAbsolutePath(), "test.jar"))
.close();
Path defaultPath = new Path(dependencyTarGz.toString());
Assert.assertFalse("Dependency tarball should not exist before the test",
fs.isFile(defaultPath));
String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
"-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_SUCCESS, runCLI(args));
Assert.assertTrue("Dependency tarball did not exist after the test",
fs.isFile(defaultPath));
File secondBaseDir = new File(dependencyTarGzBaseDir, "2");
Path secondTarGz = getDependencyTarGz(secondBaseDir);
Assert.assertFalse("Dependency tarball should not exist before the test",
fs.isFile(secondTarGz));
String[] args2 = {"app", "-D", yarnAdminNoneAclProp, "-D",
dfsAdminAclProp, "-D", dependencyTarGzProp, "-enableFastLaunch",
secondBaseDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_SUCCESS, runCLI(args2));
Assert.assertTrue("Dependency tarball did not exist after the test",
fs.isFile(secondTarGz));
}
@Test (timeout = 180000)
public void testEnableFastLaunchUserPermissions() throws Exception {
String[] args = {"app", "-D", yarnAdminNoneAclProp, "-D",
dependencyTarGzProp, "-enableFastLaunch", "-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
}
@Test (timeout = 180000)
public void testEnableFastLaunchFilePermissions() throws Exception {
File badDir = new File(dependencyTarGzBaseDir, "bad");
badDir.mkdir();
fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
new FsPermission("751"));
String[] args = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args));
badDir = new File(badDir, "child");
badDir.mkdir();
fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
new FsPermission("755"));
String[] args2 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args2));
badDir = new File(dependencyTarGzBaseDir, "badx");
badDir.mkdir();
fs.getFileSystem().setPermission(new Path(badDir.getAbsolutePath()),
new FsPermission("754"));
String[] args3 = {"app", "-D", dependencyTarGzProp, "-enableFastLaunch",
badDir.getAbsolutePath(), "-appTypes", DUMMY_APP_TYPE};
Assert.assertEquals(EXIT_UNAUTHORIZED, runCLI(args3));
}
private void checkApp(String serviceName, String compName, long count, Long
lifetime, String queue) throws IOException {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
View File

@ -100,7 +100,7 @@ System-wide service AM properties can only be configured in the cluster `yarn-si
| System-Level Config Name | Description |
| ------------ | ------------- |
|yarn.service.framework.path | HDFS parent directory where the service AM dependency tarball can be found.|
|yarn.service.framework.path | HDFS path of the service AM dependency tarball. When no file exists at this location, AM dependencies will be uploaded by the RM the first time a service is started or launched. If the RM user does not have permission to upload the file to this location or the location is not world readable, the AM dependency jars will be uploaded each time a service is started or launched. If unspecified, value will be assumed to be /yarn-services/${hadoop.version}/service-dep.tar.gz.|
|yarn.service.base.path | HDFS parent directory where service artifacts will be stored (default ${user_home_dir}/.yarn/).
|yarn.service.client-am.retry.max-wait-ms | Max retry time in milliseconds for the service client to talk to the service AM (default 900000, i.e. 15 minutes).|
|yarn.service.client-am.retry-interval-ms | Retry interval in milliseconds for the service client to talk to the service AM (default 2000, i.e. 2 seconds).|