big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi

Browse Source
This commit is contained in:
Gour Saha 2016-11-30 14:00:22 -08:00 committed by Jian He
parent 4f43ebdb9f
commit bd0e9f91b6
3 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
View File

@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys {
* Prefix for export options: {@value}
*/
String EXPORT_PREFIX = "export.";
/**
* Type suffix for config file and package options: {@value}
*/
String TYPE_SUFFIX = ".type";
/**
* Name suffix for config file and package options: {@value}
*/
String NAME_SUFFIX = ".name";
/**
* Per component suffix for config file options: {@value}
*/
String PER_COMPONENT = ".per.component";
/**
* Per group suffix for config file options: {@value}
*/
String PER_GROUP = ".per.group";
/**
* Zookeeper quorum host list: {@value}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
View File

@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
String PER_COMPONENT = "per.component";
String PER_GROUP = "per.group";
String APP_PACKAGES_DIR = "app/packages";
}

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
View File

@ -614,8 +614,16 @@ public void localizeConfigFiles(ContainerLauncher launcher,
throw new BadConfigException("Config format " + configFileType +
" doesn't exist");
}
boolean perComponent = appConf.getComponentOptBool(roleGroup,
OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
.PER_COMPONENT, false);
boolean perGroup = appConf.getComponentOptBool(roleGroup,
OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
.PER_GROUP, false);
localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),
configFormat, configFileName, configs, env, fileSystem, clusterName);
configFormat, configFileName, configs, env, fileSystem,
clusterName, perComponent, perGroup);
}
}
@ -631,6 +639,8 @@ public void localizeConfigFiles(ContainerLauncher launcher,
* @param env environment variables
* @param fileSystem file system
* @param clusterName app name
* @param perComponent true if file should be created per unique component
* @param perGroup true if file should be created per component group
* @throws IOException file cannot be uploaded
*/
public void localizeConfigFile(ContainerLauncher launcher,
@ -639,7 +649,9 @@ public void localizeConfigFile(ContainerLauncher launcher,
Map<String, Map<String, String>> configs,
MapOperations env,
SliderFileSystem fileSystem,
String clusterName)
String clusterName,
boolean perComponent,
boolean perGroup)
throws IOException {
if (launcher == null) {
return;
@ -655,9 +667,9 @@ public void localizeConfigFile(ContainerLauncher launcher,
}
String folder = null;
if ("true".equals(config.get(PER_COMPONENT))) {
if (perComponent) {
folder = roleName;
} else if ("true".equals(config.get(PER_GROUP))) {
} else if (perGroup) {
folder = roleGroup;
}
if (folder != null) {