YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev.
This commit is contained in:
Varun Vasudev
Wangda Tan
parent
239c1824a0
commit
a9b4426302
@ -884,6 +884,29 @@ public static boolean isAclEnabled(Configuration conf) {
|
||||
*/
|
||||
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
|
||||
|
||||
/**
|
||||
* Enable/disable resource profiles.
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public static final String RM_RESOURCE_PROFILES_ENABLED =
|
||||
RM_PREFIX + "resource-profiles.enabled";
|
||||
@Public
|
||||
@Unstable
|
||||
public static final boolean DEFAULT_RM_RESOURCE_PROFILES_ENABLED = false;
|
||||
|
||||
/**
|
||||
* File containing resource profiles.
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public static final String RM_RESOURCE_PROFILES_SOURCE_FILE =
|
||||
RM_PREFIX + "resource-profiles.source-file";
|
||||
@Public
|
||||
@Unstable
|
||||
public static final String DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE =
|
||||
"resource-profiles.json";
|
||||
|
||||
/**
|
||||
* Timeout in seconds for YARN node graceful decommission.
|
||||
* This is the maximal time to wait for running containers and applications
|
||||
|
||||
@ -143,6 +143,10 @@ public void initializeMemberVariables() {
|
||||
// Used as Java command line properties, not XML
|
||||
configurationPrefixToSkipCompare.add("yarn.app.container");
|
||||
|
||||
// Ignore default file name for resource profiles
|
||||
configurationPropsToSkipCompare
|
||||
.add(YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE);
|
||||
|
||||
// Ignore NodeManager "work in progress" variables
|
||||
configurationPrefixToSkipCompare
|
||||
.add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED);
|
||||
|
||||
@ -944,6 +944,22 @@
|
||||
<value>600000</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
Flag to enable/disable resource profiles
|
||||
</description>
|
||||
<name>yarn.resourcemanager.resource-profiles.enabled</name>
|
||||
<value>false</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
If resource profiles is enabled, source file for the profiles
|
||||
</description>
|
||||
<name>yarn.resourcemanager.resource-profiles.source-file</name>
|
||||
<value>resource-profiles.json</value>
|
||||
</property>
|
||||
|
||||
<!-- Node Manager Configuration -->
|
||||
|
||||
<property>
|
||||
|
||||
@ -351,6 +351,11 @@
|
||||
<exclude>src/test/resources/delete-reservation.json</exclude>
|
||||
<exclude>src/test/resources/update-reservation.json</exclude>
|
||||
<exclude>src/test/resources/invariants.txt</exclude>
|
||||
<exclude>src/test/resources/profiles/sample-profiles-1.json</exclude>
|
||||
<exclude>src/test/resources/profiles/sample-profiles-2.json</exclude>
|
||||
<exclude>src/test/resources/profiles/illegal-profiles-1.json</exclude>
|
||||
<exclude>src/test/resources/profiles/illegal-profiles-2.json</exclude>
|
||||
<exclude>src/test/resources/profiles/illegal-profiles-3.json</exclude>
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.resource;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Interface for the resource profiles manager. Provides an interface to get
|
||||
* the list of available profiles and some helper functions.
|
||||
*/
|
||||
public interface ResourceProfilesManager {
|
||||
|
||||
void init(Configuration config) throws IOException;
|
||||
|
||||
Resource getProfile(String profile);
|
||||
|
||||
Map<String, Resource> getResourceProfiles();
|
||||
|
||||
void reloadProfiles() throws IOException;
|
||||
|
||||
Resource getDefaultProfile();
|
||||
|
||||
Resource getMinimumProfile();
|
||||
|
||||
Resource getMaximumProfile();
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.resource;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.api.records.ResourceInformation;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
|
||||
import org.apache.hadoop.yarn.util.resource.Resources;
|
||||
import org.codehaus.jackson.map.ObjectMapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
|
||||
|
||||
private static final Log LOG =
|
||||
LogFactory.getLog(ResourceProfilesManagerImpl.class);
|
||||
|
||||
private final Map<String, Resource> profiles = new ConcurrentHashMap<>();
|
||||
private Configuration conf;
|
||||
|
||||
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
|
||||
private static final String VCORES = ResourceInformation.VCORES.getName();
|
||||
|
||||
private static final String DEFAULT_PROFILE = "default";
|
||||
private static final String MINIMUM_PROFILE = "minimum";
|
||||
private static final String MAXIMUM_PROFILE = "maximum";
|
||||
|
||||
private static final String[] MANDATORY_PROFILES =
|
||||
{ DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE };
|
||||
|
||||
public void init(Configuration config) throws IOException {
|
||||
conf = config;
|
||||
loadProfiles();
|
||||
}
|
||||
|
||||
private void loadProfiles() throws IOException {
|
||||
boolean profilesEnabled =
|
||||
conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
|
||||
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
|
||||
if (!profilesEnabled) {
|
||||
return;
|
||||
}
|
||||
String sourceFile =
|
||||
conf.get(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
|
||||
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE);
|
||||
String resourcesFile = sourceFile;
|
||||
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
|
||||
if (classLoader == null) {
|
||||
classLoader = ResourceProfilesManagerImpl.class.getClassLoader();
|
||||
}
|
||||
if (classLoader != null) {
|
||||
URL tmp = classLoader.getResource(sourceFile);
|
||||
if (tmp != null) {
|
||||
resourcesFile = tmp.getPath();
|
||||
}
|
||||
}
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
Map data = mapper.readValue(new File(resourcesFile), Map.class);
|
||||
Iterator iterator = data.entrySet().iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Map.Entry entry = (Map.Entry) iterator.next();
|
||||
String key = entry.getKey().toString();
|
||||
if (entry.getValue() instanceof Map) {
|
||||
Map value = (Map) entry.getValue();
|
||||
// ensure memory and vcores are specified
|
||||
if (!value.containsKey(MEMORY) || !value.containsKey(VCORES)) {
|
||||
throw new IOException(
|
||||
"Illegal resource profile definition; profile '" + key
|
||||
+ "' must contain '" + MEMORY + "' and '" + VCORES + "'");
|
||||
}
|
||||
Resource resource = parseResource(key, value);
|
||||
profiles.put(key, resource);
|
||||
LOG.info("Added profile '" + key + "' with resources " + resource);
|
||||
}
|
||||
}
|
||||
// check to make sure mandatory profiles are present
|
||||
for (String profile : MANDATORY_PROFILES) {
|
||||
if (!profiles.containsKey(profile)) {
|
||||
throw new IOException(
|
||||
"Mandatory profile missing '" + profile + "' missing. "
|
||||
+ Arrays.toString(MANDATORY_PROFILES) + " must be present");
|
||||
}
|
||||
}
|
||||
LOG.info("Loaded profiles " + profiles.keySet());
|
||||
}
|
||||
|
||||
private Resource parseResource(String key, Map value) throws IOException {
|
||||
Resource resource = Resource.newInstance(0, 0);
|
||||
Iterator iterator = value.entrySet().iterator();
|
||||
Map<String, ResourceInformation> resourceTypes =
|
||||
ResourceUtils.getResourceTypes();
|
||||
while (iterator.hasNext()) {
|
||||
Map.Entry resourceEntry = (Map.Entry) iterator.next();
|
||||
String resourceName = resourceEntry.getKey().toString();
|
||||
ResourceInformation resourceValue =
|
||||
fromString(resourceName, resourceEntry.getValue().toString());
|
||||
if (resourceName.equals(MEMORY)) {
|
||||
resource.setMemorySize(resourceValue.getValue());
|
||||
continue;
|
||||
}
|
||||
if (resourceName.equals(VCORES)) {
|
||||
resource.setVirtualCores(resourceValue.getValue().intValue());
|
||||
continue;
|
||||
}
|
||||
if (resourceTypes.containsKey(resourceName)) {
|
||||
resource.setResourceInformation(resourceName, resourceValue);
|
||||
} else {
|
||||
throw new IOException("Unrecognized resource type '" + resourceName
|
||||
+ "'. Recognized resource types are '" + resourceTypes.keySet()
|
||||
+ "'");
|
||||
}
|
||||
}
|
||||
return resource;
|
||||
}
|
||||
|
||||
public Resource getProfile(String profile) {
|
||||
return Resources.clone(profiles.get(profile));
|
||||
}
|
||||
|
||||
public Map<String, Resource> getResourceProfiles() {
|
||||
return Collections.unmodifiableMap(profiles);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public void reloadProfiles() throws IOException {
|
||||
profiles.clear();
|
||||
loadProfiles();
|
||||
}
|
||||
|
||||
public Resource getDefaultProfile() {
|
||||
return getProfile(DEFAULT_PROFILE);
|
||||
}
|
||||
|
||||
public Resource getMinimumProfile() {
|
||||
return getProfile(MINIMUM_PROFILE);
|
||||
}
|
||||
|
||||
public Resource getMaximumProfile() {
|
||||
return getProfile(MAXIMUM_PROFILE);
|
||||
}
|
||||
|
||||
private ResourceInformation fromString(String name, String value) {
|
||||
String units = ResourceUtils.getUnits(value);
|
||||
Long resourceValue =
|
||||
Long.valueOf(value.substring(0, value.length() - units.length()));
|
||||
return ResourceInformation.newInstance(name, units, resourceValue);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,142 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.resource;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.yarn.api.records.Resource;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class TestResourceProfiles {
|
||||
|
||||
@Test
|
||||
public void testProfilesEnabled() throws Exception {
|
||||
ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
|
||||
Configuration conf = new Configuration();
|
||||
// be default resource profiles should not be enabled
|
||||
manager.init(conf);
|
||||
Map<String, Resource> profiles = manager.getResourceProfiles();
|
||||
Assert.assertTrue(profiles.isEmpty());
|
||||
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
|
||||
try {
|
||||
manager.init(conf);
|
||||
Assert.fail(
|
||||
"Exception should be thrown due to missing resource profiles file");
|
||||
} catch (IOException ie) {
|
||||
}
|
||||
conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
|
||||
"profiles/sample-profiles-1.json");
|
||||
manager.init(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadProfiles() throws Exception {
|
||||
ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
|
||||
conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
|
||||
"profiles/sample-profiles-1.json");
|
||||
manager.init(conf);
|
||||
Map<String, Resource> profiles = manager.getResourceProfiles();
|
||||
Map<String, Resource> expected = new HashMap<>();
|
||||
expected.put("minimum", Resource.newInstance(1024, 1));
|
||||
expected.put("default", Resource.newInstance(2048, 2));
|
||||
expected.put("maximum", Resource.newInstance(4096, 4));
|
||||
|
||||
for (Map.Entry<String, Resource> entry : expected.entrySet()) {
|
||||
String profile = entry.getKey();
|
||||
Resource res = entry.getValue();
|
||||
Assert.assertTrue("Mandatory profile '" + profile + "' missing",
|
||||
profiles.containsKey(profile));
|
||||
Assert.assertEquals("Profile " + profile + "' resources don't match", res,
|
||||
manager.getProfile(profile));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadProfilesMissingMandatoryProfile() throws Exception {
|
||||
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
|
||||
|
||||
String[] badProfiles = { "profiles/illegal-profiles-1.json",
|
||||
"profiles/illegal-profiles-2.json",
|
||||
"profiles/illegal-profiles-3.json" };
|
||||
for (String file : badProfiles) {
|
||||
ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
|
||||
conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, file);
|
||||
try {
|
||||
manager.init(conf);
|
||||
Assert.fail("Bad profile '" + file + "' is not valid");
|
||||
} catch (IOException ie) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetProfile() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
|
||||
ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
|
||||
conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
|
||||
"profiles/sample-profiles-2.json");
|
||||
manager.init(conf);
|
||||
Map<String, Resource> expected = new HashMap<>();
|
||||
expected.put("minimum", Resource.newInstance(1024, 1));
|
||||
expected.put("default", Resource.newInstance(2048, 2));
|
||||
expected.put("maximum", Resource.newInstance(4096, 4));
|
||||
expected.put("small", Resource.newInstance(1024, 1));
|
||||
expected.put("medium", Resource.newInstance(2048, 1));
|
||||
expected.put("large", Resource.newInstance(4096, 4));
|
||||
|
||||
for (Map.Entry<String, Resource> entry : expected.entrySet()) {
|
||||
String profile = entry.getKey();
|
||||
Resource res = entry.getValue();
|
||||
Assert.assertEquals("Profile " + profile + "' resources don't match", res,
|
||||
manager.getProfile(profile));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetMandatoryProfiles() throws Exception {
|
||||
ResourceProfilesManager manager = new ResourceProfilesManagerImpl();
|
||||
Configuration conf = new Configuration();
|
||||
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
|
||||
conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE,
|
||||
"profiles/sample-profiles-1.json");
|
||||
manager.init(conf);
|
||||
Map<String, Resource> expected = new HashMap<>();
|
||||
expected.put("minimum", Resource.newInstance(1024, 1));
|
||||
expected.put("default", Resource.newInstance(2048, 2));
|
||||
expected.put("maximum", Resource.newInstance(4096, 4));
|
||||
|
||||
Assert.assertEquals("Profile 'minimum' resources don't match",
|
||||
expected.get("minimum"), manager.getMinimumProfile());
|
||||
Assert.assertEquals("Profile 'default' resources don't match",
|
||||
expected.get("default"), manager.getDefaultProfile());
|
||||
Assert.assertEquals("Profile 'maximum' resources don't match",
|
||||
expected.get("maximum"), manager.getMaximumProfile());
|
||||
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
{
|
||||
"minimum": {
|
||||
"memoryMB" : 1024,
|
||||
"vcores" : 1
|
||||
},
|
||||
"default" : {
|
||||
"memoryMB" : 2048,
|
||||
"vcores" : 2
|
||||
},
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
{
|
||||
"minimum": {
|
||||
"memoryMB" : 1024,
|
||||
"vcores" : 1
|
||||
},
|
||||
"maximum" : {
|
||||
"memoryMB": 4096,
|
||||
"vcores" : 4
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,10 @@
|
||||
{
|
||||
"default" : {
|
||||
"memoryMB" : 2048,
|
||||
"vcores" : 2
|
||||
},
|
||||
"maximum" : {
|
||||
"memoryMB": 4096,
|
||||
"vcores" : 4
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
{
|
||||
"minimum": {
|
||||
"memory-mb" : 1024,
|
||||
"vcores" : 1
|
||||
},
|
||||
"default" : {
|
||||
"memory-mb" : 2048,
|
||||
"vcores" : 2
|
||||
},
|
||||
"maximum" : {
|
||||
"memory-mb": 4096,
|
||||
"vcores" : 4
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,26 @@
|
||||
{
|
||||
"minimum": {
|
||||
"memory-mb" : 1024,
|
||||
"vcores" : 1
|
||||
},
|
||||
"default" : {
|
||||
"memory-mb" : 2048,
|
||||
"vcores" : 2
|
||||
},
|
||||
"maximum" : {
|
||||
"memory-mb": 4096,
|
||||
"vcores" : 4
|
||||
},
|
||||
"small" : {
|
||||
"memory-mb": 1024,
|
||||
"vcores": 1
|
||||
},
|
||||
"medium" : {
|
||||
"memory-mb": 2048,
|
||||
"vcores": 1
|
||||
},
|
||||
"large": {
|
||||
"memory-mb" : 4096,
|
||||
"vcores" : 4
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user