YARN-10412. Move CS placement rule related changes to a separate package. Contributed by Gergely Pollak
This commit is contained in:
Szilard Nemeth
parent
2b62b12d0d
commit
5db4c0bf70
@ -25,6 +25,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.*;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
@ -74,12 +75,12 @@ public class CSMappingPlacementRule extends PlacementRule {
|
||||
private boolean failOnConfigError = true;
|
||||
|
||||
@VisibleForTesting
|
||||
void setGroups(Groups groups) {
|
||||
public void setGroups(Groups groups) {
|
||||
this.groups = groups;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
void setFailOnConfigError(boolean failOnConfigError) {
|
||||
public void setFailOnConfigError(boolean failOnConfigError) {
|
||||
this.failOnConfigError = failOnConfigError;
|
||||
}
|
||||
|
||||
|
||||
@ -55,7 +55,7 @@ public class VariableContext {
|
||||
* @param name Name of the variable to check
|
||||
* @return true if the variable is immutable
|
||||
*/
|
||||
boolean isImmutable(String name) {
|
||||
public boolean isImmutable(String name) {
|
||||
return (immutableNames != null && immutableNames.contains(name));
|
||||
}
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
|
||||
|
||||
@ -16,9 +16,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* Mapping rule represents a single mapping setting defined by the user. All
|
||||
@ -16,9 +16,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This interface represents the action part of a MappingRule, action are
|
||||
@ -16,7 +16,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This class implements the fallback logic for MappingRuleActions, this can
|
||||
@ -16,9 +16,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
/**
|
||||
* This class contains all the actions and some helper methods to generate them.
|
||||
@ -16,7 +16,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
public interface MappingRuleMatcher {
|
||||
/**
|
||||
@ -16,7 +16,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Set;
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
/**
|
||||
* This class represents the outcome of an action.
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
public enum MappingRuleResultType {
|
||||
/**
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
* interface should contain all validation related data and functions, this way
|
||||
* schedulers or engines can be changed without changing the MappingRules.
|
||||
*/
|
||||
interface MappingRuleValidationContext {
|
||||
public interface MappingRuleValidationContext {
|
||||
/**
|
||||
* This method should determine if the provided queue path can result in
|
||||
* a possible placement. It should fail if the provided path cannot be placed
|
||||
@ -15,7 +15,7 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
@ -48,7 +48,7 @@ public class MappingRuleValidationContextImpl
|
||||
*/
|
||||
private final CapacitySchedulerQueueManager queueManager;
|
||||
|
||||
MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
|
||||
public MappingRuleValidationContextImpl(CapacitySchedulerQueueManager qm) {
|
||||
queueManager = qm;
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
|
||||
@ -18,12 +18,10 @@
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
|
||||
|
||||
import org.apache.hadoop.ipc.WeightedTimeCostProvider;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.base.Strings;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueuePlacementRuleUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.MappingRuleCreator;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@ -85,41 +83,41 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||
LoggerFactory.getLogger(CapacitySchedulerConfiguration.class);
|
||||
|
||||
private static final String CS_CONFIGURATION_FILE = "capacity-scheduler.xml";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String PREFIX = "yarn.scheduler.capacity.";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String DOT = ".";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String MAXIMUM_APPLICATIONS_SUFFIX =
|
||||
"maximum-applications";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String MAXIMUM_SYSTEM_APPLICATIONS =
|
||||
PREFIX + MAXIMUM_APPLICATIONS_SUFFIX;
|
||||
|
||||
|
||||
@Private
|
||||
public static final String MAXIMUM_AM_RESOURCE_SUFFIX =
|
||||
"maximum-am-resource-percent";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT =
|
||||
PREFIX + MAXIMUM_AM_RESOURCE_SUFFIX;
|
||||
|
||||
@Private
|
||||
public static final String QUEUES = "queues";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String CAPACITY = "capacity";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String MAXIMUM_CAPACITY = "maximum-capacity";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String USER_LIMIT = "minimum-user-limit-percent";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String USER_LIMIT_FACTOR = "user-limit-factor";
|
||||
|
||||
@ -134,17 +132,17 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||
|
||||
@Private
|
||||
public static final String STATE = "state";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String ACCESSIBLE_NODE_LABELS = "accessible-node-labels";
|
||||
|
||||
|
||||
@Private
|
||||
public static final String DEFAULT_NODE_LABEL_EXPRESSION =
|
||||
"default-node-label-expression";
|
||||
|
||||
public static final String RESERVE_CONT_LOOK_ALL_NODES = PREFIX
|
||||
+ "reservations-continue-look-all-nodes";
|
||||
|
||||
|
||||
@Private
|
||||
public static final boolean DEFAULT_RESERVE_CONT_LOOK_ALL_NODES = true;
|
||||
|
||||
@ -177,29 +175,29 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||
|
||||
public static final String DEFAULT_APP_ORDERING_POLICY =
|
||||
FIFO_APP_ORDERING_POLICY;
|
||||
|
||||
|
||||
@Private
|
||||
public static final int DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS = 10000;
|
||||
|
||||
|
||||
@Private
|
||||
public static final float
|
||||
public static final float
|
||||
DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT = 0.1f;
|
||||
|
||||
@Private
|
||||
public static final float UNDEFINED = -1;
|
||||
|
||||
|
||||
@Private
|
||||
public static final float MINIMUM_CAPACITY_VALUE = 0;
|
||||
|
||||
|
||||
@Private
|
||||
public static final float MAXIMUM_CAPACITY_VALUE = 100;
|
||||
|
||||
|
||||
@Private
|
||||
public static final float DEFAULT_MAXIMUM_CAPACITY_VALUE = -1.0f;
|
||||
|
||||
|
||||
@Private
|
||||
public static final int DEFAULT_USER_LIMIT = 100;
|
||||
|
||||
|
||||
@Private
|
||||
public static final float DEFAULT_USER_LIMIT_FACTOR = 1.0f;
|
||||
|
||||
@ -217,17 +215,17 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||
@Private public static final String RESOURCE_CALCULATOR_CLASS =
|
||||
PREFIX + "resource-calculator";
|
||||
|
||||
@Private public static final Class<? extends ResourceCalculator>
|
||||
@Private public static final Class<? extends ResourceCalculator>
|
||||
DEFAULT_RESOURCE_CALCULATOR_CLASS = DefaultResourceCalculator.class;
|
||||
|
||||
|
||||
@Private
|
||||
public static final String ROOT = "root";
|
||||
|
||||
@Private
|
||||
public static final String NODE_LOCALITY_DELAY =
|
||||
@Private
|
||||
public static final String NODE_LOCALITY_DELAY =
|
||||
PREFIX + "node-locality-delay";
|
||||
|
||||
@Private
|
||||
@Private
|
||||
public static final int DEFAULT_NODE_LOCALITY_DELAY = 40;
|
||||
|
||||
@Private
|
||||
@ -312,7 +310,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
|
||||
|
||||
@Private
|
||||
public static final Integer DEFAULT_CONFIGURATION_APPLICATION_PRIORITY = 0;
|
||||
|
||||
|
||||
@Private
|
||||
public static final String AVERAGE_CAPACITY = "average-capacity";
|
||||
|
||||
@ -421,7 +419,7 @@ public enum AbsoluteResourceType {
|
||||
public CapacitySchedulerConfiguration() {
|
||||
this(new Configuration());
|
||||
}
|
||||
|
||||
|
||||
public CapacitySchedulerConfiguration(Configuration configuration) {
|
||||
this(configuration, true);
|
||||
}
|
||||
@ -454,15 +452,15 @@ private String getNodeLabelPrefix(String queue, String label) {
|
||||
}
|
||||
return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
|
||||
}
|
||||
|
||||
|
||||
public int getMaximumSystemApplications() {
|
||||
int maxApplications =
|
||||
int maxApplications =
|
||||
getInt(MAXIMUM_SYSTEM_APPLICATIONS, DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS);
|
||||
return maxApplications;
|
||||
}
|
||||
|
||||
|
||||
public float getMaximumApplicationMasterResourcePercent() {
|
||||
return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
|
||||
return getFloat(MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
|
||||
DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT);
|
||||
}
|
||||
|
||||
@ -473,23 +471,23 @@ public float getMaximumApplicationMasterResourcePercent() {
|
||||
* @return setting specified or -1 if not set
|
||||
*/
|
||||
public int getMaximumApplicationsPerQueue(String queue) {
|
||||
int maxApplicationsPerQueue =
|
||||
getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
|
||||
(int)UNDEFINED);
|
||||
int maxApplicationsPerQueue =
|
||||
getInt(getQueuePrefix(queue) + MAXIMUM_APPLICATIONS_SUFFIX,
|
||||
(int)UNDEFINED);
|
||||
return maxApplicationsPerQueue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum am resource percent per queue setting.
|
||||
* @param queue name of the queue
|
||||
* @return per queue setting or defaults to the global am-resource-percent
|
||||
* @return per queue setting or defaults to the global am-resource-percent
|
||||
* setting if per queue setting not present
|
||||
*/
|
||||
public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
|
||||
return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
|
||||
return getFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX,
|
||||
getMaximumApplicationMasterResourcePercent());
|
||||
}
|
||||
|
||||
|
||||
public void setMaximumApplicationMasterResourcePerQueuePercent(String queue,
|
||||
float percent) {
|
||||
setFloat(getQueuePrefix(queue) + MAXIMUM_AM_RESOURCE_SUFFIX, percent);
|
||||
@ -555,7 +553,7 @@ public float getNonLabeledQueueCapacity(String queue) {
|
||||
|
||||
return capacity;
|
||||
}
|
||||
|
||||
|
||||
public void setCapacity(String queue, float capacity) {
|
||||
if (queue.equals("root")) {
|
||||
throw new IllegalArgumentException(
|
||||
@ -599,7 +597,7 @@ public float getNonLabeledQueueMaximumCapacity(String queue) {
|
||||
: maxCapacity;
|
||||
return maxCapacity;
|
||||
}
|
||||
|
||||
|
||||
public void setMaximumCapacity(String queue, float maxCapacity) {
|
||||
if (maxCapacity > MAXIMUM_CAPACITY_VALUE) {
|
||||
throw new IllegalArgumentException("Illegal " +
|
||||
@ -609,7 +607,7 @@ public void setMaximumCapacity(String queue, float maxCapacity) {
|
||||
LOG.debug("CSConf - setMaxCapacity: queuePrefix={}, maxCapacity={}",
|
||||
getQueuePrefix(queue), maxCapacity);
|
||||
}
|
||||
|
||||
|
||||
public void setCapacityByLabel(String queue, String label, float capacity) {
|
||||
setFloat(getNodeLabelPrefix(queue, label) + CAPACITY, capacity);
|
||||
}
|
||||
@ -630,7 +628,7 @@ public void setMaximumCapacityByLabel(String queue, String label,
|
||||
set(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY,
|
||||
absoluteResourceCapacity);
|
||||
}
|
||||
|
||||
|
||||
public int getUserLimit(String queue) {
|
||||
int userLimit = getInt(getQueuePrefix(queue) + USER_LIMIT,
|
||||
DEFAULT_USER_LIMIT);
|
||||
@ -643,12 +641,12 @@ public int getUserLimit(String queue) {
|
||||
@SuppressWarnings("unchecked")
|
||||
public <S extends SchedulableEntity> OrderingPolicy<S> getAppOrderingPolicy(
|
||||
String queue) {
|
||||
|
||||
|
||||
String policyType = get(getQueuePrefix(queue) + ORDERING_POLICY,
|
||||
DEFAULT_APP_ORDERING_POLICY);
|
||||
|
||||
|
||||
OrderingPolicy<S> orderingPolicy;
|
||||
|
||||
|
||||
if (policyType.trim().equals(FIFO_APP_ORDERING_POLICY)) {
|
||||
policyType = FifoOrderingPolicy.class.getName();
|
||||
}
|
||||
@ -686,18 +684,18 @@ public void setUserLimit(String queue, int userLimit) {
|
||||
LOG.debug("here setUserLimit: queuePrefix={}, userLimit={}",
|
||||
getQueuePrefix(queue), getUserLimit(queue));
|
||||
}
|
||||
|
||||
|
||||
public float getUserLimitFactor(String queue) {
|
||||
float userLimitFactor =
|
||||
getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR,
|
||||
DEFAULT_USER_LIMIT_FACTOR);
|
||||
float userLimitFactor =
|
||||
getFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR,
|
||||
DEFAULT_USER_LIMIT_FACTOR);
|
||||
return userLimitFactor;
|
||||
}
|
||||
|
||||
public void setUserLimitFactor(String queue, float userLimitFactor) {
|
||||
setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor);
|
||||
setFloat(getQueuePrefix(queue) + USER_LIMIT_FACTOR, userLimitFactor);
|
||||
}
|
||||
|
||||
|
||||
public QueueState getConfiguredState(String queue) {
|
||||
String state = get(getQueuePrefix(queue) + STATE);
|
||||
if (state == null) {
|
||||
@ -725,12 +723,12 @@ public void setAccessibleNodeLabels(String queue, Set<String> labels) {
|
||||
String str = StringUtils.join(",", labels);
|
||||
set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str);
|
||||
}
|
||||
|
||||
|
||||
public Set<String> getAccessibleNodeLabels(String queue) {
|
||||
String accessibleLabelStr =
|
||||
get(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS);
|
||||
|
||||
// When accessible-label is null,
|
||||
// When accessible-label is null,
|
||||
if (accessibleLabelStr == null) {
|
||||
// Only return null when queue is not ROOT
|
||||
if (!queue.equals(ROOT)) {
|
||||
@ -757,7 +755,7 @@ public Set<String> getAccessibleNodeLabels(String queue) {
|
||||
set.add(str.trim());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// if labels contains "*", only keep ANY behind
|
||||
if (set.contains(RMNodeLabelsManager.ANY)) {
|
||||
set.clear();
|
||||
@ -813,15 +811,15 @@ private float internalGetLabeledQueueCapacity(String queue, String label,
|
||||
}
|
||||
return capacity;
|
||||
}
|
||||
|
||||
|
||||
public float getLabeledQueueCapacity(String queue, String label) {
|
||||
return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f);
|
||||
}
|
||||
|
||||
|
||||
public float getLabeledQueueMaximumCapacity(String queue, String label) {
|
||||
return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f);
|
||||
}
|
||||
|
||||
|
||||
public String getDefaultNodeLabelExpression(String queue) {
|
||||
String defaultLabelExpression = get(getQueuePrefix(queue)
|
||||
+ DEFAULT_NODE_LABEL_EXPRESSION);
|
||||
@ -830,7 +828,7 @@ public String getDefaultNodeLabelExpression(String queue) {
|
||||
}
|
||||
return defaultLabelExpression.trim();
|
||||
}
|
||||
|
||||
|
||||
public void setDefaultNodeLabelExpression(String queue, String exp) {
|
||||
set(getQueuePrefix(queue) + DEFAULT_NODE_LABEL_EXPRESSION, exp);
|
||||
}
|
||||
@ -860,7 +858,7 @@ public boolean getReservationContinueLook() {
|
||||
return getBoolean(RESERVE_CONT_LOOK_ALL_NODES,
|
||||
DEFAULT_RESERVE_CONT_LOOK_ALL_NODES);
|
||||
}
|
||||
|
||||
|
||||
private static String getAclKey(QueueACL acl) {
|
||||
return "acl_" + StringUtils.toLowerCase(acl.toString());
|
||||
}
|
||||
@ -987,13 +985,13 @@ public String[] getQueues(String queue) {
|
||||
|
||||
return queues;
|
||||
}
|
||||
|
||||
|
||||
public void setQueues(String queue, String[] subQueues) {
|
||||
set(getQueuePrefix(queue) + QUEUES, StringUtils.arrayToString(subQueues));
|
||||
LOG.debug("CSConf - setQueues: qPrefix={}, queues={}",
|
||||
getQueuePrefix(queue), StringUtils.arrayToString(subQueues));
|
||||
}
|
||||
|
||||
|
||||
public Resource getMinimumAllocation() {
|
||||
int minimumMemory = getInt(
|
||||
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
|
||||
@ -1087,9 +1085,9 @@ public boolean getRackLocalityFullReset() {
|
||||
public ResourceCalculator getResourceCalculator() {
|
||||
return ReflectionUtils.newInstance(
|
||||
getClass(
|
||||
RESOURCE_CALCULATOR_CLASS,
|
||||
DEFAULT_RESOURCE_CALCULATOR_CLASS,
|
||||
ResourceCalculator.class),
|
||||
RESOURCE_CALCULATOR_CLASS,
|
||||
DEFAULT_RESOURCE_CALCULATOR_CLASS,
|
||||
ResourceCalculator.class),
|
||||
this);
|
||||
}
|
||||
|
||||
@ -1101,8 +1099,8 @@ public boolean getUsePortForNodeName() {
|
||||
public void setResourceComparator(
|
||||
Class<? extends ResourceCalculator> resourceCalculatorClass) {
|
||||
setClass(
|
||||
RESOURCE_CALCULATOR_CLASS,
|
||||
resourceCalculatorClass,
|
||||
RESOURCE_CALCULATOR_CLASS,
|
||||
resourceCalculatorClass,
|
||||
ResourceCalculator.class);
|
||||
}
|
||||
|
||||
@ -1488,18 +1486,18 @@ public long getEnforcementWindow(String queue) {
|
||||
* Sets the <em>disable_preemption</em> property in order to indicate
|
||||
* whether or not container preemption will be disabled for the specified
|
||||
* queue.
|
||||
*
|
||||
*
|
||||
* @param queue queue path
|
||||
* @param preemptionDisabled true if preemption is disabled on queue
|
||||
*/
|
||||
public void setPreemptionDisabled(String queue, boolean preemptionDisabled) {
|
||||
setBoolean(getQueuePrefix(queue) + QUEUE_PREEMPTION_DISABLED,
|
||||
preemptionDisabled);
|
||||
preemptionDisabled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Indicates whether preemption is disabled on the specified queue.
|
||||
*
|
||||
*
|
||||
* @param queue queue path to query
|
||||
* @param defaultVal used as default if the <em>disable_preemption</em>
|
||||
* is not set in the configuration
|
||||
@ -1533,7 +1531,7 @@ public boolean getIntraQueuePreemptionDisabled(String queue,
|
||||
public Set<String> getConfiguredNodeLabels(String queuePath) {
|
||||
Set<String> configuredNodeLabels = new HashSet<String>();
|
||||
Entry<String, String> e = null;
|
||||
|
||||
|
||||
Iterator<Entry<String, String>> iter = iterator();
|
||||
while (iter.hasNext()) {
|
||||
e = iter.next();
|
||||
@ -1551,10 +1549,10 @@ public Set<String> getConfiguredNodeLabels(String queuePath) {
|
||||
configuredNodeLabels.add(labelName);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// always add NO_LABEL
|
||||
configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL);
|
||||
|
||||
|
||||
return configuredNodeLabels;
|
||||
}
|
||||
|
||||
|
||||
@ -27,11 +27,11 @@
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleAction;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleActions;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatcher;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleMatchers;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleAction;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleActions;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatcher;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleMatchers;
|
||||
|
||||
// These are generated classes - use GeneratePojos class to create them
|
||||
// if they are missing
|
||||
|
||||
@ -22,11 +22,10 @@
|
||||
import com.fasterxml.jackson.databind.node.ArrayNode;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingQueuePath;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingQueuePath;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
|
||||
public class LegacyMappingRuleToJson {
|
||||
//Legacy rule parse helper constants
|
||||
|
||||
@ -18,6 +18,17 @@
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
@ -29,18 +40,6 @@
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableList;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
|
||||
|
||||
/**
|
||||
* Maintains a list of queues as well as scheduling parameters for each queue,
|
||||
* such as guaranteed share allocations, from the fair scheduler config file.
|
||||
|
||||
@ -36,7 +36,7 @@
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
class MockQueueHierarchyBuilder {
|
||||
public class MockQueueHierarchyBuilder {
|
||||
private static final String ROOT = "root";
|
||||
private static final String QUEUE_SEP = ".";
|
||||
private List<String> queuePaths = Lists.newArrayList();
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
|
||||
@ -35,6 +35,7 @@
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
@ -24,6 +24,9 @@
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.CSMappingPlacementRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
@ -48,12 +51,12 @@
|
||||
import static junit.framework.TestCase.assertNull;
|
||||
import static junit.framework.TestCase.assertTrue;
|
||||
import static junit.framework.TestCase.fail;
|
||||
import static org.apache.hadoop.yarn.server.resourcemanager.placement.FairQueuePlacementUtils.DOT;
|
||||
import static org.mockito.ArgumentMatchers.isNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class TestCSMappingPlacementRule {
|
||||
public static final String DOT = ".";
|
||||
private static final Logger LOG = LoggerFactory
|
||||
.getLogger(TestCSMappingPlacementRule.class);
|
||||
|
||||
@ -16,13 +16,14 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMappingRule {
|
||||
@ -16,7 +16,8 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
@ -16,10 +16,11 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import junit.framework.TestCase;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
@ -16,9 +16,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MockQueueHierarchyBuilder;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
|
||||
import org.junit.Test;
|
||||
|
||||
@ -16,9 +16,10 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement;
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
@ -20,7 +20,7 @@
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@ -42,10 +42,10 @@ public class TestQueueMappings {
|
||||
CapacitySchedulerConfiguration.ROOT + "." + Q1;
|
||||
private final static String Q2_PATH =
|
||||
CapacitySchedulerConfiguration.ROOT + "." + Q2;
|
||||
|
||||
|
||||
private CapacityScheduler cs;
|
||||
private YarnConfiguration conf;
|
||||
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
CapacitySchedulerConfiguration csConf =
|
||||
@ -70,7 +70,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
|
||||
|
||||
LOG.info("Setup top-level queues q1 and q2");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testQueueMappingSpecifyingNotExistedQueue() {
|
||||
// if the mapping specifies a queue that does not exist, reinitialize will
|
||||
@ -85,7 +85,7 @@ public void testQueueMappingSpecifyingNotExistedQueue() {
|
||||
}
|
||||
Assert.assertTrue("queue initialization failed for non-existent q", fail);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testQueueMappingTrimSpaces() throws IOException {
|
||||
// space trimming
|
||||
|
||||
@ -23,9 +23,9 @@
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResult;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRuleResultType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResult;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRuleResultType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.VariableContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.MappingRulesDescription;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema.Rule;
|
||||
|
||||
@ -19,7 +19,7 @@
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.placement.csmappingrule.MappingRule;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user