YARN-11394. Fix hadoop-yarn-server-resourcemanager module Java Doc Errors. (#5288)
Contributed by Shilun Fan
This commit is contained in:
parent
b75ced1e5d
commit
4a23389f5e
@ -615,12 +615,12 @@ public boolean isUntrackedNode(String hostName) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Refresh the nodes gracefully
|
* Refresh the nodes gracefully.
|
||||||
*
|
*
|
||||||
* @param yarnConf
|
* @param yarnConf yarn configuration.
|
||||||
* @param timeout decommission timeout, null means default timeout.
|
* @param timeout decommission timeout, null means default timeout.
|
||||||
* @throws IOException
|
* @throws IOException io error occur.
|
||||||
* @throws YarnException
|
* @throws YarnException exceptions from yarn servers.
|
||||||
*/
|
*/
|
||||||
public void refreshNodesGracefully(Configuration yarnConf, Integer timeout)
|
public void refreshNodesGracefully(Configuration yarnConf, Integer timeout)
|
||||||
throws IOException, YarnException {
|
throws IOException, YarnException {
|
||||||
@ -682,6 +682,9 @@ private int readDecommissioningTimeout(Configuration pConf) {
|
|||||||
/**
|
/**
|
||||||
* A NodeId instance needed upon startup for populating inactive nodes Map.
|
* A NodeId instance needed upon startup for populating inactive nodes Map.
|
||||||
* It only knows the hostname/ip and marks the port to -1 or invalid.
|
* It only knows the hostname/ip and marks the port to -1 or invalid.
|
||||||
|
*
|
||||||
|
* @param host host name.
|
||||||
|
* @return node id.
|
||||||
*/
|
*/
|
||||||
public static NodeId createUnknownNodeId(String host) {
|
public static NodeId createUnknownNodeId(String host) {
|
||||||
return NodeId.newInstance(host, -1);
|
return NodeId.newInstance(host, -1);
|
||||||
|
@ -489,6 +489,7 @@ public static void logFailure(String user, String operation, String perm,
|
|||||||
* @param description Some additional information as to why the operation
|
* @param description Some additional information as to why the operation
|
||||||
* failed.
|
* failed.
|
||||||
* @param appId ApplicationId in which operation was performed.
|
* @param appId ApplicationId in which operation was performed.
|
||||||
|
* @param attemptId Application Attempt Id in which operation was performed.
|
||||||
*
|
*
|
||||||
* <br><br>
|
* <br><br>
|
||||||
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
|
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
|
||||||
|
@ -43,7 +43,8 @@ public class RMSecretManagerService extends AbstractService {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Construct the service.
|
* Construct the service.
|
||||||
*
|
* @param conf Configuration.
|
||||||
|
* @param rmContext RMContext.
|
||||||
*/
|
*/
|
||||||
public RMSecretManagerService(Configuration conf, RMContextImpl rmContext) {
|
public RMSecretManagerService(Configuration conf, RMContextImpl rmContext) {
|
||||||
super(RMSecretManagerService.class.getName());
|
super(RMSecretManagerService.class.getName());
|
||||||
|
@ -241,7 +241,15 @@ private static String validateContainerIdAndVersion(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility method to validate a list resource requests, by ensuring that the
|
* Utility method to validate a list resource requests, by ensuring that the
|
||||||
* requested memory/vcore is non-negative and not greater than max
|
* requested memory/vcore is non-negative and not greater than max.
|
||||||
|
*
|
||||||
|
* @param ask resource request.
|
||||||
|
* @param maximumAllocation Maximum Allocation.
|
||||||
|
* @param queueName queue name.
|
||||||
|
* @param scheduler YarnScheduler.
|
||||||
|
* @param rmContext RMContext.
|
||||||
|
* @param nodeLabelsEnabled the node labels feature enabled.
|
||||||
|
* @throws InvalidResourceRequestException when there is invalid request.
|
||||||
*/
|
*/
|
||||||
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
|
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
|
||||||
Resource maximumAllocation, String queueName, YarnScheduler scheduler,
|
Resource maximumAllocation, String queueName, YarnScheduler scheduler,
|
||||||
@ -264,9 +272,13 @@ public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Validate increase/decrease request.
|
* Validate increase/decrease request.
|
||||||
|
*
|
||||||
* <pre>
|
* <pre>
|
||||||
* - Throw exception when any other error happens
|
* - Throw exception when any other error happens
|
||||||
* </pre>
|
* </pre>
|
||||||
|
* @param request SchedContainerChangeRequest.
|
||||||
|
* @param increase true, add container; false, decrease container.
|
||||||
|
* @throws InvalidResourceRequestException when there is invalid request.
|
||||||
*/
|
*/
|
||||||
public static void checkSchedContainerChangeRequest(
|
public static void checkSchedContainerChangeRequest(
|
||||||
SchedContainerChangeRequest request, boolean increase)
|
SchedContainerChangeRequest request, boolean increase)
|
||||||
@ -362,6 +374,7 @@ private static boolean validateIncreaseDecreaseRequest(RMContext rmContext,
|
|||||||
* application master.
|
* application master.
|
||||||
* @param appAttemptId Application attempt Id
|
* @param appAttemptId Application attempt Id
|
||||||
* @throws InvalidContainerReleaseException
|
* @throws InvalidContainerReleaseException
|
||||||
|
* an Application Master tries to release containers not belonging to it using.
|
||||||
*/
|
*/
|
||||||
public static void
|
public static void
|
||||||
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
|
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
|
||||||
@ -394,7 +407,7 @@ public static UserGroupInformation verifyAdminAccess(
|
|||||||
* @param module like AdminService or NodeLabelManager
|
* @param module like AdminService or NodeLabelManager
|
||||||
* @param LOG the logger to use
|
* @param LOG the logger to use
|
||||||
* @return {@link UserGroupInformation} of the current user
|
* @return {@link UserGroupInformation} of the current user
|
||||||
* @throws IOException
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public static UserGroupInformation verifyAdminAccess(
|
public static UserGroupInformation verifyAdminAccess(
|
||||||
YarnAuthorizationProvider authorizer, String method, String module,
|
YarnAuthorizationProvider authorizer, String method, String module,
|
||||||
@ -509,7 +522,9 @@ public static YarnApplicationAttemptState createApplicationAttemptState(
|
|||||||
/**
|
/**
|
||||||
* Find all configs whose name starts with
|
* Find all configs whose name starts with
|
||||||
* YarnConfiguration.RM_PROXY_USER_PREFIX, and add a record for each one by
|
* YarnConfiguration.RM_PROXY_USER_PREFIX, and add a record for each one by
|
||||||
* replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER
|
* replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER.
|
||||||
|
*
|
||||||
|
* @param conf Configuration.
|
||||||
*/
|
*/
|
||||||
public static void processRMProxyUsersConf(Configuration conf) {
|
public static void processRMProxyUsersConf(Configuration conf) {
|
||||||
Map<String, String> rmProxyUsers = new HashMap<String, String>();
|
Map<String, String> rmProxyUsers = new HashMap<String, String>();
|
||||||
|
@ -1589,6 +1589,7 @@ private boolean isOpportunisticSchedulingEnabled(Configuration conf) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Create RMDelegatedNodeLabelsUpdater based on configuration.
|
* Create RMDelegatedNodeLabelsUpdater based on configuration.
|
||||||
|
* @return RMDelegatedNodeLabelsUpdater.
|
||||||
*/
|
*/
|
||||||
protected RMDelegatedNodeLabelsUpdater createRMDelegatedNodeLabelsUpdater() {
|
protected RMDelegatedNodeLabelsUpdater createRMDelegatedNodeLabelsUpdater() {
|
||||||
if (conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED,
|
if (conf.getBoolean(YarnConfiguration.NODE_LABELS_ENABLED,
|
||||||
@ -1745,9 +1746,9 @@ private void setSchedulerRecoveryStartAndWaitTime(RMState state,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Retrieve RM bind address from configuration
|
* Retrieve RM bind address from configuration.
|
||||||
*
|
*
|
||||||
* @param conf
|
* @param conf Configuration.
|
||||||
* @return InetSocketAddress
|
* @return InetSocketAddress
|
||||||
*/
|
*/
|
||||||
public static InetSocketAddress getBindAddress(Configuration conf) {
|
public static InetSocketAddress getBindAddress(Configuration conf) {
|
||||||
@ -1758,8 +1759,8 @@ public static InetSocketAddress getBindAddress(Configuration conf) {
|
|||||||
/**
|
/**
|
||||||
* Deletes the RMStateStore
|
* Deletes the RMStateStore
|
||||||
*
|
*
|
||||||
* @param conf
|
* @param conf Configuration.
|
||||||
* @throws Exception
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
static void deleteRMStateStore(Configuration conf) throws Exception {
|
static void deleteRMStateStore(Configuration conf) throws Exception {
|
||||||
|
@ -194,8 +194,8 @@ protected void serviceInit(Configuration conf) throws Exception {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Load DynamicResourceConfiguration from dynamic-resources.xml.
|
* Load DynamicResourceConfiguration from dynamic-resources.xml.
|
||||||
* @param conf
|
* @param conf Configuration.
|
||||||
* @throws IOException
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public void loadDynamicResourceConfiguration(Configuration conf)
|
public void loadDynamicResourceConfiguration(Configuration conf)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
@ -219,7 +219,7 @@ public void loadDynamicResourceConfiguration(Configuration conf)
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Update DynamicResourceConfiguration with new configuration.
|
* Update DynamicResourceConfiguration with new configuration.
|
||||||
* @param conf
|
* @param conf DynamicResourceConfiguration.
|
||||||
*/
|
*/
|
||||||
public void updateDynamicResourceConfiguration(
|
public void updateDynamicResourceConfiguration(
|
||||||
DynamicResourceConfiguration conf) {
|
DynamicResourceConfiguration conf) {
|
||||||
|
@ -144,6 +144,7 @@ private static void deductPreemptableResourcePerApp(
|
|||||||
* stop preempt container when: all major resource type
|
* stop preempt container when: all major resource type
|
||||||
* {@literal <=} 0 for to-preempt.
|
* {@literal <=} 0 for to-preempt.
|
||||||
* This is default preemption behavior of inter-queue preemption
|
* This is default preemption behavior of inter-queue preemption
|
||||||
|
* @param curCandidates RMContainer Set.
|
||||||
* @return should we preempt rmContainer. If we should, deduct from
|
* @return should we preempt rmContainer. If we should, deduct from
|
||||||
* <code>resourceToObtainByPartition</code>
|
* <code>resourceToObtainByPartition</code>
|
||||||
*/
|
*/
|
||||||
|
@ -42,13 +42,22 @@ public class PreemptableResourceCalculator
|
|||||||
LoggerFactory.getLogger(PreemptableResourceCalculator.class);
|
LoggerFactory.getLogger(PreemptableResourceCalculator.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* PreemptableResourceCalculator constructor
|
* PreemptableResourceCalculator constructor.
|
||||||
*
|
*
|
||||||
* @param preemptionContext
|
* @param preemptionContext context.
|
||||||
* @param isReservedPreemptionCandidatesSelector this will be set by
|
* @param isReservedPreemptionCandidatesSelector this will be set by
|
||||||
* different implementation of candidate selectors, please refer to
|
* different implementation of candidate selectors, please refer to
|
||||||
* TempQueuePerPartition#offer for details.
|
* TempQueuePerPartition#offer for details.
|
||||||
* @param allowQueuesBalanceAfterAllQueuesSatisfied
|
* @param allowQueuesBalanceAfterAllQueuesSatisfied
|
||||||
|
* Should resources be preempted from an over-served queue when the
|
||||||
|
* requesting queues are all at or over their guarantees?
|
||||||
|
* An example is, there're 10 queues under root, guaranteed resource
|
||||||
|
* of them are all 10%.
|
||||||
|
* Assume there're two queues are using resources, queueA uses 10%
|
||||||
|
* queueB uses 90%. For all queues are guaranteed, but it's not fair
|
||||||
|
* for queueA.
|
||||||
|
* We wanna make this behavior can be configured. By default it is
|
||||||
|
* not allowed.
|
||||||
*/
|
*/
|
||||||
public PreemptableResourceCalculator(
|
public PreemptableResourceCalculator(
|
||||||
CapacitySchedulerPreemptionContext preemptionContext,
|
CapacitySchedulerPreemptionContext preemptionContext,
|
||||||
|
@ -272,9 +272,11 @@ private void replaceNodeToAttribute(String nodeHost, String prefix,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param nodeAttributeMapping
|
* Validate for attributes.
|
||||||
* @param newAttributesToBeAdded
|
*
|
||||||
* @param isRemoveOperation : to indicate whether its a remove operation.
|
* @param nodeAttributeMapping NodeAttribute Mapping
|
||||||
|
* @param newAttributesToBeAdded new Attributes ToBeAdded
|
||||||
|
* @param isRemoveOperation : to indicate whether it's a remove operation.
|
||||||
* @return Map of String to Map of NodeAttribute to AttributeValue
|
* @return Map of String to Map of NodeAttribute to AttributeValue
|
||||||
* @throws IOException : on invalid mapping in the current request or against
|
* @throws IOException : on invalid mapping in the current request or against
|
||||||
* already existing NodeAttributes.
|
* already existing NodeAttributes.
|
||||||
@ -329,12 +331,13 @@ protected Map<String, Map<NodeAttribute, AttributeValue>> validate(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* Validate For AttributeType Mismatch.
|
||||||
*
|
*
|
||||||
* @param isRemoveOperation
|
* @param isRemoveOperation to indicate whether it's a remove operation.
|
||||||
* @param attribute
|
* @param attribute NodeAttribute.
|
||||||
* @param newAttributes
|
* @param newAttributes new Attributes.
|
||||||
* @return Whether its a new Attribute added
|
* @return Whether it's a new Attribute added
|
||||||
* @throws IOException
|
* @throws IOException an I/O exception of some sort has occurred.
|
||||||
*/
|
*/
|
||||||
private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation,
|
private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation,
|
||||||
NodeAttribute attribute,
|
NodeAttribute attribute,
|
||||||
|
@ -99,7 +99,8 @@ protected void serviceStart() throws Exception {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Terminate the timer.
|
* Terminate the timer.
|
||||||
* @throws Exception
|
*
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
protected void serviceStop() throws Exception {
|
protected void serviceStop() throws Exception {
|
||||||
|
@ -46,6 +46,8 @@ private PlacementFactory() {
|
|||||||
* @param ruleStr The name of the class to instantiate
|
* @param ruleStr The name of the class to instantiate
|
||||||
* @param conf The configuration object to set for the rule
|
* @param conf The configuration object to set for the rule
|
||||||
* @return Created class instance
|
* @return Created class instance
|
||||||
|
* @throws ClassNotFoundException
|
||||||
|
* no definition for the class with the specified name could be found.
|
||||||
*/
|
*/
|
||||||
public static PlacementRule getPlacementRule(String ruleStr,
|
public static PlacementRule getPlacementRule(String ruleStr,
|
||||||
Configuration conf)
|
Configuration conf)
|
||||||
|
@ -36,6 +36,7 @@ public abstract class PlacementRule {
|
|||||||
/**
|
/**
|
||||||
* Set the config based on the passed in argument. This construct is used to
|
* Set the config based on the passed in argument. This construct is used to
|
||||||
* not pollute this abstract class with implementation specific references.
|
* not pollute this abstract class with implementation specific references.
|
||||||
|
* @param initArg initialization arguments.
|
||||||
*/
|
*/
|
||||||
public void setConfig(Object initArg) {
|
public void setConfig(Object initArg) {
|
||||||
// Default is a noop
|
// Default is a noop
|
||||||
|
@ -791,7 +791,9 @@ public ProxyCAState getProxyCAState() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Dispatcher used to send state operation completion events to
|
* Dispatcher used to send state operation completion events to
|
||||||
* ResourceManager services
|
* ResourceManager services.
|
||||||
|
*
|
||||||
|
* @param dispatcher Dispatcher.
|
||||||
*/
|
*/
|
||||||
public void setRMDispatcher(Dispatcher dispatcher) {
|
public void setRMDispatcher(Dispatcher dispatcher) {
|
||||||
this.rmDispatcher = dispatcher;
|
this.rmDispatcher = dispatcher;
|
||||||
@ -827,13 +829,18 @@ protected void serviceStart() throws Exception {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Derived classes initialize themselves using this method.
|
* Derived classes initialize themselves using this method.
|
||||||
|
*
|
||||||
|
* @param conf Configuration.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void initInternal(Configuration conf) throws Exception;
|
protected abstract void initInternal(Configuration conf) throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Derived classes start themselves using this method.
|
* Derived classes start themselves using this method.
|
||||||
* The base class is started and the event dispatcher is ready to use at
|
* The base class is started and the event dispatcher is ready to use at
|
||||||
* this point
|
* this point.
|
||||||
|
*
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void startInternal() throws Exception;
|
protected abstract void startInternal() throws Exception;
|
||||||
|
|
||||||
@ -846,7 +853,9 @@ protected void serviceStop() throws Exception {
|
|||||||
/**
|
/**
|
||||||
* Derived classes close themselves using this method.
|
* Derived classes close themselves using this method.
|
||||||
* The base class will be closed and the event dispatcher will be shutdown
|
* The base class will be closed and the event dispatcher will be shutdown
|
||||||
* after this
|
* after this.
|
||||||
|
*
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void closeInternal() throws Exception;
|
protected abstract void closeInternal() throws Exception;
|
||||||
|
|
||||||
@ -860,6 +869,8 @@ protected void serviceStop() throws Exception {
|
|||||||
* 5) Within a major upgrade, say 1.2 to 2.0:
|
* 5) Within a major upgrade, say 1.2 to 2.0:
|
||||||
* throw exception and indicate user to use a separate upgrade tool to
|
* throw exception and indicate user to use a separate upgrade tool to
|
||||||
* upgrade RM state.
|
* upgrade RM state.
|
||||||
|
*
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
public void checkVersion() throws Exception {
|
public void checkVersion() throws Exception {
|
||||||
Version loadedVersion = loadVersion();
|
Version loadedVersion = loadVersion();
|
||||||
@ -884,22 +895,28 @@ public void checkVersion() throws Exception {
|
|||||||
/**
|
/**
|
||||||
* Derived class use this method to load the version information from state
|
* Derived class use this method to load the version information from state
|
||||||
* store.
|
* store.
|
||||||
|
* @throws Exception error occur.
|
||||||
|
* @return current version.
|
||||||
*/
|
*/
|
||||||
protected abstract Version loadVersion() throws Exception;
|
protected abstract Version loadVersion() throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Derived class use this method to store the version information.
|
* Derived class use this method to store the version information.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeVersion() throws Exception;
|
protected abstract void storeVersion() throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the current version of the underlying state store.
|
* Get the current version of the underlying state store.
|
||||||
|
* @return current version.
|
||||||
*/
|
*/
|
||||||
protected abstract Version getCurrentVersion();
|
protected abstract Version getCurrentVersion();
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the current epoch of RM and increment the value.
|
* Get the current epoch of RM and increment the value.
|
||||||
|
* @throws Exception error occur.
|
||||||
|
* @return current epoch.
|
||||||
*/
|
*/
|
||||||
public abstract long getAndIncrementEpoch() throws Exception;
|
public abstract long getAndIncrementEpoch() throws Exception;
|
||||||
|
|
||||||
@ -907,6 +924,9 @@ public void checkVersion() throws Exception {
|
|||||||
* Compute the next epoch value by incrementing by one.
|
* Compute the next epoch value by incrementing by one.
|
||||||
* Wraps around if the epoch range is exceeded so that
|
* Wraps around if the epoch range is exceeded so that
|
||||||
* when federation is enabled epoch collisions can be avoided.
|
* when federation is enabled epoch collisions can be avoided.
|
||||||
|
*
|
||||||
|
* @param epoch epoch value.
|
||||||
|
* @return next epoch value.
|
||||||
*/
|
*/
|
||||||
protected long nextEpoch(long epoch){
|
protected long nextEpoch(long epoch){
|
||||||
long epochVal = epoch - baseEpoch + 1;
|
long epochVal = epoch - baseEpoch + 1;
|
||||||
@ -920,7 +940,9 @@ protected long nextEpoch(long epoch){
|
|||||||
* Blocking API
|
* Blocking API
|
||||||
* The derived class must recover state from the store and return a new
|
* The derived class must recover state from the store and return a new
|
||||||
* RMState object populated with that state
|
* RMState object populated with that state
|
||||||
* This must not be called on the dispatcher thread
|
* This must not be called on the dispatcher thread.
|
||||||
|
* @throws Exception error occur.
|
||||||
|
* @return RMState.
|
||||||
*/
|
*/
|
||||||
public abstract RMState loadState() throws Exception;
|
public abstract RMState loadState() throws Exception;
|
||||||
|
|
||||||
@ -928,7 +950,9 @@ protected long nextEpoch(long epoch){
|
|||||||
* Non-Blocking API
|
* Non-Blocking API
|
||||||
* ResourceManager services use this to store the application's state
|
* ResourceManager services use this to store the application's state
|
||||||
* This does not block the dispatcher threads
|
* This does not block the dispatcher threads
|
||||||
* RMAppStoredEvent will be sent on completion to notify the RMApp
|
* RMAppStoredEvent will be sent on completion to notify the RMApp.
|
||||||
|
*
|
||||||
|
* @param app rmApp.
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public void storeNewApplication(RMApp app) {
|
public void storeNewApplication(RMApp app) {
|
||||||
@ -969,6 +993,10 @@ public void updateFencedState() {
|
|||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the state of an
|
* Derived classes must implement this method to store the state of an
|
||||||
* application.
|
* application.
|
||||||
|
*
|
||||||
|
* @param appId application Id.
|
||||||
|
* @param appStateData application StateData.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeApplicationStateInternal(ApplicationId appId,
|
protected abstract void storeApplicationStateInternal(ApplicationId appId,
|
||||||
ApplicationStateData appStateData) throws Exception;
|
ApplicationStateData appStateData) throws Exception;
|
||||||
@ -981,7 +1009,9 @@ protected abstract void updateApplicationStateInternal(ApplicationId appId,
|
|||||||
* Non-blocking API
|
* Non-blocking API
|
||||||
* ResourceManager services call this to store state on an application attempt
|
* ResourceManager services call this to store state on an application attempt
|
||||||
* This does not block the dispatcher threads
|
* This does not block the dispatcher threads
|
||||||
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
|
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt.
|
||||||
|
*
|
||||||
|
* @param appAttempt RM AppAttempt.
|
||||||
*/
|
*/
|
||||||
public void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
|
public void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
|
||||||
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
|
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
|
||||||
@ -1011,7 +1041,11 @@ public void updateApplicationAttemptState(
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the state of an
|
* Derived classes must implement this method to store the state of an
|
||||||
* application attempt
|
* application attempt.
|
||||||
|
*
|
||||||
|
* @param attemptId Application AttemptId.
|
||||||
|
* @param attemptStateData Application AttemptStateData.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeApplicationAttemptStateInternal(
|
protected abstract void storeApplicationAttemptStateInternal(
|
||||||
ApplicationAttemptId attemptId,
|
ApplicationAttemptId attemptId,
|
||||||
@ -1023,7 +1057,10 @@ protected abstract void updateApplicationAttemptStateInternal(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* RMDTSecretManager call this to store the state of a delegation token
|
* RMDTSecretManager call this to store the state of a delegation token
|
||||||
* and sequence number
|
* and sequence number.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
|
* @param renewDate token renew date.
|
||||||
*/
|
*/
|
||||||
public void storeRMDelegationToken(
|
public void storeRMDelegationToken(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
|
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
|
||||||
@ -1034,14 +1071,20 @@ public void storeRMDelegationToken(
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the state of
|
* Derived classes must implement this method to store the state of
|
||||||
* RMDelegationToken and sequence number
|
* RMDelegationToken and sequence number.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
|
* @param renewDate token renew date.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeRMDelegationTokenState(
|
protected abstract void storeRMDelegationTokenState(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
|
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RMDTSecretManager call this to remove the state of a delegation token
|
* RMDTSecretManager call this to remove the state of a delegation token.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
*/
|
*/
|
||||||
public void removeRMDelegationToken(
|
public void removeRMDelegationToken(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier) {
|
RMDelegationTokenIdentifier rmDTIdentifier) {
|
||||||
@ -1051,14 +1094,20 @@ public void removeRMDelegationToken(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to remove the state of RMDelegationToken
|
* Derived classes must implement this method to remove the state of RMDelegationToken.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
|
* @throws Exception error occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void removeRMDelegationTokenState(
|
protected abstract void removeRMDelegationTokenState(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier) throws Exception;
|
RMDelegationTokenIdentifier rmDTIdentifier) throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RMDTSecretManager call this to update the state of a delegation token
|
* RMDTSecretManager call this to update the state of a delegation token
|
||||||
* and sequence number
|
* and sequence number.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
|
* @param renewDate token renew date.
|
||||||
*/
|
*/
|
||||||
public void updateRMDelegationToken(
|
public void updateRMDelegationToken(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
|
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate) {
|
||||||
@ -1069,14 +1118,20 @@ public void updateRMDelegationToken(
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to update the state of
|
* Derived classes must implement this method to update the state of
|
||||||
* RMDelegationToken and sequence number
|
* RMDelegationToken and sequence number.
|
||||||
|
*
|
||||||
|
* @param rmDTIdentifier RMDelegationTokenIdentifier.
|
||||||
|
* @param renewDate token renew date.
|
||||||
|
* @throws Exception error occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void updateRMDelegationTokenState(
|
protected abstract void updateRMDelegationTokenState(
|
||||||
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
|
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RMDTSecretManager call this to store the state of a master key
|
* RMDTSecretManager call this to store the state of a master key.
|
||||||
|
*
|
||||||
|
* @param delegationKey DelegationToken Master Key.
|
||||||
*/
|
*/
|
||||||
public void storeRMDTMasterKey(DelegationKey delegationKey) {
|
public void storeRMDTMasterKey(DelegationKey delegationKey) {
|
||||||
handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
|
handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
|
||||||
@ -1086,13 +1141,18 @@ public void storeRMDTMasterKey(DelegationKey delegationKey) {
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the state of
|
* Derived classes must implement this method to store the state of
|
||||||
* DelegationToken Master Key
|
* DelegationToken Master Key.
|
||||||
|
*
|
||||||
|
* @param delegationKey DelegationToken Master Key.
|
||||||
|
* @throws Exception error occur.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeRMDTMasterKeyState(DelegationKey delegationKey)
|
protected abstract void storeRMDTMasterKeyState(DelegationKey delegationKey)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RMDTSecretManager call this to remove the state of a master key
|
* RMDTSecretManager call this to remove the state of a master key.
|
||||||
|
*
|
||||||
|
* @param delegationKey DelegationToken Master Key.
|
||||||
*/
|
*/
|
||||||
public void removeRMDTMasterKey(DelegationKey delegationKey) {
|
public void removeRMDTMasterKey(DelegationKey delegationKey) {
|
||||||
handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
|
handleStoreEvent(new RMStateStoreRMDTMasterKeyEvent(delegationKey,
|
||||||
@ -1101,6 +1161,10 @@ public void removeRMDTMasterKey(DelegationKey delegationKey) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Blocking Apis to maintain reservation state.
|
* Blocking Apis to maintain reservation state.
|
||||||
|
*
|
||||||
|
* @param reservationAllocation reservation Allocation.
|
||||||
|
* @param planName plan Name.
|
||||||
|
* @param reservationIdName reservationId Name.
|
||||||
*/
|
*/
|
||||||
public void storeNewReservation(
|
public void storeNewReservation(
|
||||||
ReservationAllocationStateProto reservationAllocation, String planName,
|
ReservationAllocationStateProto reservationAllocation, String planName,
|
||||||
@ -1120,6 +1184,11 @@ public void removeReservation(String planName, String reservationIdName) {
|
|||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the state of
|
* Derived classes must implement this method to store the state of
|
||||||
* a reservation allocation.
|
* a reservation allocation.
|
||||||
|
*
|
||||||
|
* @param reservationAllocation reservation Allocation.
|
||||||
|
* @param planName plan Name.
|
||||||
|
* @param reservationIdName reservationId Name.
|
||||||
|
* @throws Exception error occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeReservationState(
|
protected abstract void storeReservationState(
|
||||||
ReservationAllocationStateProto reservationAllocation, String planName,
|
ReservationAllocationStateProto reservationAllocation, String planName,
|
||||||
@ -1129,6 +1198,10 @@ protected abstract void storeReservationState(
|
|||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to remove the state of
|
* Derived classes must implement this method to remove the state of
|
||||||
* a reservation allocation.
|
* a reservation allocation.
|
||||||
|
*
|
||||||
|
* @param planName plan Name.
|
||||||
|
* @param reservationIdName reservationId Name.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void removeReservationState(String planName,
|
protected abstract void removeReservationState(String planName,
|
||||||
String reservationIdName) throws Exception;
|
String reservationIdName) throws Exception;
|
||||||
@ -1136,21 +1209,31 @@ protected abstract void removeReservationState(String planName,
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to remove the state of
|
* Derived classes must implement this method to remove the state of
|
||||||
* DelegationToken Master Key
|
* DelegationToken Master Key.
|
||||||
|
*
|
||||||
|
* @param delegationKey DelegationKey.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void removeRMDTMasterKeyState(DelegationKey delegationKey)
|
protected abstract void removeRMDTMasterKeyState(DelegationKey delegationKey)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Blocking API Derived classes must implement this method to store or update
|
* Blocking API Derived classes must implement this method to store or update
|
||||||
* the state of AMRMToken Master Key
|
* the state of AMRMToken Master Key.
|
||||||
|
*
|
||||||
|
* @param amrmTokenSecretManagerState amrmTokenSecretManagerState.
|
||||||
|
* @param isUpdate true, update; otherwise not update.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeOrUpdateAMRMTokenSecretManagerState(
|
protected abstract void storeOrUpdateAMRMTokenSecretManagerState(
|
||||||
AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate)
|
AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Store or Update state of AMRMToken Master Key
|
* Store or Update state of AMRMToken Master Key.
|
||||||
|
*
|
||||||
|
* @param amrmTokenSecretManagerState amrmTokenSecretManagerState.
|
||||||
|
* @param isUpdate true, update; otherwise not update.
|
||||||
*/
|
*/
|
||||||
public void storeOrUpdateAMRMTokenSecretManager(
|
public void storeOrUpdateAMRMTokenSecretManager(
|
||||||
AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate) {
|
AMRMTokenSecretManagerState amrmTokenSecretManagerState, boolean isUpdate) {
|
||||||
@ -1165,6 +1248,8 @@ public void storeOrUpdateAMRMTokenSecretManager(
|
|||||||
* store
|
* store
|
||||||
* This does not block the dispatcher threads
|
* This does not block the dispatcher threads
|
||||||
* There is no notification of completion for this operation.
|
* There is no notification of completion for this operation.
|
||||||
|
*
|
||||||
|
* @param app RMApp.
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public void removeApplication(RMApp app) {
|
public void removeApplication(RMApp app) {
|
||||||
@ -1183,7 +1268,10 @@ public void removeApplication(RMApp app) {
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to remove the state of an
|
* Derived classes must implement this method to remove the state of an
|
||||||
* application and its attempts
|
* application and its attempts.
|
||||||
|
*
|
||||||
|
* @param appState ApplicationStateData.
|
||||||
|
* @throws Exception error occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void removeApplicationStateInternal(
|
protected abstract void removeApplicationStateInternal(
|
||||||
ApplicationStateData appState) throws Exception;
|
ApplicationStateData appState) throws Exception;
|
||||||
@ -1194,6 +1282,8 @@ protected abstract void removeApplicationStateInternal(
|
|||||||
* store
|
* store
|
||||||
* This does not block the dispatcher threads
|
* This does not block the dispatcher threads
|
||||||
* There is no notification of completion for this operation.
|
* There is no notification of completion for this operation.
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId applicationAttemptId.
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public synchronized void removeApplicationAttempt(
|
public synchronized void removeApplicationAttempt(
|
||||||
@ -1206,6 +1296,8 @@ public synchronized void removeApplicationAttempt(
|
|||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to remove the state of specified
|
* Derived classes must implement this method to remove the state of specified
|
||||||
* attempt.
|
* attempt.
|
||||||
|
* @param attemptId application attempt id.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void removeApplicationAttemptInternal(
|
protected abstract void removeApplicationAttemptInternal(
|
||||||
ApplicationAttemptId attemptId) throws Exception;
|
ApplicationAttemptId attemptId) throws Exception;
|
||||||
@ -1316,7 +1408,7 @@ private void notifyApplicationAttempt(RMAppAttemptEvent event) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* EventHandler implementation which forward events to the FSRMStateStore
|
* EventHandler implementation which forward events to the FSRMStateStore
|
||||||
* This hides the EventHandle methods of the store from its public interface
|
* This hides the EventHandle methods of the store from its public interface
|
||||||
*/
|
*/
|
||||||
private final class ForwardingEventHandler
|
private final class ForwardingEventHandler
|
||||||
implements EventHandler<RMStateStoreEvent> {
|
implements EventHandler<RMStateStoreEvent> {
|
||||||
@ -1328,16 +1420,18 @@ public void handle(RMStateStoreEvent event) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Derived classes must implement this method to delete the state store
|
* Derived classes must implement this method to delete the state store.
|
||||||
* @throws Exception
|
*
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
public abstract void deleteStore() throws Exception;
|
public abstract void deleteStore() throws Exception;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Derived classes must implement this method to remove application from the
|
* Derived classes must implement this method to remove application from the
|
||||||
* state store
|
* state store.
|
||||||
*
|
*
|
||||||
* @throws Exception
|
* @param removeAppId application Id.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
public abstract void removeApplication(ApplicationId removeAppId)
|
public abstract void removeApplication(ApplicationId removeAppId)
|
||||||
throws Exception;
|
throws Exception;
|
||||||
@ -1362,6 +1456,8 @@ protected EventHandler getRMStateStoreEventHandler() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* ProxyCAManager calls this to store the CA Certificate and Private Key.
|
* ProxyCAManager calls this to store the CA Certificate and Private Key.
|
||||||
|
* @param caCert X509Certificate.
|
||||||
|
* @param caPrivateKey PrivateKey.
|
||||||
*/
|
*/
|
||||||
public void storeProxyCACert(X509Certificate caCert,
|
public void storeProxyCACert(X509Certificate caCert,
|
||||||
PrivateKey caPrivateKey) {
|
PrivateKey caPrivateKey) {
|
||||||
@ -1372,7 +1468,11 @@ public void storeProxyCACert(X509Certificate caCert,
|
|||||||
/**
|
/**
|
||||||
* Blocking API
|
* Blocking API
|
||||||
* Derived classes must implement this method to store the CA Certificate
|
* Derived classes must implement this method to store the CA Certificate
|
||||||
* and Private Key
|
* and Private Key.
|
||||||
|
*
|
||||||
|
* @param caCert X509Certificate.
|
||||||
|
* @param caPrivateKey PrivateKey.
|
||||||
|
* @throws Exception error occurs.
|
||||||
*/
|
*/
|
||||||
protected abstract void storeProxyCACertState(
|
protected abstract void storeProxyCACertState(
|
||||||
X509Certificate caCert, PrivateKey caPrivateKey) throws Exception;
|
X509Certificate caCert, PrivateKey caPrivateKey) throws Exception;
|
||||||
|
@ -48,6 +48,7 @@ public class RMStateStoreUtils {
|
|||||||
* @param fsIn The {@link DataInputStream} containing RM Delegation Token data
|
* @param fsIn The {@link DataInputStream} containing RM Delegation Token data
|
||||||
* @return An {@link RMDelegationTokenIdentifierData} containing the read in
|
* @return An {@link RMDelegationTokenIdentifierData} containing the read in
|
||||||
* RM Delegation Token
|
* RM Delegation Token
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public static RMDelegationTokenIdentifierData
|
public static RMDelegationTokenIdentifierData
|
||||||
readRMDelegationTokenIdentifierData(DataInputStream fsIn)
|
readRMDelegationTokenIdentifierData(DataInputStream fsIn)
|
||||||
|
@ -16,6 +16,9 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This package contains classes to recovery the resourcemanager.
|
||||||
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
|
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
|
||||||
|
@ -51,7 +51,8 @@ public static AMRMTokenSecretManagerState newInstance(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link AMRMTokenSecretManager} current Master key
|
* {@link AMRMTokenSecretManager} current Master key.
|
||||||
|
* @return MasterKey.
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
@ -62,7 +63,8 @@ public static AMRMTokenSecretManagerState newInstance(
|
|||||||
public abstract void setCurrentMasterKey(MasterKey currentMasterKey);
|
public abstract void setCurrentMasterKey(MasterKey currentMasterKey);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link AMRMTokenSecretManager} next Master key
|
* {@link AMRMTokenSecretManager} next Master key.
|
||||||
|
* @return MasterKey.
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
|
@ -143,7 +143,7 @@ public static ApplicationAttemptStateData newInstance(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the final tracking Url of the AM.
|
* Set the final tracking Url of the AM.
|
||||||
* @param url
|
* @param url tracking url.
|
||||||
*/
|
*/
|
||||||
public abstract void setFinalTrackingUrl(String url);
|
public abstract void setFinalTrackingUrl(String url);
|
||||||
/**
|
/**
|
||||||
|
@ -192,7 +192,8 @@ public int getFirstAttemptId() {
|
|||||||
public abstract void setLaunchTime(long launchTime);
|
public abstract void setLaunchTime(long launchTime);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The application submitter
|
* The application submitter.
|
||||||
|
* @param user submitter user name.
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Unstable
|
@Unstable
|
||||||
|
@ -380,6 +380,10 @@ protected abstract List<? extends Queue> getChildReservationQueues(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Add a new reservation queue for reservation currResId for this planQueue.
|
* Add a new reservation queue for reservation currResId for this planQueue.
|
||||||
|
*
|
||||||
|
* @param planQueueName name of the reservable queue.
|
||||||
|
* @param queue the queue for the current {@link Plan}.
|
||||||
|
* @param currResId curr reservationId.
|
||||||
*/
|
*/
|
||||||
protected abstract void addReservationQueue(String planQueueName, Queue queue,
|
protected abstract void addReservationQueue(String planQueueName, Queue queue,
|
||||||
String currResId);
|
String currResId);
|
||||||
@ -399,6 +403,7 @@ protected abstract void createDefaultReservationQueue(String planQueueName,
|
|||||||
* Get plan resources for this planQueue.
|
* Get plan resources for this planQueue.
|
||||||
*
|
*
|
||||||
* @param plan the current {@link Plan} being considered
|
* @param plan the current {@link Plan} being considered
|
||||||
|
* @param queue the queue for the current {@link Plan}
|
||||||
* @param clusterResources the resources available in the cluster
|
* @param clusterResources the resources available in the cluster
|
||||||
*
|
*
|
||||||
* @return the resources allocated to the specified {@link Plan}
|
* @return the resources allocated to the specified {@link Plan}
|
||||||
|
@ -178,7 +178,6 @@ public boolean removeInterval(ReservationInterval interval,
|
|||||||
* maximum capacity.
|
* maximum capacity.
|
||||||
* @param period periodic offset at which capacities are evaluated.
|
* @param period periodic offset at which capacities are evaluated.
|
||||||
* @return the maximum {@link Resource} across the specified time instants.
|
* @return the maximum {@link Resource} across the specified time instants.
|
||||||
* @return true if removal is successful, false otherwise
|
|
||||||
*/
|
*/
|
||||||
public Resource getMaximumPeriodicCapacity(long tick, long period) {
|
public Resource getMaximumPeriodicCapacity(long tick, long period) {
|
||||||
Resource maxResource;
|
Resource maxResource;
|
||||||
|
@ -290,8 +290,10 @@ ApplicationReport createAndGetApplicationReport(String clientUserName,
|
|||||||
Map<NodeId, LogAggregationReport> getLogAggregationReportsForApp();
|
Map<NodeId, LogAggregationReport> getLogAggregationReportsForApp();
|
||||||
|
|
||||||
LogAggregationStatus getLogAggregationStatusForAppReport();
|
LogAggregationStatus getLogAggregationStatusForAppReport();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Return the node label expression of the AM container.
|
* Return the node label expression of the AM container.
|
||||||
|
* @return the node label expression.
|
||||||
*/
|
*/
|
||||||
String getAmNodeLabelExpression();
|
String getAmNodeLabelExpression();
|
||||||
|
|
||||||
|
@ -1896,10 +1896,11 @@ public Map<String, String> getApplicationSchedulingEnvs() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* catch the InvalidStateTransition.
|
* catch the InvalidStateTransition.
|
||||||
* @param state
|
*
|
||||||
* @param rmAppEventType
|
* @param state RMAppState.
|
||||||
*/
|
* @param rmAppEventType RMAppEventType.
|
||||||
|
*/
|
||||||
protected void onInvalidStateTransition(RMAppEventType rmAppEventType,
|
protected void onInvalidStateTransition(RMAppEventType rmAppEventType,
|
||||||
RMAppState state){
|
RMAppState state){
|
||||||
/* TODO fail the application on the failed transition */
|
/* TODO fail the application on the failed transition */
|
||||||
|
@ -241,6 +241,8 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
|
|||||||
* </li>
|
* </li>
|
||||||
* <li>killed by RM because of RM restart or failover.</li>
|
* <li>killed by RM because of RM restart or failover.</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
|
*
|
||||||
|
* @return attempt retry count.
|
||||||
*/
|
*/
|
||||||
boolean shouldCountTowardsMaxAttemptRetry();
|
boolean shouldCountTowardsMaxAttemptRetry();
|
||||||
|
|
||||||
@ -258,7 +260,7 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* To capture Launch diagnostics of the app.
|
* To capture Launch diagnostics of the app.
|
||||||
* @param amLaunchDiagnostics
|
* @param amLaunchDiagnostics amLaunchDiagnostics.
|
||||||
*/
|
*/
|
||||||
void updateAMLaunchDiagnostics(String amLaunchDiagnostics);
|
void updateAMLaunchDiagnostics(String amLaunchDiagnostics);
|
||||||
|
|
||||||
|
@ -922,8 +922,8 @@ public void setContainerId(ContainerId containerId) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* catch the InvalidStateTransition.
|
* catch the InvalidStateTransition.
|
||||||
* @param state
|
* @param state RMContainerState.
|
||||||
* @param rmContainerEventType
|
* @param rmContainerEventType RMContainerEventType.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected void onInvalidStateTransition(
|
protected void onInvalidStateTransition(
|
||||||
|
@ -94,7 +94,8 @@ public interface RMNode {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* the node manager version of the node received as part of the
|
* the node manager version of the node received as part of the
|
||||||
* registration with the resource manager
|
* registration with the resource manager.
|
||||||
|
* @return node manager version.
|
||||||
*/
|
*/
|
||||||
public String getNodeManagerVersion();
|
public String getNodeManagerVersion();
|
||||||
|
|
||||||
|
@ -1186,8 +1186,8 @@ public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Put a node in deactivated (decommissioned or shutdown) status.
|
* Put a node in deactivated (decommissioned or shutdown) status.
|
||||||
* @param rmNode
|
* @param rmNode RMNode.
|
||||||
* @param finalState
|
* @param finalState NodeState.
|
||||||
*/
|
*/
|
||||||
public static void deactivateNode(RMNodeImpl rmNode, NodeState finalState) {
|
public static void deactivateNode(RMNodeImpl rmNode, NodeState finalState) {
|
||||||
|
|
||||||
@ -1209,8 +1209,8 @@ public static void deactivateNode(RMNodeImpl rmNode, NodeState finalState) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Report node is RUNNING.
|
* Report node is RUNNING.
|
||||||
* @param rmNode
|
* @param rmNode RMNode.
|
||||||
* @param containers
|
* @param containers NMContainerStatus List.
|
||||||
*/
|
*/
|
||||||
public static void reportNodeRunning(RMNodeImpl rmNode,
|
public static void reportNodeRunning(RMNodeImpl rmNode,
|
||||||
List<NMContainerStatus> containers) {
|
List<NMContainerStatus> containers) {
|
||||||
@ -1225,8 +1225,8 @@ public static void reportNodeRunning(RMNodeImpl rmNode,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Report node is UNUSABLE and update metrics.
|
* Report node is UNUSABLE and update metrics.
|
||||||
* @param rmNode
|
* @param rmNode RMNode.
|
||||||
* @param finalState
|
* @param finalState NodeState.
|
||||||
*/
|
*/
|
||||||
public static void reportNodeUnusable(RMNodeImpl rmNode,
|
public static void reportNodeUnusable(RMNodeImpl rmNode,
|
||||||
NodeState finalState) {
|
NodeState finalState) {
|
||||||
|
@ -312,6 +312,7 @@ public List<Container> getTransferredContainers(
|
|||||||
* Add blacklisted NodeIds to the list that is passed.
|
* Add blacklisted NodeIds to the list that is passed.
|
||||||
*
|
*
|
||||||
* @param app application attempt.
|
* @param app application attempt.
|
||||||
|
* @return blacklisted NodeIds.
|
||||||
*/
|
*/
|
||||||
public List<N> getBlacklistedNodes(final SchedulerApplicationAttempt app) {
|
public List<N> getBlacklistedNodes(final SchedulerApplicationAttempt app) {
|
||||||
|
|
||||||
@ -836,6 +837,9 @@ public void killAllAppsInQueue(String queueName)
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Process resource update on a node.
|
* Process resource update on a node.
|
||||||
|
*
|
||||||
|
* @param nm RMNode.
|
||||||
|
* @param resourceOption resourceOption.
|
||||||
*/
|
*/
|
||||||
public void updateNodeResource(RMNode nm,
|
public void updateNodeResource(RMNode nm,
|
||||||
ResourceOption resourceOption) {
|
ResourceOption resourceOption) {
|
||||||
@ -1313,6 +1317,7 @@ protected void normalizeResourceRequests(List<ResourceRequest> asks) {
|
|||||||
* Normalize a list of resource requests
|
* Normalize a list of resource requests
|
||||||
* using queue maximum resource allocations.
|
* using queue maximum resource allocations.
|
||||||
* @param asks resource requests
|
* @param asks resource requests
|
||||||
|
* @param queueName queue Name.
|
||||||
*/
|
*/
|
||||||
protected void normalizeResourceRequests(List<ResourceRequest> asks,
|
protected void normalizeResourceRequests(List<ResourceRequest> asks,
|
||||||
String queueName) {
|
String queueName) {
|
||||||
|
@ -38,6 +38,9 @@ public class ApplicationPlacementAllocatorFactory {
|
|||||||
*
|
*
|
||||||
* @param appPlacementAllocatorName
|
* @param appPlacementAllocatorName
|
||||||
* allocator class name.
|
* allocator class name.
|
||||||
|
* @param appSchedulingInfo app SchedulingInfo.
|
||||||
|
* @param schedulerRequestKey scheduler RequestKey.
|
||||||
|
* @param rmContext RMContext.
|
||||||
* @return Specific AppPlacementAllocator instance based on type
|
* @return Specific AppPlacementAllocator instance based on type
|
||||||
*/
|
*/
|
||||||
public static AppPlacementAllocator<SchedulerNode> getAppPlacementAllocator(
|
public static AppPlacementAllocator<SchedulerNode> getAppPlacementAllocator(
|
||||||
|
@ -82,6 +82,7 @@ void confirmPendingMutation(LogMutation pendingMutation,
|
|||||||
/**
|
/**
|
||||||
* Get the last updated scheduler config version.
|
* Get the last updated scheduler config version.
|
||||||
* @return Last updated scheduler config version.
|
* @return Last updated scheduler config version.
|
||||||
|
* @throws Exception exception occurs.
|
||||||
*/
|
*/
|
||||||
long getConfigVersion() throws Exception;
|
long getConfigVersion() throws Exception;
|
||||||
|
|
||||||
|
@ -28,6 +28,9 @@ public interface NodeFilter {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Criteria to accept node in the filtered list.
|
* Criteria to accept node in the filtered list.
|
||||||
|
*
|
||||||
|
* @param node SchedulerNode.
|
||||||
|
* @return true, node accept; false, node not accept.
|
||||||
*/
|
*/
|
||||||
boolean accept(SchedulerNode node);
|
boolean accept(SchedulerNode node);
|
||||||
}
|
}
|
||||||
|
@ -42,8 +42,8 @@ public interface PreemptableResourceScheduler extends ResourceScheduler {
|
|||||||
void markContainerForPreemption(ApplicationAttemptId aid, RMContainer container);
|
void markContainerForPreemption(ApplicationAttemptId aid, RMContainer container);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ask the scheduler to forcibly interrupt the container given as input
|
* Ask the scheduler to forcibly interrupt the container given as input.
|
||||||
* @param container
|
* @param container RMContainer.
|
||||||
*/
|
*/
|
||||||
void markContainerForKillable(RMContainer container);
|
void markContainerForKillable(RMContainer container);
|
||||||
|
|
||||||
|
@ -297,7 +297,7 @@ public synchronized QueueMetrics getUserMetrics(String userName) {
|
|||||||
* QueueMetrics (B)
|
* QueueMetrics (B)
|
||||||
* metrics
|
* metrics
|
||||||
*
|
*
|
||||||
* @param partition
|
* @param partition Node Partition
|
||||||
* @return QueueMetrics
|
* @return QueueMetrics
|
||||||
*/
|
*/
|
||||||
public synchronized QueueMetrics getPartitionQueueMetrics(String partition) {
|
public synchronized QueueMetrics getPartitionQueueMetrics(String partition) {
|
||||||
@ -529,7 +529,7 @@ public void setAvailableResourcesToQueue(String partition, Resource limit) {
|
|||||||
/**
|
/**
|
||||||
* Set Available resources with support for resource vectors.
|
* Set Available resources with support for resource vectors.
|
||||||
*
|
*
|
||||||
* @param limit
|
* @param limit Resource.
|
||||||
*/
|
*/
|
||||||
public void setAvailableResources(Resource limit) {
|
public void setAvailableResources(Resource limit) {
|
||||||
availableMB.set(limit.getMemorySize());
|
availableMB.set(limit.getMemorySize());
|
||||||
@ -557,7 +557,7 @@ public void setAvailableResourcesToQueue(Resource limit) {
|
|||||||
* resources become available.
|
* resources become available.
|
||||||
*
|
*
|
||||||
* @param partition Node Partition
|
* @param partition Node Partition
|
||||||
* @param user
|
* @param user Name of the user.
|
||||||
* @param limit resource limit
|
* @param limit resource limit
|
||||||
*/
|
*/
|
||||||
public void setAvailableResourcesToUser(String partition, String user,
|
public void setAvailableResourcesToUser(String partition, String user,
|
||||||
@ -583,8 +583,8 @@ public void setAvailableResourcesToUser(String partition, String user,
|
|||||||
* Increment pending resource metrics
|
* Increment pending resource metrics
|
||||||
*
|
*
|
||||||
* @param partition Node Partition
|
* @param partition Node Partition
|
||||||
* @param user
|
* @param user Name of the user.
|
||||||
* @param containers
|
* @param containers containers count.
|
||||||
* @param res the TOTAL delta of resources note this is different from the
|
* @param res the TOTAL delta of resources note this is different from the
|
||||||
* other APIs which use per container resource
|
* other APIs which use per container resource
|
||||||
*/
|
*/
|
||||||
@ -778,8 +778,8 @@ private void computeAllocateResources(int containers, Resource res,
|
|||||||
/**
|
/**
|
||||||
* Allocate Resource for container size change.
|
* Allocate Resource for container size change.
|
||||||
* @param partition Node Partition
|
* @param partition Node Partition
|
||||||
* @param user
|
* @param user Name of the user
|
||||||
* @param res
|
* @param res Resource.
|
||||||
*/
|
*/
|
||||||
public void allocateResources(String partition, String user, Resource res) {
|
public void allocateResources(String partition, String user, Resource res) {
|
||||||
allocatedMB.incr(res.getMemorySize());
|
allocatedMB.incr(res.getMemorySize());
|
||||||
|
@ -49,7 +49,8 @@ public interface ResourceScheduler extends YarnScheduler, Recoverable {
|
|||||||
/**
|
/**
|
||||||
* Re-initialize the <code>ResourceScheduler</code>.
|
* Re-initialize the <code>ResourceScheduler</code>.
|
||||||
* @param conf configuration
|
* @param conf configuration
|
||||||
* @throws IOException
|
* @param rmContext RMContext.
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
void reinitialize(Configuration conf, RMContext rmContext) throws IOException;
|
void reinitialize(Configuration conf, RMContext rmContext) throws IOException;
|
||||||
|
|
||||||
|
@ -65,7 +65,8 @@ public RMContext getRmContext() {
|
|||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
* Delta capacity = target - before, so if it is a decrease request, delta
|
* Delta capacity = target - before, so if it is a decrease request, delta
|
||||||
* capacity will be negative
|
* capacity will be negative.
|
||||||
|
* @return delta Capacity.
|
||||||
*/
|
*/
|
||||||
public synchronized Resource getDeltaCapacity() {
|
public synchronized Resource getDeltaCapacity() {
|
||||||
// Only calculate deltaCapacity once
|
// Only calculate deltaCapacity once
|
||||||
|
@ -791,6 +791,7 @@ List<RMContainer> pullContainersToTransfer() {
|
|||||||
* Called when AM heartbeats. These containers were recovered by the RM after
|
* Called when AM heartbeats. These containers were recovered by the RM after
|
||||||
* the AM had registered. They are reported to the AM in the
|
* the AM had registered. They are reported to the AM in the
|
||||||
* <code>AllocateResponse#containersFromPreviousAttempts</code>.
|
* <code>AllocateResponse#containersFromPreviousAttempts</code>.
|
||||||
|
* @return Container List.
|
||||||
*/
|
*/
|
||||||
public List<Container> pullPreviousAttemptContainers() {
|
public List<Container> pullPreviousAttemptContainers() {
|
||||||
writeLock.lock();
|
writeLock.lock();
|
||||||
|
@ -246,6 +246,8 @@ public synchronized Resource getTotalResource() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if a container is launched by this node.
|
* Check if a container is launched by this node.
|
||||||
|
*
|
||||||
|
* @param containerId containerId.
|
||||||
* @return If the container is launched by the node.
|
* @return If the container is launched by the node.
|
||||||
*/
|
*/
|
||||||
public synchronized boolean isValidContainer(ContainerId containerId) {
|
public synchronized boolean isValidContainer(ContainerId containerId) {
|
||||||
|
@ -196,6 +196,12 @@ private static ContainerStatus createAbnormalContainerStatus(
|
|||||||
/**
|
/**
|
||||||
* Utility method to normalize a resource request, by ensuring that the
|
* Utility method to normalize a resource request, by ensuring that the
|
||||||
* requested memory is a multiple of minMemory and is not zero.
|
* requested memory is a multiple of minMemory and is not zero.
|
||||||
|
*
|
||||||
|
* @param ask resource request.
|
||||||
|
* @param resourceCalculator {@link ResourceCalculator} the resource
|
||||||
|
* calculator to use.
|
||||||
|
* @param minimumResource minimum Resource.
|
||||||
|
* @param maximumResource maximum Resource.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public static void normalizeRequest(
|
public static void normalizeRequest(
|
||||||
@ -212,6 +218,12 @@ public static void normalizeRequest(
|
|||||||
* Utility method to normalize a resource request, by ensuring that the
|
* Utility method to normalize a resource request, by ensuring that the
|
||||||
* requested memory is a multiple of increment resource and is not zero.
|
* requested memory is a multiple of increment resource and is not zero.
|
||||||
*
|
*
|
||||||
|
* @param ask resource request.
|
||||||
|
* @param resourceCalculator {@link ResourceCalculator} the resource
|
||||||
|
* calculator to use.
|
||||||
|
* @param minimumResource minimum Resource.
|
||||||
|
* @param maximumResource maximum Resource.
|
||||||
|
* @param incrementResource increment Resource.
|
||||||
* @return normalized resource
|
* @return normalized resource
|
||||||
*/
|
*/
|
||||||
public static Resource getNormalizedResource(
|
public static Resource getNormalizedResource(
|
||||||
@ -528,7 +540,14 @@ private static void checkQueueLabelInLabelManager(String labelExpression,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Check queue label expression, check if node label in queue's
|
* Check queue label expression, check if node label in queue's
|
||||||
* node-label-expression existed in clusterNodeLabels if rmContext != null
|
* node-label-expression existed in clusterNodeLabels if rmContext != null.
|
||||||
|
*
|
||||||
|
* @param queueLabels queue Labels.
|
||||||
|
* @param labelExpression label expression.
|
||||||
|
* @param rmContext rmContext.
|
||||||
|
* @return true, if node label in queue's node-label-expression existed in clusterNodeLabels;
|
||||||
|
* otherwise false.
|
||||||
|
*
|
||||||
*/
|
*/
|
||||||
public static boolean checkQueueLabelExpression(Set<String> queueLabels,
|
public static boolean checkQueueLabelExpression(Set<String> queueLabels,
|
||||||
String labelExpression, RMContext rmContext) {
|
String labelExpression, RMContext rmContext) {
|
||||||
|
@ -61,12 +61,13 @@
|
|||||||
public interface YarnScheduler extends EventHandler<SchedulerEvent> {
|
public interface YarnScheduler extends EventHandler<SchedulerEvent> {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get queue information
|
* Get queue information.
|
||||||
|
*
|
||||||
* @param queueName queue name
|
* @param queueName queue name
|
||||||
* @param includeChildQueues include child queues?
|
* @param includeChildQueues include child queues?
|
||||||
* @param recursive get children queues?
|
* @param recursive get children queues?
|
||||||
* @return queue information
|
* @return queue information
|
||||||
* @throws IOException
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
@Public
|
@Public
|
||||||
@Stable
|
@Stable
|
||||||
@ -148,7 +149,8 @@ Allocation allocate(ApplicationAttemptId appAttemptId,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get node resource usage report.
|
* Get node resource usage report.
|
||||||
* @param nodeId
|
*
|
||||||
|
* @param nodeId nodeId.
|
||||||
* @return the {@link SchedulerNodeReport} for the node or null
|
* @return the {@link SchedulerNodeReport} for the node or null
|
||||||
* if nodeId does not point to a defined node.
|
* if nodeId does not point to a defined node.
|
||||||
*/
|
*/
|
||||||
@ -186,10 +188,11 @@ ApplicationResourceUsageReport getAppResourceUsageReport(
|
|||||||
/**
|
/**
|
||||||
* Check if the user has permission to perform the operation.
|
* Check if the user has permission to perform the operation.
|
||||||
* If the user has {@link QueueACL#ADMINISTER_QUEUE} permission,
|
* If the user has {@link QueueACL#ADMINISTER_QUEUE} permission,
|
||||||
* this user can view/modify the applications in this queue
|
* this user can view/modify the applications in this queue.
|
||||||
* @param callerUGI
|
*
|
||||||
* @param acl
|
* @param callerUGI caller UserGroupInformation.
|
||||||
* @param queueName
|
* @param acl queue ACL.
|
||||||
|
* @param queueName queue Name.
|
||||||
* @return <code>true</code> if the user has the permission,
|
* @return <code>true</code> if the user has the permission,
|
||||||
* <code>false</code> otherwise
|
* <code>false</code> otherwise
|
||||||
*/
|
*/
|
||||||
@ -207,7 +210,8 @@ boolean checkAccess(UserGroupInformation callerUGI,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the container for the given containerId.
|
* Get the container for the given containerId.
|
||||||
* @param containerId
|
*
|
||||||
|
* @param containerId the given containerId.
|
||||||
* @return the container for the given containerId.
|
* @return the container for the given containerId.
|
||||||
*/
|
*/
|
||||||
@LimitedPrivate("yarn")
|
@LimitedPrivate("yarn")
|
||||||
@ -215,9 +219,9 @@ boolean checkAccess(UserGroupInformation callerUGI,
|
|||||||
public RMContainer getRMContainer(ContainerId containerId);
|
public RMContainer getRMContainer(ContainerId containerId);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Moves the given application to the given queue
|
* Moves the given application to the given queue.
|
||||||
* @param appId
|
* @param appId application Id
|
||||||
* @param newQueue
|
* @param newQueue the given queue.
|
||||||
* @return the name of the queue the application was placed into
|
* @return the name of the queue the application was placed into
|
||||||
* @throws YarnException if the move cannot be carried out
|
* @throws YarnException if the move cannot be carried out
|
||||||
*/
|
*/
|
||||||
@ -241,9 +245,9 @@ public void preValidateMoveApplication(ApplicationId appId,
|
|||||||
* Completely drain sourceQueue of applications, by moving all of them to
|
* Completely drain sourceQueue of applications, by moving all of them to
|
||||||
* destQueue.
|
* destQueue.
|
||||||
*
|
*
|
||||||
* @param sourceQueue
|
* @param sourceQueue sourceQueue.
|
||||||
* @param destQueue
|
* @param destQueue destQueue.
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
void moveAllApps(String sourceQueue, String destQueue) throws YarnException;
|
void moveAllApps(String sourceQueue, String destQueue) throws YarnException;
|
||||||
|
|
||||||
@ -251,7 +255,7 @@ public void preValidateMoveApplication(ApplicationId appId,
|
|||||||
* Terminate all applications in the specified queue.
|
* Terminate all applications in the specified queue.
|
||||||
*
|
*
|
||||||
* @param queueName the name of queue to be drained
|
* @param queueName the name of queue to be drained
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
void killAllAppsInQueue(String queueName) throws YarnException;
|
void killAllAppsInQueue(String queueName) throws YarnException;
|
||||||
|
|
||||||
@ -261,7 +265,7 @@ public void preValidateMoveApplication(ApplicationId appId,
|
|||||||
* must be a leaf, etc..).
|
* must be a leaf, etc..).
|
||||||
*
|
*
|
||||||
* @param queueName name of the queue to remove
|
* @param queueName name of the queue to remove
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
void removeQueue(String queueName) throws YarnException;
|
void removeQueue(String queueName) throws YarnException;
|
||||||
|
|
||||||
@ -271,7 +275,8 @@ public void preValidateMoveApplication(ApplicationId appId,
|
|||||||
* attached to existing parent, must have zero entitlement).
|
* attached to existing parent, must have zero entitlement).
|
||||||
*
|
*
|
||||||
* @param newQueue the queue being added.
|
* @param newQueue the queue being added.
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
|
* @throws IOException when io exception occur.
|
||||||
*/
|
*/
|
||||||
void addQueue(Queue newQueue) throws YarnException, IOException;
|
void addQueue(Queue newQueue) throws YarnException, IOException;
|
||||||
|
|
||||||
@ -284,14 +289,15 @@ public void preValidateMoveApplication(ApplicationId appId,
|
|||||||
* @param queue the queue for which we change entitlement
|
* @param queue the queue for which we change entitlement
|
||||||
* @param entitlement the new entitlement for the queue (capacity,
|
* @param entitlement the new entitlement for the queue (capacity,
|
||||||
* maxCapacity, etc..)
|
* maxCapacity, etc..)
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
void setEntitlement(String queue, QueueEntitlement entitlement)
|
void setEntitlement(String queue, QueueEntitlement entitlement)
|
||||||
throws YarnException;
|
throws YarnException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the list of names for queues managed by the Reservation System
|
* Gets the list of names for queues managed by the Reservation System.
|
||||||
* @return the list of queues which support reservations
|
* @return the list of queues which support reservations
|
||||||
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
public Set<String> getPlanQueues() throws YarnException;
|
public Set<String> getPlanQueues() throws YarnException;
|
||||||
|
|
||||||
@ -317,6 +323,7 @@ void setEntitlement(String queue, QueueEntitlement entitlement)
|
|||||||
* @param applicationId
|
* @param applicationId
|
||||||
* Application ID
|
* Application ID
|
||||||
* @return Updated Priority from scheduler
|
* @return Updated Priority from scheduler
|
||||||
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
public Priority checkAndGetApplicationPriority(Priority priorityRequestedByApp,
|
public Priority checkAndGetApplicationPriority(Priority priorityRequestedByApp,
|
||||||
UserGroupInformation user, String queuePath, ApplicationId applicationId)
|
UserGroupInformation user, String queuePath, ApplicationId applicationId)
|
||||||
@ -334,6 +341,7 @@ public Priority checkAndGetApplicationPriority(Priority priorityRequestedByApp,
|
|||||||
* @param user who submitted the application
|
* @param user who submitted the application
|
||||||
*
|
*
|
||||||
* @return updated priority
|
* @return updated priority
|
||||||
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
public Priority updateApplicationPriority(Priority newPriority,
|
public Priority updateApplicationPriority(Priority newPriority,
|
||||||
ApplicationId applicationId, SettableFuture<Object> future,
|
ApplicationId applicationId, SettableFuture<Object> future,
|
||||||
@ -350,15 +358,18 @@ public Priority updateApplicationPriority(Priority newPriority,
|
|||||||
List<Container> getTransferredContainers(ApplicationAttemptId appAttemptId);
|
List<Container> getTransferredContainers(ApplicationAttemptId appAttemptId);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the cluster max priority
|
* Set the cluster max priority.
|
||||||
*
|
*
|
||||||
* @param conf
|
* @param conf Configuration.
|
||||||
* @throws YarnException
|
* @throws YarnException when yarn exception occur.
|
||||||
*/
|
*/
|
||||||
void setClusterMaxPriority(Configuration conf) throws YarnException;
|
void setClusterMaxPriority(Configuration conf) throws YarnException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param attemptId
|
* Get pending resource request for specified application attempt.
|
||||||
|
*
|
||||||
|
* @param attemptId the id of the application attempt
|
||||||
|
* @return pending resource requests.
|
||||||
*/
|
*/
|
||||||
List<ResourceRequest> getPendingResourceRequestsForAttempt(
|
List<ResourceRequest> getPendingResourceRequestsForAttempt(
|
||||||
ApplicationAttemptId attemptId);
|
ApplicationAttemptId attemptId);
|
||||||
|
@ -60,7 +60,7 @@ public AbstractAutoCreatedLeafQueue(CapacitySchedulerContext cs,
|
|||||||
*
|
*
|
||||||
* @param entitlement the new entitlement for the queue (capacity,
|
* @param entitlement the new entitlement for the queue (capacity,
|
||||||
* maxCapacity, etc..)
|
* maxCapacity, etc..)
|
||||||
* @throws SchedulerDynamicEditException
|
* @throws SchedulerDynamicEditException when setEntitlement fails.
|
||||||
*/
|
*/
|
||||||
public void setEntitlement(QueueEntitlement entitlement)
|
public void setEntitlement(QueueEntitlement entitlement)
|
||||||
throws SchedulerDynamicEditException {
|
throws SchedulerDynamicEditException {
|
||||||
@ -69,11 +69,12 @@ public void setEntitlement(QueueEntitlement entitlement)
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* This methods to change capacity for a queue and adjusts its
|
* This methods to change capacity for a queue and adjusts its
|
||||||
* absoluteCapacity
|
* absoluteCapacity.
|
||||||
*
|
*
|
||||||
|
* @param nodeLabel nodeLabel.
|
||||||
* @param entitlement the new entitlement for the queue (capacity,
|
* @param entitlement the new entitlement for the queue (capacity,
|
||||||
* maxCapacity, etc..)
|
* maxCapacity, etc..)
|
||||||
* @throws SchedulerDynamicEditException
|
* @throws SchedulerDynamicEditException when setEntitlement fails.
|
||||||
*/
|
*/
|
||||||
public void setEntitlement(String nodeLabel, QueueEntitlement entitlement)
|
public void setEntitlement(String nodeLabel, QueueEntitlement entitlement)
|
||||||
throws SchedulerDynamicEditException {
|
throws SchedulerDynamicEditException {
|
||||||
|
@ -68,7 +68,8 @@ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
|
|||||||
/**
|
/**
|
||||||
* Add the specified child queue.
|
* Add the specified child queue.
|
||||||
* @param childQueue reference to the child queue to be added
|
* @param childQueue reference to the child queue to be added
|
||||||
* @throws SchedulerDynamicEditException
|
* @throws SchedulerDynamicEditException when addChildQueue fails.
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public void addChildQueue(CSQueue childQueue)
|
public void addChildQueue(CSQueue childQueue)
|
||||||
throws SchedulerDynamicEditException, IOException {
|
throws SchedulerDynamicEditException, IOException {
|
||||||
@ -91,7 +92,7 @@ public void addChildQueue(CSQueue childQueue)
|
|||||||
/**
|
/**
|
||||||
* Remove the specified child queue.
|
* Remove the specified child queue.
|
||||||
* @param childQueue reference to the child queue to be removed
|
* @param childQueue reference to the child queue to be removed
|
||||||
* @throws SchedulerDynamicEditException
|
* @throws SchedulerDynamicEditException when removeChildQueue fails.
|
||||||
*/
|
*/
|
||||||
public void removeChildQueue(CSQueue childQueue)
|
public void removeChildQueue(CSQueue childQueue)
|
||||||
throws SchedulerDynamicEditException {
|
throws SchedulerDynamicEditException {
|
||||||
@ -117,7 +118,8 @@ public void removeChildQueue(CSQueue childQueue)
|
|||||||
/**
|
/**
|
||||||
* Remove the specified child queue.
|
* Remove the specified child queue.
|
||||||
* @param childQueueName name of the child queue to be removed
|
* @param childQueueName name of the child queue to be removed
|
||||||
* @throws SchedulerDynamicEditException
|
* @return child queue.
|
||||||
|
* @throws SchedulerDynamicEditException when removeChildQueue fails.
|
||||||
*/
|
*/
|
||||||
public CSQueue removeChildQueue(String childQueueName)
|
public CSQueue removeChildQueue(String childQueueName)
|
||||||
throws SchedulerDynamicEditException {
|
throws SchedulerDynamicEditException {
|
||||||
|
@ -172,6 +172,8 @@ public void updateRunnabilityOnReload() {
|
|||||||
*
|
*
|
||||||
* Runs in O(n log(n)) where n is the number of queues that are under the
|
* Runs in O(n log(n)) where n is the number of queues that are under the
|
||||||
* highest queue that went from having no slack to having slack.
|
* highest queue that went from having no slack to having slack.
|
||||||
|
*
|
||||||
|
* @param app FiCaSchedulerApp.
|
||||||
*/
|
*/
|
||||||
public void updateRunnabilityOnAppRemoval(FiCaSchedulerApp app) {
|
public void updateRunnabilityOnAppRemoval(FiCaSchedulerApp app) {
|
||||||
// childqueueX might have no pending apps itself, but if a queue higher up
|
// childqueueX might have no pending apps itself, but if a queue higher up
|
||||||
|
@ -172,12 +172,16 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
|
|||||||
* @param applicationId the applicationId of the application being submitted
|
* @param applicationId the applicationId of the application being submitted
|
||||||
* @param user user who submitted the application
|
* @param user user who submitted the application
|
||||||
* @param queue queue to which the application is submitted
|
* @param queue queue to which the application is submitted
|
||||||
|
* @throws AccessControlException if any acl violation is there.
|
||||||
*/
|
*/
|
||||||
public void submitApplication(ApplicationId applicationId, String user,
|
public void submitApplication(ApplicationId applicationId, String user,
|
||||||
String queue) throws AccessControlException;
|
String queue) throws AccessControlException;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Submit an application attempt to the queue.
|
* Submit an application attempt to the queue.
|
||||||
|
*
|
||||||
|
* @param application application whose attempt is being submitted.
|
||||||
|
* @param userName userName who submitted the application.
|
||||||
*/
|
*/
|
||||||
public void submitApplicationAttempt(FiCaSchedulerApp application,
|
public void submitApplicationAttempt(FiCaSchedulerApp application,
|
||||||
String userName);
|
String userName);
|
||||||
@ -193,13 +197,16 @@ public void submitApplicationAttempt(FiCaSchedulerApp application,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* An application submitted to this queue has finished.
|
* An application submitted to this queue has finished.
|
||||||
* @param applicationId
|
* @param applicationId applicationId.
|
||||||
* @param user user who submitted the application
|
* @param user user who submitted the application
|
||||||
*/
|
*/
|
||||||
public void finishApplication(ApplicationId applicationId, String user);
|
public void finishApplication(ApplicationId applicationId, String user);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An application attempt submitted to this queue has finished.
|
* An application attempt submitted to this queue has finished.
|
||||||
|
*
|
||||||
|
* @param application application attempt.
|
||||||
|
* @param queue queue.
|
||||||
*/
|
*/
|
||||||
public void finishApplicationAttempt(FiCaSchedulerApp application,
|
public void finishApplicationAttempt(FiCaSchedulerApp application,
|
||||||
String queue);
|
String queue);
|
||||||
@ -248,6 +255,7 @@ public void completedContainer(Resource clusterResource,
|
|||||||
* Reinitialize the queue.
|
* Reinitialize the queue.
|
||||||
* @param newlyParsedQueue new queue to re-initalize from
|
* @param newlyParsedQueue new queue to re-initalize from
|
||||||
* @param clusterResource resources in the cluster
|
* @param clusterResource resources in the cluster
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
|
public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
@ -325,6 +333,10 @@ public void attachContainer(Resource clusterResource,
|
|||||||
/**
|
/**
|
||||||
* When partition of node updated, we will update queue's resource usage if it
|
* When partition of node updated, we will update queue's resource usage if it
|
||||||
* has container(s) running on that.
|
* has container(s) running on that.
|
||||||
|
*
|
||||||
|
* @param nodePartition node label.
|
||||||
|
* @param resourceToInc resource.
|
||||||
|
* @param application application.
|
||||||
*/
|
*/
|
||||||
public void incUsedResource(String nodePartition, Resource resourceToInc,
|
public void incUsedResource(String nodePartition, Resource resourceToInc,
|
||||||
SchedulerApplicationAttempt application);
|
SchedulerApplicationAttempt application);
|
||||||
@ -332,6 +344,10 @@ public void incUsedResource(String nodePartition, Resource resourceToInc,
|
|||||||
/**
|
/**
|
||||||
* When partition of node updated, we will update queue's resource usage if it
|
* When partition of node updated, we will update queue's resource usage if it
|
||||||
* has container(s) running on that.
|
* has container(s) running on that.
|
||||||
|
*
|
||||||
|
* @param nodePartition node label.
|
||||||
|
* @param resourceToDec resource.
|
||||||
|
* @param application application.
|
||||||
*/
|
*/
|
||||||
public void decUsedResource(String nodePartition, Resource resourceToDec,
|
public void decUsedResource(String nodePartition, Resource resourceToDec,
|
||||||
SchedulerApplicationAttempt application);
|
SchedulerApplicationAttempt application);
|
||||||
|
@ -170,6 +170,11 @@ private static void updateAbsoluteCapacitiesByNodeLabels(
|
|||||||
/**
|
/**
|
||||||
* Update partitioned resource usage, if nodePartition == null, will update
|
* Update partitioned resource usage, if nodePartition == null, will update
|
||||||
* used resource for all partitions of this queue.
|
* used resource for all partitions of this queue.
|
||||||
|
*
|
||||||
|
* @param rc resource calculator.
|
||||||
|
* @param totalPartitionResource total Partition Resource.
|
||||||
|
* @param nodePartition node label.
|
||||||
|
* @param childQueue child queue.
|
||||||
*/
|
*/
|
||||||
public static void updateUsedCapacity(final ResourceCalculator rc,
|
public static void updateUsedCapacity(final ResourceCalculator rc,
|
||||||
final Resource totalPartitionResource, String nodePartition,
|
final Resource totalPartitionResource, String nodePartition,
|
||||||
@ -285,6 +290,12 @@ private static Resource getMaxAvailableResourceToQueuePartition(
|
|||||||
* When nodePartition is null, all partition of
|
* When nodePartition is null, all partition of
|
||||||
* used-capacity/absolute-used-capacity will be updated.
|
* used-capacity/absolute-used-capacity will be updated.
|
||||||
* </p>
|
* </p>
|
||||||
|
*
|
||||||
|
* @param rc resource calculator.
|
||||||
|
* @param cluster cluster resource.
|
||||||
|
* @param childQueue child queue.
|
||||||
|
* @param nlm RMNodeLabelsManager.
|
||||||
|
* @param nodePartition node label.
|
||||||
*/
|
*/
|
||||||
@Lock(CSQueue.class)
|
@Lock(CSQueue.class)
|
||||||
public static void updateQueueStatistics(
|
public static void updateQueueStatistics(
|
||||||
|
@ -116,6 +116,8 @@ public static void validateVCores(Configuration conf) {
|
|||||||
*
|
*
|
||||||
* @param queues existing queues
|
* @param queues existing queues
|
||||||
* @param newQueues new queues
|
* @param newQueues new queues
|
||||||
|
* @param newConf Capacity Scheduler Configuration.
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public static void validateQueueHierarchy(
|
public static void validateQueueHierarchy(
|
||||||
CSQueueStore queues,
|
CSQueueStore queues,
|
||||||
|
@ -1344,7 +1344,10 @@ public boolean getIntraQueuePreemptionDisabled(String queue,
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get configured node labels in a given queuePath
|
* Get configured node labels in a given queuePath.
|
||||||
|
*
|
||||||
|
* @param queuePath queue path.
|
||||||
|
* @return configured node labels.
|
||||||
*/
|
*/
|
||||||
public Set<String> getConfiguredNodeLabels(String queuePath) {
|
public Set<String> getConfiguredNodeLabels(String queuePath) {
|
||||||
Set<String> configuredNodeLabels = new HashSet<String>();
|
Set<String> configuredNodeLabels = new HashSet<String>();
|
||||||
|
@ -58,6 +58,7 @@ public interface CapacitySchedulerContext {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the yarn configuration.
|
* Get the yarn configuration.
|
||||||
|
* @return yarn configuration.
|
||||||
*/
|
*/
|
||||||
Configuration getConf();
|
Configuration getConf();
|
||||||
|
|
||||||
|
@ -279,9 +279,11 @@ public boolean shouldFailAutoCreationWhenGuaranteedCapacityExceeded() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Asynchronously called from scheduler to apply queue management changes
|
* Asynchronously called from scheduler to apply queue management changes.
|
||||||
*
|
*
|
||||||
* @param queueManagementChanges
|
* @param queueManagementChanges QueueManagementChange List.
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
|
* @throws SchedulerDynamicEditException when validate and apply QueueManagementChanges fails.
|
||||||
*/
|
*/
|
||||||
public void validateAndApplyQueueManagementChanges(
|
public void validateAndApplyQueueManagementChanges(
|
||||||
List<QueueManagementChange> queueManagementChanges)
|
List<QueueManagementChange> queueManagementChanges)
|
||||||
|
@ -165,7 +165,8 @@ public float getUserLimitFactor() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Determine whether to hide/show the ReservationQueues
|
* Determine whether to hide/show the ReservationQueues.
|
||||||
|
* @return true, show ReservationQueues; false, hide ReservationQueues.
|
||||||
*/
|
*/
|
||||||
public boolean showReservationsAsQueues() {
|
public boolean showReservationsAsQueues() {
|
||||||
return showReservationsAsQueues;
|
return showReservationsAsQueues;
|
||||||
|
@ -112,7 +112,7 @@ protected LinkedList<LogMutation> getLogs() {
|
|||||||
* As there is no configuration store for versioning purposes,
|
* As there is no configuration store for versioning purposes,
|
||||||
* a conf store version is not applicable.
|
* a conf store version is not applicable.
|
||||||
* @return null Conf store version not applicable for this store.
|
* @return null Conf store version not applicable for this store.
|
||||||
* @throws Exception
|
* @throws Exception if any exception occurs during getConfStoreVersion.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public Version getConfStoreVersion() throws Exception {
|
public Version getConfStoreVersion() throws Exception {
|
||||||
@ -123,7 +123,7 @@ public Version getConfStoreVersion() throws Exception {
|
|||||||
/**
|
/**
|
||||||
* Configuration mutations not logged (i.e. not persisted). As such, they are
|
* Configuration mutations not logged (i.e. not persisted). As such, they are
|
||||||
* not persisted and not versioned. Hence, no version information to store.
|
* not persisted and not versioned. Hence, no version information to store.
|
||||||
* @throws Exception
|
* @throws Exception if any exception occurs during store Version.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void storeVersion() throws Exception {
|
public void storeVersion() throws Exception {
|
||||||
|
@ -126,6 +126,7 @@ public abstract void confirmMutation(LogMutation pendingMutation,
|
|||||||
/**
|
/**
|
||||||
* Retrieve the persisted configuration.
|
* Retrieve the persisted configuration.
|
||||||
* @return configuration as key-value
|
* @return configuration as key-value
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public abstract Configuration retrieve() throws IOException;
|
public abstract Configuration retrieve() throws IOException;
|
||||||
|
|
||||||
@ -139,6 +140,7 @@ public abstract void confirmMutation(LogMutation pendingMutation,
|
|||||||
/**
|
/**
|
||||||
* Get the last updated config version.
|
* Get the last updated config version.
|
||||||
* @return Last updated config version.
|
* @return Last updated config version.
|
||||||
|
* @throws Exception On version fetch failure.
|
||||||
*/
|
*/
|
||||||
public abstract long getConfigVersion() throws Exception;
|
public abstract long getConfigVersion() throws Exception;
|
||||||
|
|
||||||
|
@ -350,7 +350,7 @@ private void initializeLeafQueueTemplate(ManagedParentQueue parentQueue)
|
|||||||
*
|
*
|
||||||
* @return List of Queue Management change suggestions which could potentially
|
* @return List of Queue Management change suggestions which could potentially
|
||||||
* be committed/rejected by the scheduler due to validation failures
|
* be committed/rejected by the scheduler due to validation failures
|
||||||
* @throws SchedulerDynamicEditException
|
* @throws SchedulerDynamicEditException when compute queueManagement changes fails.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public List<QueueManagementChange> computeQueueManagementChanges()
|
public List<QueueManagementChange> computeQueueManagementChanges()
|
||||||
|
@ -405,9 +405,10 @@ public void removeContainer(NodeId nodeId,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Helper method to just remove the tags associated with a container.
|
* Helper method to just remove the tags associated with a container.
|
||||||
* @param nodeId
|
*
|
||||||
* @param applicationId
|
* @param nodeId nodeId.
|
||||||
* @param allocationTags
|
* @param applicationId application Id
|
||||||
|
* @param allocationTags application Tags.
|
||||||
*/
|
*/
|
||||||
public void removeTags(NodeId nodeId, ApplicationId applicationId,
|
public void removeTags(NodeId nodeId, ApplicationId applicationId,
|
||||||
Set<String> allocationTags) {
|
Set<String> allocationTags) {
|
||||||
@ -644,7 +645,7 @@ public long getRackCardinalityByOp(String rack, AllocationTags tags,
|
|||||||
* Returns a map whose key is the allocation tag and value is the
|
* Returns a map whose key is the allocation tag and value is the
|
||||||
* count of allocations with this tag.
|
* count of allocations with this tag.
|
||||||
*
|
*
|
||||||
* @param nodeId
|
* @param nodeId nodeId.
|
||||||
* @return allocation tag to count mapping
|
* @return allocation tag to count mapping
|
||||||
*/
|
*/
|
||||||
public Map<String, Long> getAllocationTagsWithCount(NodeId nodeId) {
|
public Map<String, Long> getAllocationTagsWithCount(NodeId nodeId) {
|
||||||
|
@ -32,7 +32,7 @@ public interface Evaluable<T> {
|
|||||||
* of current class.
|
* of current class.
|
||||||
*
|
*
|
||||||
* @param target a generic type target that impacts this evaluation.
|
* @param target a generic type target that impacts this evaluation.
|
||||||
* @throws YarnException
|
* @throws YarnException if evaluate error.
|
||||||
*/
|
*/
|
||||||
void evaluate(T target) throws YarnException;
|
void evaluate(T target) throws YarnException;
|
||||||
}
|
}
|
||||||
|
@ -347,7 +347,7 @@ private static boolean canSatisfyConstraints(ApplicationId appId,
|
|||||||
* @param atm allocation tags manager
|
* @param atm allocation tags manager
|
||||||
* @param dcOpt optional diagnostics collector
|
* @param dcOpt optional diagnostics collector
|
||||||
* @return true if the given node satisfies the constraint of the request
|
* @return true if the given node satisfies the constraint of the request
|
||||||
* @throws InvalidAllocationTagsQueryException
|
* @throws InvalidAllocationTagsQueryException if given string is not in valid format.
|
||||||
*/
|
*/
|
||||||
public static boolean canSatisfyConstraints(ApplicationId applicationId,
|
public static boolean canSatisfyConstraints(ApplicationId applicationId,
|
||||||
SchedulingRequest request, SchedulerNode schedulerNode,
|
SchedulingRequest request, SchedulerNode schedulerNode,
|
||||||
|
@ -92,7 +92,7 @@ public Set<ApplicationId> getNamespaceScope() {
|
|||||||
* in this method.
|
* in this method.
|
||||||
*
|
*
|
||||||
* @param target a generic type target that impacts this evaluation.
|
* @param target a generic type target that impacts this evaluation.
|
||||||
* @throws InvalidAllocationTagsQueryException
|
* @throws InvalidAllocationTagsQueryException if given string is not in valid format.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void evaluate(TargetApplications target)
|
public void evaluate(TargetApplications target)
|
||||||
|
@ -77,6 +77,7 @@ public CentralizedOpportunisticContainerAllocator(
|
|||||||
* @param tokenSecretManager TokenSecretManager
|
* @param tokenSecretManager TokenSecretManager
|
||||||
* @param maxAllocationsPerAMHeartbeat max number of containers to be
|
* @param maxAllocationsPerAMHeartbeat max number of containers to be
|
||||||
* allocated in one AM heartbeat
|
* allocated in one AM heartbeat
|
||||||
|
* @param nodeQueueLoadMonitor NodeQueueLoadMonitor.
|
||||||
*/
|
*/
|
||||||
public CentralizedOpportunisticContainerAllocator(
|
public CentralizedOpportunisticContainerAllocator(
|
||||||
BaseContainerTokenSecretManager tokenSecretManager,
|
BaseContainerTokenSecretManager tokenSecretManager,
|
||||||
|
@ -108,7 +108,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
|
|||||||
* configuration.
|
* configuration.
|
||||||
* @param allocationFileParser The allocation file parser
|
* @param allocationFileParser The allocation file parser
|
||||||
* @param globalReservationQueueConfig The reservation queue config
|
* @param globalReservationQueueConfig The reservation queue config
|
||||||
* @throws AllocationConfigurationException
|
* @throws AllocationConfigurationException for any errors.
|
||||||
*/
|
*/
|
||||||
public AllocationConfiguration(QueueProperties queueProperties,
|
public AllocationConfiguration(QueueProperties queueProperties,
|
||||||
AllocationFileParser allocationFileParser,
|
AllocationFileParser allocationFileParser,
|
||||||
@ -202,6 +202,9 @@ public Map<ReservationACL, AccessControlList> getReservationAcls(String
|
|||||||
/**
|
/**
|
||||||
* Get a queue's min share preemption timeout configured in the allocation
|
* Get a queue's min share preemption timeout configured in the allocation
|
||||||
* file, in milliseconds. Return -1 if not set.
|
* file, in milliseconds. Return -1 if not set.
|
||||||
|
*
|
||||||
|
* @param queueName queue name.
|
||||||
|
* @return min share preemption timeout, return -1f if not set.
|
||||||
*/
|
*/
|
||||||
public long getMinSharePreemptionTimeout(String queueName) {
|
public long getMinSharePreemptionTimeout(String queueName) {
|
||||||
Long minSharePreemptionTimeout = minSharePreemptionTimeouts.get(queueName);
|
Long minSharePreemptionTimeout = minSharePreemptionTimeouts.get(queueName);
|
||||||
@ -211,6 +214,9 @@ public long getMinSharePreemptionTimeout(String queueName) {
|
|||||||
/**
|
/**
|
||||||
* Get a queue's fair share preemption timeout configured in the allocation
|
* Get a queue's fair share preemption timeout configured in the allocation
|
||||||
* file, in milliseconds. Return -1 if not set.
|
* file, in milliseconds. Return -1 if not set.
|
||||||
|
*
|
||||||
|
* @param queueName queue Name.
|
||||||
|
* @return fair share preemption timeout, return -1f if not set.
|
||||||
*/
|
*/
|
||||||
public long getFairSharePreemptionTimeout(String queueName) {
|
public long getFairSharePreemptionTimeout(String queueName) {
|
||||||
Long fairSharePreemptionTimeout = fairSharePreemptionTimeouts.get(queueName);
|
Long fairSharePreemptionTimeout = fairSharePreemptionTimeouts.get(queueName);
|
||||||
@ -221,6 +227,9 @@ public long getFairSharePreemptionTimeout(String queueName) {
|
|||||||
/**
|
/**
|
||||||
* Get a queue's fair share preemption threshold in the allocation file.
|
* Get a queue's fair share preemption threshold in the allocation file.
|
||||||
* Return -1f if not set.
|
* Return -1f if not set.
|
||||||
|
*
|
||||||
|
* @param queueName queue Name.
|
||||||
|
* @return preemption threshold, return -1f if not set.
|
||||||
*/
|
*/
|
||||||
public float getFairSharePreemptionThreshold(String queueName) {
|
public float getFairSharePreemptionThreshold(String queueName) {
|
||||||
Float fairSharePreemptionThreshold =
|
Float fairSharePreemptionThreshold =
|
||||||
|
@ -185,6 +185,11 @@ public void serviceStop() throws Exception {
|
|||||||
* Path to XML file containing allocations. If the
|
* Path to XML file containing allocations. If the
|
||||||
* path is relative, it is searched for in the
|
* path is relative, it is searched for in the
|
||||||
* classpath, but loaded like a regular File.
|
* classpath, but loaded like a regular File.
|
||||||
|
*
|
||||||
|
* @param conf configuration.
|
||||||
|
* @return Allocation File Path.
|
||||||
|
* @throws UnsupportedFileSystemException
|
||||||
|
* File system for a given file system name/scheme is not supported.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
public Path getAllocationFile(Configuration conf)
|
public Path getAllocationFile(Configuration conf)
|
||||||
|
@ -131,6 +131,7 @@ public FSAppAttempt(FairScheduler scheduler,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get metrics reference from containing queue.
|
* Get metrics reference from containing queue.
|
||||||
|
* @return metrics reference from containing queue.
|
||||||
*/
|
*/
|
||||||
public QueueMetrics getMetrics() {
|
public QueueMetrics getMetrics() {
|
||||||
return queue.getMetrics();
|
return queue.getMetrics();
|
||||||
|
@ -473,6 +473,7 @@ public boolean isEmpty() {
|
|||||||
/**
|
/**
|
||||||
* TODO: Based on how frequently this is called, we might want to club
|
* TODO: Based on how frequently this is called, we might want to club
|
||||||
* counting pending and active apps in the same method.
|
* counting pending and active apps in the same method.
|
||||||
|
* @return active apps.
|
||||||
*/
|
*/
|
||||||
public int getNumActiveApps() {
|
public int getNumActiveApps() {
|
||||||
int numActiveApps = 0;
|
int numActiveApps = 0;
|
||||||
|
@ -304,7 +304,10 @@ public void setFairShare(Resource fairShare) {
|
|||||||
LOG.debug("The updated fairShare for {} is {}", getName(), fairShare);
|
LOG.debug("The updated fairShare for {} is {}", getName(), fairShare);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Get the steady fair share assigned to this Schedulable. */
|
/**
|
||||||
|
* Get the steady fair share assigned to this Schedulable.
|
||||||
|
* @return the steady fair share assigned to this Schedulable.
|
||||||
|
*/
|
||||||
public Resource getSteadyFairShare() {
|
public Resource getSteadyFairShare() {
|
||||||
return steadyFairShare;
|
return steadyFairShare;
|
||||||
}
|
}
|
||||||
@ -364,7 +367,7 @@ public boolean isPreemptable() {
|
|||||||
*
|
*
|
||||||
* To be called holding the scheduler writelock.
|
* To be called holding the scheduler writelock.
|
||||||
*
|
*
|
||||||
* @param fairShare
|
* @param fairShare queue's fairshare.
|
||||||
*/
|
*/
|
||||||
public void update(Resource fairShare) {
|
public void update(Resource fairShare) {
|
||||||
setFairShare(fairShare);
|
setFairShare(fairShare);
|
||||||
@ -407,6 +410,8 @@ private void updatePreemptionVariables() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the children of this queue, if any.
|
* Gets the children of this queue, if any.
|
||||||
|
*
|
||||||
|
* @return the children of this queue.
|
||||||
*/
|
*/
|
||||||
public abstract List<FSQueue> getChildQueues();
|
public abstract List<FSQueue> getChildQueues();
|
||||||
|
|
||||||
@ -420,6 +425,8 @@ public abstract void collectSchedulerApplications(
|
|||||||
/**
|
/**
|
||||||
* Return the number of apps for which containers can be allocated.
|
* Return the number of apps for which containers can be allocated.
|
||||||
* Includes apps in subqueues.
|
* Includes apps in subqueues.
|
||||||
|
*
|
||||||
|
* @return the number of apps.
|
||||||
*/
|
*/
|
||||||
public abstract int getNumRunnableApps();
|
public abstract int getNumRunnableApps();
|
||||||
|
|
||||||
@ -447,6 +454,8 @@ boolean assignContainerPreCheck(FSSchedulerNode node) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns true if queue has at least one app running.
|
* Returns true if queue has at least one app running.
|
||||||
|
*
|
||||||
|
* @return true, if queue has at least one app running; otherwise, false;
|
||||||
*/
|
*/
|
||||||
public boolean isActive() {
|
public boolean isActive() {
|
||||||
return getNumRunnableApps() > 0;
|
return getNumRunnableApps() > 0;
|
||||||
|
@ -463,6 +463,12 @@ public int getContinuousSchedulingSleepMs() {
|
|||||||
* Add a new application to the scheduler, with a given id, queue name, and
|
* Add a new application to the scheduler, with a given id, queue name, and
|
||||||
* user. This will accept a new app even if the user or queue is above
|
* user. This will accept a new app even if the user or queue is above
|
||||||
* configured limits, but the app will not be marked as runnable.
|
* configured limits, but the app will not be marked as runnable.
|
||||||
|
*
|
||||||
|
* @param applicationId applicationId.
|
||||||
|
* @param queueName queue name.
|
||||||
|
* @param user submit application user.
|
||||||
|
* @param isAppRecovering true, app recover; false, app not recover.
|
||||||
|
* @param placementContext application placement context.
|
||||||
*/
|
*/
|
||||||
protected void addApplication(ApplicationId applicationId,
|
protected void addApplication(ApplicationId applicationId,
|
||||||
String queueName, String user, boolean isAppRecovering,
|
String queueName, String user, boolean isAppRecovering,
|
||||||
@ -582,6 +588,10 @@ protected void addApplication(ApplicationId applicationId,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Add a new application attempt to the scheduler.
|
* Add a new application attempt to the scheduler.
|
||||||
|
*
|
||||||
|
* @param applicationAttemptId application AttemptId.
|
||||||
|
* @param transferStateFromPreviousAttempt transferStateFromPreviousAttempt.
|
||||||
|
* @param isAttemptRecovering true, attempt recovering;false, attempt not recovering.
|
||||||
*/
|
*/
|
||||||
protected void addApplicationAttempt(
|
protected void addApplicationAttempt(
|
||||||
ApplicationAttemptId applicationAttemptId,
|
ApplicationAttemptId applicationAttemptId,
|
||||||
|
@ -119,6 +119,8 @@ public boolean exceedQueueMaxRunningApps(FSQueue queue) {
|
|||||||
/**
|
/**
|
||||||
* Tracks the given new runnable app for purposes of maintaining max running
|
* Tracks the given new runnable app for purposes of maintaining max running
|
||||||
* app limits.
|
* app limits.
|
||||||
|
*
|
||||||
|
* @param app FSAppAttempt, the given application.
|
||||||
*/
|
*/
|
||||||
public void trackRunnableApp(FSAppAttempt app) {
|
public void trackRunnableApp(FSAppAttempt app) {
|
||||||
String user = app.getUser();
|
String user = app.getUser();
|
||||||
@ -138,6 +140,8 @@ public void trackRunnableApp(FSAppAttempt app) {
|
|||||||
/**
|
/**
|
||||||
* Tracks the given new non runnable app so that it can be made runnable when
|
* Tracks the given new non runnable app so that it can be made runnable when
|
||||||
* it would not violate max running app limits.
|
* it would not violate max running app limits.
|
||||||
|
*
|
||||||
|
* @param app FSAppAttempt, the given application.
|
||||||
*/
|
*/
|
||||||
public void trackNonRunnableApp(FSAppAttempt app) {
|
public void trackNonRunnableApp(FSAppAttempt app) {
|
||||||
String user = app.getUser();
|
String user = app.getUser();
|
||||||
@ -170,6 +174,9 @@ public void updateRunnabilityOnReload() {
|
|||||||
*
|
*
|
||||||
* Runs in O(n log(n)) where n is the number of queues that are under the
|
* Runs in O(n log(n)) where n is the number of queues that are under the
|
||||||
* highest queue that went from having no slack to having slack.
|
* highest queue that went from having no slack to having slack.
|
||||||
|
*
|
||||||
|
* @param queue FSLeafQueue, the given queue.
|
||||||
|
* @param app FSAppAttempt, the given application.
|
||||||
*/
|
*/
|
||||||
public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) {
|
public void updateRunnabilityOnAppRemoval(FSAppAttempt app, FSLeafQueue queue) {
|
||||||
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
|
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
|
||||||
@ -271,6 +278,8 @@ private void updateAppsRunnability(List<List<FSAppAttempt>>
|
|||||||
/**
|
/**
|
||||||
* Updates the relevant tracking variables after a runnable app with the given
|
* Updates the relevant tracking variables after a runnable app with the given
|
||||||
* queue and user has been removed.
|
* queue and user has been removed.
|
||||||
|
*
|
||||||
|
* @param app FSAppAttempt.
|
||||||
*/
|
*/
|
||||||
public void untrackRunnableApp(FSAppAttempt app) {
|
public void untrackRunnableApp(FSAppAttempt app) {
|
||||||
// Update usersRunnableApps
|
// Update usersRunnableApps
|
||||||
@ -292,7 +301,9 @@ public void untrackRunnableApp(FSAppAttempt app) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stops tracking the given non-runnable app
|
* Stops tracking the given non-runnable app.
|
||||||
|
*
|
||||||
|
* @param app FSAppAttempt.
|
||||||
*/
|
*/
|
||||||
public void untrackNonRunnableApp(FSAppAttempt app) {
|
public void untrackNonRunnableApp(FSAppAttempt app) {
|
||||||
usersNonRunnableApps.remove(app.getUser(), app);
|
usersNonRunnableApps.remove(app.getUser(), app);
|
||||||
@ -301,6 +312,9 @@ public void untrackNonRunnableApp(FSAppAttempt app) {
|
|||||||
/**
|
/**
|
||||||
* Traverses the queue hierarchy under the given queue to gather all lists
|
* Traverses the queue hierarchy under the given queue to gather all lists
|
||||||
* of non-runnable applications.
|
* of non-runnable applications.
|
||||||
|
*
|
||||||
|
* @param queue FSQueue
|
||||||
|
* @param appLists FSAppAttempt List.
|
||||||
*/
|
*/
|
||||||
private void gatherPossiblyRunnableAppLists(FSQueue queue,
|
private void gatherPossiblyRunnableAppLists(FSQueue queue,
|
||||||
List<List<FSAppAttempt>> appLists) {
|
List<List<FSAppAttempt>> appLists) {
|
||||||
|
@ -548,6 +548,8 @@ private void removeQueue(FSQueue queue) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets a queue by name.
|
* Gets a queue by name.
|
||||||
|
* @param name queue name.
|
||||||
|
* @return queue objects, FSQueue.
|
||||||
*/
|
*/
|
||||||
public FSQueue getQueue(String name) {
|
public FSQueue getQueue(String name) {
|
||||||
name = ensureRootPrefix(name);
|
name = ensureRootPrefix(name);
|
||||||
@ -558,6 +560,10 @@ public FSQueue getQueue(String name) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Return whether a queue exists already.
|
* Return whether a queue exists already.
|
||||||
|
*
|
||||||
|
* @param name queue name.
|
||||||
|
* @return Returns true if the queue exists,
|
||||||
|
* otherwise returns false.
|
||||||
*/
|
*/
|
||||||
public boolean exists(String name) {
|
public boolean exists(String name) {
|
||||||
name = ensureRootPrefix(name);
|
name = ensureRootPrefix(name);
|
||||||
@ -568,6 +574,7 @@ public boolean exists(String name) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a collection of all leaf queues.
|
* Get a collection of all leaf queues.
|
||||||
|
* @return a collection of all leaf queues.
|
||||||
*/
|
*/
|
||||||
public Collection<FSLeafQueue> getLeafQueues() {
|
public Collection<FSLeafQueue> getLeafQueues() {
|
||||||
synchronized (queues) {
|
synchronized (queues) {
|
||||||
@ -577,6 +584,7 @@ public Collection<FSLeafQueue> getLeafQueues() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get a collection of all queues.
|
* Get a collection of all queues.
|
||||||
|
* @return a collection of all queues.
|
||||||
*/
|
*/
|
||||||
public Collection<FSQueue> getQueues() {
|
public Collection<FSQueue> getQueues() {
|
||||||
synchronized (queues) {
|
synchronized (queues) {
|
||||||
|
@ -48,6 +48,7 @@ public interface Schedulable {
|
|||||||
/**
|
/**
|
||||||
* Name of job/queue, used for debugging as well as for breaking ties in
|
* Name of job/queue, used for debugging as well as for breaking ties in
|
||||||
* scheduling order deterministically.
|
* scheduling order deterministically.
|
||||||
|
* @return Name of job/queue.
|
||||||
*/
|
*/
|
||||||
String getName();
|
String getName();
|
||||||
|
|
||||||
@ -55,16 +56,26 @@ public interface Schedulable {
|
|||||||
* Maximum number of resources required by this Schedulable. This is defined as
|
* Maximum number of resources required by this Schedulable. This is defined as
|
||||||
* number of currently utilized resources + number of unlaunched resources (that
|
* number of currently utilized resources + number of unlaunched resources (that
|
||||||
* are either not yet launched or need to be speculated).
|
* are either not yet launched or need to be speculated).
|
||||||
|
* @return resources required by this Schedulable.
|
||||||
*/
|
*/
|
||||||
Resource getDemand();
|
Resource getDemand();
|
||||||
|
|
||||||
/** Get the aggregate amount of resources consumed by the schedulable. */
|
/**
|
||||||
|
* Get the aggregate amount of resources consumed by the schedulable.
|
||||||
|
* @return aggregate amount of resources.
|
||||||
|
*/
|
||||||
Resource getResourceUsage();
|
Resource getResourceUsage();
|
||||||
|
|
||||||
/** Minimum Resource share assigned to the schedulable. */
|
/**
|
||||||
|
* Minimum Resource share assigned to the schedulable.
|
||||||
|
* @return Minimum Resource share.
|
||||||
|
*/
|
||||||
Resource getMinShare();
|
Resource getMinShare();
|
||||||
|
|
||||||
/** Maximum Resource share assigned to the schedulable. */
|
/**
|
||||||
|
* Maximum Resource share assigned to the schedulable.
|
||||||
|
* @return Maximum Resource share.
|
||||||
|
*/
|
||||||
Resource getMaxShare();
|
Resource getMaxShare();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -77,10 +88,16 @@ public interface Schedulable {
|
|||||||
*/
|
*/
|
||||||
float getWeight();
|
float getWeight();
|
||||||
|
|
||||||
/** Start time for jobs in FIFO queues; meaningless for QueueSchedulables.*/
|
/**
|
||||||
|
* Start time for jobs in FIFO queues; meaningless for QueueSchedulables.
|
||||||
|
* @return Start time for jobs.
|
||||||
|
*/
|
||||||
long getStartTime();
|
long getStartTime();
|
||||||
|
|
||||||
/** Job priority for jobs in FIFO queues; meaningless for QueueSchedulables. */
|
/**
|
||||||
|
* Job priority for jobs in FIFO queues; meaningless for QueueSchedulables.
|
||||||
|
* @return Job priority.
|
||||||
|
*/
|
||||||
Priority getPriority();
|
Priority getPriority();
|
||||||
|
|
||||||
/** Refresh the Schedulable's demand and those of its children if any. */
|
/** Refresh the Schedulable's demand and those of its children if any. */
|
||||||
@ -89,13 +106,22 @@ public interface Schedulable {
|
|||||||
/**
|
/**
|
||||||
* Assign a container on this node if possible, and return the amount of
|
* Assign a container on this node if possible, and return the amount of
|
||||||
* resources assigned.
|
* resources assigned.
|
||||||
|
*
|
||||||
|
* @param node FSSchedulerNode.
|
||||||
|
* @return the amount of resources assigned.
|
||||||
*/
|
*/
|
||||||
Resource assignContainer(FSSchedulerNode node);
|
Resource assignContainer(FSSchedulerNode node);
|
||||||
|
|
||||||
/** Get the fair share assigned to this Schedulable. */
|
/**
|
||||||
|
* Get the fair share assigned to this Schedulable.
|
||||||
|
* @return the fair share assigned to this Schedulable.
|
||||||
|
*/
|
||||||
Resource getFairShare();
|
Resource getFairShare();
|
||||||
|
|
||||||
/** Assign a fair share to this Schedulable. */
|
/**
|
||||||
|
* Assign a fair share to this Schedulable.
|
||||||
|
* @param fairShare a fair share to this Schedulable.
|
||||||
|
*/
|
||||||
void setFairShare(Resource fairShare);
|
void setFairShare(Resource fairShare);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -84,7 +84,7 @@ public static SchedulingPolicy getInstance(
|
|||||||
*
|
*
|
||||||
* @param policy canonical class name or "drf" or "fair" or "fifo"
|
* @param policy canonical class name or "drf" or "fair" or "fifo"
|
||||||
* @return a {@link SchedulingPolicy} instance parsed from given policy
|
* @return a {@link SchedulingPolicy} instance parsed from given policy
|
||||||
* @throws AllocationConfigurationException
|
* @throws AllocationConfigurationException for any errors.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
|
@ -45,9 +45,9 @@ private ComputeFairShares() {
|
|||||||
* shares considering only active schedulables ie schedulables which have
|
* shares considering only active schedulables ie schedulables which have
|
||||||
* running apps.
|
* running apps.
|
||||||
*
|
*
|
||||||
* @param schedulables
|
* @param schedulables given schedulables.
|
||||||
* @param totalResources
|
* @param totalResources totalResources.
|
||||||
* @param type
|
* @param type type of the resource.
|
||||||
*/
|
*/
|
||||||
public static void computeShares(
|
public static void computeShares(
|
||||||
Collection<? extends Schedulable> schedulables, Resource totalResources,
|
Collection<? extends Schedulable> schedulables, Resource totalResources,
|
||||||
@ -60,9 +60,9 @@ public static void computeShares(
|
|||||||
* share is an allocation of shares considering all queues, i.e.,
|
* share is an allocation of shares considering all queues, i.e.,
|
||||||
* active and inactive.
|
* active and inactive.
|
||||||
*
|
*
|
||||||
* @param queues
|
* @param queues {@link FSQueue}s whose shares are to be updated.
|
||||||
* @param totalResources
|
* @param totalResources totalResources.
|
||||||
* @param type
|
* @param type type of the resource.
|
||||||
*/
|
*/
|
||||||
public static void computeSteadyShares(
|
public static void computeSteadyShares(
|
||||||
Collection<? extends FSQueue> queues, Resource totalResources,
|
Collection<? extends FSQueue> queues, Resource totalResources,
|
||||||
|
@ -51,6 +51,8 @@ public interface MultiNodeLookupPolicy<N extends SchedulerNode> {
|
|||||||
*
|
*
|
||||||
* @param nodes
|
* @param nodes
|
||||||
* a collection working nm's.
|
* a collection working nm's.
|
||||||
|
* @param partition
|
||||||
|
* node label
|
||||||
*/
|
*/
|
||||||
void addAndRefreshNodesSet(Collection<N> nodes, String partition);
|
void addAndRefreshNodesSet(Collection<N> nodes, String partition);
|
||||||
|
|
||||||
|
@ -29,35 +29,43 @@
|
|||||||
public interface SchedulableEntity {
|
public interface SchedulableEntity {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Id - each entity must have a unique id
|
* Id - each entity must have a unique id.
|
||||||
|
* @return id.
|
||||||
*/
|
*/
|
||||||
public String getId();
|
public String getId();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compare the passed SchedulableEntity to this one for input order.
|
* Compare the passed SchedulableEntity to this one for input order.
|
||||||
* Input order is implementation defined and should reflect the
|
* Input order is implementation defined and should reflect the
|
||||||
* correct ordering for first-in first-out processing
|
* correct ordering for first-in first-out processing.
|
||||||
|
*
|
||||||
|
* @param other SchedulableEntity.
|
||||||
|
* @return correct ordering.
|
||||||
*/
|
*/
|
||||||
public int compareInputOrderTo(SchedulableEntity other);
|
public int compareInputOrderTo(SchedulableEntity other);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* View of Resources wanted and consumed by the entity
|
* View of Resources wanted and consumed by the entity.
|
||||||
|
* @return ResourceUsage.
|
||||||
*/
|
*/
|
||||||
public ResourceUsage getSchedulingResourceUsage();
|
public ResourceUsage getSchedulingResourceUsage();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the priority of the application
|
* Get the priority of the application.
|
||||||
|
* @return priority of the application.
|
||||||
*/
|
*/
|
||||||
public Priority getPriority();
|
public Priority getPriority();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether application was running before RM restart.
|
* Whether application was running before RM restart.
|
||||||
|
* @return true, application was running before RM restart;
|
||||||
|
* otherwise false.
|
||||||
*/
|
*/
|
||||||
public boolean isRecovering();
|
public boolean isRecovering();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get partition corresponding to this entity.
|
* Get partition corresponding to this entity.
|
||||||
* @return partition
|
* @return partition node label.
|
||||||
*/
|
*/
|
||||||
String getPartition();
|
String getPartition();
|
||||||
|
|
||||||
|
@ -76,7 +76,9 @@ public class AMRMTokenSecretManager extends
|
|||||||
new HashSet<ApplicationAttemptId>();
|
new HashSet<ApplicationAttemptId>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create an {@link AMRMTokenSecretManager}
|
* Create an {@link AMRMTokenSecretManager}.
|
||||||
|
* @param conf configuration.
|
||||||
|
* @param rmContext rm context.
|
||||||
*/
|
*/
|
||||||
public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
|
public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
|
||||||
this.rmContext = rmContext;
|
this.rmContext = rmContext;
|
||||||
@ -219,6 +221,8 @@ public MasterKeyData getMasterKey() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Populate persisted password of AMRMToken back to AMRMTokenSecretManager.
|
* Populate persisted password of AMRMToken back to AMRMTokenSecretManager.
|
||||||
|
* @param token AMRMTokenIdentifier.
|
||||||
|
* @throws IOException an I/O exception has occurred.
|
||||||
*/
|
*/
|
||||||
public void addPersistedPassword(Token<AMRMTokenIdentifier> token)
|
public void addPersistedPassword(Token<AMRMTokenIdentifier> token)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
@ -623,7 +623,9 @@ private boolean skipTokenRenewal(Token<?> token)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* set task to renew the token
|
* set task to renew the token.
|
||||||
|
* @param token DelegationTokenToRenew.
|
||||||
|
* @throws IOException if an IO error occurred.
|
||||||
*/
|
*/
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
protected void setTimerForTokenRenewal(DelegationTokenToRenew token)
|
protected void setTimerForTokenRenewal(DelegationTokenToRenew token)
|
||||||
|
@ -262,7 +262,7 @@ public void unregisterApplicationAttempt(ApplicationAttemptId appAttemptId) {
|
|||||||
/**
|
/**
|
||||||
* This is to be called when NodeManager reconnects or goes down. This will
|
* This is to be called when NodeManager reconnects or goes down. This will
|
||||||
* remove if NMTokens if present for any running application from cache.
|
* remove if NMTokens if present for any running application from cache.
|
||||||
* @param nodeId
|
* @param nodeId Node Id.
|
||||||
*/
|
*/
|
||||||
public void removeNodeKey(NodeId nodeId) {
|
public void removeNodeKey(NodeId nodeId) {
|
||||||
this.writeLock.lock();
|
this.writeLock.lock();
|
||||||
|
@ -193,6 +193,7 @@ public Token createContainerToken(ContainerId containerId,
|
|||||||
* @param containerType Container Type
|
* @param containerType Container Type
|
||||||
* @param execType Execution Type
|
* @param execType Execution Type
|
||||||
* @param allocationRequestId allocationRequestId
|
* @param allocationRequestId allocationRequestId
|
||||||
|
* @param allocationTags allocation Tags
|
||||||
* @return the container-token
|
* @return the container-token
|
||||||
*/
|
*/
|
||||||
public Token createContainerToken(ContainerId containerId,
|
public Token createContainerToken(ContainerId containerId,
|
||||||
|
@ -43,30 +43,32 @@ public interface VolumeManager {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Start to supervise on a volume.
|
* Start to supervise on a volume.
|
||||||
* @param volume
|
* @param volume volume.
|
||||||
* @return the volume being managed by the manager.
|
* @return the volume being managed by the manager.
|
||||||
*/
|
*/
|
||||||
Volume addOrGetVolume(Volume volume);
|
Volume addOrGetVolume(Volume volume);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute volume provisioning tasks as backend threads.
|
* Execute volume provisioning tasks as backend threads.
|
||||||
* @param volumeProvisioningTask
|
* @param volumeProvisioningTask A provisioning task encapsulates
|
||||||
* @param delaySecond
|
* all the logic required by a storage system to provision a volume.
|
||||||
|
* @param delaySecond delay Second.
|
||||||
|
* @return ScheduledFuture.
|
||||||
*/
|
*/
|
||||||
ScheduledFuture<VolumeProvisioningResults> schedule(
|
ScheduledFuture<VolumeProvisioningResults> schedule(
|
||||||
VolumeProvisioningTask volumeProvisioningTask, int delaySecond);
|
VolumeProvisioningTask volumeProvisioningTask, int delaySecond);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Register a csi-driver-adaptor to the volume manager.
|
* Register a csi-driver-adaptor to the volume manager.
|
||||||
* @param driverName
|
* @param driverName driver name.
|
||||||
* @param client
|
* @param client csi adaptor protocol client.
|
||||||
*/
|
*/
|
||||||
void registerCsiDriverAdaptor(String driverName, CsiAdaptorProtocol client);
|
void registerCsiDriverAdaptor(String driverName, CsiAdaptorProtocol client);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the csi-driver-adaptor client from cache by the given driver name.
|
* Returns the csi-driver-adaptor client from cache by the given driver name.
|
||||||
* If the client is not found, null is returned.
|
* If the client is not found, null is returned.
|
||||||
* @param driverName
|
* @param driverName driver name.
|
||||||
* @return a csi-driver-adaptor client working for given driver or null
|
* @return a csi-driver-adaptor client working for given driver or null
|
||||||
* if the adaptor could not be found.
|
* if the adaptor could not be found.
|
||||||
*/
|
*/
|
||||||
|
@ -49,7 +49,7 @@ public FairSchedulerInfo(FairScheduler fs) {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the fair share assigned to the appAttemptId.
|
* Get the fair share assigned to the appAttemptId.
|
||||||
* @param appAttemptId
|
* @param appAttemptId the application attempt id
|
||||||
* @return The fair share assigned to the appAttemptId,
|
* @return The fair share assigned to the appAttemptId,
|
||||||
* <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does
|
* <code>FairSchedulerInfo#INVALID_FAIR_SHARE</code> if the scheduler does
|
||||||
* not know about this application attempt.
|
* not know about this application attempt.
|
||||||
|
@ -154,6 +154,7 @@ protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the steady fair share as a fraction of the entire cluster capacity.
|
* Returns the steady fair share as a fraction of the entire cluster capacity.
|
||||||
|
* @return steady fairshare memoryfraction.
|
||||||
*/
|
*/
|
||||||
public float getSteadyFairShareMemoryFraction() {
|
public float getSteadyFairShareMemoryFraction() {
|
||||||
return fractionMemSteadyFairShare;
|
return fractionMemSteadyFairShare;
|
||||||
@ -161,6 +162,7 @@ public float getSteadyFairShareMemoryFraction() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the fair share as a fraction of the entire cluster capacity.
|
* Returns the fair share as a fraction of the entire cluster capacity.
|
||||||
|
* @return fair share memory fraction.
|
||||||
*/
|
*/
|
||||||
public float getFairShareMemoryFraction() {
|
public float getFairShareMemoryFraction() {
|
||||||
return fractionMemFairShare;
|
return fractionMemFairShare;
|
||||||
@ -168,13 +170,15 @@ public float getFairShareMemoryFraction() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the steady fair share of this queue in megabytes.
|
* Returns the steady fair share of this queue in megabytes.
|
||||||
|
* @return steady fair share.
|
||||||
*/
|
*/
|
||||||
public ResourceInfo getSteadyFairShare() {
|
public ResourceInfo getSteadyFairShare() {
|
||||||
return steadyFairResources;
|
return steadyFairResources;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the fair share of this queue in megabytes
|
* Returns the fair share of this queue in megabytes.
|
||||||
|
* @return fair share.
|
||||||
*/
|
*/
|
||||||
public ResourceInfo getFairShare() {
|
public ResourceInfo getFairShare() {
|
||||||
return fairResources;
|
return fairResources;
|
||||||
@ -232,6 +236,7 @@ public ResourceInfo getDemandResources() {
|
|||||||
/**
|
/**
|
||||||
* Returns the memory used by this queue as a fraction of the entire
|
* Returns the memory used by this queue as a fraction of the entire
|
||||||
* cluster capacity.
|
* cluster capacity.
|
||||||
|
* @return used memory fraction.
|
||||||
*/
|
*/
|
||||||
public float getUsedMemoryFraction() {
|
public float getUsedMemoryFraction() {
|
||||||
return fractionMemUsed;
|
return fractionMemUsed;
|
||||||
@ -240,6 +245,7 @@ public float getUsedMemoryFraction() {
|
|||||||
/**
|
/**
|
||||||
* Returns the capacity of this queue as a fraction of the entire cluster
|
* Returns the capacity of this queue as a fraction of the entire cluster
|
||||||
* capacity.
|
* capacity.
|
||||||
|
* @return max resources fraction.
|
||||||
*/
|
*/
|
||||||
public float getMaxResourcesFraction() {
|
public float getMaxResourcesFraction() {
|
||||||
return fractionMemMaxShare;
|
return fractionMemMaxShare;
|
||||||
@ -247,6 +253,7 @@ public float getMaxResourcesFraction() {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the name of the scheduling policy used by this queue.
|
* Returns the name of the scheduling policy used by this queue.
|
||||||
|
* @return SchedulingPolicy.
|
||||||
*/
|
*/
|
||||||
public String getSchedulingPolicy() {
|
public String getSchedulingPolicy() {
|
||||||
return schedulingPolicy;
|
return schedulingPolicy;
|
||||||
|
Loading…
Reference in New Issue
Block a user