YARN-11362: Fix several typos in YARN codebase of misspelled resource (#6474) Contributed by EremenkoValentin.
Reviewed-by: Shilun Fan <slfan1989@apache.org> Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
parent
20d8596af2
commit
141627778d
@ -29,18 +29,18 @@
|
|||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public class SchedulerInvalidResoureRequestException extends YarnRuntimeException {
|
public class SchedulerInvalidResourceRequestException extends YarnRuntimeException {
|
||||||
private static final long serialVersionUID = 10081123982L;
|
private static final long serialVersionUID = 10081123982L;
|
||||||
|
|
||||||
public SchedulerInvalidResoureRequestException(String message) {
|
public SchedulerInvalidResourceRequestException(String message) {
|
||||||
super(message);
|
super(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SchedulerInvalidResoureRequestException(Throwable cause) {
|
public SchedulerInvalidResourceRequestException(Throwable cause) {
|
||||||
super(cause);
|
super(cause);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SchedulerInvalidResoureRequestException(String message,
|
public SchedulerInvalidResourceRequestException(String message,
|
||||||
Throwable cause) {
|
Throwable cause) {
|
||||||
super(message, cause);
|
super(message, cause);
|
||||||
}
|
}
|
@ -133,17 +133,17 @@ public void testGetApplicationReport() throws Exception {
|
|||||||
ApplicationCLI cli = createAndGetAppCLI();
|
ApplicationCLI cli = createAndGetAppCLI();
|
||||||
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
|
||||||
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
||||||
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
|
Map<String, Long> preemptedResourceSecondsMap = new HashMap<>();
|
||||||
resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
|
resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
|
||||||
resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
|
resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
|
||||||
preemptedResoureSecondsMap
|
preemptedResourceSecondsMap
|
||||||
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
|
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
|
||||||
preemptedResoureSecondsMap
|
preemptedResourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), 2222L);
|
.put(ResourceInformation.VCORES.getName(), 2222L);
|
||||||
ApplicationResourceUsageReport usageReport = i == 0 ? null :
|
ApplicationResourceUsageReport usageReport = i == 0 ? null :
|
||||||
ApplicationResourceUsageReport
|
ApplicationResourceUsageReport
|
||||||
.newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
|
.newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
|
||||||
preemptedResoureSecondsMap);
|
preemptedResourceSecondsMap);
|
||||||
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
|
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
|
||||||
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
|
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
|
||||||
"user", "queue", "appname", "host", 124, null,
|
"user", "queue", "appname", "host", 124, null,
|
||||||
|
@ -395,19 +395,19 @@ public static ApplicationReport convertToApplicationReport(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
||||||
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
|
Map<String, Long> preemptedResourceSecondsMap = new HashMap<>();
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
|
.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
|
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
|
||||||
preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
preemptedResourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
||||||
preemptedMemorySeconds);
|
preemptedMemorySeconds);
|
||||||
preemptedResoureSecondsMap
|
preemptedResourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
|
.put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
|
||||||
|
|
||||||
appResources = ApplicationResourceUsageReport
|
appResources = ApplicationResourceUsageReport
|
||||||
.newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
|
.newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
|
||||||
preemptedResoureSecondsMap);
|
preemptedResourceSecondsMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
NavigableSet<TimelineEvent> events = entity.getEvents();
|
NavigableSet<TimelineEvent> events = entity.getEvents();
|
||||||
|
@ -345,19 +345,19 @@ private static ApplicationReportExt convertToApplicationReport(
|
|||||||
long preemptedVcoreSeconds = parseLong(entityInfo,
|
long preemptedVcoreSeconds = parseLong(entityInfo,
|
||||||
ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
|
ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
|
||||||
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
||||||
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
|
Map<String, Long> preemptedResourceSecondsMap = new HashMap<>();
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
|
.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
|
.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
|
||||||
preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
preemptedResourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
||||||
preemptedMemorySeconds);
|
preemptedMemorySeconds);
|
||||||
preemptedResoureSecondsMap
|
preemptedResourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
|
.put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
|
||||||
|
|
||||||
appResources = ApplicationResourceUsageReport
|
appResources = ApplicationResourceUsageReport
|
||||||
.newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
|
.newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
|
||||||
preemptedResoureSecondsMap);
|
preemptedResourceSecondsMap);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
|
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
|
||||||
|
@ -55,7 +55,7 @@
|
|||||||
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
|
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
|
||||||
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
|
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
|
||||||
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
|
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
@ -301,7 +301,7 @@ public void allocate(ApplicationAttemptId appAttemptId,
|
|||||||
allocation = getScheduler().allocate(appAttemptId, ask,
|
allocation = getScheduler().allocate(appAttemptId, ask,
|
||||||
request.getSchedulingRequests(), release,
|
request.getSchedulingRequests(), release,
|
||||||
blacklistAdditions, blacklistRemovals, containerUpdateRequests);
|
blacklistAdditions, blacklistRemovals, containerUpdateRequests);
|
||||||
} catch (SchedulerInvalidResoureRequestException e) {
|
} catch (SchedulerInvalidResourceRequestException e) {
|
||||||
LOG.warn("Exceptions caught when scheduler handling requests");
|
LOG.warn("Exceptions caught when scheduler handling requests");
|
||||||
throw new YarnException(e);
|
throw new YarnException(e);
|
||||||
}
|
}
|
||||||
|
@ -43,8 +43,7 @@
|
|||||||
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
|
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
|
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
|
||||||
.InvalidResourceType;
|
.InvalidResourceType;
|
||||||
import org.apache.hadoop.yarn.exceptions
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
.SchedulerInvalidResoureRequestException;
|
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.security.AccessType;
|
import org.apache.hadoop.yarn.security.AccessType;
|
||||||
@ -430,7 +429,7 @@ static void checkResourceRequestAgainstAvailableResource(Resource reqResource,
|
|||||||
public static MaxResourceValidationResult
|
public static MaxResourceValidationResult
|
||||||
validateResourceRequestsAgainstQueueMaxResource(
|
validateResourceRequestsAgainstQueueMaxResource(
|
||||||
ResourceRequest resReq, Resource availableResource)
|
ResourceRequest resReq, Resource availableResource)
|
||||||
throws SchedulerInvalidResoureRequestException {
|
throws SchedulerInvalidResourceRequestException {
|
||||||
final Resource reqResource = resReq.getCapability();
|
final Resource reqResource = resReq.getCapability();
|
||||||
Map<String, ResourceInformation> resourcesWithZeroAmount =
|
Map<String, ResourceInformation> resourcesWithZeroAmount =
|
||||||
getZeroResources(availableResource);
|
getZeroResources(availableResource);
|
||||||
|
@ -370,11 +370,11 @@ private ContainerAllocation assignNodeLocalContainers(
|
|||||||
Resource clusterResource, PendingAsk nodeLocalAsk,
|
Resource clusterResource, PendingAsk nodeLocalAsk,
|
||||||
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
||||||
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
||||||
ResourceLimits currentResoureLimits) {
|
ResourceLimits currentResourceLimits) {
|
||||||
if (canAssign(schedulerKey, node, NodeType.NODE_LOCAL, reservedContainer)) {
|
if (canAssign(schedulerKey, node, NodeType.NODE_LOCAL, reservedContainer)) {
|
||||||
return assignContainer(clusterResource, node, schedulerKey,
|
return assignContainer(clusterResource, node, schedulerKey,
|
||||||
nodeLocalAsk, NodeType.NODE_LOCAL, reservedContainer,
|
nodeLocalAsk, NodeType.NODE_LOCAL, reservedContainer,
|
||||||
schedulingMode, currentResoureLimits);
|
schedulingMode, currentResourceLimits);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip node-local request, go to rack-local request
|
// Skip node-local request, go to rack-local request
|
||||||
@ -385,11 +385,11 @@ private ContainerAllocation assignRackLocalContainers(
|
|||||||
Resource clusterResource, PendingAsk rackLocalAsk,
|
Resource clusterResource, PendingAsk rackLocalAsk,
|
||||||
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
||||||
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
||||||
ResourceLimits currentResoureLimits) {
|
ResourceLimits currentResourceLimits) {
|
||||||
if (canAssign(schedulerKey, node, NodeType.RACK_LOCAL, reservedContainer)) {
|
if (canAssign(schedulerKey, node, NodeType.RACK_LOCAL, reservedContainer)) {
|
||||||
return assignContainer(clusterResource, node, schedulerKey,
|
return assignContainer(clusterResource, node, schedulerKey,
|
||||||
rackLocalAsk, NodeType.RACK_LOCAL, reservedContainer,
|
rackLocalAsk, NodeType.RACK_LOCAL, reservedContainer,
|
||||||
schedulingMode, currentResoureLimits);
|
schedulingMode, currentResourceLimits);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip rack-local request, go to off-switch request
|
// Skip rack-local request, go to off-switch request
|
||||||
@ -400,11 +400,11 @@ private ContainerAllocation assignOffSwitchContainers(
|
|||||||
Resource clusterResource, PendingAsk offSwitchAsk,
|
Resource clusterResource, PendingAsk offSwitchAsk,
|
||||||
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
||||||
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
||||||
ResourceLimits currentResoureLimits) {
|
ResourceLimits currentResourceLimits) {
|
||||||
if (canAssign(schedulerKey, node, NodeType.OFF_SWITCH, reservedContainer)) {
|
if (canAssign(schedulerKey, node, NodeType.OFF_SWITCH, reservedContainer)) {
|
||||||
return assignContainer(clusterResource, node, schedulerKey,
|
return assignContainer(clusterResource, node, schedulerKey,
|
||||||
offSwitchAsk, NodeType.OFF_SWITCH, reservedContainer,
|
offSwitchAsk, NodeType.OFF_SWITCH, reservedContainer,
|
||||||
schedulingMode, currentResoureLimits);
|
schedulingMode, currentResourceLimits);
|
||||||
}
|
}
|
||||||
|
|
||||||
application.updateAppSkipNodeDiagnostics(
|
application.updateAppSkipNodeDiagnostics(
|
||||||
@ -419,7 +419,7 @@ private ContainerAllocation assignOffSwitchContainers(
|
|||||||
private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
||||||
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
||||||
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
RMContainer reservedContainer, SchedulingMode schedulingMode,
|
||||||
ResourceLimits currentResoureLimits) {
|
ResourceLimits currentResourceLimits) {
|
||||||
ContainerAllocation allocation;
|
ContainerAllocation allocation;
|
||||||
NodeType requestLocalityType = null;
|
NodeType requestLocalityType = null;
|
||||||
|
|
||||||
@ -431,7 +431,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
|||||||
allocation =
|
allocation =
|
||||||
assignNodeLocalContainers(clusterResource, nodeLocalAsk,
|
assignNodeLocalContainers(clusterResource, nodeLocalAsk,
|
||||||
node, schedulerKey, reservedContainer, schedulingMode,
|
node, schedulerKey, reservedContainer, schedulingMode,
|
||||||
currentResoureLimits);
|
currentResourceLimits);
|
||||||
if (Resources.greaterThan(rc, clusterResource,
|
if (Resources.greaterThan(rc, clusterResource,
|
||||||
allocation.getResourceToBeAllocated(), Resources.none())) {
|
allocation.getResourceToBeAllocated(), Resources.none())) {
|
||||||
allocation.requestLocalityType = requestLocalityType;
|
allocation.requestLocalityType = requestLocalityType;
|
||||||
@ -458,7 +458,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
|||||||
allocation =
|
allocation =
|
||||||
assignRackLocalContainers(clusterResource, rackLocalAsk,
|
assignRackLocalContainers(clusterResource, rackLocalAsk,
|
||||||
node, schedulerKey, reservedContainer, schedulingMode,
|
node, schedulerKey, reservedContainer, schedulingMode,
|
||||||
currentResoureLimits);
|
currentResourceLimits);
|
||||||
if (Resources.greaterThan(rc, clusterResource,
|
if (Resources.greaterThan(rc, clusterResource,
|
||||||
allocation.getResourceToBeAllocated(), Resources.none())) {
|
allocation.getResourceToBeAllocated(), Resources.none())) {
|
||||||
allocation.requestLocalityType = requestLocalityType;
|
allocation.requestLocalityType = requestLocalityType;
|
||||||
@ -485,7 +485,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
|||||||
allocation =
|
allocation =
|
||||||
assignOffSwitchContainers(clusterResource, offSwitchAsk,
|
assignOffSwitchContainers(clusterResource, offSwitchAsk,
|
||||||
node, schedulerKey, reservedContainer, schedulingMode,
|
node, schedulerKey, reservedContainer, schedulingMode,
|
||||||
currentResoureLimits);
|
currentResourceLimits);
|
||||||
|
|
||||||
// When a returned allocation is LOCALITY_SKIPPED, since we're in
|
// When a returned allocation is LOCALITY_SKIPPED, since we're in
|
||||||
// off-switch request now, we will skip this app w.r.t priorities
|
// off-switch request now, we will skip this app w.r.t priorities
|
||||||
@ -507,7 +507,7 @@ private ContainerAllocation assignContainersOnNode(Resource clusterResource,
|
|||||||
private ContainerAllocation assignContainer(Resource clusterResource,
|
private ContainerAllocation assignContainer(Resource clusterResource,
|
||||||
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
FiCaSchedulerNode node, SchedulerRequestKey schedulerKey,
|
||||||
PendingAsk pendingAsk, NodeType type, RMContainer rmContainer,
|
PendingAsk pendingAsk, NodeType type, RMContainer rmContainer,
|
||||||
SchedulingMode schedulingMode, ResourceLimits currentResoureLimits) {
|
SchedulingMode schedulingMode, ResourceLimits currentResourceLimits) {
|
||||||
|
|
||||||
if (LOG.isDebugEnabled()) {
|
if (LOG.isDebugEnabled()) {
|
||||||
LOG.debug("assignContainers: node=" + node.getNodeName()
|
LOG.debug("assignContainers: node=" + node.getNodeName()
|
||||||
@ -547,8 +547,8 @@ private ContainerAllocation assignContainer(Resource clusterResource,
|
|||||||
// max(required - headroom, amountNeedUnreserve)
|
// max(required - headroom, amountNeedUnreserve)
|
||||||
Resource resourceNeedToUnReserve =
|
Resource resourceNeedToUnReserve =
|
||||||
Resources.max(rc, clusterResource,
|
Resources.max(rc, clusterResource,
|
||||||
Resources.subtract(capability, currentResoureLimits.getHeadroom()),
|
Resources.subtract(capability, currentResourceLimits.getHeadroom()),
|
||||||
currentResoureLimits.getAmountNeededUnreserve());
|
currentResourceLimits.getAmountNeededUnreserve());
|
||||||
|
|
||||||
boolean needToUnreserve =
|
boolean needToUnreserve =
|
||||||
rc.isAnyMajorResourceAboveZero(resourceNeedToUnReserve);
|
rc.isAnyMajorResourceAboveZero(resourceNeedToUnReserve);
|
||||||
@ -559,7 +559,7 @@ private ContainerAllocation assignContainer(Resource clusterResource,
|
|||||||
|
|
||||||
// Check if we need to kill some containers to allocate this one
|
// Check if we need to kill some containers to allocate this one
|
||||||
List<RMContainer> toKillContainers = null;
|
List<RMContainer> toKillContainers = null;
|
||||||
if (availableContainers == 0 && currentResoureLimits.isAllowPreemption()) {
|
if (availableContainers == 0 && currentResourceLimits.isAllowPreemption()) {
|
||||||
Resource availableAndKillable = Resources.clone(available);
|
Resource availableAndKillable = Resources.clone(available);
|
||||||
for (RMContainer killableContainer : node
|
for (RMContainer killableContainer : node
|
||||||
.getKillableContainers().values()) {
|
.getKillableContainers().values()) {
|
||||||
@ -590,7 +590,7 @@ private ContainerAllocation assignContainer(Resource clusterResource,
|
|||||||
if (!needToUnreserve) {
|
if (!needToUnreserve) {
|
||||||
// If we shouldn't allocate/reserve new container then we should
|
// If we shouldn't allocate/reserve new container then we should
|
||||||
// unreserve one the same size we are asking for since the
|
// unreserve one the same size we are asking for since the
|
||||||
// currentResoureLimits.getAmountNeededUnreserve could be zero. If
|
// currentResourceLimits.getAmountNeededUnreserve could be zero. If
|
||||||
// the limit was hit then use the amount we need to unreserve to be
|
// the limit was hit then use the amount we need to unreserve to be
|
||||||
// under the limit.
|
// under the limit.
|
||||||
resourceNeedToUnReserve = capability;
|
resourceNeedToUnReserve = capability;
|
||||||
|
@ -41,8 +41,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
||||||
import org.apache.hadoop.yarn.api.records.Priority;
|
import org.apache.hadoop.yarn.api.records.Priority;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
.SchedulerInvalidResoureRequestException;
|
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
|
||||||
@ -912,7 +911,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId,
|
|||||||
// scheduler would clear them right away and AM
|
// scheduler would clear them right away and AM
|
||||||
// would not get this information.
|
// would not get this information.
|
||||||
if (!invalidAsks.isEmpty()) {
|
if (!invalidAsks.isEmpty()) {
|
||||||
throw new SchedulerInvalidResoureRequestException(String.format(
|
throw new SchedulerInvalidResourceRequestException(String.format(
|
||||||
"Resource request is invalid for application %s because queue %s "
|
"Resource request is invalid for application %s because queue %s "
|
||||||
+ "has 0 amount of resource for a resource type! "
|
+ "has 0 amount of resource for a resource type! "
|
||||||
+ "Validation result: %s",
|
+ "Validation result: %s",
|
||||||
|
@ -23,7 +23,7 @@
|
|||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
import org.apache.hadoop.yarn.api.records.ResourceRequest;
|
||||||
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
||||||
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
||||||
@ -172,8 +172,8 @@ public PendingAskUpdateResult updatePendingAsk(
|
|||||||
SchedulerRequestKey schedulerRequestKey,
|
SchedulerRequestKey schedulerRequestKey,
|
||||||
SchedulingRequest schedulingRequest,
|
SchedulingRequest schedulingRequest,
|
||||||
boolean recoverPreemptedRequestForAContainer)
|
boolean recoverPreemptedRequestForAContainer)
|
||||||
throws SchedulerInvalidResoureRequestException {
|
throws SchedulerInvalidResourceRequestException {
|
||||||
throw new SchedulerInvalidResoureRequestException(this.getClass().getName()
|
throw new SchedulerInvalidResourceRequestException(this.getClass().getName()
|
||||||
+ " not be able to handle SchedulingRequest, there exists a "
|
+ " not be able to handle SchedulingRequest, there exists a "
|
||||||
+ "ResourceRequest with the same scheduler key=" + schedulerRequestKey
|
+ "ResourceRequest with the same scheduler key=" + schedulerRequestKey
|
||||||
+ ", please send SchedulingRequest with a different allocationId and "
|
+ ", please send SchedulingRequest with a different allocationId and "
|
||||||
|
@ -28,7 +28,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.SchedulingRequestPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.SchedulingRequestPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
|
import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
|
||||||
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
||||||
@ -81,7 +81,7 @@ public PendingAskUpdateResult updatePendingAsk(
|
|||||||
Collection<ResourceRequest> requests,
|
Collection<ResourceRequest> requests,
|
||||||
boolean recoverPreemptedRequestForAContainer) {
|
boolean recoverPreemptedRequestForAContainer) {
|
||||||
if (requests != null && !requests.isEmpty()) {
|
if (requests != null && !requests.isEmpty()) {
|
||||||
throw new SchedulerInvalidResoureRequestException(
|
throw new SchedulerInvalidResourceRequestException(
|
||||||
this.getClass().getName()
|
this.getClass().getName()
|
||||||
+ " not be able to handle ResourceRequest, there exists a "
|
+ " not be able to handle ResourceRequest, there exists a "
|
||||||
+ "SchedulingRequest with the same scheduler key="
|
+ "SchedulingRequest with the same scheduler key="
|
||||||
@ -98,7 +98,7 @@ private PendingAskUpdateResult internalUpdatePendingAsk(
|
|||||||
SchedulingRequest newSchedulingRequest, boolean recoverContainer) {
|
SchedulingRequest newSchedulingRequest, boolean recoverContainer) {
|
||||||
// When it is a recover container, there must exists an schedulingRequest.
|
// When it is a recover container, there must exists an schedulingRequest.
|
||||||
if (recoverContainer && schedulingRequest == null) {
|
if (recoverContainer && schedulingRequest == null) {
|
||||||
throw new SchedulerInvalidResoureRequestException("Trying to recover a "
|
throw new SchedulerInvalidResourceRequestException("Trying to recover a "
|
||||||
+ "container request=" + newSchedulingRequest.toString() + ", however"
|
+ "container request=" + newSchedulingRequest.toString() + ", however"
|
||||||
+ "there's no existing scheduling request, this should not happen.");
|
+ "there's no existing scheduling request, this should not happen.");
|
||||||
}
|
}
|
||||||
@ -127,7 +127,7 @@ private PendingAskUpdateResult internalUpdatePendingAsk(
|
|||||||
if (!schedulingRequest.equals(newSchedulingRequest)) {
|
if (!schedulingRequest.equals(newSchedulingRequest)) {
|
||||||
// Rollback #numAllocations
|
// Rollback #numAllocations
|
||||||
sizing.setNumAllocations(newNumAllocations);
|
sizing.setNumAllocations(newNumAllocations);
|
||||||
throw new SchedulerInvalidResoureRequestException(
|
throw new SchedulerInvalidResourceRequestException(
|
||||||
"Invalid updated SchedulingRequest added to scheduler, "
|
"Invalid updated SchedulingRequest added to scheduler, "
|
||||||
+ " we only allows changing numAllocations for the updated "
|
+ " we only allows changing numAllocations for the updated "
|
||||||
+ "SchedulingRequest. Old=" + schedulingRequest.toString()
|
+ "SchedulingRequest. Old=" + schedulingRequest.toString()
|
||||||
@ -148,7 +148,7 @@ private PendingAskUpdateResult internalUpdatePendingAsk(
|
|||||||
|
|
||||||
// Basic sanity check
|
// Basic sanity check
|
||||||
if (newNumAllocations < 0) {
|
if (newNumAllocations < 0) {
|
||||||
throw new SchedulerInvalidResoureRequestException(
|
throw new SchedulerInvalidResourceRequestException(
|
||||||
"numAllocation in ResourceSizing field must be >= 0, "
|
"numAllocation in ResourceSizing field must be >= 0, "
|
||||||
+ "updating schedulingRequest failed.");
|
+ "updating schedulingRequest failed.");
|
||||||
}
|
}
|
||||||
@ -197,12 +197,12 @@ private String throwExceptionWithMetaInfo(String message) {
|
|||||||
sb.append("AppId=").append(appSchedulingInfo.getApplicationId()).append(
|
sb.append("AppId=").append(appSchedulingInfo.getApplicationId()).append(
|
||||||
" Key=").append(this.schedulerRequestKey).append(". Exception message:")
|
" Key=").append(this.schedulerRequestKey).append(". Exception message:")
|
||||||
.append(message);
|
.append(message);
|
||||||
throw new SchedulerInvalidResoureRequestException(sb.toString());
|
throw new SchedulerInvalidResourceRequestException(sb.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
private void validateAndSetSchedulingRequest(SchedulingRequest
|
private void validateAndSetSchedulingRequest(SchedulingRequest
|
||||||
newSchedulingRequest)
|
newSchedulingRequest)
|
||||||
throws SchedulerInvalidResoureRequestException {
|
throws SchedulerInvalidResourceRequestException {
|
||||||
// Check sizing exists
|
// Check sizing exists
|
||||||
if (newSchedulingRequest.getResourceSizing() == null
|
if (newSchedulingRequest.getResourceSizing() == null
|
||||||
|| newSchedulingRequest.getResourceSizing().getResources() == null) {
|
|| newSchedulingRequest.getResourceSizing().getResources() == null) {
|
||||||
|
@ -814,19 +814,19 @@ private static ApplicationAttemptStateData createFinishedAttempt(
|
|||||||
ApplicationAttemptId attemptId, Container container, long startTime,
|
ApplicationAttemptId attemptId, Container container, long startTime,
|
||||||
int amExitStatus) {
|
int amExitStatus) {
|
||||||
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
Map<String, Long> resourceSecondsMap = new HashMap<>();
|
||||||
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
|
Map<String, Long> preemptedResourceSecondsMap = new HashMap<>();
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.MEMORY_MB.getName(), 0L);
|
.put(ResourceInformation.MEMORY_MB.getName(), 0L);
|
||||||
resourceSecondsMap
|
resourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), 0L);
|
.put(ResourceInformation.VCORES.getName(), 0L);
|
||||||
preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
preemptedResourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
|
||||||
0L);
|
0L);
|
||||||
preemptedResoureSecondsMap
|
preemptedResourceSecondsMap
|
||||||
.put(ResourceInformation.VCORES.getName(), 0L);
|
.put(ResourceInformation.VCORES.getName(), 0L);
|
||||||
return ApplicationAttemptStateData.newInstance(attemptId,
|
return ApplicationAttemptStateData.newInstance(attemptId,
|
||||||
container, null, startTime, RMAppAttemptState.FINISHED,
|
container, null, startTime, RMAppAttemptState.FINISHED,
|
||||||
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
|
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
|
||||||
amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap, 0);
|
amExitStatus, 0, resourceSecondsMap, preemptedResourceSecondsMap, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
private ApplicationAttemptId storeAttempt(RMStateStore store,
|
private ApplicationAttemptId storeAttempt(RMStateStore store,
|
||||||
|
@ -1486,7 +1486,7 @@ public void testHierarchyQueuesCurrentLimits() throws Exception {
|
|||||||
|
|
||||||
waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1);
|
waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1);
|
||||||
|
|
||||||
// Maximum resoure of b1 is 100 * 0.895 * 0.792 = 71 GB
|
// Maximum resource of b1 is 100 * 0.895 * 0.792 = 71 GB
|
||||||
// 2 GBs used by am, so it's 71 - 2 = 69G.
|
// 2 GBs used by am, so it's 71 - 2 = 69G.
|
||||||
Assert.assertEquals(69 * GB,
|
Assert.assertEquals(69 * GB,
|
||||||
am1.doHeartbeat().getAvailableResources().getMemorySize());
|
am1.doHeartbeat().getAvailableResources().getMemorySize());
|
||||||
|
@ -46,7 +46,7 @@
|
|||||||
import org.apache.hadoop.yarn.event.Dispatcher;
|
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||||
import org.apache.hadoop.yarn.event.Event;
|
import org.apache.hadoop.yarn.event.Event;
|
||||||
import org.apache.hadoop.yarn.event.EventHandler;
|
import org.apache.hadoop.yarn.event.EventHandler;
|
||||||
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
|
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
|
||||||
@ -5513,7 +5513,7 @@ private void testSchedulingRejectedToQueueZeroCapacityOfResource(
|
|||||||
+ resource + " and requested resource capabilities are: "
|
+ resource + " and requested resource capabilities are: "
|
||||||
+ requests.stream().map(ResourceRequest::getCapability)
|
+ requests.stream().map(ResourceRequest::getCapability)
|
||||||
.collect(Collectors.toList()));
|
.collect(Collectors.toList()));
|
||||||
} catch (SchedulerInvalidResoureRequestException e) {
|
} catch (SchedulerInvalidResourceRequestException e) {
|
||||||
assertTrue(
|
assertTrue(
|
||||||
"The thrown exception is not the expected one. Exception message: "
|
"The thrown exception is not the expected one. Exception message: "
|
||||||
+ e.getMessage(),
|
+ e.getMessage(),
|
||||||
|
@ -21,7 +21,7 @@
|
|||||||
import org.apache.hadoop.yarn.api.records.*;
|
import org.apache.hadoop.yarn.api.records.*;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTags;
|
||||||
import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
|
import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
|
||||||
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
|
import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResourceRequestException;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
|
||||||
@ -104,7 +104,7 @@ private void assertInvalidSchedulingRequest(
|
|||||||
allocator.initialize(appSchedulingInfo, schedulerRequestKey, rmContext);
|
allocator.initialize(appSchedulingInfo, schedulerRequestKey, rmContext);
|
||||||
}
|
}
|
||||||
allocator.updatePendingAsk(schedulerRequestKey, schedulingRequest, false);
|
allocator.updatePendingAsk(schedulerRequestKey, schedulingRequest, false);
|
||||||
} catch (SchedulerInvalidResoureRequestException e) {
|
} catch (SchedulerInvalidResourceRequestException e) {
|
||||||
// Expected
|
// Expected
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user