diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index d2e81a50d9..9d2b0586dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -877,6 +877,16 @@ public List getPendingResourceRequestsForAttempt( return null; } + @Override + public List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId) { + SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId); + if (attempt != null) { + return attempt.getAppSchedulingInfo().getAllSchedulingRequests(); + } + return null; + } + @Override public Priority checkAndGetApplicationPriority( Priority priorityRequestedByApp, UserGroupInformation user, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 8074f06969..d63d2b82fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -476,6 +476,23 @@ public List getAllResourceRequests() { return ret; } + /** + * Fetch SchedulingRequests. + * @return All pending SchedulingRequests. + */ + public List getAllSchedulingRequests() { + List ret = new ArrayList<>(); + try { + this.readLock.lock(); + schedulerKeyToAppPlacementAllocator.values().stream() + .filter(ap -> ap.getSchedulingRequest() != null) + .forEach(ap -> ret.add(ap.getSchedulingRequest())); + } finally { + this.readLock.unlock(); + } + return ret; + } + public PendingAsk getNextPendingAsk() { try { readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java index 43d55c4f1d..0f7a5b5b3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java @@ -363,6 +363,16 @@ public Priority updateApplicationPriority(Priority newPriority, List getPendingResourceRequestsForAttempt( ApplicationAttemptId attemptId); + /** + * Get pending scheduling request for specified application attempt. + * + * @param attemptId the id of the application attempt + * + * @return pending scheduling requests + */ + List getPendingSchedulingRequestsForAttempt( + ApplicationAttemptId attemptId); + /** * Get cluster max priority. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java index df58157efe..088b3dd9c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java @@ -192,4 +192,10 @@ public void initialize(AppSchedulingInfo appSchedulingInfo, this.rmContext = rmContext; this.schedulerRequestKey = schedulerRequestKey; } + + /** + * Get pending SchedulingRequest. + * @return SchedulingRequest + */ + public abstract SchedulingRequest getSchedulingRequest(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java index e1239a9db3..f1df3432a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java @@ -425,4 +425,9 @@ public ContainerRequest allocate(SchedulerRequestKey schedulerKey, writeLock.unlock(); } } + + @Override + public SchedulingRequest getSchedulingRequest() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 2b610f2fe7..914f35d7bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -505,8 +505,8 @@ public void showRequests() { } } - @VisibleForTesting - SchedulingRequest getSchedulingRequest() { + @Override + public SchedulingRequest getSchedulingRequest() { return schedulingRequest; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java index 43a6ac94b0..99a569af89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java @@ -87,10 +87,14 @@ private void createResourceRequestsTable(Block html) { .h3("Total Outstanding Resource Requests: " + getTotalResource(resourceRequests)) .table("#resourceRequests").thead().tr().th(".priority", "Priority") + .th(".allocationRequestId", "AllocationRequestId") .th(".resource", "ResourceName").th(".capacity", "Capability") .th(".containers", "NumContainers") .th(".relaxlocality", "RelaxLocality") - .th(".labelexpression", "NodeLabelExpression").__().__().tbody(); + .th(".labelexpression", "NodeLabelExpression") + .th(".executiontype", "ExecutionType") + .th(".allocationTags", "AllocationTags") + .th(".placementConstraint", "PlacementConstraint").__().__().tbody(); StringBuilder resourceRequestTableData = new StringBuilder("[\n"); for (ResourceRequestInfo resourceRequest : resourceRequests) { @@ -99,7 +103,11 @@ private void createResourceRequestsTable(Block html) { } resourceRequestTableData.append("[\"") .append(String.valueOf(resourceRequest.getPriority())).append("\",\"") - .append(resourceRequest.getResourceName()).append("\",\"") + .append(String.valueOf(resourceRequest.getAllocationRequestId())) + .append("\",\"") + .append(resourceRequest.getResourceName() == null ? "N/A" + : resourceRequest.getResourceName()) + .append("\",\"") .append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils .escapeHtml4(String.valueOf(resourceRequest.getCapability())))) .append("\",\"") @@ -109,6 +117,15 @@ private void createResourceRequestsTable(Block html) { .append("\",\"") .append(resourceRequest.getNodeLabelExpression() == null ? "N/A" : resourceRequest.getNodeLabelExpression()) + .append("\",\"") + .append(resourceRequest.getExecutionTypeRequest() == null ? "N/A" + : resourceRequest.getExecutionTypeRequest().getExecutionType()) + .append("\",\"") + .append(resourceRequest.getAllocationTags() == null ? "N/A" : + StringUtils.join(resourceRequest.getAllocationTags(), ",")) + .append("\",\"") + .append(resourceRequest.getPlacementConstraint() == null ? "N/A" + : resourceRequest.getPlacementConstraint()) .append("\"],\n"); } if (resourceRequestTableData @@ -132,7 +149,8 @@ private Resource getTotalResource(List requests) { if (request.getNumContainers() == 0) { continue; } - if (request.getResourceName().equals(ResourceRequest.ANY)) { + if (request.getResourceName() == null || request.getResourceName() + .equals(ResourceRequest.ANY)) { Resources.addTo( totalResource, Resources.multiply(request.getCapability().getResource(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9d82bc78c7..63b6fe072d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -230,6 +231,15 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, resourceRequests.add(new ResourceRequestInfo(req)); } } + + List schedulingRequestsRaw = rm.getRMContext() + .getScheduler().getPendingSchedulingRequestsForAttempt( + attempt.getAppAttemptId()); + if (schedulingRequestsRaw != null) { + for (SchedulingRequest req : schedulingRequestsRaw) { + resourceRequests.add(new ResourceRequestInfo(req)); + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java index 030af45cce..beab9d4f24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java @@ -21,11 +21,13 @@ import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import java.util.Set; /** * Simple class representing a resource request. @@ -36,6 +38,8 @@ public class ResourceRequestInfo { @XmlElement(name = "priority") private int priority; + @XmlElement(name = "allocationRequestId") + private long allocationRequestId; @XmlElement(name = "resourceName") private String resourceName; @XmlElement(name = "capability") @@ -50,16 +54,17 @@ public class ResourceRequestInfo { @XmlElement(name = "executionTypeRequest") private ExecutionTypeRequestInfo executionTypeRequest; - @XmlElement(name = "executionType") - private String executionType; - @XmlElement(name = "enforceExecutionType") - private boolean enforceExecutionType; + @XmlElement(name = "placementConstraint") + private String placementConstraint; + @XmlElement(name = "allocationTags") + private Set allocationTags; public ResourceRequestInfo() { } public ResourceRequestInfo(ResourceRequest request) { priority = request.getPriority().getPriority(); + allocationRequestId = request.getAllocationRequestId(); resourceName = request.getResourceName(); capability = new ResourceInfo(request.getCapability()); numContainers = request.getNumContainers(); @@ -71,6 +76,21 @@ public ResourceRequestInfo(ResourceRequest request) { } } + public ResourceRequestInfo(SchedulingRequest request) { + priority = request.getPriority().getPriority(); + allocationRequestId = request.getAllocationRequestId(); + capability = new ResourceInfo(request.getResourceSizing().getResources()); + numContainers = request.getResourceSizing().getNumAllocations(); + if (request.getExecutionType() != null) { + executionTypeRequest = + new ExecutionTypeRequestInfo(request.getExecutionType()); + } + allocationTags = request.getAllocationTags(); + if (request.getPlacementConstraint() != null) { + placementConstraint = request.getPlacementConstraint().toString(); + } + } + public Priority getPriority() { return Priority.newInstance(priority); } @@ -128,4 +148,28 @@ public void setExecutionTypeRequest( public ExecutionTypeRequestInfo getExecutionTypeRequest() { return executionTypeRequest; } + + public String getPlacementConstraint() { + return placementConstraint; + } + + public void setPlacementConstraint(String placementConstraint) { + this.placementConstraint = placementConstraint; + } + + public Set getAllocationTags() { + return allocationTags; + } + + public void setAllocationTags(Set allocationTags) { + this.allocationTags = allocationTags; + } + + public long getAllocationRequestId() { + return allocationRequestId; + } + + public void setAllocationRequestId(long allocationRequestId) { + this.allocationRequestId = allocationRequestId; + } }