YARN-8517. getContainer and getContainers ResourceManager REST API methods are not documented (snemeth via rkanter)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidResourceRequestException.java

@@ -30,19 +30,55 @@
  * 
  */
 public class InvalidResourceRequestException extends YarnException {
+  public static final String LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE =
+          "Invalid resource request! Cannot allocate containers as "
+                  + "requested resource is less than 0! "
+                  + "Requested resource type=[%s], " + "Requested resource=%s";
+
+  public static final String GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE =
+          "Invalid resource request! Cannot allocate containers as "
+                  + "requested resource is greater than " +
+                  "maximum allowed allocation. "
+                  + "Requested resource type=[%s], "
+                  + "Requested resource=%s, maximum allowed allocation=%s, "
+                  + "please note that maximum allowed allocation is calculated "
+                  + "by scheduler based on maximum resource of registered "
+                  + "NodeManagers, which might be less than configured "
+                  + "maximum allocation=%s";
+
+  public static final String UNKNOWN_REASON_MESSAGE_TEMPLATE =
+          "Invalid resource request! "
+                  + "Cannot allocate containers for an unknown reason! "
+                  + "Requested resource type=[%s], Requested resource=%s";
+
+  public enum InvalidResourceType {
+    LESS_THAN_ZERO, GREATER_THEN_MAX_ALLOCATION, UNKNOWN;
+  }
 
   private static final long serialVersionUID = 13498237L;
+  private final InvalidResourceType invalidResourceType;
 
   public InvalidResourceRequestException(Throwable cause) {
     super(cause);
+    this.invalidResourceType = InvalidResourceType.UNKNOWN;
   }
 
   public InvalidResourceRequestException(String message) {
+    this(message, InvalidResourceType.UNKNOWN);
+  }
+
+  public InvalidResourceRequestException(String message,
+      InvalidResourceType invalidResourceType) {
     super(message);
+    this.invalidResourceType = invalidResourceType;
   }
 
   public InvalidResourceRequestException(String message, Throwable cause) {
     super(message, cause);
+    this.invalidResourceType = InvalidResourceType.UNKNOWN;
   }
 
+  public InvalidResourceType getInvalidResourceType() {
+    return invalidResourceType;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java

@@ -53,6 +53,8 @@
 import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
+        .InvalidResourceType;
 import org.apache.hadoop.yarn.exceptions.SchedulerInvalidResoureRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -89,6 +91,12 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException.InvalidResourceType
+        .GREATER_THEN_MAX_ALLOCATION;
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException.InvalidResourceType.LESS_THAN_ZERO;
+
 /**
  * This is the default Application Master Service processor. It has be the
  * last processor in the @{@link AMSProcessingChain}.
@@ -231,8 +239,8 @@ public void allocate(ApplicationAttemptId appAttemptId,
           maximumCapacity, app.getQueue(),
           getScheduler(), getRmContext());
     } catch (InvalidResourceRequestException e) {
-      LOG.warn("Invalid resource ask by application " + appAttemptId, e);
+      RMAppAttempt rmAppAttempt = app.getRMAppAttempt(appAttemptId);
-      throw e;
+      handleInvalidResourceException(e, rmAppAttempt);
     }
 
     try {
@@ -336,6 +344,17 @@ public void allocate(ApplicationAttemptId appAttemptId,
         allocation.getPreviousAttemptContainers());
   }
 
+  private void handleInvalidResourceException(InvalidResourceRequestException e,
+          RMAppAttempt rmAppAttempt) throws InvalidResourceRequestException {
+    if (e.getInvalidResourceType() == LESS_THAN_ZERO ||
+            e.getInvalidResourceType() == GREATER_THEN_MAX_ALLOCATION) {
+      rmAppAttempt.updateAMLaunchDiagnostics(e.getMessage());
+    }
+    LOG.warn("Invalid resource ask by application " +
+            rmAppAttempt.getAppAttemptId(), e);
+    throw e;
+  }
+
   private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) {
     Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>();
     if(app.pullRMNodeUpdates(updatedNodes) > 0) {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java

@@ -45,6 +45,8 @@
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
+        .InvalidResourceType;
 import org.apache.hadoop.yarn.exceptions
         .SchedulerInvalidResoureRequestException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -61,6 +63,15 @@
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
 
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException
+        .GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE;
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException
+        .LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE;
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException.UNKNOWN_REASON_MESSAGE_TEMPLATE;
+
 /**
  * Utilities shared by schedulers. 
  */
@@ -257,7 +268,7 @@ public static void normalizeAndValidateRequest(ResourceRequest resReq,
   }
 
 
-  public static void normalizeAndValidateRequest(ResourceRequest resReq,
+  private static void normalizeAndValidateRequest(ResourceRequest resReq,
           Resource maximumResource, String queueName, YarnScheduler scheduler,
           boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
       throws InvalidResourceRequestException {
@@ -384,13 +395,13 @@ static void checkResourceRequestAgainstAvailableResource(Resource reqResource,
 
       if (requestedRI.getValue() < 0) {
         throwInvalidResourceException(reqResource, availableResource,
-            reqResourceName);
+            reqResourceName, InvalidResourceType.LESS_THAN_ZERO);
       }
 
       boolean valid = checkResource(requestedRI, availableResource);
       if (!valid) {
         throwInvalidResourceException(reqResource, availableResource,
-            reqResourceName);
+            reqResourceName, InvalidResourceType.GREATER_THEN_MAX_ALLOCATION);
       }
     }
   }
@@ -470,18 +481,30 @@ private static boolean checkResource(
   }
 
   private static void throwInvalidResourceException(Resource reqResource,
-      Resource availableResource, String reqResourceName)
+          Resource maxAllowedAllocation, String reqResourceName,
+          InvalidResourceType invalidResourceType)
       throws InvalidResourceRequestException {
-    throw new InvalidResourceRequestException(
+    final String message;
-        "Invalid resource request, requested resource type=[" + reqResourceName
+
-            + "] < 0 or greater than maximum allowed allocation. Requested "
+    if (invalidResourceType == InvalidResourceType.LESS_THAN_ZERO) {
-            + "resource=" + reqResource + ", maximum allowed allocation="
+      message = String.format(LESS_THAN_ZERO_RESOURCE_MESSAGE_TEMPLATE,
-            + availableResource
+          reqResourceName, reqResource);
-            + ", please note that maximum allowed allocation is calculated "
+    } else if (invalidResourceType ==
-            + "by scheduler based on maximum resource of registered "
+            InvalidResourceType.GREATER_THEN_MAX_ALLOCATION) {
-            + "NodeManagers, which might be less than configured "
+      message = String.format(GREATER_THAN_MAX_RESOURCE_MESSAGE_TEMPLATE,
-            + "maximum allocation="
+          reqResourceName, reqResource, maxAllowedAllocation,
-            + ResourceUtils.getResourceTypesMaximumAllocation());
+          ResourceUtils.getResourceTypesMaximumAllocation());
+    } else if (invalidResourceType == InvalidResourceType.UNKNOWN) {
+      message = String.format(UNKNOWN_REASON_MESSAGE_TEMPLATE, reqResourceName,
+          reqResource);
+    } else {
+      throw new IllegalArgumentException(String.format(
+          "InvalidResourceType argument should be either " + "%s, %s or %s",
+          InvalidResourceType.LESS_THAN_ZERO,
+          InvalidResourceType.GREATER_THEN_MAX_ALLOCATION,
+          InvalidResourceType.UNKNOWN));
+    }
+    throw new InvalidResourceRequestException(message, invalidResourceType);
   }
 
   private static void checkQueueLabelInLabelManager(String labelExpression,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java

@@ -18,6 +18,11 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
 
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException.InvalidResourceType
+        .GREATER_THEN_MAX_ALLOCATION;
+import static org.apache.hadoop.yarn.exceptions
+        .InvalidResourceRequestException.InvalidResourceType.LESS_THAN_ZERO;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -67,6 +72,8 @@
 import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
 import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException
+        .InvalidResourceType;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
@@ -162,7 +169,7 @@ public void setUp() {
                     .build());
   }
 
-  @Test (timeout = 30000)
+  @Test(timeout = 30000)
   public void testNormalizeRequest() {
     ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
 
@@ -227,7 +234,7 @@ public void testNormalizeRequest() {
             ask.getCapability().getMemorySize());
   }
 
-  @Test (timeout = 30000)
+  @Test(timeout = 30000)
   public void testNormalizeRequestWithDominantResourceCalculator() {
     ResourceCalculator resourceCalculator = new DominantResourceCalculator();
 
@@ -406,7 +413,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
       e.printStackTrace();
       fail("Should be valid when request labels is empty");
     }
-    boolean invalidlabelexception=false;
+    boolean invalidlabelexception = false;
     // queue doesn't have label, failed (when request any label)
     try {
       // set queue accessible node labels to empty
@@ -425,7 +432,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
               scheduler, rmContext);
       fail("Should fail");
     } catch (InvalidLabelResourceRequestException e) {
-      invalidlabelexception=true;
+      invalidlabelexception = true;
     } catch (InvalidResourceRequestException e) {
     } finally {
       rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
@@ -563,7 +570,7 @@ public void testValidateResourceRequestWithErrorLabelsPermission()
     }
   }
 
-  @Test (timeout = 30000)
+  @Test(timeout = 30000)
   public void testValidateResourceRequest() {
     YarnScheduler mockScheduler = mock(YarnScheduler.class);
 
@@ -642,7 +649,7 @@ public void testValidateResourceRequest() {
               mockScheduler, rmContext);
       fail("Negative memory should not be accepted");
     } catch (InvalidResourceRequestException e) {
-      // expected
+      assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
     }
 
     // negative vcores
@@ -657,7 +664,7 @@ public void testValidateResourceRequest() {
               mockScheduler, rmContext);
       fail("Negative vcores should not be accepted");
     } catch (InvalidResourceRequestException e) {
-      // expected
+      assertEquals(LESS_THAN_ZERO, e.getInvalidResourceType());
     }
 
     // more than max memory
@@ -673,7 +680,7 @@ public void testValidateResourceRequest() {
               mockScheduler, rmContext);
       fail("More than max memory should not be accepted");
     } catch (InvalidResourceRequestException e) {
-      // expected
+      assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
     }
 
     // more than max vcores
@@ -688,7 +695,7 @@ public void testValidateResourceRequest() {
               mockScheduler, rmContext);
       fail("More than max vcores should not be accepted");
     } catch (InvalidResourceRequestException e) {
-      // expected
+      assertEquals(GREATER_THEN_MAX_ALLOCATION, e.getInvalidResourceType());
     }
   }
 
@@ -772,7 +779,7 @@ private void waitForLaunchedState(RMAppAttempt attempt)
   }
 
   @Test
-  public void testComparePriorities(){
+  public void testComparePriorities() {
     Priority high = Priority.newInstance(1);
     Priority low = Priority.newInstance(2);
     assertTrue(high.compareTo(low) > 0);
@@ -794,7 +801,7 @@ public void testCreatePreemptedContainerStatus() {
     Assert.assertEquals(ContainerExitStatus.PREEMPTED, cd.getExitStatus());
   }
 
-  @Test (timeout = 30000)
+  @Test(timeout = 30000)
   public void testNormalizeNodeLabelExpression()
           throws IOException {
     // mock queue and scheduler
@@ -857,7 +864,9 @@ public void testCustomResourceRequestedUnitIsSmallerThanAvailableUnit()
             .create().withRequestedResourceType("custom-resource-1")
             .withRequestedResource(requestedResource)
             .withAvailableAllocation(availableResource)
-        .withMaxAllocation(configuredMaxAllocation).build());
+            .withMaxAllocation(configuredMaxAllocation)
+            .withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
+            .build());
 
     SchedulerUtils.checkResourceRequestAgainstAvailableResource(
             requestedResource, availableResource);
@@ -891,7 +900,8 @@ public void testCustomResourceRequestedUnitIsGreaterThanAvailableUnit()
                     ImmutableMap.of("custom-resource-1", "1M"));
 
     Resource availableResource = ResourceTypesTestHelper.newResource(1, 1,
-        ImmutableMap.<String, String> builder().put("custom-resource-1", "120k")
+            ImmutableMap.<String, String>builder().put("custom-resource-1",
+                    "120k")
                     .build());
 
     exception.expect(InvalidResourceRequestException.class);
@@ -899,7 +909,9 @@ ImmutableMap.<String, String> builder().put("custom-resource-1", "120k")
             .create().withRequestedResourceType("custom-resource-1")
             .withRequestedResource(requestedResource)
             .withAvailableAllocation(availableResource)
-        .withMaxAllocation(configuredMaxAllocation).build());
+            .withMaxAllocation(configuredMaxAllocation)
+            .withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
+            .build());
     SchedulerUtils.checkResourceRequestAgainstAvailableResource(
             requestedResource, availableResource);
   }
@@ -907,7 +919,7 @@ ImmutableMap.<String, String> builder().put("custom-resource-1", "120k")
   @Test
   public void testCustomResourceRequestedUnitIsGreaterThanAvailableUnit2() {
     Resource requestedResource = ResourceTypesTestHelper.newResource(1, 1,
-        ImmutableMap.<String, String> builder().put("custom-resource-1", "11M")
+            ImmutableMap.<String, String>builder().put("custom-resource-1", "11M")
                     .build());
 
     Resource availableResource =
@@ -956,7 +968,9 @@ public void testCustomResourceRequestedUnitIsSameAsAvailableUnit2()
             .create().withRequestedResourceType("custom-resource-1")
             .withRequestedResource(requestedResource)
             .withAvailableAllocation(availableResource)
-        .withMaxAllocation(configuredMaxAllocation).build());
+            .withInvalidResourceType(GREATER_THEN_MAX_ALLOCATION)
+            .withMaxAllocation(configuredMaxAllocation)
+            .build());
 
     SchedulerUtils.checkResourceRequestAgainstAvailableResource(
             requestedResource, availableResource);
@@ -1026,6 +1040,7 @@ private static class InvalidResourceRequestExceptionMessageGenerator {
     private Resource availableAllocation;
     private Resource configuredMaxAllowedAllocation;
     private String resourceType;
+    private InvalidResourceType invalidResourceType;
 
     InvalidResourceRequestExceptionMessageGenerator(StringBuilder sb) {
       this.sb = sb;
@@ -1060,18 +1075,41 @@ InvalidResourceRequestExceptionMessageGenerator withMaxAllocation(
       return this;
     }
 
+    InvalidResourceRequestExceptionMessageGenerator
+    withInvalidResourceType(InvalidResourceType invalidResourceType) {
+      this.invalidResourceType = invalidResourceType;
+      return this;
+    }
+
     public String build() {
-      return sb
+      if (invalidResourceType == LESS_THAN_ZERO) {
-          .append("Invalid resource request, requested resource type=[")
+        return sb.append("Invalid resource request! " +
+                "Cannot allocate containers as " +
+                "requested resource is less than 0! ")
+                .append("Requested resource type=[")
                 .append(resourceType).append("]")
-          .append(" < 0 or greater than maximum allowed allocation. ")
+                .append(", Requested resource=")
-          .append("Requested resource=").append(requestedResource).append(", ")
+                .append(requestedResource).toString();
-          .append("maximum allowed allocation=").append(availableAllocation)
+
-          .append(", please note that maximum allowed allocation is calculated "
+      } else if (invalidResourceType == GREATER_THEN_MAX_ALLOCATION) {
-              + "by scheduler based on maximum resource of " +
+        return sb.append("Invalid resource request! " +
-                  "registered NodeManagers, which might be less than " +
+                "Cannot allocate containers as "
-                  "configured maximum allocation=")
+                + "requested resource is greater than " +
-          .append(configuredMaxAllowedAllocation).toString();
+                "maximum allowed allocation. ")
+                .append("Requested resource type=[").append(resourceType)
+                .append("], ")
+                .append("Requested resource=").append(requestedResource)
+                .append(", maximum allowed allocation=")
+                .append(availableAllocation)
+                .append(", please note that maximum allowed allocation is " +
+                        "calculated by scheduler based on maximum resource " +
+                        "of registered NodeManagers, which might be less " +
+                        "than configured maximum allocation=")
+                .append(configuredMaxAllowedAllocation)
+                .toString();
+      }
+      throw new IllegalStateException("Wrong type of InvalidResourceType is " +
+              "detected!");
     }
   }
 }