YARN-553. Replaced YarnClient.getNewApplication with YarnClient.createApplication which provides a directly usable ApplicationSubmissionContext to simplify the api. Contributed by Karthik Kambatla.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1494476 13f79535-47bb-0310-9956-ffa450edef68
2013-06-19 07:06:54 +00:00 · 2013-06-19 07:06:54 +00:00 · d3198dddc8
commit d3198dddc8
parent 571aa3900c
9 changed files with 102 additions and 68 deletions
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@ -41,7 +41,6 @@
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -49,6 +48,7 @@
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@ -59,7 +59,7 @@ public class ResourceMgrDelegate extends YarnClient {
  private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
      
  private YarnConfiguration conf;
-  private GetNewApplicationResponse application;
+  private ApplicationSubmissionContext application;
  private ApplicationId applicationId;
  @Private
  @VisibleForTesting
@ -178,7 +178,7 @@ public String getFilesystemName() throws IOException, InterruptedException {

  public JobID getNewJobID() throws IOException, InterruptedException {
    try {
-      this.application = client.getNewApplication();
+      this.application = client.createApplication().getApplicationSubmissionContext();
      this.applicationId = this.application.getApplicationId();
      return TypeConverter.fromYarn(applicationId);
    } catch (YarnException e) {
@ -272,9 +272,9 @@ public ApplicationId getApplicationId() {
  }

  @Override
-  public GetNewApplicationResponse getNewApplication() throws YarnException,
-      IOException {
-    return client.getNewApplication();
+  public YarnClientApplication createApplication() throws
+      YarnException, IOException {
+    return client.createApplication();
  }

  @Override
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@ -202,6 +202,11 @@ Release 2.1.0-beta - UNRELEASED
    YARN-694. Starting to use NMTokens to authenticate all communication with
    NodeManagers. (Omkar Vinit Joshi via vinodkv) 

+    YARN-553. Replaced YarnClient.getNewApplication with
+    YarnClient.createApplication which provides a directly usable
+    ApplicationSubmissionContext to simplify the api. (Karthik Kambatla via
+    acmurthy) 
+
  NEW FEATURES

    YARN-482. FS: Extend SchedulingMode to intermediate queues. 
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@ -61,6 +61,7 @@
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
@ -217,7 +218,6 @@ public Client() throws Exception  {

  /**
   * Helper function to print out usage
-   * @param opts Parsed command line options 
   */
  private void printUsage() {
    new HelpFormatter().printHelp("Client", opts);
@ -351,16 +351,15 @@ public boolean run() throws IOException, YarnException {
      }
    }		

-    // Get a new application id 
-    GetNewApplicationResponse newApp = yarnClient.getNewApplication();
-    ApplicationId appId = newApp.getApplicationId();
-
+    // Get a new application id
+    YarnClientApplication app = yarnClient.createApplication();
+    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
    // TODO get min/max resource capabilities from RM and change memory ask if needed
    // If we do not have min/max, we may not be able to correctly request 
    // the required resources from the RM for the app master
    // Memory ask has to be a multiple of min and less than max. 
    // Dump out information about cluster capability as seen by the resource manager
-    int maxMem = newApp.getMaximumResourceCapability().getMemory();
+    int maxMem = appResponse.getMaximumResourceCapability().getMemory();
    LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

    // A resource ask cannot exceed the max. 
@ -371,13 +370,9 @@ public boolean run() throws IOException, YarnException {
      amMemory = maxMem;
    }				

-    // Create launch context for app master
-    LOG.info("Setting up application submission context for ASM");
-    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
-
-    // set the application id 
-    appContext.setApplicationId(appId);
    // set the application name
+    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
+    ApplicationId appId = appContext.getApplicationId();
    appContext.setApplicationName(appName);

    // Set up the container launch context for the application master
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
@ -274,17 +274,12 @@ public boolean run() throws IOException, YarnException {
    // Connect to ResourceManager
    rmClient.start();
    try {  
-      // Get a new application id
-      GetNewApplicationResponse newApp = rmClient.getNewApplication();
-      ApplicationId appId = newApp.getApplicationId();
-  
      // Create launch context for app master
      LOG.info("Setting up application submission context for ASM");
-      ApplicationSubmissionContext appContext = Records
-          .newRecord(ApplicationSubmissionContext.class);
-  
-      // set the application id
-      appContext.setApplicationId(appId);
+      ApplicationSubmissionContext appContext = rmClient.createApplication()
+          .getApplicationSubmissionContext();
+      ApplicationId appId = appContext.getApplicationId();
+
      // set the application name
      appContext.setApplicationName(appName);
  
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@ -28,7 +28,6 @@
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -79,27 +78,18 @@ protected YarnClient(String name) {

  /**
   * <p>
-   * Obtain a new {@link ApplicationId} for submitting new applications.
+   * Obtain a {@link YarnClientApplication} for a new application,
+   * which in turn contains the {@link ApplicationSubmissionContext} and
+   * {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
+   * objects.
   * </p>
-   * 
-   * <p>
-   * Returns a response which contains {@link ApplicationId} that can be used to
-   * submit a new application. See
-   * {@link #submitApplication(ApplicationSubmissionContext)}.
-   * </p>
-   * 
-   * <p>
-   * See {@link GetNewApplicationResponse} for other information that is
-   * returned.
-   * </p>
-   * 
-   * @return response containing the new <code>ApplicationId</code> to be used
-   *         to submit an application
+   *
+   * @return {@link YarnClientApplication} built for a new application
   * @throws YarnException
   * @throws IOException
   */
-  public abstract GetNewApplicationResponse getNewApplication() throws YarnException,
-      IOException;
+  public abstract YarnClientApplication createApplication()
+      throws YarnException, IOException;

  /**
   * <p>
@ -114,7 +104,7 @@ public abstract GetNewApplicationResponse getNewApplication() throws YarnExcepti
   * @return {@link ApplicationId} of the accepted application
   * @throws YarnException
   * @throws IOException
-   * @see #getNewApplication()
+   * @see #createApplication()
   */
  public abstract ApplicationId submitApplication(ApplicationSubmissionContext appContext)
      throws YarnException, IOException;
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java
@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+
+/**
+ * Holder for the {@link GetNewApplicationResponse} and {@link
+ * ApplicationSubmissionContext} objects created via {@link org.apache.hadoop
+ * .yarn.client.api.YarnClient#createApplication()}
+ */
+public  class YarnClientApplication {
+  private final GetNewApplicationResponse newAppResponse;
+  private final ApplicationSubmissionContext appSubmissionContext;
+
+  public YarnClientApplication(GetNewApplicationResponse newAppResponse,
+                               ApplicationSubmissionContext appContext) {
+    this.newAppResponse = newAppResponse;
+    this.appSubmissionContext = appContext;
+  }
+
+  public GetNewApplicationResponse getNewApplicationResponse() {
+    return newAppResponse;
+  }
+
+  public ApplicationSubmissionContext getApplicationSubmissionContext() {
+    return appSubmissionContext;
+  }
+}
+
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@ -57,6 +57,7 @@
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
 import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
@ -125,14 +126,24 @@ protected void serviceStop() throws Exception {
    super.serviceStop();
  }

-  @Override
-  public GetNewApplicationResponse getNewApplication()
+  private GetNewApplicationResponse getNewApplication()
      throws YarnException, IOException {
    GetNewApplicationRequest request =
        Records.newRecord(GetNewApplicationRequest.class);
    return rmClient.getNewApplication(request);
  }

+  @Override
+  public YarnClientApplication createApplication()
+      throws YarnException, IOException {
+    ApplicationSubmissionContext context = Records.newRecord
+        (ApplicationSubmissionContext.class);
+    GetNewApplicationResponse newApp = getNewApplication();
+    ApplicationId appId = newApp.getApplicationId();
+    context.setApplicationId(appId);
+    return new YarnClientApplication(newApp, context);
+  }
+
  @Override
  public ApplicationId
      submitApplication(ApplicationSubmissionContext appContext)
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@ -39,7 +39,6 @@
 import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -61,7 +60,6 @@
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
 import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@ -117,13 +115,9 @@ public static void setup() throws Exception {
  @Before
  public void startApp() throws Exception {
    // submit new app
-    GetNewApplicationResponse newApp = yarnClient.getNewApplication();
-    ApplicationId appId = newApp.getApplicationId();
-
-    ApplicationSubmissionContext appContext = Records
-        .newRecord(ApplicationSubmissionContext.class);
-    // set the application id
-    appContext.setApplicationId(appId);
+    ApplicationSubmissionContext appContext = 
+        yarnClient.createApplication().getApplicationSubmissionContext();
+    ApplicationId appId = appContext.getApplicationId();
    // set the application name
    appContext.setApplicationName("Test");
    // Set the priority for the application master
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@ -36,7 +36,6 @@
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.service.Service.STATE;
 import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -58,9 +57,6 @@
 import org.apache.hadoop.yarn.client.api.NMClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.client.api.impl.NMClientImpl;
-import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.MiniYARNCluster;
@ -102,13 +98,9 @@ public void setup() throws YarnException, IOException {
    nodeReports = yarnClient.getNodeReports();

    // submit new app
-    GetNewApplicationResponse newApp = yarnClient.getNewApplication();
-    ApplicationId appId = newApp.getApplicationId();
-
-    ApplicationSubmissionContext appContext = Records
-        .newRecord(ApplicationSubmissionContext.class);
-    // set the application id
-    appContext.setApplicationId(appId);
+    ApplicationSubmissionContext appContext = 
+        yarnClient.createApplication().getApplicationSubmissionContext();
+    ApplicationId appId = appContext.getApplicationId();
    // set the application name
    appContext.setApplicationName("Test");
    // Set the priority for the application master