YARN-553. Replaced YarnClient.getNewApplication with YarnClient.createApplication which provides a directly usable ApplicationSubmissionContext to simplify the api. Contributed by Karthik Kambatla.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1494476 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Arun Murthy 2013-06-19 07:06:54 +00:00
parent 571aa3900c
commit d3198dddc8
9 changed files with 102 additions and 68 deletions

View File

@ -41,7 +41,6 @@
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -49,6 +48,7 @@
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
@ -59,7 +59,7 @@ public class ResourceMgrDelegate extends YarnClient {
private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class);
private YarnConfiguration conf; private YarnConfiguration conf;
private GetNewApplicationResponse application; private ApplicationSubmissionContext application;
private ApplicationId applicationId; private ApplicationId applicationId;
@Private @Private
@VisibleForTesting @VisibleForTesting
@ -178,7 +178,7 @@ public String getFilesystemName() throws IOException, InterruptedException {
public JobID getNewJobID() throws IOException, InterruptedException { public JobID getNewJobID() throws IOException, InterruptedException {
try { try {
this.application = client.getNewApplication(); this.application = client.createApplication().getApplicationSubmissionContext();
this.applicationId = this.application.getApplicationId(); this.applicationId = this.application.getApplicationId();
return TypeConverter.fromYarn(applicationId); return TypeConverter.fromYarn(applicationId);
} catch (YarnException e) { } catch (YarnException e) {
@ -272,9 +272,9 @@ public ApplicationId getApplicationId() {
} }
@Override @Override
public GetNewApplicationResponse getNewApplication() throws YarnException, public YarnClientApplication createApplication() throws
IOException { YarnException, IOException {
return client.getNewApplication(); return client.createApplication();
} }
@Override @Override

View File

@ -202,6 +202,11 @@ Release 2.1.0-beta - UNRELEASED
YARN-694. Starting to use NMTokens to authenticate all communication with YARN-694. Starting to use NMTokens to authenticate all communication with
NodeManagers. (Omkar Vinit Joshi via vinodkv) NodeManagers. (Omkar Vinit Joshi via vinodkv)
YARN-553. Replaced YarnClient.getNewApplication with
YarnClient.createApplication which provides a directly usable
ApplicationSubmissionContext to simplify the api. (Karthik Kambatla via
acmurthy)
NEW FEATURES NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues. YARN-482. FS: Extend SchedulingMode to intermediate queues.

View File

@ -61,6 +61,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
@ -217,7 +218,6 @@ public Client() throws Exception {
/** /**
* Helper function to print out usage * Helper function to print out usage
* @param opts Parsed command line options
*/ */
private void printUsage() { private void printUsage() {
new HelpFormatter().printHelp("Client", opts); new HelpFormatter().printHelp("Client", opts);
@ -352,15 +352,14 @@ public boolean run() throws IOException, YarnException {
} }
// Get a new application id // Get a new application id
GetNewApplicationResponse newApp = yarnClient.getNewApplication(); YarnClientApplication app = yarnClient.createApplication();
ApplicationId appId = newApp.getApplicationId(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
// TODO get min/max resource capabilities from RM and change memory ask if needed // TODO get min/max resource capabilities from RM and change memory ask if needed
// If we do not have min/max, we may not be able to correctly request // If we do not have min/max, we may not be able to correctly request
// the required resources from the RM for the app master // the required resources from the RM for the app master
// Memory ask has to be a multiple of min and less than max. // Memory ask has to be a multiple of min and less than max.
// Dump out information about cluster capability as seen by the resource manager // Dump out information about cluster capability as seen by the resource manager
int maxMem = newApp.getMaximumResourceCapability().getMemory(); int maxMem = appResponse.getMaximumResourceCapability().getMemory();
LOG.info("Max mem capabililty of resources in this cluster " + maxMem); LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
// A resource ask cannot exceed the max. // A resource ask cannot exceed the max.
@ -371,13 +370,9 @@ public boolean run() throws IOException, YarnException {
amMemory = maxMem; amMemory = maxMem;
} }
// Create launch context for app master
LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name // set the application name
ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
appContext.setApplicationName(appName); appContext.setApplicationName(appName);
// Set up the container launch context for the application master // Set up the container launch context for the application master

View File

@ -274,17 +274,12 @@ public boolean run() throws IOException, YarnException {
// Connect to ResourceManager // Connect to ResourceManager
rmClient.start(); rmClient.start();
try { try {
// Get a new application id
GetNewApplicationResponse newApp = rmClient.getNewApplication();
ApplicationId appId = newApp.getApplicationId();
// Create launch context for app master // Create launch context for app master
LOG.info("Setting up application submission context for ASM"); LOG.info("Setting up application submission context for ASM");
ApplicationSubmissionContext appContext = Records ApplicationSubmissionContext appContext = rmClient.createApplication()
.newRecord(ApplicationSubmissionContext.class); .getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
// set the application id
appContext.setApplicationId(appId);
// set the application name // set the application name
appContext.setApplicationName(appName); appContext.setApplicationName(appName);

View File

@ -28,7 +28,6 @@
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -79,27 +78,18 @@ protected YarnClient(String name) {
/** /**
* <p> * <p>
* Obtain a new {@link ApplicationId} for submitting new applications. * Obtain a {@link YarnClientApplication} for a new application,
* which in turn contains the {@link ApplicationSubmissionContext} and
* {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse}
* objects.
* </p> * </p>
* *
* <p> * @return {@link YarnClientApplication} built for a new application
* Returns a response which contains {@link ApplicationId} that can be used to
* submit a new application. See
* {@link #submitApplication(ApplicationSubmissionContext)}.
* </p>
*
* <p>
* See {@link GetNewApplicationResponse} for other information that is
* returned.
* </p>
*
* @return response containing the new <code>ApplicationId</code> to be used
* to submit an application
* @throws YarnException * @throws YarnException
* @throws IOException * @throws IOException
*/ */
public abstract GetNewApplicationResponse getNewApplication() throws YarnException, public abstract YarnClientApplication createApplication()
IOException; throws YarnException, IOException;
/** /**
* <p> * <p>
@ -114,7 +104,7 @@ public abstract GetNewApplicationResponse getNewApplication() throws YarnExcepti
* @return {@link ApplicationId} of the accepted application * @return {@link ApplicationId} of the accepted application
* @throws YarnException * @throws YarnException
* @throws IOException * @throws IOException
* @see #getNewApplication() * @see #createApplication()
*/ */
public abstract ApplicationId submitApplication(ApplicationSubmissionContext appContext) public abstract ApplicationId submitApplication(ApplicationSubmissionContext appContext)
throws YarnException, IOException; throws YarnException, IOException;

View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.client.api;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@InterfaceAudience.Public
@InterfaceStability.Stable
/**
* Holder for the {@link GetNewApplicationResponse} and {@link
* ApplicationSubmissionContext} objects created via {@link org.apache.hadoop
* .yarn.client.api.YarnClient#createApplication()}
*/
public class YarnClientApplication {
private final GetNewApplicationResponse newAppResponse;
private final ApplicationSubmissionContext appSubmissionContext;
public YarnClientApplication(GetNewApplicationResponse newAppResponse,
ApplicationSubmissionContext appContext) {
this.newAppResponse = newAppResponse;
this.appSubmissionContext = appContext;
}
public GetNewApplicationResponse getNewApplicationResponse() {
return newAppResponse;
}
public ApplicationSubmissionContext getApplicationSubmissionContext() {
return appSubmissionContext;
}
}

View File

@ -57,6 +57,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
@ -125,14 +126,24 @@ protected void serviceStop() throws Exception {
super.serviceStop(); super.serviceStop();
} }
@Override private GetNewApplicationResponse getNewApplication()
public GetNewApplicationResponse getNewApplication()
throws YarnException, IOException { throws YarnException, IOException {
GetNewApplicationRequest request = GetNewApplicationRequest request =
Records.newRecord(GetNewApplicationRequest.class); Records.newRecord(GetNewApplicationRequest.class);
return rmClient.getNewApplication(request); return rmClient.getNewApplication(request);
} }
@Override
public YarnClientApplication createApplication()
throws YarnException, IOException {
ApplicationSubmissionContext context = Records.newRecord
(ApplicationSubmissionContext.class);
GetNewApplicationResponse newApp = getNewApplication();
ApplicationId appId = newApp.getApplicationId();
context.setApplicationId(appId);
return new YarnClientApplication(newApp, context);
}
@Override @Override
public ApplicationId public ApplicationId
submitApplication(ApplicationSubmissionContext appContext) submitApplication(ApplicationSubmissionContext appContext)

View File

@ -39,7 +39,6 @@
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -61,7 +60,6 @@
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.MiniYARNCluster;
@ -117,13 +115,9 @@ public static void setup() throws Exception {
@Before @Before
public void startApp() throws Exception { public void startApp() throws Exception {
// submit new app // submit new app
GetNewApplicationResponse newApp = yarnClient.getNewApplication(); ApplicationSubmissionContext appContext =
ApplicationId appId = newApp.getApplicationId(); yarnClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name // set the application name
appContext.setApplicationName("Test"); appContext.setApplicationName("Test");
// Set the priority for the application master // Set the priority for the application master

View File

@ -36,7 +36,6 @@
import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -58,9 +57,6 @@
import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.client.api.impl.NMClientImpl;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.server.MiniYARNCluster;
@ -102,13 +98,9 @@ public void setup() throws YarnException, IOException {
nodeReports = yarnClient.getNodeReports(); nodeReports = yarnClient.getNodeReports();
// submit new app // submit new app
GetNewApplicationResponse newApp = yarnClient.getNewApplication(); ApplicationSubmissionContext appContext =
ApplicationId appId = newApp.getApplicationId(); yarnClient.createApplication().getApplicationSubmissionContext();
ApplicationId appId = appContext.getApplicationId();
ApplicationSubmissionContext appContext = Records
.newRecord(ApplicationSubmissionContext.class);
// set the application id
appContext.setApplicationId(appId);
// set the application name // set the application name
appContext.setApplicationName("Test"); appContext.setApplicationName("Test");
// Set the priority for the application master // Set the priority for the application master