big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-4164. Changed updateApplicationPriority API to return the updated application priority. Contributed by Rohith Sharma K S

Browse Source
This commit is contained in:
Jian He 2015-12-18 14:13:48 -08:00
parent e63388fdf2
commit 85c2466048
10 changed files with 140 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
View File

@ -476,9 +476,9 @@ public List<NodeLabel> getClusterNodeLabels()
}
@Override
public void updateApplicationPriority(ApplicationId applicationId,
public Priority updateApplicationPriority(ApplicationId applicationId,
Priority priority) throws YarnException, IOException {
client.updateApplicationPriority(applicationId, priority);
return client.updateApplicationPriority(applicationId, priority);
}
@Override

3
hadoop-yarn-project/CHANGES.txt
View File

@ -629,6 +629,9 @@ Release 2.8.0 - UNRELEASED
YARN-4293. ResourceUtilization should be a part of yarn node CLI. (Sunil G via wangda)
YARN-4164. Changed updateApplicationPriority API to return the updated
application priority. (Rohith Sharma K S via jianhe)
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/UpdateApplicationPriorityResponse.java
View File

@ -21,6 +21,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.util.Records;
/**
@ -39,9 +40,24 @@
@Unstable
public abstract class UpdateApplicationPriorityResponse {
public static UpdateApplicationPriorityResponse newInstance() {
public static UpdateApplicationPriorityResponse newInstance(
Priority priority) {
UpdateApplicationPriorityResponse response =
Records.newRecord(UpdateApplicationPriorityResponse.class);
response.setApplicationPriority(priority);
return response;
}
/**
* Get the <code>Priority</code> of the application to be set.
* @return Updated <code>Priority</code> of the application.
*/
public abstract Priority getApplicationPriority();
/**
* Set the <code>Priority</code> of the application.
*
* @param priority <code>Priority</code> of the application
*/
public abstract void setApplicationPriority(Priority priority);
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
View File

@ -231,6 +231,7 @@ message UpdateApplicationPriorityRequestProto {
}
message UpdateApplicationPriorityResponseProto {
optional PriorityProto applicationPriority = 1;
}
message SignalContainerRequestProto {

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
View File

@ -692,12 +692,14 @@ public abstract List<NodeLabel> getClusterNodeLabels()
* </p>
* @param applicationId
* @param priority
* @return updated priority of an application.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
public abstract void updateApplicationPriority(ApplicationId applicationId,
public abstract Priority updateApplicationPriority(
ApplicationId applicationId,
Priority priority) throws YarnException, IOException;
/**

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
View File

@ -832,11 +832,11 @@ public List<NodeLabel> getClusterNodeLabels() throws YarnException, IOException
}
@Override
public void updateApplicationPriority(ApplicationId applicationId,
public Priority updateApplicationPriority(ApplicationId applicationId,
Priority priority) throws YarnException, IOException {
UpdateApplicationPriorityRequest request =
UpdateApplicationPriorityRequest.newInstance(applicationId, priority);
rmClient.updateApplicationPriority(request);
return rmClient.updateApplicationPriority(request).getApplicationPriority();
}
@Override

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -713,8 +713,17 @@ private void updateApplicationPriority(String applicationId, String priority)
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
Priority newAppPriority = Priority.newInstance(Integer.parseInt(priority));
sysout.println("Updating priority of an aplication " + applicationId);
client.updateApplicationPriority(appId, newAppPriority);
sysout.println("Successfully updated the application with id "
+ applicationId + " with priority '" + priority + "'");
Priority updateApplicationPriority =
client.updateApplicationPriority(appId, newAppPriority);
if (newAppPriority.equals(updateApplicationPriority)) {
sysout.println("Successfully updated the application "
+ applicationId + " with priority '" + priority + "'");
} else {
sysout
.println("Updated priority of an application "
+ applicationId
+ " to cluster max priority OR keeping old priority"
+ " as application is in final states");
}
}
}

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/UpdateApplicationPriorityResponsePBImpl.java
View File

@ -19,7 +19,11 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationPriorityResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateApplicationPriorityResponseProtoOrBuilder;
import com.google.protobuf.TextFormat;
@ -31,6 +35,8 @@ public class UpdateApplicationPriorityResponsePBImpl extends
UpdateApplicationPriorityResponseProto.Builder builder = null;
boolean viaProto = false;
private Priority updatedAppPriority = null;
public UpdateApplicationPriorityResponsePBImpl() {
builder = UpdateApplicationPriorityResponseProto.newBuilder();
}
@ -42,11 +48,68 @@ public UpdateApplicationPriorityResponsePBImpl(
}
public UpdateApplicationPriorityResponseProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
private void mergeLocalToProto() {
if (viaProto) {
maybeInitBuilder();
}
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
}
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = UpdateApplicationPriorityResponseProto.newBuilder(proto);
}
viaProto = false;
}
private void mergeLocalToBuilder() {
if (this.updatedAppPriority != null) {
builder
.setApplicationPriority(
convertToProtoFormat(this.updatedAppPriority));
}
}
@Override
public Priority getApplicationPriority() {
UpdateApplicationPriorityResponseProtoOrBuilder p =
viaProto ? proto : builder;
if (this.updatedAppPriority != null) {
return this.updatedAppPriority;
}
if (!p.hasApplicationPriority()) {
return null;
}
this.updatedAppPriority =
convertFromProtoFormat(p.getApplicationPriority());
return this.updatedAppPriority;
}
@Override
public void setApplicationPriority(Priority priority) {
maybeInitBuilder();
if (priority == null) {
builder.clearApplicationPriority();
}
this.updatedAppPriority = priority;
}
private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
return new PriorityPBImpl(p);
}
private PriorityProto convertToProtoFormat(Priority t) {
return ((PriorityPBImpl) t).getProto();
}
@Override
public int hashCode() {
return getProto().hashCode();

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -1449,6 +1449,8 @@ public UpdateApplicationPriorityResponse updateApplicationPriority(
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService",
applicationId);
response.setApplicationPriority(application
.getApplicationSubmissionContext().getPriority());
return response;
}
String msg = "Application in " + application.getState()
@ -1472,6 +1474,8 @@ public UpdateApplicationPriorityResponse updateApplicationPriority(
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.UPDATE_APP_PRIORITY, "ClientRMService", applicationId);
response.setApplicationPriority(application
.getApplicationSubmissionContext().getPriority());
return response;
}
@ -1480,6 +1484,7 @@ public UpdateApplicationPriorityResponse updateApplicationPriority(
* After the request passes some sanity check, it will be delivered
* to RMNodeImpl so that the next NM heartbeat will pick up the signal request
*/
@SuppressWarnings("unchecked")
@Override
public SignalContainerResponse signalContainer(
SignalContainerRequest request) throws YarnException, IOException {

58
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -81,6 +81,7 @@
import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -1330,49 +1331,54 @@ public void testUpdateApplicationPriorityRequest() throws Exception {
appPriority, app1.getApplicationSubmissionContext().getPriority()
.getPriority());
appPriority = 9;
appPriority = 11;
ClientRMService rmService = rm.getClientRMService();
UpdateApplicationPriorityRequest updateRequest =
UpdateApplicationPriorityRequest.newInstance(app1.getApplicationId(),
Priority.newInstance(appPriority));
testAplicationPriorityUpdation(rmService, app1, appPriority, maxPriority);
rmService.updateApplicationPriority(updateRequest);
Assert.assertEquals("Incorrect priority has been set to application",
appPriority, app1.getApplicationSubmissionContext().getPriority()
.getPriority());
appPriority = 9;
testAplicationPriorityUpdation(rmService, app1, appPriority, appPriority);
rm.killApp(app1.getApplicationId());
rm.waitForState(app1.getApplicationId(), RMAppState.KILLED);
appPriority = 8;
UpdateApplicationPriorityRequest updateRequestNew =
UpdateApplicationPriorityRequest.newInstance(app1.getApplicationId(),
Priority.newInstance(appPriority));
// Update priority request for application in KILLED state
rmService.updateApplicationPriority(updateRequestNew);
// Hence new priority should not be updated
Assert.assertNotEquals("Priority should not be updated as app is in KILLED state",
appPriority, app1.getApplicationSubmissionContext().getPriority()
.getPriority());
Assert.assertEquals("Priority should be same as old one before update",
9, app1.getApplicationSubmissionContext().getPriority()
.getPriority());
// Update priority request for invalid application id.
ApplicationId invalidAppId = ApplicationId.newInstance(123456789L, 3);
updateRequest =
UpdateApplicationPriorityRequest updateRequest =
UpdateApplicationPriorityRequest.newInstance(invalidAppId,
Priority.newInstance(appPriority));
try {
rmService.updateApplicationPriority(updateRequest);
Assert
.fail("ApplicationNotFoundException should be thrown for invalid application id");
Assert.fail("ApplicationNotFoundException should be thrown "
+ "for invalid application id");
} catch (ApplicationNotFoundException e) {
// Expected
}
updateRequest =
UpdateApplicationPriorityRequest.newInstance(app1.getApplicationId(),
Priority.newInstance(11));
Assert.assertEquals("Incorrect priority has been set to application",
appPriority, rmService.updateApplicationPriority(updateRequest)
.getApplicationPriority().getPriority());
rm.stop();
}
private void testAplicationPriorityUpdation(ClientRMService rmService,
RMApp app1, int tobeUpdatedPriority, int expected) throws YarnException,
IOException {
UpdateApplicationPriorityRequest updateRequest =
UpdateApplicationPriorityRequest.newInstance(app1.getApplicationId(),
Priority.newInstance(tobeUpdatedPriority));
UpdateApplicationPriorityResponse updateApplicationPriority =
rmService.updateApplicationPriority(updateRequest);
Assert.assertEquals("Incorrect priority has been set to application",
expected, app1.getApplicationSubmissionContext().getPriority()
.getPriority());
Assert.assertEquals("Incorrect priority has been returned", expected,
updateApplicationPriority.getApplicationPriority().getPriority());
}
}