big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-2301. Improved yarn container command. Contributed by Naganarasimha G R

Browse Source
This commit is contained in:
Jian He 2014-12-04 12:51:15 -08:00
parent 565b0e60a8
commit 258623ff8b
9 changed files with 63 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/CHANGES.txt
View File

@ -121,6 +121,8 @@ Release 2.7.0 - UNRELEASED
YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics
for aggregation of decimal values. (Tsuyoshi OZAWA via junping_du)
YARN-2301. Improved yarn container command. (Naganarasimha G R via jianhe)
OPTIMIZATIONS
BUG FIXES

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
@ -536,8 +537,11 @@ private void listContainers(String appAttemptId) throws YarnException,
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
"Finish Time", "State", "Host", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(),
containerReport.getCreationTime(), containerReport.getFinishTime(),
writer.printf(
CONTAINER_PATTERN,
containerReport.getContainerId(),
Times.format(containerReport.getCreationTime()),
Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
.getAssignedNode(), containerReport.getLogUrl());
}

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
View File

@ -32,19 +32,17 @@
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -65,9 +63,7 @@
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.util.Records;
import org.jboss.netty.logging.CommonsLoggerFactory;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@ -257,25 +253,31 @@ public void testGetContainers() throws Exception {
applicationId, 1);
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerId containerId1 = ContainerId.newContainerId(attemptId, 2);
ContainerId containerId2 = ContainerId.newContainerId(attemptId, 3);
long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
"diagnosticInfo", "", 0, ContainerState.RUNNING);
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
reports.add(container2);
DateFormat dateFormat=new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy");
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
verify(client).getContainers(attemptId);
Log.info(sysOutStream.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Total number of containers :2");
pw.println("Total number of containers :3");
pw.print(" Container-Id");
pw.print("\t Start Time");
pw.print("\t Finish Time");
@ -283,19 +285,30 @@ public void testGetContainers() throws Exception {
pw.print("\t Host");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
pw.print("\t 1234");
pw.print("\t 5678");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
pw.print("\t 1234");
pw.print("\t 5678");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000003");
pw.print("\t"+dateFormat.format(new Date(time1)));
pw.print("\t N/A");
pw.print("\t RUNNING");
pw.print("\t host:1234");
pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");
Log.info("ExpectedOutput");
Log.info("["+appReportStr+"]");
Log.info("OutputFrom command");
String actualOutput = sysOutStream.toString();
Log.info("["+actualOutput+"]");
Assert.assertEquals(appReportStr, sysOutStream.toString());
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
View File

@ -21,6 +21,7 @@
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
@ -121,4 +122,6 @@ void setRMApplicationHistoryWriter(
ReservationSystem getReservationSystem();
boolean isSchedulerReadyForAllocatingContainers();
Configuration getYarnConfiguration();
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
View File

@ -22,6 +22,7 @@
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
@ -65,6 +66,8 @@ public class RMContextImpl implements RMContext {
private RMActiveServiceContext activeServiceContext;
private Configuration yarnConfiguration;
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@ -399,4 +402,12 @@ void setActiveServiceContext(RMActiveServiceContext activeServiceContext) {
this.activeServiceContext = activeServiceContext;
}
@Override
public Configuration getYarnConfiguration() {
return this.yarnConfiguration;
}
public void setYarnConfiguration(Configuration yarnConfiguration) {
this.yarnConfiguration=yarnConfiguration;
}
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -246,6 +246,8 @@ protected void serviceInit(Configuration conf) throws Exception {
addService(adminService);
rmContext.setRMAdminService(adminService);
rmContext.setYarnConfiguration(conf);
createAndInitActiveServices();
webAppAddress = WebAppUtils.getWebAppBindURL(this.conf,

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
View File

@ -285,8 +285,13 @@ public String getDiagnosticsInfo() {
public String getLogURL() {
try {
readLock.lock();
return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(),
ConverterUtils.toString(containerId), user);
StringBuilder logURL = new StringBuilder();
logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
.getYarnConfiguration()));
logURL.append(WebAppUtils.getRunningLogURL(
container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
user));
return logURL.toString();
} finally {
readLock.unlock();
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -443,6 +443,7 @@ public ClientRMService createRMService() throws IOException {
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);
when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
View File

@ -107,6 +107,7 @@ public void testReleaseWhileRunning() {
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getRMApps()).thenReturn(rmApps);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@ -130,7 +131,7 @@ public void testReleaseWhileRunning() {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
@ -196,6 +197,7 @@ public void testExpireWhileRunning() {
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@ -220,7 +222,7 @@ public void testExpireWhileRunning() {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.