YARN-5339. Passing file to -out for YARN log CLI doesnt give warning or error code. Contributed by Xuan Gong.

This commit is contained in:
Junping Du 2016-07-15 09:23:11 -07:00
parent b5ee7dbd8d
commit 7e5355c14e
2 changed files with 38 additions and 6 deletions

View File

@ -26,7 +26,9 @@
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.TreeMap;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
@ -51,6 +53,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -249,7 +252,7 @@ private int runCommand(String[] args) throws Exception {
ContainerLogsRequest request = new ContainerLogsRequest(appId, ContainerLogsRequest request = new ContainerLogsRequest(appId,
isApplicationFinished(appState), appOwner, nodeAddress, null, isApplicationFinished(appState), appOwner, nodeAddress, null,
containerIdStr, localDir, logs, bytes); containerIdStr, localDir, logs, bytes, null);
if (showContainerLogInfo) { if (showContainerLogInfo) {
return showContainerLogInfo(request, logCliHelper); return showContainerLogInfo(request, logCliHelper);
@ -470,9 +473,14 @@ public int printContainerLogsFromRunningApplication(Configuration conf,
.queryParam("size", Long.toString(request.getBytes())) .queryParam("size", Long.toString(request.getBytes()))
.accept(MediaType.TEXT_PLAIN).get(ClientResponse.class); .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
out.println(response.getEntity(String.class)); out.println(response.getEntity(String.class));
out.println("End of LogType:" + logFile + ". This log file belongs" StringBuilder sb = new StringBuilder();
+ " to a running container (" + containerIdStr + ") and so may" sb.append("End of LogType:" + logFile + ".");
+ " not be complete."); if (request.getContainerState() == ContainerState.RUNNING) {
sb.append(" This log file belongs"
+ " to a running container (" + containerIdStr + ") and so may"
+ " not be complete.");
}
out.println(sb.toString());
out.flush(); out.flush();
foundAnyLogs = true; foundAnyLogs = true;
} catch (ClientHandlerException | UniformInterfaceException ex) { } catch (ClientHandlerException | UniformInterfaceException ex) {
@ -645,6 +653,9 @@ private void outputAMContainerLogs(ContainerLogsRequest request,
} else { } else {
if (nodeHttpAddress != null && containerId != null if (nodeHttpAddress != null && containerId != null
&& !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) { && !nodeHttpAddress.isEmpty() && !containerId.isEmpty()) {
ContainerState containerState = getContainerReport(containerId)
.getContainerState();
request.setContainerState(containerState);
printContainerLogsFromRunningApplication(conf, printContainerLogsFromRunningApplication(conf,
request, logCliHelper); request, logCliHelper);
} }
@ -880,6 +891,7 @@ private int fetchContainerLogs(ContainerLogsRequest request,
} }
nodeId = report.getAssignedNode().toString(); nodeId = report.getAssignedNode().toString();
request.setNodeId(nodeId); request.setNodeId(nodeId);
request.setContainerState(report.getContainerState());
} catch (IOException | YarnException ex) { } catch (IOException | YarnException ex) {
if (isAppFinished) { if (isAppFinished) {
return printContainerLogsForFinishedApplicationWithoutNodeId( return printContainerLogsForFinishedApplicationWithoutNodeId(
@ -1020,6 +1032,7 @@ private boolean isFileMatching(String fileType,
newOptions.setNodeHttpAddress(httpAddress newOptions.setNodeHttpAddress(httpAddress
.replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), "")); .replaceFirst(WebAppUtils.getHttpSchemePrefix(getConf()), ""));
} }
newOptions.setContainerState(container.getContainerState());
newOptionsList.add(newOptions); newOptionsList.add(newOptions);
} }
return newOptionsList; return newOptionsList;
@ -1030,11 +1043,18 @@ private List<ContainerReport> getContainerReportsFromRunningApplication(
List<ContainerReport> reports = new ArrayList<ContainerReport>(); List<ContainerReport> reports = new ArrayList<ContainerReport>();
List<ApplicationAttemptReport> attempts = List<ApplicationAttemptReport> attempts =
yarnClient.getApplicationAttempts(options.getAppId()); yarnClient.getApplicationAttempts(options.getAppId());
Map<ContainerId, ContainerReport> containerMap = new TreeMap<
ContainerId, ContainerReport>();
for (ApplicationAttemptReport attempt : attempts) { for (ApplicationAttemptReport attempt : attempts) {
List<ContainerReport> containers = yarnClient.getContainers( List<ContainerReport> containers = yarnClient.getContainers(
attempt.getApplicationAttemptId()); attempt.getApplicationAttemptId());
reports.addAll(containers); for (ContainerReport container : containers) {
if (!containerMap.containsKey(container.getContainerId())) {
containerMap.put(container.getContainerId(), container);
}
}
} }
reports.addAll(containerMap.values());
return reports; return reports;
} }

View File

@ -20,6 +20,7 @@
import java.util.List; import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerState;
public class ContainerLogsRequest { public class ContainerLogsRequest {
private ApplicationId appId; private ApplicationId appId;
@ -31,6 +32,7 @@ public class ContainerLogsRequest {
private String outputLocalDir; private String outputLocalDir;
private List<String> logTypes; private List<String> logTypes;
private long bytes; private long bytes;
private ContainerState containerState;
public ContainerLogsRequest() {} public ContainerLogsRequest() {}
@ -44,12 +46,13 @@ public ContainerLogsRequest(ContainerLogsRequest request) {
this.setOutputLocalDir(request.getOutputLocalDir()); this.setOutputLocalDir(request.getOutputLocalDir());
this.setLogTypes(request.getLogTypes()); this.setLogTypes(request.getLogTypes());
this.setBytes(request.getBytes()); this.setBytes(request.getBytes());
this.setContainerState(request.getContainerState());
} }
public ContainerLogsRequest(ApplicationId applicationId, public ContainerLogsRequest(ApplicationId applicationId,
boolean isAppFinished, String owner, boolean isAppFinished, String owner,
String address, String httpAddress, String container, String localDir, String address, String httpAddress, String container, String localDir,
List<String> logs, long bytes) { List<String> logs, long bytes, ContainerState containerState) {
this.setAppId(applicationId); this.setAppId(applicationId);
this.setAppFinished(isAppFinished); this.setAppFinished(isAppFinished);
this.setAppOwner(owner); this.setAppOwner(owner);
@ -59,6 +62,7 @@ public ContainerLogsRequest(ApplicationId applicationId,
this.setOutputLocalDir(localDir); this.setOutputLocalDir(localDir);
this.setLogTypes(logs); this.setLogTypes(logs);
this.setBytes(bytes); this.setBytes(bytes);
this.setContainerState(containerState);
} }
public ApplicationId getAppId() { public ApplicationId getAppId() {
@ -132,4 +136,12 @@ public long getBytes() {
public void setBytes(long bytes) { public void setBytes(long bytes) {
this.bytes = bytes; this.bytes = bytes;
} }
public ContainerState getContainerState() {
return containerState;
}
public void setContainerState(ContainerState containerState) {
this.containerState = containerState;
}
} }