YARN-5470. Differentiate exactly match with regex in yarn log CLI. Contributed by Xuan Gong.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java

@@ -25,6 +25,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -90,6 +91,7 @@ public class LogsCLI extends Configured implements Tool {
       = "show_container_log_info";
   private static final String OUT_OPTION = "out";
   private static final String SIZE_OPTION = "size";
+  private static final String REGEX_OPTION = "regex";
   public static final String HELP_CMD = "help";
   private PrintStream outStream = System.out;
   private YarnClient yarnClient = null;
@@ -126,6 +128,7 @@ private int runCommand(String[] args) throws Exception {
     boolean nodesList = false;
     boolean showApplicationLogInfo = false;
     boolean showContainerLogInfo = false;
+    boolean useRegex = false;
     String[] logFiles = null;
     List<String> amContainersList = new ArrayList<String>();
     String localDir = null;
@@ -142,6 +145,7 @@ private int runCommand(String[] args) throws Exception {
       showApplicationLogInfo = commandLine.hasOption(
           SHOW_APPLICATION_LOG_INFO);
       showContainerLogInfo = commandLine.hasOption(SHOW_CONTAINER_LOG_INFO);
+      useRegex = commandLine.hasOption(REGEX_OPTION);
       if (getAMContainerLogs) {
         try {
           amContainersList = parseAMContainer(commandLine, printOpts);
@@ -243,11 +247,11 @@ private int runCommand(String[] args) throws Exception {
       }
     }
 
-    List<String> logs = new ArrayList<String>();
+    Set<String> logs = new HashSet<String>();
-    if (fetchAllLogFiles(logFiles)) {
+    if (fetchAllLogFiles(logFiles, useRegex)) {
-      logs.add(".*");
+      logs.add("ALL");
     } else if (logFiles != null && logFiles.length > 0) {
-      logs = Arrays.asList(logFiles);
+      logs.addAll(Arrays.asList(logFiles));
     }
 
     ContainerLogsRequest request = new ContainerLogsRequest(appId,
@@ -268,15 +272,15 @@ private int runCommand(String[] args) throws Exception {
     // To get am logs
     if (getAMContainerLogs) {
       return fetchAMContainerLogs(request, amContainersList,
-          logCliHelper);
+          logCliHelper, useRegex);
     }
 
     int resultCode = 0;
     if (containerIdStr != null) {
-      return fetchContainerLogs(request, logCliHelper);
+      return fetchContainerLogs(request, logCliHelper, useRegex);
     } else {
       if (nodeAddress == null) {
-        resultCode = fetchApplicationLogs(request, logCliHelper);
+        resultCode = fetchApplicationLogs(request, logCliHelper, useRegex);
       } else {
         System.err.println("Should at least provide ContainerId!");
         printHelpMessage(printOpts);
@@ -362,7 +366,7 @@ private List<JSONObject> getAMContainerInfoForAHSWebService(
     return amContainersList;
   }
 
-  private boolean fetchAllLogFiles(String[] logFiles) {
+  private boolean fetchAllLogFiles(String[] logFiles, boolean useRegex) {
 
     // If no value is specified for the PER_CONTAINER_LOG_FILES_OPTION option,
     // we will assume all logs.
@@ -371,7 +375,8 @@ private boolean fetchAllLogFiles(String[] logFiles) {
     }
 
     List<String> logs = Arrays.asList(logFiles);
-    if (logs.contains("ALL") || logs.contains(".*")) {
+    if (logs.contains("ALL") || logs.contains("*")||
+        (logs.contains(".*") && useRegex)) {
       return true;
     }
 
@@ -417,8 +422,8 @@ private List<PerLogFileInfo> getContainerLogFiles(Configuration conf,
   @Private
   @VisibleForTesting
   public int printContainerLogsFromRunningApplication(Configuration conf,
-      ContainerLogsRequest request, LogCLIHelpers logCliHelper)
+      ContainerLogsRequest request, LogCLIHelpers logCliHelper,
-      throws IOException {
+      boolean useRegex) throws IOException {
     String containerIdStr = request.getContainerId().toString();
     String localDir = request.getOutputLocalDir();
     String nodeHttpAddress = request.getNodeHttpAddress();
@@ -441,7 +446,8 @@ public int printContainerLogsFromRunningApplication(Configuration conf,
       for (PerLogFileInfo fileInfo : allLogFileInfos) {
         fileNames.add(fileInfo.getFileName());
       }
-      List<String> matchedFiles = getMatchedLogFiles(request, fileNames);
+      Set<String> matchedFiles = getMatchedLogFiles(request, fileNames,
+          useRegex);
       if (matchedFiles.isEmpty()) {
         System.err.println("Can not find any log file matching the pattern: "
             + request.getLogTypes() + " for the container: " + containerIdStr
@@ -503,10 +509,10 @@ public int printContainerLogsFromRunningApplication(Configuration conf,
   }
 
   private int printContainerLogsForFinishedApplication(
-      ContainerLogsRequest request, LogCLIHelpers logCliHelper)
+      ContainerLogsRequest request, LogCLIHelpers logCliHelper,
-      throws IOException {
+      boolean useRegex) throws IOException {
     ContainerLogsRequest newOptions = getMatchedLogOptions(
-        request, logCliHelper);
+        request, logCliHelper, useRegex);
     if (newOptions == null) {
       System.err.println("Can not find any log file matching the pattern: "
           + request.getLogTypes() + " for the container: "
@@ -518,10 +524,10 @@ private int printContainerLogsForFinishedApplication(
   }
 
   private int printContainerLogsForFinishedApplicationWithoutNodeId(
-      ContainerLogsRequest request, LogCLIHelpers logCliHelper)
+      ContainerLogsRequest request, LogCLIHelpers logCliHelper,
-      throws IOException {
+      boolean useRegex) throws IOException {
     ContainerLogsRequest newOptions = getMatchedLogOptions(
-        request, logCliHelper);
+        request, logCliHelper, useRegex);
     if (newOptions == null) {
       System.err.println("Can not find any log file matching the pattern: "
           + request.getLogTypes() + " for the container: "
@@ -549,7 +555,7 @@ private boolean isApplicationFinished(YarnApplicationState appState) {
 
   private int printAMContainerLogs(Configuration conf,
       ContainerLogsRequest request, List<String> amContainers,
-      LogCLIHelpers logCliHelper) throws Exception {
+      LogCLIHelpers logCliHelper, boolean useRegex) throws Exception {
     List<JSONObject> amContainersList = null;
     List<ContainerLogsRequest> requests =
         new ArrayList<ContainerLogsRequest>();
@@ -613,7 +619,7 @@ private int printAMContainerLogs(Configuration conf,
 
     if (amContainers.contains("ALL")) {
       for (ContainerLogsRequest amRequest : requests) {
-        outputAMContainerLogs(amRequest, conf, logCliHelper);
+        outputAMContainerLogs(amRequest, conf, logCliHelper, useRegex);
       }
       outStream.println();
       outStream.println("Specified ALL for -am option. "
@@ -623,11 +629,11 @@ private int printAMContainerLogs(Configuration conf,
         int amContainerId = Integer.parseInt(amContainer.trim());
         if (amContainerId == -1) {
           outputAMContainerLogs(requests.get(requests.size() - 1), conf,
-              logCliHelper);
+              logCliHelper, useRegex);
         } else {
           if (amContainerId <= requests.size()) {
             outputAMContainerLogs(requests.get(amContainerId - 1), conf,
-                logCliHelper);
+                logCliHelper, useRegex);
           } else {
             System.err.println(String.format("ERROR: Specified AM containerId"
                 + " (%s) exceeds the number of AM containers (%s).",
@@ -641,7 +647,8 @@ private int printAMContainerLogs(Configuration conf,
   }
 
   private void outputAMContainerLogs(ContainerLogsRequest request,
-      Configuration conf, LogCLIHelpers logCliHelper) throws Exception {
+      Configuration conf, LogCLIHelpers logCliHelper, boolean useRegex)
+      throws Exception {
     String nodeHttpAddress = request.getNodeHttpAddress();
     String containerId = request.getContainerId();
     String nodeId = request.getNodeId();
@@ -650,10 +657,10 @@ private void outputAMContainerLogs(ContainerLogsRequest request,
       if (containerId != null && !containerId.isEmpty()) {
         if (nodeId != null && !nodeId.isEmpty()) {
           printContainerLogsForFinishedApplication(request,
-              logCliHelper);
+              logCliHelper, useRegex);
         } else {
           printContainerLogsForFinishedApplicationWithoutNodeId(
-              request, logCliHelper);
+              request, logCliHelper, useRegex);
         }
       }
     } else {
@@ -663,7 +670,7 @@ private void outputAMContainerLogs(ContainerLogsRequest request,
             .getContainerState();
         request.setContainerState(containerState);
         printContainerLogsFromRunningApplication(conf,
-            request, logCliHelper);
+            request, logCliHelper, useRegex);
       }
     }
   }
@@ -746,12 +753,15 @@ private Options createCommandOpts() {
     opts.addOption(amOption);
     Option logFileOpt = new Option(PER_CONTAINER_LOG_FILES_OPTION, true,
         "Specify comma-separated value "
-        + "to get specified container log files. Use \"ALL\" to fetch all the "
+        + "to get exact matched log files. Use \"ALL\" or \"*\"to "
-        + "log files for the container. It also supports Java Regex.");
+        + "fetch all the log files for the container. Specific -regex "
+        + "for using java regex to find matched log files.");
     logFileOpt.setValueSeparator(',');
     logFileOpt.setArgs(Option.UNLIMITED_VALUES);
     logFileOpt.setArgName("Log File Name");
     opts.addOption(logFileOpt);
+    opts.addOption(REGEX_OPTION, false, "Work with -log_files to find "
+        + "matched files by using java regex.");
     opts.addOption(SHOW_CONTAINER_LOG_INFO, false,
         "Show the container log metadata, "
         + "including log-file names, the size of the log files. "
@@ -794,6 +804,7 @@ private Options createPrintOpts(Options commandOpts) {
     printOpts.addOption(commandOpts.getOption(SHOW_CONTAINER_LOG_INFO));
     printOpts.addOption(commandOpts.getOption(OUT_OPTION));
     printOpts.addOption(commandOpts.getOption(SIZE_OPTION));
+    printOpts.addOption(commandOpts.getOption(REGEX_OPTION));
     return printOpts;
   }
 
@@ -829,14 +840,14 @@ private List<String> parseAMContainer(CommandLine commandLine,
   }
 
   private int fetchAMContainerLogs(ContainerLogsRequest request,
-      List<String> amContainersList, LogCLIHelpers logCliHelper)
+      List<String> amContainersList, LogCLIHelpers logCliHelper,
-      throws Exception {
+      boolean useRegex) throws Exception {
     return printAMContainerLogs(getConf(), request, amContainersList,
-        logCliHelper);
+        logCliHelper, useRegex);
   }
 
   private int fetchContainerLogs(ContainerLogsRequest request,
-      LogCLIHelpers logCliHelper) throws IOException {
+      LogCLIHelpers logCliHelper, boolean useRegex) throws IOException {
     int resultCode = 0;
     String appIdStr = request.getAppId().toString();
     String containerIdStr = request.getContainerId();
@@ -850,10 +861,10 @@ private int fetchContainerLogs(ContainerLogsRequest request,
       // to logCliHelper so that it fetches all the logs
       if (nodeAddress != null && !nodeAddress.isEmpty()) {
         return printContainerLogsForFinishedApplication(
-            request, logCliHelper);
+            request, logCliHelper, useRegex);
       } else {
         return printContainerLogsForFinishedApplicationWithoutNodeId(
-            request, logCliHelper);
+            request, logCliHelper, useRegex);
       }
     }
     String nodeHttpAddress = null;
@@ -875,7 +886,7 @@ private int fetchContainerLogs(ContainerLogsRequest request,
     } catch (IOException | YarnException ex) {
       if (isAppFinished) {
         return printContainerLogsForFinishedApplicationWithoutNodeId(
-            request, logCliHelper);
+            request, logCliHelper, useRegex);
       } else {
         System.err.println("Unable to get logs for this container:"
             + containerIdStr + "for the application:" + appIdStr
@@ -892,18 +903,19 @@ private int fetchContainerLogs(ContainerLogsRequest request,
     // by calling NodeManager webservice.
     if (!isAppFinished) {
       resultCode = printContainerLogsFromRunningApplication(getConf(), request,
-          logCliHelper);
+          logCliHelper, useRegex);
     } else {
       // If the application is in the final state, we will directly
       // get the container logs from HDFS.
       resultCode = printContainerLogsForFinishedApplication(
-          request, logCliHelper);
+          request, logCliHelper, useRegex);
     }
     return resultCode;
   }
 
   private int fetchApplicationLogs(ContainerLogsRequest options,
-      LogCLIHelpers logCliHelper) throws IOException, YarnException {
+      LogCLIHelpers logCliHelper, boolean useRegex) throws IOException,
+      YarnException {
     // If the application has finished, we would fetch the logs
     // from HDFS.
     // If the application is still running, we would get the full
@@ -912,7 +924,7 @@ private int fetchApplicationLogs(ContainerLogsRequest options,
     int resultCode = -1;
     if (options.isAppFinished()) {
       ContainerLogsRequest newOptions = getMatchedLogOptions(
-          options, logCliHelper);
+          options, logCliHelper, useRegex);
       if (newOptions == null) {
         System.err.println("Can not find any log file matching the pattern: "
             + options.getLogTypes() + " for the application: "
@@ -926,7 +938,7 @@ private int fetchApplicationLogs(ContainerLogsRequest options,
           getContainersLogRequestForRunningApplication(options);
       for (ContainerLogsRequest container : containerLogRequests) {
         int result = printContainerLogsFromRunningApplication(getConf(),
-            container, logCliHelper);
+            container, logCliHelper, useRegex);
         if (result == 0) {
           resultCode = 0;
         }
@@ -955,14 +967,14 @@ private String guessAppOwner(ApplicationReport appReport,
   }
 
   private ContainerLogsRequest getMatchedLogOptions(
-      ContainerLogsRequest request, LogCLIHelpers logCliHelper)
+      ContainerLogsRequest request, LogCLIHelpers logCliHelper,
-      throws IOException {
+      boolean useRegex) throws IOException {
     ContainerLogsRequest newOptions = new ContainerLogsRequest(request);
     if (request.getLogTypes() != null && !request.getLogTypes().isEmpty()) {
-      List<String> matchedFiles = new ArrayList<String>();
+      Set<String> matchedFiles = new HashSet<String>();
-      if (!request.getLogTypes().contains(".*")) {
+      if (!request.getLogTypes().contains("ALL")) {
         Set<String> files = logCliHelper.listContainerLogs(request);
-        matchedFiles = getMatchedLogFiles(request, files);
+        matchedFiles = getMatchedLogFiles(request, files, useRegex);
         if (matchedFiles.isEmpty()) {
           return null;
         }
@@ -972,20 +984,29 @@ private ContainerLogsRequest getMatchedLogOptions(
     return newOptions;
   }
 
-  private List<String> getMatchedLogFiles(ContainerLogsRequest options,
+  private Set<String> getMatchedLogFiles(ContainerLogsRequest options,
-      Collection<String> candidate) throws IOException {
+      Collection<String> candidate, boolean useRegex) throws IOException {
-    List<String> matchedFiles = new ArrayList<String>();
+    Set<String> matchedFiles = new HashSet<String>();
-    List<String> filePattern = options.getLogTypes();
+    Set<String> filePattern = options.getLogTypes();
+    if (options.getLogTypes().contains("ALL")) {
+      return new HashSet<String>(candidate);
+    }
     for (String file : candidate) {
-      if (isFileMatching(file, filePattern)) {
+      if (useRegex) {
-        matchedFiles.add(file);
+        if (isFileMatching(file, filePattern)) {
+          matchedFiles.add(file);
+        }
+      } else {
+        if (filePattern.contains(file)) {
+          matchedFiles.add(file);
+        }
       }
     }
     return matchedFiles;
   }
 
   private boolean isFileMatching(String fileType,
-      List<String> logTypes) {
+      Set<String> logTypes) {
     for (String logType : logTypes) {
       Pattern filterPattern = Pattern.compile(logType);
       boolean match = filterPattern.matcher(fileType).find();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java

@@ -22,6 +22,7 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
@@ -46,7 +47,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-
+import java.util.Set;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -211,14 +212,17 @@ public void testHelpMessage() throws Exception {
     pw.println("                                 aggregated logs. This option can only be");
     pw.println("                                 used with finished applications.");
     pw.println(" -log_files <Log File Name>      Specify comma-separated value to get");
-    pw.println("                                 specified container log files. Use \"ALL\"");
+    pw.println("                                 exact matched log files. Use \"ALL\" or");
-    pw.println("                                 to fetch all the log files for the");
+    pw.println("                                 \"*\"to fetch all the log files for the");
-    pw.println("                                 container. It also supports Java Regex.");
+    pw.println("                                 container. Specific -regex for using java");
+    pw.println("                                 regex to find matched log files.");
     pw.println(" -nodeAddress <Node Address>     NodeAddress in the format nodename:port");
     pw.println(" -out <Local Directory>          Local directory for storing individual");
     pw.println("                                 container logs. The container logs will");
     pw.println("                                 be stored based on the node the container");
     pw.println("                                 ran on.");
+    pw.println(" -regex                          Work with -log_files to find matched");
+    pw.println("                                 files by using java regex.");
     pw.println(" -show_application_log_info      Show the containerIds which belong to the");
     pw.println("                                 specific Application. You can combine");
     pw.println("                                 this with --nodeAddress to get");
@@ -286,6 +290,7 @@ public void testFetchFinishedApplictionLogs() throws Exception {
 
     // create two logs for container3 in localLogDir
     logTypes.add("stdout");
+    logTypes.add("stdout1234");
     createContainerLogInLocalDir(appLogsDir, containerId3, fs, logTypes);
 
     Path path =
@@ -336,10 +341,12 @@ public ContainerReport getContainerReport(String containerIdStr)
         logMessage(containerId3, "syslog")));
     assertTrue(sysOutStream.toString().contains(
         logMessage(containerId3, "stdout")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout1234")));
     sysOutStream.reset();
 
     exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
-        "-log_files", ".*"});
+        "-log_files", ".*", "-regex"});
     assertTrue(exitCode == 0);
     assertTrue(sysOutStream.toString().contains(
         logMessage(containerId1, "syslog")));
@@ -349,11 +356,28 @@ public ContainerReport getContainerReport(String containerIdStr)
         logMessage(containerId3, "syslog")));
     assertTrue(sysOutStream.toString().contains(
         logMessage(containerId3, "stdout")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout1234")));
+    sysOutStream.reset();
+
+    exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
+        "-log_files", "*"});
+    assertTrue(exitCode == 0);
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId1, "syslog")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId2, "syslog")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "syslog")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout1234")));
     int fullSize = sysOutStream.toByteArray().length;
     sysOutStream.reset();
 
     exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
-        "-log_files", "std*"});
+        "-log_files", "stdout"});
     assertTrue(exitCode == 0);
     assertFalse(sysOutStream.toString().contains(
         logMessage(containerId1, "syslog")));
@@ -363,6 +387,23 @@ public ContainerReport getContainerReport(String containerIdStr)
         logMessage(containerId3, "syslog")));
     assertTrue(sysOutStream.toString().contains(
         logMessage(containerId3, "stdout")));
+    assertFalse(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout1234")));
+    sysOutStream.reset();
+
+    exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
+        "-log_files", "std*", "-regex"});
+    assertTrue(exitCode == 0);
+    assertFalse(sysOutStream.toString().contains(
+        logMessage(containerId1, "syslog")));
+    assertFalse(sysOutStream.toString().contains(
+        logMessage(containerId2, "syslog")));
+    assertFalse(sysOutStream.toString().contains(
+        logMessage(containerId3, "syslog")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout")));
+    assertTrue(sysOutStream.toString().contains(
+        logMessage(containerId3, "stdout1234")));
     sysOutStream.reset();
 
     exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
@@ -376,7 +417,7 @@ public ContainerReport getContainerReport(String containerIdStr)
     // specify the bytes which is larger than the actual file size,
     // we would get the full logs
     exitCode = cli.run(new String[] {"-applicationId", appId.toString(),
-        "-log_files", ".*", "-size", "10000" });
+        "-log_files", "*", "-size", "10000" });
     assertTrue(exitCode == 0);
     assertTrue(sysOutStream.toByteArray().length == fullSize);
     sysOutStream.reset();
@@ -568,7 +609,7 @@ public void testFetchRunningApplicationLogs() throws Exception {
     LogsCLI cli = spy(new LogsCLIForTest(mockYarnClient));
     doReturn(0).when(cli).printContainerLogsFromRunningApplication(
         any(Configuration.class), any(ContainerLogsRequest.class),
-        any(LogCLIHelpers.class));
+        any(LogCLIHelpers.class), anyBoolean());
 
     cli.setConf(new YarnConfiguration());
     int exitCode = cli.run(new String[] {"-applicationId", appId.toString()});
@@ -581,16 +622,16 @@ public void testFetchRunningApplicationLogs() throws Exception {
     // printContainerLogsFromRunningApplication twice
     verify(cli, times(2)).printContainerLogsFromRunningApplication(
         any(Configuration.class), logsRequestCaptor.capture(),
-        any(LogCLIHelpers.class));
+        any(LogCLIHelpers.class), anyBoolean());
 
-    // Verify that the log-type is *
+    // Verify that the log-type is "ALL"
     List<ContainerLogsRequest> capturedRequests =
         logsRequestCaptor.getAllValues();
     Assert.assertEquals(2, capturedRequests.size());
-    List<String> logTypes0 = capturedRequests.get(0).getLogTypes();
+    Set<String> logTypes0 = capturedRequests.get(0).getLogTypes();
-    List<String> logTypes1 = capturedRequests.get(1).getLogTypes();
+    Set<String> logTypes1 = capturedRequests.get(1).getLogTypes();
-    Assert.assertEquals(".*", logTypes0.get(0));
+    Assert.assertTrue(logTypes0.contains("ALL") && (logTypes0.size() == 1));
-    Assert.assertEquals(".*", logTypes1.get(0));
+    Assert.assertTrue(logTypes1.contains("ALL") && (logTypes1.size() == 1));
   }
 
   @Test (timeout = 15000)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/ContainerLogsRequest.java

@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.yarn.logaggregation;
 
-import java.util.List;
+import java.util.Set;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 
@@ -30,7 +30,7 @@ public class ContainerLogsRequest {
   private String appOwner;
   private boolean appFinished;
   private String outputLocalDir;
-  private List<String> logTypes;
+  private Set<String> logTypes;
   private long bytes;
   private ContainerState containerState;
 
@@ -52,7 +52,7 @@ public ContainerLogsRequest(ContainerLogsRequest request) {
   public ContainerLogsRequest(ApplicationId applicationId,
       boolean isAppFinished, String owner,
       String address, String httpAddress, String container, String localDir,
-      List<String> logs, long bytes, ContainerState containerState) {
+      Set<String> logs, long bytes, ContainerState containerState) {
     this.setAppId(applicationId);
     this.setAppFinished(isAppFinished);
     this.setAppOwner(owner);
@@ -121,11 +121,11 @@ public void setOutputLocalDir(String outputLocalDir) {
     this.outputLocalDir = outputLocalDir;
   }
 
-  public List<String> getLogTypes() {
+  public Set<String> getLogTypes() {
     return logTypes;
   }
 
-  public void setLogTypes(List<String> logTypes) {
+  public void setLogTypes(Set<String> logTypes) {
     this.logTypes = logTypes;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java

@@ -67,7 +67,7 @@ public int dumpAContainersLogs(String appId, String containerId,
     options.setContainerId(containerId);
     options.setNodeId(nodeId);
     options.setAppOwner(jobOwner);
-    List<String> logs = new ArrayList<String>();
+    Set<String> logs = new HashSet<String>();
     options.setLogTypes(logs);
     options.setBytes(Long.MAX_VALUE);
     return dumpAContainerLogsForLogType(options, false);
@@ -136,7 +136,7 @@ public int dumpAContainerLogsForLogType(ContainerLogsRequest options,
     String nodeId = options.getNodeId();
     String containerId = options.getContainerId();
     String localDir = options.getOutputLocalDir();
-    List<String> logType = options.getLogTypes();
+    List<String> logType = new ArrayList<String>(options.getLogTypes());
     RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
         applicationId, jobOwner);
     if (nodeFiles == null) {
@@ -208,7 +208,7 @@ public int dumpAContainerLogsForLogTypeWithoutNodeId(
     String jobOwner = options.getAppOwner();
     String containerId = options.getContainerId();
     String localDir = options.getOutputLocalDir();
-    List<String> logType = options.getLogTypes();
+    List<String> logType = new ArrayList<String>(options.getLogTypes());
     RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
         applicationId, jobOwner);
     if (nodeFiles == null) {
@@ -344,7 +344,7 @@ public int dumpAllContainersLogs(ContainerLogsRequest options)
     ApplicationId appId = options.getAppId();
     String appOwner = options.getAppOwner();
     String localDir = options.getOutputLocalDir();
-    List<String> logTypes = options.getLogTypes();
+    List<String> logTypes = new ArrayList<String>(options.getLogTypes());
     RemoteIterator<FileStatus> nodeFiles = getRemoteNodeFileDir(
         appId, appOwner);
     if (nodeFiles == null) {