Merge branch 'trunk' into HADOOP-12756

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java

@@ -260,7 +260,7 @@ public void preempted(TaskAttemptID taskAttemptID, TaskStatus taskStatus)
 
   @Override
   public void done(TaskAttemptID taskAttemptID) throws IOException {
-    LOG.info("Done acknowledgement from " + taskAttemptID.toString());
+    LOG.info("Done acknowledgment from " + taskAttemptID.toString());
 
     org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
         TypeConverter.toYarn(taskAttemptID);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -400,7 +400,7 @@ protected void serviceStop() throws Exception {
         }
         mi.shutDownTimer();
       } catch (IOException e) {
-        LOG.info("Exception while cancelling delayed flush timer. "
+        LOG.info("Exception while canceling delayed flush timer. "
             + "Likely caused by a failed flush " + e.getMessage());
       }
     }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java

@@ -1312,7 +1312,7 @@ void taskCleanup(TaskUmbilicalProtocol umbilical)
     setPhase(TaskStatus.Phase.CLEANUP);
     getProgress().setStatus("cleanup");
     statusUpdate(umbilical);
-    LOG.info("Runnning cleanup for the task");
+    LOG.info("Running cleanup for the task");
     // do the cleanup
     committer.abortTask(taskContext);
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java

@@ -672,7 +672,7 @@ private void checkVersion() throws IOException {
       return;
     }
     if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
-      LOG.info("Storing state DB schedma version info " + getCurrentVersion());
+      LOG.info("Storing state DB schema version info " + getCurrentVersion());
       storeVersion();
     } else {
       throw new IOException(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java

@@ -20,10 +20,13 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumMap;
@@ -37,6 +40,10 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import org.apache.commons.cli.CommandLine;
@@ -50,7 +57,12 @@
 import org.apache.commons.lang.time.DurationFormatUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig.Policy;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
 import org.apache.hadoop.util.Time;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -60,12 +72,17 @@
 import org.apache.hadoop.yarn.api.records.QueueStatistics;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 
 public class TopCLI extends YarnCLI {
 
+  private static final String CLUSTER_INFO_URL = "/ws/v1/cluster/info";
+
   private static final Log LOG = LogFactory.getLog(TopCLI.class);
   private String CLEAR = "\u001b[2J";
   private String CLEAR_LINE = "\u001b[2K";
@@ -742,23 +759,91 @@ protected QueueMetrics getQueueMetrics() {
 
   long getRMStartTime() {
     try {
-      URL url =
+      // connect with url
-          new URL("http://"
+      URL url = getClusterUrl();
-              + client.getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)
+      if (null == url) {
-              + "/ws/v1/cluster/info");
+        return -1;
-      URLConnection conn = url.openConnection();
+      }
-      conn.connect();
+      JSONObject clusterInfo = getJSONObject(connect(url));
+      return clusterInfo.getLong("startedOn");
+    } catch (Exception e) {
+      LOG.error("Could not fetch RM start time", e);
+    }
+    return -1;
+  }
+
+  private JSONObject getJSONObject(URLConnection conn)
+      throws IOException, JSONException {
     InputStream in = conn.getInputStream();
     String encoding = conn.getContentEncoding();
     encoding = encoding == null ? "UTF-8" : encoding;
     String body = IOUtils.toString(in, encoding);
     JSONObject obj = new JSONObject(body);
     JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
-      return clusterInfo.getLong("startedOn");
+    return clusterInfo;
-    } catch (Exception e) {
-      LOG.error("Could not fetch RM start time", e);
   }
-    return -1;
+
+  private URL getClusterUrl() throws Exception {
+    URL url = null;
+    Configuration conf = getConf();
+    if (HAUtil.isHAEnabled(conf)) {
+      Collection<String> haids = HAUtil.getRMHAIds(conf);
+      for (String rmhid : haids) {
+        try {
+          url = getHAClusterUrl(conf, rmhid);
+          if (isActive(url)) {
+            break;
+          }
+        } catch (ConnectException e) {
+          // ignore and try second one when one of RM is down
+        }
+      }
+    } else {
+      url = new URL(
+          WebAppUtils.getRMWebAppURLWithScheme(conf) + CLUSTER_INFO_URL);
+    }
+    return url;
+  }
+
+  private boolean isActive(URL url) throws Exception {
+    URLConnection connect = connect(url);
+    JSONObject clusterInfo = getJSONObject(connect);
+    return clusterInfo.getString("haState").equals("ACTIVE");
+  }
+
+  @VisibleForTesting
+  public URL getHAClusterUrl(Configuration conf, String rmhid)
+      throws MalformedURLException {
+    return new URL(WebAppUtils.getHttpSchemePrefix(conf)
+        + WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf,
+            YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY
+                : Policy.HTTP_ONLY,
+            rmhid)
+        + CLUSTER_INFO_URL);
+  }
+
+  private URLConnection connect(URL url) throws Exception {
+    AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+    AuthenticatedURL authUrl;
+    SSLFactory clientSslFactory;
+    URLConnection connection;
+    // If https is chosen, configures SSL client.
+    if (YarnConfiguration.useHttps(getConf())) {
+      clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, getConf());
+      clientSslFactory.init();
+      SSLSocketFactory sslSocktFact = clientSslFactory.createSSLSocketFactory();
+
+      authUrl =
+          new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory);
+      connection = authUrl.openConnection(url, token);
+      HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+      httpsConn.setSSLSocketFactory(sslSocktFact);
+    } else {
+      authUrl = new AuthenticatedURL(new KerberosAuthenticator());
+      connection = authUrl.openConnection(url, token);
+    }
+    connection.connect();
+    return connection;
   }
 
   String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) {
@@ -768,7 +853,10 @@ String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) {
       queue = StringUtils.join(queues, ",");
     }
     long now = Time.now();
-    long uptime = now - rmStartTime;
+    long uptime = 0L;
+    if (rmStartTime != -1) {
+      uptime = now - rmStartTime;
+    }
     long days = TimeUnit.MILLISECONDS.toDays(uptime);
     long hours =
         TimeUnit.MILLISECONDS.toHours(uptime)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java

@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test class for TopCli.
+ *
+ */
+public class TestTopCLI {
+
+  private static final String RM1_NODE_ID = "rm1";
+  private static final String RM2_NODE_ID = "rm2";
+
+  private static List<String> dummyHostNames =
+      Arrays.asList("host1", "host2", "host3");
+
+  private static Map<String, String> savedStaticResolution = new HashMap<>();
+
+  @BeforeClass
+  public static void initializeDummyHostnameResolution() throws Exception {
+    String previousIpAddress;
+    for (String hostName : dummyHostNames) {
+      previousIpAddress = NetUtils.getStaticResolution(hostName);
+      if (null != previousIpAddress) {
+        savedStaticResolution.put(hostName, previousIpAddress);
+      }
+      NetUtils.addStaticResolution(hostName, "10.20.30.40");
+    }
+  }
+
+  @AfterClass
+  public static void restoreDummyHostnameResolution() throws Exception {
+    for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution
+        .entrySet()) {
+      NetUtils.addStaticResolution(hostnameToIpEntry.getKey(),
+          hostnameToIpEntry.getValue());
+    }
+  }
+
+  @Test
+  public void testHAClusterInfoURL() throws IOException, InterruptedException {
+    TopCLI topcli = new TopCLI();
+    // http
+    String rm1Address = "host2:8088";
+    String rm2Address = "host3:8088";
+    Configuration conf = topcli.getConf();
+    conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID,
+        rm1Address);
+    topcli.getConf().set(
+        YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
+    topcli.getConf().setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    topcli.getConf().set(YarnConfiguration.RM_HA_IDS,
+        RM1_NODE_ID + "," + RM2_NODE_ID);
+    URL clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID);
+    Assert.assertEquals("http", clusterUrl.getProtocol());
+    Assert.assertEquals(rm1Address, clusterUrl.getAuthority());
+    clusterUrl = topcli.getHAClusterUrl(conf, RM2_NODE_ID);
+    Assert.assertEquals("http", clusterUrl.getProtocol());
+    Assert.assertEquals(rm2Address, clusterUrl.getAuthority());
+    // https
+    rm1Address = "host2:9088";
+    rm2Address = "host3:9088";
+    conf = topcli.getConf();
+    conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM1_NODE_ID,
+        rm1Address);
+    conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM2_NODE_ID,
+        rm2Address);
+    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+    conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
+    clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID);
+    Assert.assertEquals("https", clusterUrl.getProtocol());
+    Assert.assertEquals(rm1Address, clusterUrl.getAuthority());
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java

@@ -142,7 +142,7 @@ public void setDrainEventsOnStop() {
   protected void serviceStop() throws Exception {
     if (drainEventsOnStop) {
       blockNewEvents = true;
-      LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+      LOG.info("AsyncDispatcher is draining to stop, ignoring any new events.");
       long endTime = System.currentTimeMillis() + getConfig()
           .getLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT,
               YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java

@@ -54,7 +54,7 @@ public static YarnAuthorizationProvider getInstance(Configuration conf) {
             (YarnAuthorizationProvider) ReflectionUtils.newInstance(
               authorizerClass, conf);
         authorizer.init(conf);
-        LOG.info(authorizerClass.getName() + " is instiantiated.");
+        LOG.info(authorizerClass.getName() + " is instantiated.");
       }
     }
     return authorizer;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java

@@ -40,7 +40,6 @@
 import org.apache.hadoop.yarn.conf.HAUtil;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.RMHAUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -183,30 +182,30 @@ public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
 
   public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
       Policy httpPolicy) {
-    InetSocketAddress address = null;
     String rmId = null;
     if (HAUtil.isHAEnabled(conf)) {
       // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
       // the Active RM
       rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
     }
+    return getResolvedRemoteRMWebAppURLWithoutScheme(conf, httpPolicy, rmId);
+  }
+
+  public static String getResolvedRemoteRMWebAppURLWithoutScheme(
+      Configuration conf, Policy httpPolicy, String rmId) {
+    InetSocketAddress address = null;
 
     if (httpPolicy == Policy.HTTPS_ONLY) {
-      address =
+      address = conf.getSocketAddr(
-          conf.getSocketAddr(
+          rmId == null ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
-              rmId == null
+              : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
-                  ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
+                  rmId),
-                  : HAUtil.addSuffix(
-                  YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
           YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
           YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
     } else {
-      address =
+      address = conf.getSocketAddr(
-          conf.getSocketAddr(
+          rmId == null ? YarnConfiguration.RM_WEBAPP_ADDRESS
-              rmId == null
+              : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
-                  ? YarnConfiguration.RM_WEBAPP_ADDRESS
-                  : HAUtil.addSuffix(
-                  YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
           YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
           YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java

@@ -405,7 +405,7 @@ public Map<ContainerId, ContainerHistoryData> getContainers(
           }
         }
       }
-      LOG.info("Completed reading history information of all conatiners"
+      LOG.info("Completed reading history information of all containers"
           + " of application attempt " + appAttemptId);
     } catch (IOException e) {
       LOG.info("Error when reading history information of some containers"

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java

@@ -216,7 +216,7 @@ private boolean isEnabled() {
             return false;
     }
     if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
-      LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
+      LOG.info("Neither virtual-memory nor physical-memory monitoring is " +
           "needed. Not running the monitor-thread");
       return false;
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java

@@ -22,6 +22,7 @@
 import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
@@ -386,8 +387,8 @@ public Object answer(InvocationOnMock invocationOnMock)
           }
         }
         return null;
-      }
+      }}).when(mockUtil).copy(any(Path.class), any(Path.class),
-    }).when(mockUtil).copy(any(Path.class), any(Path.class));
+        anyBoolean(), anyBoolean());
 
     doAnswer(new Answer() {
       @Override
@@ -478,7 +479,8 @@ public ContainerLocalizer createContainerLocalizer(String user,
     }
 
     // Verify that the calls happen the expected number of times
-    verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class));
+    verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class),
+        anyBoolean(), anyBoolean());
     verify(mockLfs, times(2)).getFsStatus(any(Path.class));
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java

@@ -415,7 +415,7 @@ protected Plan initializePlan(String planQueueName) throws YarnException {
             minAllocation, maxAllocation, planQueueName,
             getReplanner(planQueuePath), getReservationSchedulerConfiguration()
             .getMoveOnExpiry(planQueuePath), rmContext);
-    LOG.info("Intialized plan {} based on reservable queue {}",
+    LOG.info("Initialized plan {} based on reservable queue {}",
         plan.toString(), planQueueName);
     return plan;
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java

@@ -230,7 +230,7 @@ protected synchronized void setupQueueConfigs(Resource clusterResource)
     LOG.info("Initializing " + queueName + "\n" +
         "capacity = " + queueCapacities.getCapacity() +
         " [= (float) configuredCapacity / 100 ]" + "\n" + 
-        "asboluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
+        "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
         " [= parentAbsoluteCapacity * capacity ]" + "\n" +
         "maxCapacity = " + queueCapacities.getMaximumCapacity() +
         " [= configuredMaxCapacity ]" + "\n" +

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java

@@ -125,9 +125,9 @@ synchronized void setupQueueConfigs(Resource clusterResource)
 
     LOG.info(queueName +
         ", capacity=" + this.queueCapacities.getCapacity() +
-        ", asboluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
+        ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
         ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() +
-        ", asboluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() + 
+        ", absoluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
         ", state=" + state +
         ", acls=" + aclsString + 
         ", labels=" + labelStrBuilder.toString() + "\n" +

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java

@@ -944,7 +944,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId,
     FSAppAttempt application = getSchedulerApp(appAttemptId);
     if (application == null) {
       LOG.info("Calling allocate on removed " +
-          "or non existant application " + appAttemptId);
+          "or non existent application " + appAttemptId);
       return EMPTY_ALLOCATION;
     }