diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index 49a00c50a0..2c0ea2b455 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -260,7 +260,7 @@ public void preempted(TaskAttemptID taskAttemptID, TaskStatus taskStatus) @Override public void done(TaskAttemptID taskAttemptID) throws IOException { - LOG.info("Done acknowledgement from " + taskAttemptID.toString()); + LOG.info("Done acknowledgment from " + taskAttemptID.toString()); org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java index 53d8a366e2..e61bbaa368 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java @@ -400,7 +400,7 @@ protected void serviceStop() throws Exception { } mi.shutDownTimer(); } catch (IOException e) { - LOG.info("Exception while cancelling delayed flush timer. " + LOG.info("Exception while canceling delayed flush timer. " + "Likely caused by a failed flush " + e.getMessage()); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java index 3c58a671d1..d20e9bdb7a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java @@ -1312,7 +1312,7 @@ void taskCleanup(TaskUmbilicalProtocol umbilical) setPhase(TaskStatus.Phase.CLEANUP); getProgress().setStatus("cleanup"); statusUpdate(umbilical); - LOG.info("Runnning cleanup for the task"); + LOG.info("Running cleanup for the task"); // do the cleanup committer.abortTask(taskContext); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index 7818c8181c..558ee38548 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -672,7 +672,7 @@ private void checkVersion() throws IOException { return; } if (loadedVersion.isCompatibleTo(getCurrentVersion())) { - LOG.info("Storing state DB schedma version info " + getCurrentVersion()); + LOG.info("Storing state DB schema version info " + getCurrentVersion()); storeVersion(); } else { throw new IOException( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java index eca1266e44..f99dd4885e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java @@ -20,10 +20,13 @@ import java.io.IOException; import java.io.InputStream; +import java.net.ConnectException; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; @@ -37,6 +40,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLSocketFactory; + +import com.google.common.annotations.VisibleForTesting; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.commons.cli.CommandLine; @@ -50,7 +57,12 @@ import org.apache.commons.lang.time.DurationFormatUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; +import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; @@ -60,12 +72,17 @@ import org.apache.hadoop.yarn.api.records.QueueStatistics; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.webapp.util.WebAppUtils; +import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; public class TopCLI extends YarnCLI { + private static final String CLUSTER_INFO_URL = "/ws/v1/cluster/info"; + private static final Log LOG = LogFactory.getLog(TopCLI.class); private String CLEAR = "\u001b[2J"; private String CLEAR_LINE = "\u001b[2K"; @@ -742,18 +759,12 @@ protected QueueMetrics getQueueMetrics() { long getRMStartTime() { try { - URL url = - new URL("http://" - + client.getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS) - + "/ws/v1/cluster/info"); - URLConnection conn = url.openConnection(); - conn.connect(); - InputStream in = conn.getInputStream(); - String encoding = conn.getContentEncoding(); - encoding = encoding == null ? "UTF-8" : encoding; - String body = IOUtils.toString(in, encoding); - JSONObject obj = new JSONObject(body); - JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); + // connect with url + URL url = getClusterUrl(); + if (null == url) { + return -1; + } + JSONObject clusterInfo = getJSONObject(connect(url)); return clusterInfo.getLong("startedOn"); } catch (Exception e) { LOG.error("Could not fetch RM start time", e); @@ -761,6 +772,80 @@ long getRMStartTime() { return -1; } + private JSONObject getJSONObject(URLConnection conn) + throws IOException, JSONException { + InputStream in = conn.getInputStream(); + String encoding = conn.getContentEncoding(); + encoding = encoding == null ? "UTF-8" : encoding; + String body = IOUtils.toString(in, encoding); + JSONObject obj = new JSONObject(body); + JSONObject clusterInfo = obj.getJSONObject("clusterInfo"); + return clusterInfo; + } + + private URL getClusterUrl() throws Exception { + URL url = null; + Configuration conf = getConf(); + if (HAUtil.isHAEnabled(conf)) { + Collection haids = HAUtil.getRMHAIds(conf); + for (String rmhid : haids) { + try { + url = getHAClusterUrl(conf, rmhid); + if (isActive(url)) { + break; + } + } catch (ConnectException e) { + // ignore and try second one when one of RM is down + } + } + } else { + url = new URL( + WebAppUtils.getRMWebAppURLWithScheme(conf) + CLUSTER_INFO_URL); + } + return url; + } + + private boolean isActive(URL url) throws Exception { + URLConnection connect = connect(url); + JSONObject clusterInfo = getJSONObject(connect); + return clusterInfo.getString("haState").equals("ACTIVE"); + } + + @VisibleForTesting + public URL getHAClusterUrl(Configuration conf, String rmhid) + throws MalformedURLException { + return new URL(WebAppUtils.getHttpSchemePrefix(conf) + + WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf, + YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY + : Policy.HTTP_ONLY, + rmhid) + + CLUSTER_INFO_URL); + } + + private URLConnection connect(URL url) throws Exception { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + AuthenticatedURL authUrl; + SSLFactory clientSslFactory; + URLConnection connection; + // If https is chosen, configures SSL client. + if (YarnConfiguration.useHttps(getConf())) { + clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, getConf()); + clientSslFactory.init(); + SSLSocketFactory sslSocktFact = clientSslFactory.createSSLSocketFactory(); + + authUrl = + new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory); + connection = authUrl.openConnection(url, token); + HttpsURLConnection httpsConn = (HttpsURLConnection) connection; + httpsConn.setSSLSocketFactory(sslSocktFact); + } else { + authUrl = new AuthenticatedURL(new KerberosAuthenticator()); + connection = authUrl.openConnection(url, token); + } + connection.connect(); + return connection; + } + String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) { StringBuilder ret = new StringBuilder(); String queue = "root"; @@ -768,7 +853,10 @@ String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) { queue = StringUtils.join(queues, ","); } long now = Time.now(); - long uptime = now - rmStartTime; + long uptime = 0L; + if (rmStartTime != -1) { + uptime = now - rmStartTime; + } long days = TimeUnit.MILLISECONDS.toDays(uptime); long hours = TimeUnit.MILLISECONDS.toHours(uptime) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java new file mode 100644 index 0000000000..706400f80d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.cli; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test class for TopCli. + * + */ +public class TestTopCLI { + + private static final String RM1_NODE_ID = "rm1"; + private static final String RM2_NODE_ID = "rm2"; + + private static List dummyHostNames = + Arrays.asList("host1", "host2", "host3"); + + private static Map savedStaticResolution = new HashMap<>(); + + @BeforeClass + public static void initializeDummyHostnameResolution() throws Exception { + String previousIpAddress; + for (String hostName : dummyHostNames) { + previousIpAddress = NetUtils.getStaticResolution(hostName); + if (null != previousIpAddress) { + savedStaticResolution.put(hostName, previousIpAddress); + } + NetUtils.addStaticResolution(hostName, "10.20.30.40"); + } + } + + @AfterClass + public static void restoreDummyHostnameResolution() throws Exception { + for (Map.Entry hostnameToIpEntry : savedStaticResolution + .entrySet()) { + NetUtils.addStaticResolution(hostnameToIpEntry.getKey(), + hostnameToIpEntry.getValue()); + } + } + + @Test + public void testHAClusterInfoURL() throws IOException, InterruptedException { + TopCLI topcli = new TopCLI(); + // http + String rm1Address = "host2:8088"; + String rm2Address = "host3:8088"; + Configuration conf = topcli.getConf(); + conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, + rm1Address); + topcli.getConf().set( + YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address); + topcli.getConf().setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + topcli.getConf().set(YarnConfiguration.RM_HA_IDS, + RM1_NODE_ID + "," + RM2_NODE_ID); + URL clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID); + Assert.assertEquals("http", clusterUrl.getProtocol()); + Assert.assertEquals(rm1Address, clusterUrl.getAuthority()); + clusterUrl = topcli.getHAClusterUrl(conf, RM2_NODE_ID); + Assert.assertEquals("http", clusterUrl.getProtocol()); + Assert.assertEquals(rm2Address, clusterUrl.getAuthority()); + // https + rm1Address = "host2:9088"; + rm2Address = "host3:9088"; + conf = topcli.getConf(); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM1_NODE_ID, + rm1Address); + conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM2_NODE_ID, + rm2Address); + conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); + conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY"); + clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID); + Assert.assertEquals("https", clusterUrl.getProtocol()); + Assert.assertEquals(rm1Address, clusterUrl.getAuthority()); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java index 89b58614fb..df542ed24e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java @@ -142,7 +142,7 @@ public void setDrainEventsOnStop() { protected void serviceStop() throws Exception { if (drainEventsOnStop) { blockNewEvents = true; - LOG.info("AsyncDispatcher is draining to stop, igonring any new events."); + LOG.info("AsyncDispatcher is draining to stop, ignoring any new events."); long endTime = System.currentTimeMillis() + getConfig() .getLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java index dd81ebd917..4b43ea1860 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java @@ -54,7 +54,7 @@ public static YarnAuthorizationProvider getInstance(Configuration conf) { (YarnAuthorizationProvider) ReflectionUtils.newInstance( authorizerClass, conf); authorizer.init(conf); - LOG.info(authorizerClass.getName() + " is instiantiated."); + LOG.info(authorizerClass.getName() + " is instantiated."); } } return authorizer; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java index 0139500b97..89f0551239 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RMHAUtils; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -183,32 +182,32 @@ public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf, public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf, Policy httpPolicy) { - InetSocketAddress address = null; String rmId = null; if (HAUtil.isHAEnabled(conf)) { // If HA enabled, pick one of the RM-IDs and rely on redirect to go to // the Active RM rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0]; } + return getResolvedRemoteRMWebAppURLWithoutScheme(conf, httpPolicy, rmId); + } + + public static String getResolvedRemoteRMWebAppURLWithoutScheme( + Configuration conf, Policy httpPolicy, String rmId) { + InetSocketAddress address = null; if (httpPolicy == Policy.HTTPS_ONLY) { - address = - conf.getSocketAddr( - rmId == null - ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS - : HAUtil.addSuffix( - YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId), - YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); + address = conf.getSocketAddr( + rmId == null ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, + rmId), + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); } else { - address = - conf.getSocketAddr( - rmId == null - ? YarnConfiguration.RM_WEBAPP_ADDRESS - : HAUtil.addSuffix( - YarnConfiguration.RM_WEBAPP_ADDRESS, rmId), - YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, - YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); + address = conf.getSocketAddr( + rmId == null ? YarnConfiguration.RM_WEBAPP_ADDRESS + : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, rmId), + YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); } return getResolvedAddress(address); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java index 295b8ab635..bb52b5566e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java @@ -405,7 +405,7 @@ public Map getContainers( } } } - LOG.info("Completed reading history information of all conatiners" + LOG.info("Completed reading history information of all containers" + " of application attempt " + appAttemptId); } catch (IOException e) { LOG.info("Error when reading history information of some containers" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index fb8cfc4054..c456bde361 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -216,7 +216,7 @@ private boolean isEnabled() { return false; } if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) { - LOG.info("Neither virutal-memory nor physical-memory monitoring is " + + LOG.info("Neither virtual-memory nor physical-memory monitoring is " + "needed. Not running the monitor-thread"); return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java index e2ec693913..3bb8008f06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -386,8 +387,8 @@ public Object answer(InvocationOnMock invocationOnMock) } } return null; - } - }).when(mockUtil).copy(any(Path.class), any(Path.class)); + }}).when(mockUtil).copy(any(Path.class), any(Path.class), + anyBoolean(), anyBoolean()); doAnswer(new Answer() { @Override @@ -478,7 +479,8 @@ public ContainerLocalizer createContainerLocalizer(String user, } // Verify that the calls happen the expected number of times - verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class)); + verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class), + anyBoolean(), anyBoolean()); verify(mockLfs, times(2)).getFsStatus(any(Path.class)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 601a2a76bf..8769ca139c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -415,7 +415,7 @@ protected Plan initializePlan(String planQueueName) throws YarnException { minAllocation, maxAllocation, planQueueName, getReplanner(planQueuePath), getReservationSchedulerConfiguration() .getMoveOnExpiry(planQueuePath), rmContext); - LOG.info("Intialized plan {} based on reservable queue {}", + LOG.info("Initialized plan {} based on reservable queue {}", plan.toString(), planQueueName); return plan; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1ca69be737..922d7115d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -230,7 +230,7 @@ protected synchronized void setupQueueConfigs(Resource clusterResource) LOG.info("Initializing " + queueName + "\n" + "capacity = " + queueCapacities.getCapacity() + " [= (float) configuredCapacity / 100 ]" + "\n" + - "asboluteCapacity = " + queueCapacities.getAbsoluteCapacity() + + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() + " [= parentAbsoluteCapacity * capacity ]" + "\n" + "maxCapacity = " + queueCapacities.getMaximumCapacity() + " [= configuredMaxCapacity ]" + "\n" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index a245e3b9d0..3e9785fc92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -125,9 +125,9 @@ synchronized void setupQueueConfigs(Resource clusterResource) LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity() + - ", asboluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() + - ", asboluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() + + ", absoluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() + ", state=" + state + ", acls=" + aclsString + ", labels=" + labelStrBuilder.toString() + "\n" + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 15e2318705..310f2f92e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -944,7 +944,7 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, FSAppAttempt application = getSchedulerApp(appAttemptId); if (application == null) { LOG.info("Calling allocate on removed " + - "or non existant application " + appAttemptId); + "or non existent application " + appAttemptId); return EMPTY_ALLOCATION; }