From 959d1d01817e4d710e32120c70930f95494c0a3e Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 20 Jan 2012 19:49:05 +0000 Subject: [PATCH 1/8] HADOOP-7986. Adding config for MapReduce History Server protocol in hadoop-policy.xml for service level authorization. Contributed by Mahadev Konar. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234097 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/packages/templates/conf/hadoop-policy.xml | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c295c8855c..481cb6df93 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -282,6 +282,9 @@ Release 0.23.1 - Unreleased HADOOP-7982. UserGroupInformation fails to login if thread's context classloader can't load HadoopLoginModule. (todd) + HADOOP-7986. Adding config for MapReduce History Server protocol in + hadoop-policy.xml for service level authorization. (Mahadev Konar via vinodkv) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml index 8ac761e58e..b3e12d14e2 100644 --- a/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml +++ b/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/hadoop-policy.xml @@ -217,4 +217,14 @@ A special value of "*" means all users are allowed. + + security.mrhs.client.protocol.acl + * + ACL for HSClientProtocol, used by job clients to + communciate with the MR History Server job status etc. + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + From c6923061d0384cc22d459ee570f3626d98b3ef69 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Fri, 20 Jan 2012 20:44:17 +0000 Subject: [PATCH 2/8] MAPREDUCE-3698. Client cannot talk to the history server in secure mode. (mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234120 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../authorize/ClientHSPolicyProvider.java | 45 +++++++++++++++++++ .../client/HSClientProtocolPBClientImpl.java | 9 +++- .../client/MRClientProtocolPBClientImpl.java | 4 +- .../v2/jobhistory/JHAdminConfig.java | 5 +++ .../security/client/ClientHSSecurityInfo.java | 4 +- .../mapreduce/v2/hs/HistoryClientService.java | 6 +-- .../yarn/ipc/ProtoOverHadoopRpcEngine.java | 1 - 8 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1290899182..b4daae7ab9 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -517,6 +517,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3582. Move successfully passing MR1 tests to MR2 maven tree. (ahmed via tucu) + MAPREDUCE-3698. Client cannot talk to the history server in secure mode. + (mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java new file mode 100644 index 0000000000..968d0423a7 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapreduce.v2.app.security.authorize; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.Service; +import org.apache.hadoop.yarn.proto.HSClientProtocol; + +/** + * {@link PolicyProvider} for YARN MapReduce protocols. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class ClientHSPolicyProvider extends PolicyProvider { + + private static final Service[] mrHSServices = + new Service[] { + new Service( + JHAdminConfig.MR_HS_SECURITY_SERVICE_AUTHORIZATION, + HSClientProtocol.HSClientProtocolService.BlockingInterface.class) + }; + + @Override + public Service[] getServices() { + return mrHSServices; + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/HSClientProtocolPBClientImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/HSClientProtocolPBClientImpl.java index c9b745002c..aa5d40e8e7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/HSClientProtocolPBClientImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/HSClientProtocolPBClientImpl.java @@ -22,13 +22,20 @@ import java.io.IOException; import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; +import org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine; +import org.apache.hadoop.yarn.proto.HSClientProtocol.HSClientProtocolService; public class HSClientProtocolPBClientImpl extends MRClientProtocolPBClientImpl implements HSClientProtocol { public HSClientProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { - super(clientVersion, addr, conf); + super(); + RPC.setProtocolEngine(conf, HSClientProtocolService.BlockingInterface.class, + ProtoOverHadoopRpcEngine.class); + proxy = (HSClientProtocolService.BlockingInterface)RPC.getProxy( + HSClientProtocolService.BlockingInterface.class, clientVersion, addr, conf); } } \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java index 4a37c46630..1fb57f972c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java @@ -93,7 +93,9 @@ import com.google.protobuf.ServiceException; public class MRClientProtocolPBClientImpl implements MRClientProtocol { - private MRClientProtocolService.BlockingInterface proxy; + protected MRClientProtocolService.BlockingInterface proxy; + + public MRClientProtocolPBClientImpl() {}; public MRClientProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException { RPC.setProtocolEngine(conf, MRClientProtocolService.BlockingInterface.class, ProtoOverHadoopRpcEngine.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index cb529243d1..a89f70c901 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -111,4 +111,9 @@ public class JHAdminConfig { public static final int DEFAULT_MR_HISTORY_WEBAPP_PORT = 19888; public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT; + /* + * HS Service Authorization + */ + public static final String MR_HS_SECURITY_SERVICE_AUTHORIZATION = + "security.mrhs.client.protocol.acl"; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/client/ClientHSSecurityInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/client/ClientHSSecurityInfo.java index f3893a99a1..4eb5e9fee9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/client/ClientHSSecurityInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/security/client/ClientHSSecurityInfo.java @@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.v2.security.client; import java.lang.annotation.Annotation; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.security.KerberosInfo; @@ -30,7 +32,7 @@ import org.apache.hadoop.security.token.TokenSelector; import org.apache.hadoop.yarn.proto.HSClientProtocol; public class ClientHSSecurityInfo extends SecurityInfo { - + @Override public KerberosInfo getKerberosInfo(Class protocol, Configuration conf) { if (!protocol diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java index ab3eb5c150..ca933b4e10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java @@ -66,7 +66,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; -import org.apache.hadoop.mapreduce.v2.app.security.authorize.MRAMPolicyProvider; +import org.apache.hadoop.mapreduce.v2.app.security.authorize.ClientHSPolicyProvider; import org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebApp; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.net.NetUtils; @@ -136,9 +136,9 @@ public class HistoryClientService extends AbstractService { if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { - server.refreshServiceAcl(conf, new MRAMPolicyProvider()); + server.refreshServiceAcl(conf, new ClientHSPolicyProvider()); } - + server.start(); this.bindAddress = NetUtils.createSocketAddr(hostNameResolved.getHostAddress() diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java index 5f973a2578..2c56d318af 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java @@ -68,7 +68,6 @@ public class ProtoOverHadoopRpcEngine implements RpcEngine { public ProtocolProxy getProxy(Class protocol, long clientVersion, InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException { - return new ProtocolProxy(protocol, (T) Proxy.newProxyInstance(protocol .getClassLoader(), new Class[] { protocol }, new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout)), false); From 654b90fa239c0471f8a50d342284aff1a58fc105 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 20 Jan 2012 21:04:24 +0000 Subject: [PATCH 3/8] HDFS-2816. Fix missing license header in httpfs findbugsExcludeFile.xml. (hitesh via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234135 13f79535-47bb-0310-9956-ffa450edef68 --- .../dev-support/findbugsExcludeFile.xml | 16 ++++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 2 files changed, 19 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml index 1f5a4f5bc4..94c1d76bf3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/dev-support/findbugsExcludeFile.xml @@ -1,3 +1,19 @@ + diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f7bc7e10bb..73e9251adc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -343,6 +343,9 @@ Release 0.23.1 - UNRELEASED HDFS-2751. Datanode may incorrectly drop OS cache behind reads even for short reads. (todd) + HDFS-2816. Fix missing license header in httpfs findbugsExcludeFile.xml. + (hitesh via tucu) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES From b49aae0635869dd0deace166e430267ee6b08de1 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Fri, 20 Jan 2012 21:27:39 +0000 Subject: [PATCH 4/8] MAPREDUCE-3689. RM web UI doesn't handle newline in job name. (Thomas Graves via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234148 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/yarn/webapp/view/Jsons.java | 2 +- .../hadoop/yarn/server/resourcemanager/webapp/AppsList.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b4daae7ab9..70b75f092c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -520,6 +520,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3698. Client cannot talk to the history server in secure mode. (mahadev) + MAPREDUCE-3689. RM web UI doesn't handle newline in job name. + (Thomas Graves via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/Jsons.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/Jsons.java index 06e5d062c7..8e1794062b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/Jsons.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/Jsons.java @@ -40,7 +40,7 @@ public class Jsons { public static PrintWriter appendProgressBar(PrintWriter out, float progress) { - return appendProgressBar(out, String.format("%.1f", progress * 100)); + return appendProgressBar(out, String.format("%.1f", progress)); } public static PrintWriter appendSortable(PrintWriter out, Object value) { diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java index f3378d2747..a7b35abaaa 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java @@ -66,7 +66,7 @@ class AppsList implements ToJSON { appendLink(out, appInfo.getAppId(), rc.prefix(), "app", appInfo.getAppId()).append(_SEP). append(escapeHtml(appInfo.getUser())).append(_SEP). - append(escapeHtml(appInfo.getName())).append(_SEP). + append(escapeJavaScript(escapeHtml(appInfo.getName()))).append(_SEP). append(escapeHtml(appInfo.getQueue())).append(_SEP). append(appInfo.getState()).append(_SEP). append(appInfo.getFinalStatus()).append(_SEP); From c840548d6be09c5048e34b6222e800b0df0aa1dd Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Sat, 21 Jan 2012 00:42:35 +0000 Subject: [PATCH 5/8] HDFS-2817. Combine the two TestSafeMode test suites. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234221 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/TestSafeMode.java | 49 ++++++++++- .../hdfs/server/namenode/TestSafeMode.java | 82 ------------------- 3 files changed, 50 insertions(+), 83 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 73e9251adc..50b40c03dc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -272,6 +272,8 @@ Release 0.23.1 - UNRELEASED HDFS-2803. Add logging to LeaseRenewer for better lease expiration debugging. (Jimmy Xiang via todd) + HDFS-2817. Combine the two TestSafeMode test suites. (todd) + OPTIMIZATIONS HDFS-2130. Switch default checksum to CRC32C. (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java index 73adf8efcf..6ec5f8bf52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java @@ -113,6 +113,21 @@ public class TestSafeMode { dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE)); } + /** + * Test that, if there are no blocks in the filesystem, + * the NameNode doesn't enter the "safemode extension" period. + */ + @Test(timeout=45000) + public void testNoExtensionIfNoBlocks() throws IOException { + cluster.getConfiguration(0).setInt( + DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 60000); + cluster.restartNameNode(); + // Even though we have safemode extension set high, we should immediately + // exit safemode on startup because there are no blocks in the namespace. + String status = cluster.getNameNode().getNamesystem().getSafemode(); + assertEquals("", status); + } + public interface FSRun { public abstract void run(FileSystem fs) throws IOException; } @@ -193,5 +208,37 @@ public class TestSafeMode { assertFalse("Could not leave SM", dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE)); } - + + /** + * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min + * is set to a number greater than the number of live datanodes. + */ + @Test + public void testDatanodeThreshold() throws IOException { + cluster.shutdown(); + Configuration conf = cluster.getConfiguration(0); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1); + + cluster.restartNameNode(); + fs = (DistributedFileSystem)cluster.getFileSystem(); + + String tipMsg = cluster.getNamesystem().getSafemode(); + assertTrue("Safemode tip message looks right: " + tipMsg, + tipMsg.contains("The number of live datanodes 0 needs an additional " + + "2 live datanodes to reach the minimum number 1. " + + "Safe mode will be turned off automatically.")); + + // Start a datanode + cluster.startDataNodes(conf, 1, true, null, null); + + // Wait long enough for safemode check to refire + try { + Thread.sleep(1000); + } catch (InterruptedException ignored) {} + + // We now should be out of safe mode. + assertEquals("", cluster.getNamesystem().getSafemode()); + } + } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java deleted file mode 100644 index 88a1d0d955..0000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSafeMode.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hdfs.server.namenode; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; - -import org.junit.Test; -import static org.junit.Assert.*; - -/** - * Tests to verify safe mode correctness. - */ -public class TestSafeMode { - - /** - * Verify that the NameNode stays in safemode when dfs.safemode.datanode.min - * is set to a number greater than the number of live datanodes. - */ - @Test - public void testDatanodeThreshold() throws IOException { - MiniDFSCluster cluster = null; - DistributedFileSystem fs = null; - try { - Configuration conf = new HdfsConfiguration(); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_EXTENSION_KEY, 0); - conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1); - - // bring up a cluster with no datanodes - cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build(); - cluster.waitActive(); - fs = (DistributedFileSystem)cluster.getFileSystem(); - - assertTrue("No datanode started, but we require one - safemode expected", - fs.setSafeMode(SafeModeAction.SAFEMODE_GET)); - - String tipMsg = cluster.getNamesystem().getSafeModeTip(); - assertTrue("Safemode tip message looks right", - tipMsg.contains("The number of live datanodes 0 needs an additional " + - "2 live datanodes to reach the minimum number 1. " + - "Safe mode will be turned off automatically.")); - - // Start a datanode - cluster.startDataNodes(conf, 1, true, null, null); - - // Wait long enough for safemode check to refire - try { - Thread.sleep(1000); - } catch (InterruptedException ignored) {} - - // We now should be out of safe mode. - assertFalse( - "Out of safe mode after starting datanode.", - fs.setSafeMode(SafeModeAction.SAFEMODE_GET)); - } finally { - if (fs != null) fs.close(); - if (cluster != null) cluster.shutdown(); - } - } -} From 8a1719e99979d77278d0580ae5365b1b4d12635c Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Sat, 21 Jan 2012 00:53:02 +0000 Subject: [PATCH 6/8] MAPREDUCE-3549. write api documentation for web service apis for RM, NM, mapreduce app master, and job history server (Thomas Graves via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234222 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../webapp/dao/JobTaskAttemptCounterInfo.java | 2 +- .../app/webapp/TestAMWebServicesAttempts.java | 4 +- .../hs/webapp/TestHsWebServicesAttempts.java | 4 +- .../src/site/apt/HistoryServerRest.apt.vm | 2733 +++++++++++++++++ .../src/site/apt/MapredAppMasterRest.apt.vm | 2701 ++++++++++++++++ .../src/site/apt/NodeManagerRest.apt.vm | 635 ++++ .../src/site/apt/ResourceManagerRest.apt.vm | 1469 +++++++++ .../src/site/apt/WebServicesIntro.apt.vm | 595 ++++ hadoop-project/src/site/site.xml | 8 + 10 files changed, 8149 insertions(+), 5 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 70b75f092c..dc6220e80a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -523,6 +523,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3689. RM web UI doesn't handle newline in job name. (Thomas Graves via mahadev) + MAPREDUCE-3549. write api documentation for web service apis for RM, NM, + mapreduce app master, and job history server (Thomas Graves via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java index 2026c76ddb..f61b930430 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java @@ -30,7 +30,7 @@ import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.util.MRApps; -@XmlRootElement(name = "JobTaskAttemptCounters") +@XmlRootElement(name = "jobTaskAttemptCounters") @XmlAccessorType(XmlAccessType.FIELD) public class JobTaskAttemptCounterInfo { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java index ee824ee10a..e33a50671c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java @@ -629,7 +629,7 @@ public class TestAMWebServicesAttempts extends JerseyTest { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("JobTaskAttemptCounters"); + JSONObject info = json.getJSONObject("jobTaskAttemptCounters"); verifyAMJobTaskAttemptCounters(info, att); } } @@ -661,7 +661,7 @@ public class TestAMWebServicesAttempts extends JerseyTest { InputSource is = new InputSource(); is.setCharacterStream(new StringReader(xml)); Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("JobTaskAttemptCounters"); + NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters"); verifyAMTaskCountersXML(nodes, att); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java index 6fdb94d902..7ba200fcc5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java @@ -642,7 +642,7 @@ public class TestHsWebServicesAttempts extends JerseyTest { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("JobTaskAttemptCounters"); + JSONObject info = json.getJSONObject("jobTaskAttemptCounters"); verifyHsJobTaskAttemptCounters(info, att); } } @@ -674,7 +674,7 @@ public class TestHsWebServicesAttempts extends JerseyTest { InputSource is = new InputSource(); is.setCharacterStream(new StringReader(xml)); Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("JobTaskAttemptCounters"); + NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters"); verifyHsTaskCountersXML(nodes, att); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm new file mode 100644 index 0000000000..a7dda193df --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -0,0 +1,2733 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + History Server REST API's. + --- + --- + ${maven.build.timestamp} + +History Server REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=3} + +* Overview + + The history server REST API's allow the user to get status on finished applications. Currently it only supports MapReduce and provides information on finished jobs. + +* History Server Information API + + The history server information resource provides overall information about the history server. + +** URI + + Both of the following URI's give you the history server information, from an application id identified by the appid value. + +------ + * http:///ws/v1/history + * http:///ws/v1/history/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "historyInfo" : { + "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012", + "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7", + "hadoopVersion" : "0.23.1-SNAPSHOT" + } +} ++---+ + + <> + + HTTP Request: + +----- + GET http:///ws/v1/history/info + Accept: application/xml +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 330 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7 + Wed Jan 11 21:18:36 UTC 2012 + ++---+ + +* MapReduce API's + + The following list of resources apply to MapReduce. + +** Jobs API + + The jobs resource provides a list of the MapReduce jobs that have finished. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + + Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all jobs that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + +------ + * user - user name + * queue - queue name + * limit - total number of app objects to be returned + * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch + * startedTimeEnd - jobs with start time ending with this time, specified in ms since epoch + * finishedTimeBegin - jobs with finish time beginning with this time, specified in ms since epoch + * finishedTimeEnd - jobs with finish time ending with this time, specified in ms since epoch +------ + +*** Elements of the object + + When you make a request for the list of jobs, the information will be returned as an array of job objects. + See also {{Job API}} for syntax of the job object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| job | array of job objects(json)/zero or more job objects(XML) | The collection of job objects | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobs" : { + "job" : [ + { + "avgReduceTime" : 833, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381344489, + "id" : "job_1326381300833_1_1", + "avgMapTime" : 2671, + "successfulMapAttempts" : 1, + "name" : "word count", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2570, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381356010 + }, + { + "avgReduceTime" : 124961, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381446529, + "id" : "job_1326381300833_2_2", + "avgMapTime" : 2638, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2589, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381582106 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1922 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381344489 + 1326381356010 + job_1326381300833_1_1 + word count + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2671 + 833 + 2540 + 2570 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + + 1326381446529 + 1326381582106 + job_1326381300833_2_2 + Sleep job + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2638 + 124961 + 2540 + 2589 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + ++---+ + +** {Job API} + + A Job resource contains information about a particular job identified by {jobid}. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id| +*---------------+--------------+-------------------------------+ +| name | string | The job name | +*---------------+--------------+-------------------------------+ +| queue | string | The queue the job was submitted to| +*---------------+--------------+-------------------------------+ +| user | string | The user name | +*---------------+--------------+-------------------------------+ +| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR| +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message | +*---------------+--------------+-------------------------------+ +| startTime | long | The time the job started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| finishTime | long | The time the job finished (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| mapsTotal | int | The total number of maps | +*---------------+--------------+-------------------------------+ +| mapsCompleted | int | The number of completed maps | +*---------------+--------------+-------------------------------+ +| reducesTotal | int | The total number of reduces | +*---------------+--------------+-------------------------------+ +| reducesCompleted | int | The number of completed reduces| +*---------------+--------------+-------------------------------+ +| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master| +*---------------+--------------+-------------------------------+ +| avgMapTime | long | The average time of a map task (in ms)| +*---------------+--------------+-------------------------------+ +| avgReduceTime | long | The average time of the reduce (in ms)| +*---------------+--------------+-------------------------------+ +| avgShuffleTime | long | The average time of the shuffle (in ms)| +*---------------+--------------+-------------------------------+ +| avgMergeTime | long | The average time of the merge (in ms)| +*---------------+--------------+-------------------------------+ +| failedReduceAttempts | int | The number of failed reduce attempts | +*---------------+--------------+-------------------------------+ +| killedReduceAttempts | int | The number of killed reduce attempts | +*---------------+--------------+-------------------------------+ +| successfulReduceAttempts | int | The number of successful reduce attempts | +*---------------+--------------+-------------------------------+ +| failedMapAttempts | int | The number of failed map attempts | +*---------------+--------------+-------------------------------+ +| killedMapAttempts | int | The number of killed map attempts | +*---------------+--------------+-------------------------------+ +| successfulMapAttempts | int | The number of successful map attempts | +*---------------+--------------+-------------------------------+ +| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| value | string | The acl value| +*---------------+--------------+-------------------------------+ +| name | string | The acl name | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Server: Jetty(6.1.26) + Content-Length: 720 ++---+ + + Response Body: + ++---+ +{ + "job" : { + "avgReduceTime" : 124961, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381446529, + "id" : "job_1326381300833_2_2", + "avgMapTime" : 2638, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2589, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381582106 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 983 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381446529 + 1326381582106 + job_1326381300833_2_2 + Sleep job + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2638 + 124961 + 2540 + 2589 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + ++---+ + +** Job Attempts API + + With the job attempts API, you can obtain a collection of resources that represent a job attempt. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/jobattempts +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + + When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects. + + jobAttempts: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects | +*---------------+--------------+--------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job attempt id | +*---------------+--------------+--------------------------------+ +| nodeId | string | The node id of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The node http address of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| logsLink | string | The http link to the job attempt logs | +*---------------+--------------+--------------------------------+ +| containerId | string | The id of the container for the job attempt | +*---------------+--------------+--------------------------------+ +| startTime | long | The start time of the attempt (in ms since epoch)| +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobAttempts" : { + "jobAttempt" : [ + { + "nodeId" : "host.domain.com:45454", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381444693, + "id" : 1, + "logsLink" : "http://host.domain.com:19888/jobhistory/logs/host.domain.com:45454/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1", + "containerId" : "container_1326381300833_0002_01_000001" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattmpts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 575 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + host.domain.com:9999 + host.domain.com:45454 + 1 + 1326381444693 + container_1326381300833_0002_01_000001 + http://host.domain.com:19888/jobhistory/logs/host.domain.com:45454/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1 + + ++---+ + +** Job Counters API + + With the job counters API, you can object a collection of resources that represent al the counters for that job. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id | +*---------------+--------------+-------------------------------+ +| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the objecs + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| reduceCounterValue | long | The counter value of reduce tasks | +*---------------+--------------+-------------------------------+ +| mapCounterValue | long | The counter value of map tasks | +*---------------+--------------+-------------------------------+ +| totalCounterValue | long | The counter value of all tasks | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobCounters" : { + "id" : "job_1326381300833_2_2", + "counterGroup" : [ + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BAD_ID" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "CONNECTION" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "IO_ERROR" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_LENGTH" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_MAP" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2483, + "name" : "FILE_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 108525, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "HDFS_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "HDFS_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MAP_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "MAP_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 4800, + "name" : "MAP_OUTPUT_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "MAP_OUTPUT_MATERIALIZED_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "SPLIT_RAW_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2400, + "name" : "SPILLED_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 113, + "name" : "GC_TIME_MILLIS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1830, + "name" : "CPU_MILLISECONDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 478068736, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2159284224, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 378863616, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_READ" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 7030 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + job_1326381300833_2_2 + + Shuffle Errors + + BAD_ID + 0 + 0 + 0 + + + CONNECTION + 0 + 0 + 0 + + + IO_ERROR + 0 + 0 + 0 + + + WRONG_LENGTH + 0 + 0 + 0 + + + WRONG_MAP + 0 + 0 + 0 + + + WRONG_REDUCE + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2483 + 0 + 0 + + + FILE_BYTES_WRITTEN + 108525 + 0 + 0 + + + FILE_READ_OPS + 0 + 0 + 0 + + + FILE_LARGE_READ_OPS + 0 + 0 + 0 + + + FILE_WRITE_OPS + 0 + 0 + 0 + + + HDFS_BYTES_READ + 48 + 0 + 0 + + + HDFS_BYTES_WRITTEN + 0 + 0 + 0 + + + HDFS_READ_OPS + 1 + 0 + 0 + + + HDFS_LARGE_READ_OPS + 0 + 0 + 0 + + + HDFS_WRITE_OPS + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + MAP_INPUT_RECORDS + 1 + 0 + 0 + + + MAP_OUTPUT_RECORDS + 1200 + 0 + 0 + + + MAP_OUTPUT_BYTES + 4800 + 0 + 0 + + + MAP_OUTPUT_MATERIALIZED_BYTES + 2235 + 0 + 0 + + + SPLIT_RAW_BYTES + 48 + 0 + 0 + + + COMBINE_INPUT_RECORDS + 0 + 0 + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + 0 + 0 + + + REDUCE_INPUT_GROUPS + 1200 + 0 + 0 + + + REDUCE_SHUFFLE_BYTES + 2235 + 0 + 0 + + + REDUCE_INPUT_RECORDS + 1200 + 0 + 0 + + + REDUCE_OUTPUT_RECORDS + 0 + 0 + 0 + + + SPILLED_RECORDS + 2400 + 0 + 0 + + + SHUFFLED_MAPS + 1 + 0 + 0 + + + FAILED_SHUFFLE + 0 + 0 + 0 + + + MERGED_MAP_OUTPUTS + 1 + 0 + 0 + + + GC_TIME_MILLIS + 113 + 0 + 0 + + + CPU_MILLISECONDS + 1830 + 0 + 0 + + + PHYSICAL_MEMORY_BYTES + 478068736 + 0 + 0 + + + VIRTUAL_MEMORY_BYTES + 2159284224 + 0 + 0 + + + COMMITTED_HEAP_BYTES + 378863616 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter + + BYTES_READ + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + 0 + 0 + + + ++---+ + + +** Job Conf API + + A job configuration resource contains information about the job configuration for this job. + +*** URI + + Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/conf +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| path | string | The path to the job configuration file| +*---------------+--------------+-------------------------------+ +| property | array of the configuration properties(JSON)/zero or more configuration properties(XML) | Collection of configuration property objects| +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the configuration property | +*---------------+--------------+-------------------------------+ +| value | string | The value of the configuration property | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + + This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file. + ++---+ +{ + "conf" : { + "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml", + "property" : [ + { + "value" : "/home/hadoop/hdfs/data", + "name" : "dfs.datanode.data.dir" + }, + { + "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", + "name" : "hadoop.http.filter.initializers" + }, + { + "value" : "/home/hadoop/tmp", + "name" : "mapreduce.cluster.temp.dir" + }, + ... + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml + + dfs.datanode.data.dir + /home/hadoop/hdfs/data + + + hadoop.http.filter.initializers + org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + + + mapreduce.cluster.temp.dir + /home/hadoop/tmp + + ... + ++---+ + +** Tasks API + + With the tasks API, you can obtain a collection of resources that represent a task within a job. When you run a GET operation on this resource, you obtain a collection of Task Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + * type - type of task, valid values are m or r. m for map task or r for reduce task. +------ + +*** Elements of the object + + When you make a request for the list of tasks , the information will be returned as an array of task objects. + See also {{Task API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects. | +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 6777, + "state" : "SUCCEEDED", + "startTime" : 1326381446541, + "id" : "task_1326381300833_2_2_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0", + "finishTime" : 1326381453318 + }, + { + "progress" : 100, + "elapsedTime" : 135559, + "state" : "SUCCEEDED", + "startTime" : 1326381446544, + "id" : "task_1326381300833_2_2_r_0", + "type" : "REDUCE", + "successfulAttempt" : "attempt_1326381300833_2_2_r_0_0", + "finishTime" : 1326381582103 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 653 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381446541 + 1326381453318 + 6777 + 100.0 + task_1326381300833_2_2_m_0 + SUCCEEDED + MAP + attempt_1326381300833_2_2_m_0_0 + + + 1326381446544 + 1326381582103 + 135559 + 100.0 + task_1326381300833_2_2_r_0 + SUCCEEDED + REDUCE + attempt_1326381300833_2_2_r_0_0 + + ++---+ + +** {Task API} + + A Task resource contains information about a particular task within a job. + +*** URI + + Use the following URI to obtain an Task Object, from a task identified by the {taskid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED +*---------------+--------------+--------------------------------+ +| type | string | The task type - MAP or REDUCE| +*---------------+--------------+--------------------------------+ +| successfulAttempt | string | The id of the last successful attempt | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ + + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "task" : { + "progress" : 100, + "elapsedTime" : 6777, + "state" : "SUCCEEDED", + "startTime" : 1326381446541, + "id" : "task_1326381300833_2_2_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0", + "finishTime" : 1326381453318 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 299 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381446541 + 1326381453318 + 6777 + 100.0 + task_1326381300833_2_2_m_0 + SUCCEEDED + MAP + attempt_1326381300833_2_2_m_0_0 + ++---+ + +** Task Counters API + + With the task counters API, you can object a collection of resources that represent al the counters for that task. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+-------------------------------+ +| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskCounters" : { + "id" : "task_1326381300833_2_2_m_0", + "taskCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2660 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + task_1326381300833_2_2_m_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + +** Task Attempts API + + With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + + When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. + See also {{Task Attempt API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects | +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "assignedContainerId" : "container_1326381300833_0002_01_000002", + "progress" : 100, + "elapsedTime" : 2638, + "state" : "SUCCEEDED", + "diagnostics" : "", + "rack" : "/98.139.92.0", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381450680, + "id" : "attempt_1326381300833_2_2_m_0_0", + "type" : "MAP", + "finishTime" : 1326381453318 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 537 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381450680 + 1326381453318 + 2638 + 100.0 + attempt_1326381300833_2_2_m_0_0 + /98.139.92.0 + SUCCEEDED + host.domain.com:9999 + + MAP + container_1326381300833_0002_01_000002 + + ++---+ + +** {Task Attempt API} + + A Task Attempt resource contains information about a particular task attempt within a job. + +*** URI + + Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| rack | string | The rack | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED | +*---------------+--------------+--------------------------------+ +| type | string | The type of task | +*---------------+--------------+--------------------------------+ +| assignedContainerId | string | The container id this attempt is assigned to| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The http address of the node this task attempt ran on | +*---------------+--------------+--------------------------------+ +| diagnostics| string | A diagnostics message | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task attempt as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task attempt started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task attempt finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the task attempt started (in ms)| +*---------------+--------------+--------------------------------+ + + For reduce task attempts you also have the following fields: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| mergeFinishTime | long | The time at which merge finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)| +*---------------+--------------+--------------------------------+ +| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)| +*---------------+--------------+--------------------------------+ +| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)| +*---------------+--------------+--------------------------------+ + + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempt" : { + "assignedContainerId" : "container_1326381300833_0002_01_000002", + "progress" : 100, + "elapsedTime" : 2638, + "state" : "SUCCEEDED", + "diagnostics" : "", + "rack" : "/98.139.92.0", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381450680, + "id" : "attempt_1326381300833_2_2_m_0_0", + "type" : "MAP", + "finishTime" : 1326381453318 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 691 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381450680 + 1326381453318 + 2638 + 100.0 + attempt_1326381300833_2_2_m_0_0 + /98.139.92.0 + SUCCEEDED + host.domain.com:9999 + + MAP + container_1326381300833_0002_01_000002 + ++---+ + +** Task Attempt Counters API + + With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task attempt id | +*---------------+--------------+-------------------------------+ +| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326381300833_2_2_m_0_0" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2735 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + attempt_1326381300833_2_2_m_0_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm new file mode 100644 index 0000000000..186f044c22 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm @@ -0,0 +1,2701 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + MapReduce Application Master REST API's. + --- + --- + ${maven.build.timestamp} + +MapReduce Application Master REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The MapReduce Application Master REST API's allow the user to get status on the running MapReduce application master. Currently this is the equivalent to a running MapReduce job. The information includes the jobs the app master is running and all the job particulars like tasks, counters, configuration, attempts, etc. The application master should be accessed via the proxy. This proxy is configurable to run either on the resource manager or on a separate host. The proxy URL usually looks like: http:///proxy/{appid}. + +* Mapreduce Application Master Info API + + The MapReduce application master information resource provides overall information about that mapreduce application master. This includes application id, time it was started, user, name, etc. + +** URI + + Both of the following URI's give you the MapReduce application master information, from an application id identified by the appid value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce + * http:///proxy/{appid}/ws/v1/mapreduce/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the mapreduce application master information, the information will be returned as an info object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| appId | long | The application id | +*---------------+--------------+-------------------------------+ +| startedOn | long | The time the application started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| name | string | The name of the application | +*---------------+--------------+-------------------------------+ +| user | string | The user name of the user who started the application | +*---------------+--------------+-------------------------------+ +| elapsedTime | long | The time since the application was started (in ms)| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0003/ws/v1/mapreduce/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "info" : { + "appId" : "application_1326232085508_0003", + "startedOn" : 1326238244047, + "user" : "user1", + "name" : "Sleep job", + "elapsedTime" : 32374 + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///proxy/application_1326232085508_0003/ws/v1/mapreduce/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 223 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326232085508_0003 + Sleep job + user1 + 1326238244047 + 32407 + ++---+ + +* Jobs API + + The jobs resource provides a list of the jobs running on this application master. See also {{Job API}} for syntax of the job object. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of jobs, the information will be returned as a collection of job objects. See also {{Job API}} for syntax of the job object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| job | array of job objects(JSON)/Zero or more job objects(XML) | The collection of job objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobs" : { + "job" : [ + { + "runningReduceAttempts" : 1, + "reduceProgress" : 100, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326238769379, + "id" : "job_1326232085508_4_4", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 59377, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } + ] + } + } ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1214 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238769379 + 0 + 59416 + job_1326232085508_4_4 + Sleep job + user1 + RUNNING + 1 + 1 + 1 + 0 + 100.0 + 100.0 + 0 + 0 + 0 + 1 + false + + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + ++---+ + +* {Job API} + + A job resource contains information about a particular job that was started by this application master. Certain fields are only accessible if user has permissions - depends on acl settings. + +** URI + + Use the following URI to obtain a job object, for a job identified by the jobid value. + +------ + * http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id| +*---------------+--------------+-------------------------------+ +| name | string | The job name | +*---------------+--------------+-------------------------------+ +| user | string | The user name | +*---------------+--------------+-------------------------------+ +| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR| +*---------------+--------------+-------------------------------+ +| startTime | long | The time the job started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| finishTime | long | The time the job finished (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| elapsedTime | long | The elapsed time since job started (in ms)| +*---------------+--------------+-------------------------------+ +| mapsTotal | int | The total number of maps | +*---------------+--------------+-------------------------------+ +| mapsCompleted | int | The number of completed maps | +*---------------+--------------+-------------------------------+ +| reducesTotal | int | The total number of reduces | +*---------------+--------------+-------------------------------+ +| reducesCompleted | int | The number of completed reduces| +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message | +*---------------+--------------+-------------------------------+ +| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master| +*---------------+--------------+-------------------------------+ +| mapsPending | int | The number of maps still to be run| +*---------------+--------------+-------------------------------+ +| mapsRunning | int | The number of running maps | +*---------------+--------------+-------------------------------+ +| reducesPending | int | The number of reduces still to be run | +*---------------+--------------+-------------------------------+ +| reducesRunning | int | The number of running reduces| +*---------------+--------------+-------------------------------+ +| newReduceAttempts | int | The number of new reduce attempts | +*---------------+--------------+-------------------------------+ +| runningReduceAttempts | int | The number of running reduce attempts | +*---------------+--------------+-------------------------------+ +| failedReduceAttempts | int | The number of failed reduce attempts | +*---------------+--------------+-------------------------------+ +| killedReduceAttempts | int | The number of killed reduce attempts | +*---------------+--------------+-------------------------------+ +| successfulReduceAttempts | int | The number of successful reduce attempts | +*---------------+--------------+-------------------------------+ +| newMapAttempts | int | The number of new map attempts | +*---------------+--------------+-------------------------------+ +| runningMapAttempts | int | The number of running map attempts | +*---------------+--------------+-------------------------------+ +| failedMapAttempts | int | The number of failed map attempts | +*---------------+--------------+-------------------------------+ +| killedMapAttempts | int | The number of killed map attempts | +*---------------+--------------+-------------------------------+ +| successfulMapAttempts | int | The number of successful map attempts | +*---------------+--------------+-------------------------------+ +| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| value | string | The acl value| +*---------------+--------------+-------------------------------+ +| name | string | The acl name | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Server: Jetty(6.1.26) + Content-Length: 720 ++---+ + + Response Body: + ++---+ +{ + "job" : { + "runningReduceAttempts" : 1, + "reduceProgress" : 100, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326238769379, + "id" : "job_1326232085508_4_4", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 59437, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1201 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238769379 + 0 + 59474 + job_1326232085508_4_4 + Sleep job + user1 + RUNNING + 1 + 1 + 1 + 0 + 100.0 + 100.0 + 0 + 0 + 0 + 1 + false + + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + ++---+ + +* Job Attempts API + + With the job attempts API, you can obtain a collection of resources that represent the job attempts. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects. + +** URI + +------ + * http:///ws/v1/history/jobs/{jobid}/jobattempts +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects | +*---------------+--------------+--------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job attempt id | +*---------------+--------------+--------------------------------+ +| nodeId | string | The node id of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The node http address of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| logsLink | string | The http link to the job attempt logs | +*---------------+--------------+--------------------------------+ +| containerId | string | The id of the container for the job attempt | +*---------------+--------------+--------------------------------+ +| startTime | long | The start time of the attempt (in ms since epoch)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobAttempts" : { + "jobAttempt" : [ + { + "nodeId" : "host.domain.com:45454", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326238773493, + "id" : 1, + "logsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326232085508_0004_01_000001", + "containerId" : "container_1326232085508_0004_01_000001" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 498 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + host.domain.com:9999 + host.domain.com:45454 + 1 + 1326238773493 + container_1326232085508_0004_01_000001 + http://host.domain.com:9999/node/containerlogs/container_1326232085508_0004_01_000001 + + ++---+ + +* Job Counters API + + With the job counters API, you can object a collection of resources that represent all the counters for that job. + +** URI + +------ + * http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id | +*---------------+--------------+-------------------------------+ +| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| reduceCounterValue | long | The counter value of reduce tasks | +*---------------+--------------+-------------------------------+ +| mapCounterValue | long | The counter value of map tasks | +*---------------+--------------+-------------------------------+ +| totalCounterValue | long | The counter value of all tasks | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobCounters" : { + "id" : "job_1326232085508_4_4", + "counterGroup" : [ + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BAD_ID" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "CONNECTION" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "IO_ERROR" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_LENGTH" + }, { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_MAP" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2483, + "name" : "FILE_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 108763, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "HDFS_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "HDFS_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MAP_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "MAP_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 4800, + "name" : "MAP_OUTPUT_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "MAP_OUTPUT_MATERIALIZED_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "SPLIT_RAW_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "SPILLED_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 58, + "name" : "GC_TIME_MILLIS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1580, + "name" : "CPU_MILLISECONDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 462643200, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2149728256, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 357957632, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_READ" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 7027 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + job_1326232085508_4_4 + + Shuffle Errors + + BAD_ID + 0 + 0 + 0 + + + CONNECTION + 0 + 0 + 0 + + + IO_ERROR + 0 + 0 + 0 + + + WRONG_LENGTH + 0 + 0 + 0 + + + WRONG_MAP + 0 + 0 + 0 + + + WRONG_REDUCE + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2483 + 0 + 0 + + + FILE_BYTES_WRITTEN + 108763 + 0 + 0 + + + FILE_READ_OPS + 0 + 0 + 0 + + + FILE_LARGE_READ_OPS + 0 + 0 + 0 + + + FILE_WRITE_OPS + 0 + 0 + 0 + + + HDFS_BYTES_READ + 48 + 0 + 0 + + + HDFS_BYTES_WRITTEN + 0 + 0 + 0 + + + HDFS_READ_OPS + 1 + 0 + 0 + + + HDFS_LARGE_READ_OPS + 0 + 0 + 0 + + + HDFS_WRITE_OPS + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + MAP_INPUT_RECORDS + 1 + 0 + 0 + + + MAP_OUTPUT_RECORDS + 1200 + 0 + 0 + + + MAP_OUTPUT_BYTES + 4800 + 0 + 0 + + + MAP_OUTPUT_MATERIALIZED_BYTES + 2235 + 0 + 0 + + + SPLIT_RAW_BYTES + 48 + 0 + 0 + + + COMBINE_INPUT_RECORDS + 0 + 0 + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + 0 + 0 + + + REDUCE_INPUT_GROUPS + 460 + 0 + 0 + + + REDUCE_SHUFFLE_BYTES + 2235 + 0 + 0 + + + REDUCE_INPUT_RECORDS + 460 + 0 + 0 + + + REDUCE_OUTPUT_RECORDS + 0 + 0 + 0 + + + SPILLED_RECORDS + 1200 + 0 + 0 + + + SHUFFLED_MAPS + 1 + 0 + 0 + + + FAILED_SHUFFLE + 0 + 0 + 0 + + + MERGED_MAP_OUTPUTS + 1 + 0 + 0 + + + GC_TIME_MILLIS + 58 + 0 + 0 + + + CPU_MILLISECONDS + 1580 + 0 + 0 + + + PHYSICAL_MEMORY_BYTES + 462643200 + 0 + 0 + + + VIRTUAL_MEMORY_BYTES + 2149728256 + 0 + 0 + + + COMMITTED_HEAP_BYTES + 357957632 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter + + BYTES_READ + 0 + 0 + 0 + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + BYTES_WRITTEN + 0 + 0 + 0 + + + ++---+ + +* Job Conf API + + A job configuration resource contains information about the job configuration for this job. + +** URI + + Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/conf +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| path | string | The path to the job configuration file| +*---------------+--------------+-------------------------------+ +| property | array of the configuration properties(JSON)/zero or more property objects(XML) | Collection of property objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the configuration property | +*---------------+--------------+-------------------------------+ +| value | string | The value of the configuration property | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + + This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file. + ++---+ +{ + "conf" : { + "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml", + "property" : [ + { + "value" : "/home/hadoop/hdfs/data", + "name" : "dfs.datanode.data.dir" + }, + { + "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", + "name" : "hadoop.http.filter.initializers" + }, + { + "value" : "/home/hadoop/tmp", + "name" : "mapreduce.cluster.temp.dir" + }, + ... + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml + + dfs.datanode.data.dir + /home/hadoop/hdfs/data + + + hadoop.http.filter.initializers + org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + + + mapreduce.cluster.temp.dir + /home/hadoop/tmp + + ... + ++---+ + +* Tasks API + + With the tasks API, you can obtain a collection of resources that represent all the tasks for a job. When you run a GET operation on this resource, you obtain a collection of Task Objects. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + * type - type of task, valid values are m or r. m for map task or r for reduce task. +------ + +** Elements of the object + + When you make a request for the list of tasks , the information will be returned as an array of task objects. + See also {{Task API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 2768, + "state" : "SUCCEEDED", + "startTime" : 1326238773493, + "id" : "task_1326232085508_4_4_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326232085508_4_4_m_0_0", + "finishTime" : 1326238776261 + }, + { + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326238777460, + "id" : "task_1326232085508_4_4_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 603 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238773493 + 1326238776261 + 2768 + 100.0 + task_1326232085508_4_4_m_0 + SUCCEEDED + MAP + attempt_1326232085508_4_4_m_0_0 + + + 1326238777460 + 0 + 0 + 100.0 + task_1326232085508_4_4_r_0 + RUNNING + REDUCE + + + ++---+ + +* {Task API} + + A Task resource contains information about a particular task within a job. + +** URI + + Use the following URI to obtain an Task Object, from a task identified by the {taskid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED | +*---------------+--------------+--------------------------------+ +| type | string | The task type - MAP or REDUCE| +*---------------+--------------+--------------------------------+ +| successfulAttempt | string | The the id of the last successful attempt | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "task" : { + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326238777460, + "id" : "task_1326232085508_4_4_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 299 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238777460 + 0 + 0 + 100.0 + task_1326232085508_4_4_r_0 + RUNNING + REDUCE + + ++---+ + +* Task Counters API + + With the task counters API, you can object a collection of resources that represent all the counters for that task. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+-------------------------------+ +| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskCounters" : { + "id" : "task_1326232085508_4_4_r_0", + "taskCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2660 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + task_1326232085508_4_4_r_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + +* Task Attempts API + + With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. + See also {{Task Attempt API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "elapsedMergeTime" : 47, + "shuffleFinishTime" : 1326238780052, + "assignedContainerId" : "container_1326232085508_0004_01_000003", + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2592, + "mergeFinishTime" : 1326238780099, + "rack" : "/98.139.92.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "type" : "REDUCE", + "startTime" : 1326238777460, + "id" : "attempt_1326232085508_4_4_r_0_0", + "finishTime" : 0 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 807 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238777460 + 0 + 0 + 100.0 + attempt_1326232085508_4_4_r_0_0 + /98.139.92.0 + RUNNING + host.domain.com:9999 + REDUCE + container_1326232085508_0004_01_000003 + 1326238780052 + 1326238780099 + 2592 + 47 + 0 + + ++---+ + +* {Task Attempt API} + + A Task Attempt resource contains information about a particular task attempt within a job. + +** URI + + Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| rack | string | The rack | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED| +*---------------+--------------+--------------------------------+ +| type | string | The type of task | +*---------------+--------------+--------------------------------+ +| assignedContainerId | string | The container id this attempt is assigned to| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The http address of the node this task attempt ran on | +*---------------+--------------+--------------------------------+ +| diagnostics| string | The diagnostics message | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task attempt as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task attempt started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task attempt finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the task attempt started (in ms)| +*---------------+--------------+--------------------------------+ + + For reduce task attempts you also have the following fields: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| mergeFinishTime | long | The time at which merge finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)| +*---------------+--------------+--------------------------------+ +| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)| +*---------------+--------------+--------------------------------+ +| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempt" : { + "elapsedMergeTime" : 47, + "shuffleFinishTime" : 1326238780052, + "assignedContainerId" : "container_1326232085508_0004_01_000003", + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2592, + "mergeFinishTime" : 1326238780099, + "rack" : "/98.139.92.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326238777460, + "id" : "attempt_1326232085508_4_4_r_0_0", + "type" : "REDUCE", + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 691 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238777460 + 0 + 0 + 100.0 + attempt_1326232085508_4_4_r_0_0 + /98.139.92.0 + RUNNING + host.domain.com:9999 + REDUCE + container_1326232085508_0004_01_000003 + 1326238780052 + 1326238780099 + 2592 + 47 + 0 + ++---+ + +* Task Attempt Counters API + + With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task attempt id | +*---------------+--------------+-------------------------------+ +| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326232085508_4_4_r_0_0" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2735 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + attempt_1326232085508_4_4_r_0_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm new file mode 100644 index 0000000000..a733d71378 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm @@ -0,0 +1,635 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + NodeManager REST API's. + --- + --- + ${maven.build.timestamp} + +NodeManager REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The NodeManager REST API's allow the user to get status on the node and information about applications and containers running on that node. + +* NodeManager Information API + + The node information resource provides overall information about that particular node. + +** URI + + Both of the following URI's give you the cluster information. + +------ + * http:///ws/v1/node + * http:///ws/v1/node/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | long | The NodeManager id | +*---------------+--------------+-------------------------------+ +| nodeHostName | string | The host name of the NodeManager | +*---------------+--------------+-------------------------------+ +| totalPmemAllocatedContainersMB | long | The amount of physical memory allocated for use by containers in MB | +*---------------+--------------+-------------------------------+ +| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB | +*---------------+--------------+-------------------------------+ +| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| healthReport | string | The diagnostic health report of the node | +*---------------+--------------+-------------------------------+ +| nodeHealthy | boolean | true/false indicator of if the node is healthy| +*---------------+--------------+-------------------------------+ +| nodeManagerVersion | string | Version of the NodeManager | +*---------------+--------------+-------------------------------+ +| nodeManagerBuildVersion | string | NodeManager build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| nodeManagerVersionBuiltOn | string | Timestamp when NodeManager was built(in ms since epoch) | +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built(in ms since epoch) | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "nodeInfo" : { + "hadoopVersionBuiltOn" : "Mon Jan 9 14:58:42 UTC 2012", + "nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c", + "lastNodeUpdateTime" : 1326222266126, + "totalVmemAllocatedContainersMB" : 17203, + "nodeHealthy" : true, + "healthReport" : "", + "totalPmemAllocatedContainersMB" : 8192, + "nodeManagerVersionBuiltOn" : "Mon Jan 9 15:01:59 UTC 2012", + "nodeManagerVersion" : "0.23.1-SNAPSHOT", + "id" : "host.domain.com:45454", + "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1228292 by user1 source checksum 3eba233f2248a089e9b28841a784dd00", + "nodeHostName" : "host.domain.com", + "hadoopVersion" : "0.23.1-SNAPSHOT" + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/node/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 983 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 17203 + 8192 + 1326222386134 + true + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c + Mon Jan 9 15:01:59 UTC 2012 + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1228292 by user1 source checksum 3eba233f2248a089e9b28841a784dd00 + Mon Jan 9 14:58:42 UTC 2012 + host.domain.com:45454 + host.domain.com + ++---+ + +* Applications API + + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. See also {{Application API}} for syntax of the application object. + +** URI + +------ + * http:///ws/v1/node/apps +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + + Multiple paramters can be specified. + +------ + * state - application state + * user - user name +------ + +** Elements of the (Applications) object + + When you make a request for the list of applications, the information will be returned as a collection of app objects. + See also {{Application API}} for syntax of the app object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| app | array of app objects(JSON)/zero or more app objects(XML) | A collection of application objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "apps" : { + "app" : [ + { + "containerids" : [ + "container_1326121700862_0003_01_000001", + "container_1326121700862_0003_01_000002" + ], + "user" : "user1", + "id" : "application_1326121700862_0003", + "state" : "RUNNING" + }, + { + "user" : "user1", + "id" : "application_1326121700862_0002", + "state" : "FINISHED" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 400 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + application_1326121700862_0002 + FINISHED + user1 + + + application_1326121700862_0003 + RUNNING + user1 + container_1326121700862_0003_01_000002 + container_1326121700862_0003_01_000001 + + + ++---+ + +* {Application API} + + An application resource contains information about a particular application that was run or is running on this NodeManager. + +** URI + + Use the following URI to obtain an app Object, for a application identified by the {appid} value. + +------ + * http:///ws/v1/node/apps/{appid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the (Application) object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The application id | +*---------------+--------------+--------------------------------+ +| user | string | The user who started the application | +*---------------+--------------+--------------------------------+ +| state | string | The state of the application - valid states are: NEW, INITING, RUNNING, FINISHING_CONTAINERS_WAIT, APPLICATION_RESOURCES_CLEANINGUP, FINISHED | +*---------------+--------------+--------------------------------+ +| containerids | array of containerids(JSON)/zero or more containerids(XML) | The list of containerids currently being used by the application on this node. If not present then no containers are currently running for this application.| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps/application_1326121700862_0005 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "app" : { + "containerids" : [ + "container_1326121700862_0005_01_000003", + "container_1326121700862_0005_01_000001" + ], + "user" : "user1", + "id" : "application_1326121700862_0005", + "state" : "RUNNING" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps/application_1326121700862_0005 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 281 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326121700862_0005 + RUNNING + user1 + container_1326121700862_0005_01_000003 + container_1326121700862_0005_01_000001 + ++---+ + + +* Containers API + + With the containers API, you can obtain a collection of resources, each of which represents a container. When you run a GET operation on this resource, you obtain a collection of Container Objects. See also {{Container API}} for syntax of the container object. + +** URI + +------ + * http:///ws/v1/node/containers +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of containers, the information will be returned as collection of container objects. + See also {{Container API}} for syntax of the container object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| containers | array of container objects(JSON)/zero or more container objects(XML) | A collection of container objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "containers" : { + "container" : [ + { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000001/user1", + "user" : "user1", + "id" : "container_1326121700862_0006_01_000001", + "exitCode" : -1000 + }, + { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000003/user1", + "user" : "user1", + "id" : "container_1326121700862_0006_01_000003", + "exitCode" : -1000 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 988 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + container_1326121700862_0006_01_000001 + RUNNING + -1000 + + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000001/user1 + host.domain.com:45454 + + + container_1326121700862_0006_01_000003 + DONE + 0 + Container killed by the ApplicationMaster. + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000003/user1 + host.domain.com:45454 + + ++---+ + + +* {Container API} + + A container resource contains information about a particular container that is running on this NodeManager. + +** URI + + Use the following URI to obtain a Container Object, from a container identified by the {containerid} value. + +------ + * http:///ws/v1/node/containers/{containerid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The container id | +*---------------+--------------+-------------------------------+ +| state | string | State of the container - valid states are: NEW, LOCALIZING, LOCALIZATION_FAILED, LOCALIZED, RUNNING, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING, CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE| +*---------------+--------------+-------------------------------+ +| nodeId | string | The id of the node the container is on| +*---------------+--------------+-------------------------------+ +| containerLogsLink | string | The http link to the container logs | +*---------------+--------------+-------------------------------+ +| user | string | The user name of the user which started the container| +*---------------+--------------+-------------------------------+ +| exitCode | int | Exit code of the container | +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message for failed containers | +*---------------+--------------+-------------------------------+ +| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/nodes/containers/container_1326121700862_0007_01_000001 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "container" : { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0007_01_000001/user1", + "user" : "user1", + "id" : "container_1326121700862_0007_01_000001", + "exitCode" : -1000 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers/container_1326121700862_0007_01_000001 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 491 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + container_1326121700862_0007_01_000001 + RUNNING + -1000 + + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0007_01_000001/user1 + host.domain.com:45454 + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm new file mode 100644 index 0000000000..e762594af8 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -0,0 +1,1469 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + ResourceManager REST API's. + --- + --- + ${maven.build.timestamp} + +ResourceManager REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The ResourceManager REST API's allow the user to get information about the cluster - status on the cluster, metrics on the cluster, scheduler information, information about nodes in the cluster, and information about applications on the cluster. + +* Cluster Information API + + The cluster information resource provides overall information about the cluster. + +** URI + + Both of the following URI's give you the cluster information. + +------ + * http:///ws/v1/cluster + * http:///ws/v1/cluster/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | long | The cluster id | +*---------------+--------------+-------------------------------+ +| startedOn | long | The time the cluster started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| state | string | The ResourceManager state - valid values are: NOTINITED, INITED, STARTED, STOPPED| +*---------------+--------------+-------------------------------+ +| resourceManagerVersion | string | Version of the ResourceManager | +*---------------+--------------+-------------------------------+ +| resourceManagerBuildVersion | string | ResourceManager build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| resourceManagerVersionBuiltOn | string | Timestamp when ResourceManager was built (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built(in ms since epoch)| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "clusterInfo": + { + "id":1324053971963, + "startedOn":1324053971963, + "state":"STARTED", + "resourceManagerVersion":"0.23.1-SNAPSHOT", + "resourceManagerBuildVersion":"0.23.1-SNAPSHOT from 1214049 by user1 source checksum 050cd664439d931c8743a6428fd6a693", + "resourceManagerVersionBuiltOn":"Tue Dec 13 22:12:48 CST 2011", + "hadoopVersion":"0.23.1-SNAPSHOT", + "hadoopBuildVersion":"0.23.1-SNAPSHOT from 1214049 by user1 source checksum 11458df3bb77342dca5f917198fad328", + "hadoopVersionBuiltOn":"Tue Dec 13 22:12:26 CST 2011" + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/cluster/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 712 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1324053971963 + 1324053971963 + STARTED + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1214049 by user1 source checksum 050cd664439d931c8743a6428fd6a693 + Tue Dec 13 22:12:48 CST 2011 + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1214049 by user1 source checksum 11458df3bb77342dca5f917198fad328 + Tue Dec 13 22:12:48 CST 2011 + ++---+ + +* Cluster Metrics API + + The cluster metrics resource provides some overall metrics about the cluster. More detailed metrics should be retrieved from the jmx interface. + +** URI + +------ + * http:///ws/v1/cluster/metrics +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| appsSubmitted | int | The number of applications submitted | +*---------------+--------------+-------------------------------+ +| reservedMB | long | The amount of memory reserved in MB | +*---------------+--------------+-------------------------------+ +| availableMB | long | The amount of memory available in MB | +*---------------+--------------+-------------------------------+ +| allocatedMB | long | The amount of memory allocated in MB | +*---------------+--------------+-------------------------------+ +| totalMB | long | The amount of total memory in MB | +*---------------+--------------+-------------------------------+ +| containersAllocated | int | The number of containers allocated | +*---------------+--------------+-------------------------------+ +| totalNodes | int | The total number of nodes | +*---------------+--------------+-------------------------------+ +| activeNodes | int | The number of active nodes | +*---------------+--------------+-------------------------------+ +| lostNodes | int | The number of lost nodes | +*---------------+--------------+-------------------------------+ +| unhealthyNodes | int | The number of unhealthy nodes | +*---------------+--------------+-------------------------------+ +| decommissionedNodes | int | The number of nodes decommissioned | +*---------------+--------------+-------------------------------+ +| rebootedNodes | int | The number of nodes rebooted | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/metrics +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + { + "clusterMetrics": + { + "appsSubmitted":4, + "reservedMB":0, + "availableMB":8192, + "allocatedMB":0, + "totalMB":8192, + "containersAllocated":0, + "totalNodes":1, + "activeNodes":1, + "lostNodes":0, + "unhealthyNodes":0, + "decommissionedNodes":0, + "rebootedNodes":0 + } + } ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/metrics + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 432 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 4 + 0 + 8192 + 0 + 0 + 8192 + 1 + 1 + 0 + 0 + 0 + 0 + ++---+ + +* Cluster Scheduler API + + A scheduler resource contains information about the current scheduler configured in a cluster. It currently supports both the Fifo and Capacity Scheduler. You will get different information depending on which scheduler is configured so be sure to look at the type information. + +** URI + +------ + * http:///ws/v1/cluster/scheduler +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Capacity Scheduler API + + The capacity scheduler supports hierarchical queues. This one request will print information about all the queues and any subqueues they have. + Queues that can actually have jobs submitted to them are referred to as leaf queues. These queues have additional data associated with them. + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | string | Scheduler type - capacityScheduler| +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +*---------------+--------------+-------------------------------+ +| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| queueName | string | Name of the queue | +*---------------+--------------+-------------------------------+ +| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of queue resources| +*---------------+--------------+-------------------------------+ + +** Elements of the queues/subQueues object for a Parent queue + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +*---------------+--------------+-------------------------------+ +| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster | +*---------------+--------------+-------------------------------+ +| absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | +*---------------+--------------+-------------------------------+ +| utilization | float | Queue utilization percentage relative to the entire cluster | +*---------------+--------------+-------------------------------+ +| numApplications | int | The number of applications currently in the queue | +*---------------+--------------+-------------------------------+ +| usedResources | string | A string describing the current resources used by the queue | +*---------------+--------------+-------------------------------+ +| queueName | string | The name of the queue | +*---------------+--------------+-------------------------------+ +| state | string of QueueState | The state of the queue | +*---------------+--------------+-------------------------------+ +| subQueues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information| +*---------------+--------------+-------------------------------+ + +** Elements of the queues/subQueues object for a Leaf queue - contains all elements in parent plus the following: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | String | type of the queue - capacitySchedulerLeafQueueInfo | +*---------------+--------------+-------------------------------+ +| numActiveApplications | int | The number of active applications in this queue | +*---------------+--------------+-------------------------------+ +| numPendingApplications | int | The number of pending applications in this queue | +*---------------+--------------+-------------------------------+ +| numContainers | int | The number of containers being used | +*---------------+--------------+-------------------------------+ +| maxApplications | int | The maximum number of applications this queue can have | +*---------------+--------------+-------------------------------+ +| maxApplicationsPerUser | int | The maximum number of applications per user this queue can have | +*---------------+--------------+-------------------------------+ +| maxActiveApplications | int | The maximum number of active applications this queue can have | +*---------------+--------------+-------------------------------+ +| maxActiveApplicationsPerUser | int | The maximum number of active applications per user this queue can have| +*---------------+--------------+-------------------------------+ +| userLimit | int | The minimum user limit percent set in the configuration | +*---------------+--------------+-------------------------------+ +| userLimitFactor | float | The user limit factor set in the configuration | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "scheduler" : { + "schedulerInfo" : { + "queueName" : "root", + "maxCapacity" : 100, + "type" : "capacityScheduler", + "queues" : [ + { + "numPendingApplications" : 0, + "queueName" : "default", + "userLimitFactor" : 1, + "maxApplications" : 7000, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 90, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 90, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 70, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 70, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 7000 + }, + { + "queueName" : "test", + "utilization" : 0, + "absoluteCapacity" : 20, + "usedCapacity" : 0, + "capacity" : 20, + "subQueues" : [ + { + "numPendingApplications" : 0, + "queueName" : "a1", + "userLimitFactor" : 1, + "maxApplications" : 1200, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 80, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 16.000002, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 12, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 60.000004, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 1200 + }, + { + "numPendingApplications" : 0, + "queueName" : "a2", + "userLimitFactor" : 1, + "maxApplications" : 800, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 8.000001, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 40, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 800 + } + ], + "state" : "RUNNING", + "maxCapacity" : 80, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 80 + }, + { + "queueName" : "test2", + "utilization" : 0, + "absoluteCapacity" : 10, + "usedCapacity" : 0, + "capacity" : 10, + "subQueues" : [ + { + "numPendingApplications" : 0, + "queueName" : "a5", + "userLimitFactor" : 1, + "maxApplications" : 500, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 5, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 50, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 500 + }, + { + "numPendingApplications" : 0, + "queueName" : "a3", + "userLimitFactor" : 1, + "maxApplications" : 400, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 4.0000005, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 40, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 400 + }, + { + "numPendingApplications" : 0, + "queueName" : "a4", + "userLimitFactor" : 1, + "maxApplications" : 100, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 1.0000001, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 10, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 100 + } + ], + "state" : "RUNNING", + "maxCapacity" : 15.000001, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 15.000001 + } + ], + "usedCapacity" : 0, + "capacity" : 100 + } + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/cluster/scheduler +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 5778 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 100.0 + 0.0 + 100.0 + root + + 70.0 + 0.0 + 90.0 + 70.0 + 90.0 + 0.0 + 0 + memory: 0 + default + RUNNING + 0 + 0 + 0 + 7000 + 7000 + 1 + 1 + 100 + 1.0 + + + 20.0 + 0.0 + 80.0 + 20.0 + 80.0 + 0.0 + 0 + memory: 0 + test + RUNNING + + 60.000004 + 0.0 + 80.0 + 12.0 + 16.000002 + 0.0 + 0 + memory: 0 + a1 + RUNNING + 0 + 0 + 0 + 1200 + 1200 + 1 + 1 + 100 + 1.0 + + + 40.0 + 0.0 + 100.0 + 8.000001 + 100.0 + 0.0 + 0 + memory: 0 + a2 + RUNNING + 0 + 0 + 0 + 800 + 800 + 1 + 1 + 100 + 1.0 + + + + 10.0 + 0.0 + 15.000001 + 10.0 + 15.000001 + 0.0 + 0 + memory: 0 + test2 + RUNNING + + 50.0 + 0.0 + 100.0 + 5.0 + 100.0 + 0.0 + 0 + memory: 0 + A4 + RUNNING + 0 + 0 + 0 + 500 + 500 + 1 + 1 + 100 + 1.0 + + + 40.0 + 0.0 + 100.0 + 4.0000005 + 100.0 + 0.0 + 0 + memory: 0 + a3 + RUNNING + 0 + 0 + 0 + 400 + 400 + 1 + 1 + 100 + 1.0 + + + 10.0 + 0.0 + 100.0 + 1.0000001 + 100.0 + 0.0 + 0 + memory: 0 + a4 + RUNNING + 0 + 0 + 0 + 100 + 100 + 1 + 1 + 100 + 1.0 + + + + ++---+ + +** Fifo Scheduler API + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | string | Scheduler type - fifoScheduler | +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage | +*---------------+--------------+-------------------------------+ +| qstate | string | State of the queue - valid values are: STOPPED, RUNNING| +*---------------+--------------+-------------------------------+ +| minQueueMemoryCapacity | int | Minimum queue memory capacity | +*---------------+--------------+-------------------------------+ +| maxQueueMemoryCapacity | int | Maximum queue memory capacity | +*---------------+--------------+-------------------------------+ +| numNodes | int | The total number of nodes | +*---------------+--------------+-------------------------------+ +| usedNodeCapacity | int | The used node capacity | +*---------------+--------------+-------------------------------+ +| availNodeCapacity | int | The available node capacity | +*---------------+--------------+-------------------------------+ +| totalNodeCapacity | int | The total node capacity | +*---------------+--------------+-------------------------------+ +| numContainers | int | The number of containers | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "scheduler": + { + "schedulerInfo": + { + "type":"fifoScheduler", + "capacity":1, + "usedCapacity":"NaN", + "qstate":"RUNNING", + "minQueueMemoryCapacity":1024, + "maxQueueMemoryCapacity":10240, + "numNodes":0, + "usedNodeCapacity":0, + "availNodeCapacity":0, + "totalNodeCapacity":0, + "numContainers":0 + } + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 432 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1.0 + NaN + RUNNING + 1024 + 10240 + 0 + 0 + 0 + 0 + 0 + + ++---+ + +* Cluster Applications API + + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. + +** URI + +------ + * http:///ws/v1/cluster/apps +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + + Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + +------ + * state - state of the application + * user - user name + * queue - queue name + * limit - total number of app objects to be returned + * startedTimeBegin - applications with start time beginning with this time, specified in ms since epoch + * startedTimeEnd - applications with start time ending with this time, specified in ms since epoch + * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch + * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch +------ + +** Elements of the (Applications) object + + When you make a request for the list of applications, the information will be returned as a collection of app objects. + See also {{Application API}} for syntax of the app object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "apps": + { + "app": + [ + { + "finishedTime" : 1326815598530, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326815542473_0001_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326815542473_0001", + "clusterId" : 1326815542473, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "word count", + "startedTime" : 1326815573334, + "elapsedTime" : 25196, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job/job_1326815542473_1_1", + "queue" : "default" + }, + { + "finishedTime" : 1326815789546, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326815542473_0002_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326815542473_0002", + "clusterId" : 1326815542473, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326815641380, + "elapsedTime" : 148166, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2", + "queue" : "default" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2459 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + application_1326815542473_0001 + user1 + word count + default + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job +/job_1326815542473_1_1 + + 1326815542473 + 1326815573334 + 1326815598530 + 25196 + http://host.domain.com:9999/node/containerlogs/container_1326815542473_0001 +_01_000001 + host.domain.com:9999 + + + application_1326815542473_0002 + user1 + Sleep job + default + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2 + + 1326815542473 + 1326815641380 + 1326815789546 + 148166 + http://host.domain.com:9999/node/containerlogs/container_1326815542473_0002_01_000001 + host.domain.com:9999 + + + ++---+ + +* Cluster {Application API} + + An application resource contains information about a particular application that was submitted to a cluster. + +** URI + + Use the following URI to obtain an app object, from a application identified by the {appid} value. + +------ + * http:///ws/v1/cluster/apps/{appid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the (Application) object + + Note that depending on security settings a user might not be able to see all the fields. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The application id | +*---------------+--------------+--------------------------------+ +| user | string | The user who started the application | +*---------------+--------------+--------------------------------+ +| name | string | The application name | +*---------------+--------------+--------------------------------+ +| queue | string | The queue the application was submitted to| +*---------------+--------------+--------------------------------+ +| state | string | The application state according to the ResourceManager - valid values are: NEW, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED| +*---------------+--------------+--------------------------------+ +| finalStatus | string | The final status of the application if finished - reported by the application itself - valid values are: UNDEFINED, SUCCEEDED, FAILED, KILLED| +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the application as a percent | +*---------------+--------------+--------------------------------+ +| trackingUI | string | Where the tracking url is currently pointing - History (for history server) or ApplicationMaster | +*---------------+--------------+--------------------------------+ +| trackingUrl | string | The web URL that can be used to track the application | +*---------------+--------------+--------------------------------+ +| diagnostics | string | Detailed diagnostics information | +*---------------+--------------+--------------------------------+ +| clusterId | long | The cluster id | +*---------------+--------------+--------------------------------+ +| startedTime | long | The time in which application started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishedTime | long | The time in which the application finished (in ms since epoch) | +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ +| amContainerLogs | string | The URL of the application master container logs| +*---------------+--------------+--------------------------------+ +| amHostHttpAddress | string | The nodes http address of the application master | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "app" : { + "finishedTime" : 1326824991300, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0005_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326821518301_0005", + "clusterId" : 1326821518301, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326824544552, + "elapsedTime" : 446748, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5", + "queue" : "a1" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 847 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326821518301_0005 + user1 + Sleep job + a1 + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5 + + 1326821518301 + 1326824544552 + 1326824991300 + 446748 + http://host.domain.com:9999/node/containerlogs/container_1326821518301_0005_01_000001 + host.domain.com:9999 + ++---+ + +* Cluster Nodes API + + With the Nodes API, you can obtain a collection of resources, each of which represents a node. When you run a GET operation on this resource, you obtain a collection of Node Objects. + +** URI + +------ + * http:///ws/v1/cluster/nodes +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + * state - the state of the node + * healthy - true or false +------ + +** Elements of the object + + When you make a request for the list of nodes, the information will be returned as a collection of node objects. + See also {{Node API}} for syntax of the node object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| node | array of node objects(JSON)/zero or more node objects(XML) | A collection of node objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "nodes": + { + "node": + [ + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h2:1235", + "nodeHostName":"h2", + "nodeHTTPAddress":"h2:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895432, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0 + "availMemoryMB":8192 + }, + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h1:1234", + "nodeHostName":"h1", + "nodeHTTPAddress":"h1:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895092, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0, + "availMemoryMB":8192 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1104 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + /default-rack + RUNNING + h2:1234 + h2 + h2:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + + + /default-rack + RUNNING + h1:1234 + h1 + h1:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + + ++---+ + + +* Cluster {Node API} + + A node resource contains information about a node in the cluster. + +** URI + + Use the following URI to obtain a Node Object, from a node identified by the {nodeid} value. + +------ + * http:///ws/v1/cluster/nodes/{nodeid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| rack | string | The rack location of this node | +*---------------+--------------+-------------------------------+ +| state | string | State of the node - valid values are: NEW, RUNNING, UNHEALTHY, DECOMMISSIONED, LOST, REBOOTED | +*---------------+--------------+-------------------------------+ +| id | string | The node id | +*---------------+--------------+-------------------------------+ +| nodeHostName | string | The host name of the node| +*---------------+--------------+-------------------------------+ +| nodeHTTPAddress | string | The nodes HTTP address| +*---------------+--------------+-------------------------------+ +| healthStatus | string | The health status of the node - Healthy or Unhealthy | +*---------------+--------------+-------------------------------+ +| healthReport | string | A detailed health report | +*---------------+--------------+-------------------------------+ +| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| usedMemoryMB | long | The total about of memory currently used on the node (in MB)| +*---------------+--------------+-------------------------------+ +| availMemoryMB | long | The total amount of memory currently available on the node (in MB)| +*---------------+--------------+-------------------------------+ +| numContainers | int | The total number of containers currently running on the node| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes/h2:1235 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "node": + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h2:1235", + "nodeHostName":"h2", + "nodeHTTPAddress":"h2:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895432, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0, + "availMemoryMB":5120 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/node/h2:1235 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + /default-rack + NEW + h2:1235 + h2 + h2:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm new file mode 100644 index 0000000000..2cdbfbe472 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm @@ -0,0 +1,595 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + Hadoop YARN - Introduction to the web services REST API's. + --- + --- + ${maven.build.timestamp} + +Hadoop YARN - Introduction to the web services REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0} + +* Overview + + The Hadoop YARN web service REST APIs are a set of URI resources that give access to the cluster, nodes, applications, and application historical information. The URI resources are grouped into APIs based on the type of information returned. Some URI resources return collections while others return singletons. + +* URI's + + The URIs for the REST-based Web services have the following syntax: + +------ + http://{http address of service}/ws/{version}/{resourcepath} +------ + + The elements in this syntax are as follows: + +------ + {http address of service} - The http address of the service to get information about. + Currently supported are the ResourceManager, NodeManager, + MapReduce application master, and history server. + {version} - The version of the APIs. In this release, the version is v1. + {resourcepath} - A path that defines a singleton resource or a collection of resources. +------ + +* HTTP Requests + + To invoke a REST API, your application calls an HTTP operation on the URI associated with a resource. + +** Summary of HTTP operations + + Currently only GET is supported. It retrieves information about the resource specified. + +** Security + + The web service REST API's go through the same security as the web ui. If your cluster adminstrators have filters enabled you must authenticate via the mechanism they specified. + +** Headers Supported + +----- + * Accept + * Accept-Encoding +----- + + Currently the only fields used in the header is Accept and Accept-Encoding. Accept currently supports XML and JSON for the response type you accept. Accept-Encoding currently only supports gzip format and will return gzip compressed output if this is specified, otherwise output is uncompressed. All other header fields are ignored. + +* HTTP Responses + + The next few sections describe some of the syntax and other details of the HTTP Responses of the web service REST APIs. + +** Compression + + This release supports gzip compression if you specify gzip in the Accept-Encoding header of the HTTP request (Accept-Encoding: gzip). + +** Response Formats + + This release of the web service REST APIs supports responses in JSON and XML formats. JSON is the default. To set the response format, you can specify the format in the Accept header of the HTTP request. + + As specified in HTTP Response Codes, the response body can contain the data that represents the resource or an error message. In the case of success, the response body is in the selected format, either JSON or XML. In the case of error, the resonse body is in either JSON or XML based on the format requested. The Content-Type header of the response contains the format requested. If the application requests an unsupported format, the response status code is 500. +Note that the order of the fields within response body is not specified and might change. Also, additional fields might be added to a response body. Therefore, your applications should use parsing routines that can extract data from a response body in any order. + +** Response Errors + + After calling an HTTP request, an application should check the response status code to verify success or detect an error. If the response status code indicates an error, the response body contains an error message. The first field is the exception type, currently only RemoteException is returned. The following table lists the items within the RemoteException error message: + +*---------------*--------------*-------------------------------* +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| exception | String | Exception type | +*---------------+--------------+-------------------------------+ +| javaClassName | String | Java class name of exception | +*---------------+--------------+-------------------------------+ +| message | String | Detailed message of exception | +*---------------+--------------+-------------------------------+ + +** Response Examples + +*** JSON response with single resource + + HTTP Request: + GET http://rmhost.domain:8088/ws/v1/cluster/app/application_1324057493980_0001 + + Response Status Line: + HTTP/1.1 200 OK + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + app": + { + "id":"application_1324057493980_0001", + "user":"user1", + "name":"", + "queue":"default", + "state":"ACCEPTED", + "finalStatus":"UNDEFINED", + "progress":0, + "trackingUI":"UNASSIGNED", + "diagnostics":"", + "clusterId":1324057493980, + "startedTime":1324057495921, + "finishedTime":0, + "elapsedTime":2063, + "amContainerLogs":"http:\/\/amNM:2\/node\/containerlogs\/container_1324057493980_0001_01_000001", + "amHostHttpAddress":"amNM:2" + } +} ++---+ + +*** JSON response with Error response + + Here we request information about an application that doesn't exist yet. + + HTTP Request: + GET http://rmhost.domain:8088/ws/v1/cluster/app/application_1324057493980_9999 + + Response Status Line: + HTTP/1.1 404 Not Found + + Response Header: + ++---+ + HTTP/1.1 404 Not Found + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "RemoteException" : { + "javaClassName" : "org.apache.hadoop.yarn.webapp.NotFoundException", + "exception" : "NotFoundException", + "message" : "java.lang.Exception: app with id: application_1324057493980_9999 not found" + } +} ++---+ + +* Example usage + + You can use any number of ways/languages to use the web services REST API's. This example uses the curl command line interface to do the REST GET calls. + + In this example, a user submits a MapReduce application to the ResourceManager using a command like: + ++---+ + hadoop jar hadoop-mapreduce-test.jar sleep -Dmapred.job.queue.name=a1 -m 1 -r 1 -rt 1200000 -mt 20 ++---+ + + The client prints information about the job submitted along with the application id, similar to: + ++---+ +12/01/18 04:25:15 INFO mapred.ResourceMgrDelegate: Submitted application application_1326821518301_0010 to ResourceManager at host.domain.com/10.10.10.10:8040 +12/01/18 04:25:15 INFO mapreduce.Job: Running job: job_1326821518301_0010 +12/01/18 04:25:21 INFO mapred.ClientServiceDelegate: The url to track the job: host.domain.com:8088/proxy/application_1326821518301_0010/ +12/01/18 04:25:22 INFO mapreduce.Job: Job job_1326821518301_0010 running in uber mode : false +12/01/18 04:25:22 INFO mapreduce.Job: map 0% reduce 0% ++---+ + + The user then wishes to track the application. The users starts by getting the information about the application from the ResourceManager. Use the --comopressed option to request output compressed. curl handles uncompressing on client side. + ++---+ +curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/ws/v1/cluster/apps/application_1326821518301_0010" ++---+ + + Output: + ++---+ +{ + "app" : { + "finishedTime" : 0, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0010_01_000001", + "trackingUI" : "ApplicationMaster", + "state" : "RUNNING", + "user" : "user1", + "id" : "application_1326821518301_0010", + "clusterId" : 1326821518301, + "finalStatus" : "UNDEFINED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 82.44703, + "name" : "Sleep job", + "startedTime" : 1326860715335, + "elapsedTime" : 31814, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0010/", + "queue" : "a1" + } +} ++---+ + + The user then wishes to get more details about the running application and goes directly to the MapReduce application master for this application. The ResourceManager lists the trackingUrl that can be used for this application: http://host.domain.com:8088/proxy/application_1326821518301_0010. This could either go to the web browser or use the web service REST API's. The user uses the web services REST API's to get the list of jobs this MapReduce application master is running: + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs" ++---+ + + Output: + ++---+ +{ + "jobs" : { + "job" : [ + { + "runningReduceAttempts" : 1, + "reduceProgress" : 72.104515, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326860720902, + "id" : "job_1326821518301_10_10", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 64432, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } + ] + } +} ++---+ + + The user then wishes to get the task details about the job with job id job_1326821518301_10_10 that was listed above. + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks" ++---+ + + Output: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 5059, + "state" : "SUCCEEDED", + "startTime" : 1326860725014, + "id" : "task_1326821518301_10_10_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326821518301_10_10_m_0_0", + "finishTime" : 1326860730073 + }, + { + "progress" : 72.104515, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326860732984, + "id" : "task_1326821518301_10_10_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } + ] + } +} ++---+ + + The map task has finished but the reduce task is still running. The users wishes to get the task attempt information for the reduce task task_1326821518301_10_10_r_0, note that the Accept header isn't really required here since JSON is the default output format: + ++---+ + curl --compressed -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks/task_1326821518301_10_10_r_0/attempts" ++---+ + + Output: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "elapsedMergeTime" : 158, + "shuffleFinishTime" : 1326860735378, + "assignedContainerId" : "container_1326821518301_0010_01_000003", + "progress" : 72.104515, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2394, + "mergeFinishTime" : 1326860735536, + "rack" : "/10.10.10.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "type" : "REDUCE", + "startTime" : 1326860732984, + "id" : "attempt_1326821518301_10_10_r_0_0", + "finishTime" : 0 + } + ] + } +} ++---+ + + The reduce attempt is still running and the user wishes to see the current counter values for that attempt: + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks/task_1326821518301_10_10_r_0/attempts/attempt_1326821518301_10_10_r_0_0/counters" ++---+ + + Output: + ++---+ +{ + "JobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 4216, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 77151, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 1767, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 25104, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 1767, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 50, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 1580, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 141320192, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1118552064, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 73728000, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326821518301_10_10_r_0_0" + } +} ++---+ + + The job finishes and the user wishes to get the final job information from the history server for this job. + ++---+ + curl --compressed -X GET "http://host.domain.com:19888/ws/v1/history/mapreduce/jobs/job_1326821518301_10_10" ++---+ + + Output: + ++---+ +{ + "job" : { + "avgReduceTime" : 1250784, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326860720902, + "id" : "job_1326821518301_10_10", + "avgMapTime" : 5059, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2394, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2552, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "a1", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326861986164 + } +} ++---+ + + The user also gets the final applications information from the ResourceManager. + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/ws/v1/cluster/apps/application_1326821518301_0010" ++---+ + + Output: + ++---+ +{ + "app" : { + "finishedTime" : 1326861991282, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0010_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326821518301_0010", + "clusterId" : 1326821518301, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326860715335, + "elapsedTime" : 1275947, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0010/jobhistory/job/job_1326821518301_10_10", + "queue" : "a1" + } +} ++---+ diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 6b966883e0..95c3325775 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -62,6 +62,14 @@ + + + + + + + + From 355ba013747637e71936eab499055446616ed9d3 Mon Sep 17 00:00:00 2001 From: Mahadev Konar Date: Sat, 21 Jan 2012 01:15:24 +0000 Subject: [PATCH 7/8] MAPREDUCE-3705. ant build fails on 0.23 branch. (Thomas Graves via mahadev) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234227 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + hadoop-mapreduce-project/build.xml | 2 - hadoop-mapreduce-project/ivy.xml | 2 + .../src/contrib/gridmix/ivy.xml | 2 + .../tools/rumen/TestConcurrentRead.java | 136 -- .../hadoop/tools/rumen/TestParsedLine.java | 105 - .../tools/rumen/TestRumenAnonymization.java | 1940 ----------------- .../hadoop/tools/rumen/TestRumenFolder.java | 196 -- .../tools/rumen/TestRumenJobTraces.java | 1259 ----------- .../hadoop/tools/rumen/TestZombieJob.java | 338 --- 10 files changed, 7 insertions(+), 3976 deletions(-) delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestConcurrentRead.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestParsedLine.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenAnonymization.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenFolder.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java delete mode 100644 hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestZombieJob.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dc6220e80a..a7a73340da 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -526,6 +526,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3549. write api documentation for web service apis for RM, NM, mapreduce app master, and job history server (Thomas Graves via mahadev) + MAPREDUCE-3705. ant build fails on 0.23 branch. (Thomas Graves via + mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/build.xml b/hadoop-mapreduce-project/build.xml index 9df60e98bb..2eb5459be4 100644 --- a/hadoop-mapreduce-project/build.xml +++ b/hadoop-mapreduce-project/build.xml @@ -575,8 +575,6 @@ - - diff --git a/hadoop-mapreduce-project/ivy.xml b/hadoop-mapreduce-project/ivy.xml index e04da7019b..95042252e9 100644 --- a/hadoop-mapreduce-project/ivy.xml +++ b/hadoop-mapreduce-project/ivy.xml @@ -99,6 +99,8 @@ rev="${yarn.version}" conf="compile->default"> + diff --git a/hadoop-mapreduce-project/src/contrib/gridmix/ivy.xml b/hadoop-mapreduce-project/src/contrib/gridmix/ivy.xml index 4ab7b62065..d587a7b875 100644 --- a/hadoop-mapreduce-project/src/contrib/gridmix/ivy.xml +++ b/hadoop-mapreduce-project/src/contrib/gridmix/ivy.xml @@ -70,6 +70,8 @@ + cachedTrace = new ArrayList(); - static final String traceFile = - "rumen/small-trace-test/job-tracker-logs-trace-output.gz"; - - static Configuration conf; - static FileSystem lfs; - static Path path; - - @BeforeClass - static public void globalSetUp() throws IOException { - conf = new Configuration(); - lfs = FileSystem.getLocal(conf); - Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", "")) - .makeQualified(lfs.getUri(), lfs.getWorkingDirectory()); - path = new Path(rootInputDir, traceFile); - JobTraceReader reader = new JobTraceReader(path, conf); - try { - LoggedJob job; - while ((job = reader.getNext()) != null) { - cachedTrace.add(job); - } - } finally { - reader.close(); - } - } - - void readAndCompare() throws IOException { - JobTraceReader reader = new JobTraceReader(path, conf); - try { - for (Iterator it = cachedTrace.iterator(); it.hasNext();) { - LoggedJob jobExpected = it.next(); - LoggedJob jobRead = reader.getNext(); - assertNotNull(jobRead); - try { - jobRead.deepCompare(jobExpected, null); - } catch (DeepInequalityException e) { - fail(e.toString()); - } - } - assertNull(reader.getNext()); - } finally { - reader.close(); - } - } - - class TestThread extends Thread { - final int repeat; - final CountDownLatch startSignal, doneSignal; - final Map errors; - - TestThread(int id, int repeat, CountDownLatch startSignal, CountDownLatch doneSignal, Map errors) { - super(String.format("TestThread-%d", id)); - this.repeat = repeat; - this.startSignal = startSignal; - this.doneSignal = doneSignal; - this.errors = errors; - } - - @Override - public void run() { - try { - startSignal.await(); - for (int i = 0; i < repeat; ++i) { - try { - readAndCompare(); - } catch (Throwable e) { - errors.put(getName(), e); - break; - } - } - doneSignal.countDown(); - } catch (Throwable e) { - errors.put(getName(), e); - } - } - } - - @Test - public void testConcurrentRead() throws InterruptedException { - int nThr = conf.getInt("test.rumen.concurrent-read.threads", 4); - int repeat = conf.getInt("test.rumen.concurrent-read.repeat", 10); - CountDownLatch startSignal = new CountDownLatch(1); - CountDownLatch doneSignal = new CountDownLatch(nThr); - Map errors = Collections - .synchronizedMap(new TreeMap()); - for (int i = 0; i < nThr; ++i) { - new TestThread(i, repeat, startSignal, doneSignal, errors).start(); - } - startSignal.countDown(); - doneSignal.await(); - if (!errors.isEmpty()) { - StringBuilder sb = new StringBuilder(); - for (Map.Entry e : errors.entrySet()) { - sb.append(String.format("%s:\n%s\n", e.getKey(), e.getValue().toString())); - } - fail(sb.toString()); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestParsedLine.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestParsedLine.java deleted file mode 100644 index 446484869c..0000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestParsedLine.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.tools.rumen; - -import org.apache.hadoop.util.StringUtils; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestParsedLine { - static final char[] CHARS_TO_ESCAPE = new char[]{'=', '"', '.'}; - - String buildLine(String type, String[] kvseq) { - StringBuilder sb = new StringBuilder(); - sb.append(type); - for (int i=0; i defaultSerializer = new DefaultRumenSerializer(); - - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - // test username - UserName uname = new UserName("bob"); - assertEquals("Username error!", "bob", uname.getValue()); - - // test username serialization - // test with no anonymization - // test bob - testSerializer(new UserName("bob"), "bob", defaultSerializer); - // test alice - testSerializer(new UserName("alice"), "alice", defaultSerializer); - - // test user-name serialization - // test with anonymization - // test bob - testSerializer(new UserName("bob"), "user0", anonymizingSerializer); - // test alice - testSerializer(new UserName("alice"), "user1", anonymizingSerializer); - } - - /** - * Test {@link JobName}, serialization and anonymization. - */ - @Test - public void testJobNameSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - - // test jobname - JobName jname = new JobName("job-secret"); - assertEquals("Jobname error!", "job-secret", jname.getValue()); - - // test job-name serialization - // test with no anonymization - // test job1 - testSerializer(new JobName("job-myjob"), "job-myjob", defaultSerializer); - // test job2 - testSerializer(new JobName("job-yourjob"), "job-yourjob", - defaultSerializer); - - // test job-name serialization - // test with anonymization - // test queue1 - testSerializer(new JobName("secret-job-1"), "job0", anonymizingSerializer); - // test queue2 - testSerializer(new JobName("secret-job-2"), "job1", anonymizingSerializer); - } - - /** - * Test {@link QueueName}, serialization and anonymization. - */ - @Test - public void testQueueNameSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - - // test queuename - QueueName qname = new QueueName("queue-secret"); - assertEquals("Queuename error!", "queue-secret", qname.getValue()); - - // test queuename serialization - // test with no anonymization - // test queue1 - testSerializer(new QueueName("project1-queue"), - "project1-queue", defaultSerializer); - // test queue2 - testSerializer(new QueueName("project2-queue"), - "project2-queue", defaultSerializer); - - // test queue-name serialization - // test with anonymization - // test queue1 - testSerializer(new QueueName("project1-queue"), - "queue0", anonymizingSerializer); - // test queue2 - testSerializer(new QueueName("project2-queue"), - "queue1", anonymizingSerializer); - } - - /** - * Test {@link NodeName}. - */ - @Test - public void testNodeNameDataType() throws IOException { - // test hostname - // test only hostname - NodeName hname = new NodeName("host1.myorg.com"); - assertNull("Expected missing rack name", hname.getRackName()); - assertEquals("Hostname's test#1 hostname error!", - "host1.myorg.com", hname.getHostName()); - assertEquals("Hostname test#1 error!", "host1.myorg.com", hname.getValue()); - - // test rack/hostname - hname = new NodeName("/rack1.myorg.com/host1.myorg.com"); - assertEquals("Hostname's test#2 rackname error!", - "rack1.myorg.com", hname.getRackName()); - assertEquals("Hostname test#2 hostname error!", - "host1.myorg.com", hname.getHostName()); - assertEquals("Hostname test#2 error!", - "/rack1.myorg.com/host1.myorg.com", hname.getValue()); - - // test hostname and rackname separately - hname = new NodeName("rack1.myorg.com", "host1.myorg.com"); - assertEquals("Hostname's test#3 rackname error!", - "rack1.myorg.com", hname.getRackName()); - assertEquals("Hostname test#3 hostname error!", - "host1.myorg.com", hname.getHostName()); - assertEquals("Hostname test#3 error!", - "/rack1.myorg.com/host1.myorg.com", hname.getValue()); - - // test hostname with no rackname - hname = new NodeName(null, "host1.myorg.com"); - assertNull("Hostname's test#4 rackname error!", hname.getRackName()); - assertEquals("Hostname test#4 hostname error!", - "host1.myorg.com", hname.getHostName()); - assertEquals("Hostname test#4 error!", - "host1.myorg.com", hname.getValue()); - - // test rackname with no hostname - hname = new NodeName("rack1.myorg.com", null); - assertEquals("Hostname test#4 rackname error!", - "rack1.myorg.com", hname.getRackName()); - assertNull("Hostname's test#5 hostname error!", hname.getHostName()); - assertEquals("Hostname test#5 error!", - "rack1.myorg.com", hname.getValue()); - } - - /** - * Test {@link NodeName} serialization. - */ - @Test - public void testNodeNameDefaultSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - // test hostname serialization - // test with no anonymization - // test hostname - testSerializer(new NodeName("hostname.myorg.com"), "hostname.myorg.com", - defaultSerializer); - // test rack/hostname - testSerializer(new NodeName("/rackname.myorg.com/hostname.myorg.com"), - "/rackname.myorg.com/hostname.myorg.com", - defaultSerializer); - // test rack,hostname - testSerializer(new NodeName("rackname.myorg.com", "hostname.myorg.com"), - "/rackname.myorg.com/hostname.myorg.com", - defaultSerializer); - // test -,hostname - testSerializer(new NodeName(null, "hostname.myorg.com"), - "hostname.myorg.com", defaultSerializer); - // test rack,- - testSerializer(new NodeName("rackname.myorg.com", null), - "rackname.myorg.com", defaultSerializer); - } - - /** - * Test {@link NodeName} anonymization. - */ - @Test - public void testNodeNameAnonymization() throws IOException { - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - - // test hostname serializer - // test with anonymization - // test hostname - testSerializer(new NodeName("hostname.myorg.com"), "host0", - anonymizingSerializer); - // test hostname reuse - testSerializer(new NodeName("hostname213.myorg.com"), "host1", - anonymizingSerializer); - // test rack/hostname - testSerializer(new NodeName("/rackname.myorg.com/hostname.myorg.com"), - "/rack0/host0", anonymizingSerializer); - // test rack/hostname (hostname reuse) - testSerializer(new NodeName("/rackname654.myorg.com/hostname.myorg.com"), - "/rack1/host0", anonymizingSerializer); - // test rack/hostname (rack reuse) - testSerializer(new NodeName("/rackname654.myorg.com/hostname765.myorg.com"), - "/rack1/host2", anonymizingSerializer); - // test rack,hostname (rack & hostname reuse) - testSerializer(new NodeName("rackname.myorg.com", "hostname.myorg.com"), - "/rack0/host0", anonymizingSerializer); - // test rack,hostname (rack reuse) - testSerializer(new NodeName("rackname.myorg.com", "hostname543.myorg.com"), - "/rack0/host3", anonymizingSerializer); - // test rack,hostname (hostname reuse) - testSerializer(new NodeName("rackname987.myorg.com", "hostname.myorg.com"), - "/rack2/host0", anonymizingSerializer); - // test rack,hostname (rack reuse) - testSerializer(new NodeName("rackname.myorg.com", "hostname654.myorg.com"), - "/rack0/host4", anonymizingSerializer); - // test rack,hostname (host reuse) - testSerializer(new NodeName("rackname876.myorg.com", "hostname.myorg.com"), - "/rack3/host0", anonymizingSerializer); - // test rack,hostname (rack & hostname reuse) - testSerializer(new NodeName("rackname987.myorg.com", - "hostname543.myorg.com"), - "/rack2/host3", anonymizingSerializer); - // test -,hostname (hostname reuse) - testSerializer(new NodeName(null, "hostname.myorg.com"), - "host0", anonymizingSerializer); - // test -,hostname - testSerializer(new NodeName(null, "hostname15.myorg.com"), - "host5", anonymizingSerializer); - // test rack,- (rack reuse) - testSerializer(new NodeName("rackname987.myorg.com", null), - "rack2", anonymizingSerializer); - // test rack,- - testSerializer(new NodeName("rackname15.myorg.com", null), - "rack4", anonymizingSerializer); - } - - /** - * Test {@link JobProperties}. - */ - @Test - public void testJobPropertiesDataType() throws IOException { - // test job properties - Properties properties = new Properties(); - JobProperties jp = new JobProperties(properties); - - // test empty job properties - assertEquals("Job Properties (default) store error", - 0, jp.getValue().size()); - // test by adding some data - properties.put("test-key", "test-value"); // user config - properties.put(MRJobConfig.USER_NAME, "bob"); // job config - properties.put(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx1G"); // deprecated - jp = new JobProperties(properties); - assertEquals("Job Properties (default) store error", - 3, jp.getValue().size()); - assertEquals("Job Properties (default) key#1 error", - "test-value", jp.getValue().get("test-key")); - assertEquals("Job Properties (default) key#2 error", - "bob", jp.getValue().get(MRJobConfig.USER_NAME)); - assertEquals("Job Properties (default) key#3 error", - "-Xmx1G", jp.getValue().get(JobConf.MAPRED_TASK_JAVA_OPTS)); - } - - /** - * Test {@link JobProperties} serialization. - */ - @Test - public void testJobPropertiesSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - // test job properties - Properties properties = new Properties(); - properties.put("test-key", "test-value"); // user config - properties.put(MRJobConfig.USER_NAME, "bob"); // job config - properties.put(JobConf.MAPRED_TASK_JAVA_OPTS, "-Xmx1G"); // deprecated - JobProperties jp = new JobProperties(properties); - - testSerializer(jp, "{test-key:test-value," - + "mapreduce.job.user.name:bob," - + "mapred.child.java.opts:-Xmx1G}", defaultSerializer); - } - - /** - * Test {@link JobProperties} anonymization. - */ - @Test - public void testJobPropertiesAnonymization() throws IOException { - // test job properties - Properties properties = new Properties(); - Configuration conf = new Configuration(); - - properties.put("test-key", "test-value"); // user config - properties.put(MRJobConfig.USER_NAME, "bob"); // job config - // deprecated - properties.put("mapred.map.child.java.opts", - "-Xmx2G -Xms500m -Dsecret=secret"); - // deprecated and not supported - properties.put(JobConf.MAPRED_TASK_JAVA_OPTS, - "-Xmx1G -Xms200m -Dsecret=secret"); - JobProperties jp = new JobProperties(properties); - - // define a module - SimpleModule module = new SimpleModule("Test Anonymization Serializer", - new Version(0, 0, 0, "TEST")); - // add various serializers to the module - module.addSerializer(DataType.class, new DefaultRumenSerializer()); - module.addSerializer(AnonymizableDataType.class, - new DefaultAnonymizingRumenSerializer(new StatePool(), - conf)); - - //TODO Support deprecated and un-supported keys - testSerializer(jp, "{mapreduce.job.user.name:user0," - + "mapred.map.child.java.opts:-Xmx2G -Xms500m}", module); - } - - /** - * Test {@link ClassName}, serialization and anonymization. - */ - @Test - public void testClassNameSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - - // test classname - ClassName cName = new ClassName(TestRumenAnonymization.class.getName()); - assertEquals("Classname error!", TestRumenAnonymization.class.getName(), - cName.getValue()); - - // test classname serialization - // test with no anonymization - // test class1 - testSerializer(new ClassName("org.apache.hadoop.Test"), - "org.apache.hadoop.Test", defaultSerializer); - // test class2 - testSerializer(new ClassName("org.apache.hadoop.Test2"), - "org.apache.hadoop.Test2", defaultSerializer); - - // test class-name serialization - // test with anonymization - // test class1 - testSerializer(new ClassName("org.apache.hadoop.Test1"), - "class0", anonymizingSerializer); - // test class2 - testSerializer(new ClassName("org.apache.hadoop.Test2"), - "class1", anonymizingSerializer); - - // test classnames with preserves - Configuration conf = new Configuration(); - conf.set(ClassName.CLASSNAME_PRESERVE_CONFIG, "org.apache.hadoop."); - anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), conf); - // test word with prefix - testSerializer(new ClassName("org.apache.hadoop.Test3"), - "org.apache.hadoop.Test3", anonymizingSerializer); - // test word without prefix - testSerializer(new ClassName("org.apache.hadoop2.Test4"), - "class0", anonymizingSerializer); - } - - /** - * Test {@link FileName}. - */ - @Test - public void testFileName() throws IOException { - // test file on hdfs - FileName hFile = new FileName("hdfs://testnn:123/user/test.json"); - assertEquals("Filename error!", "hdfs://testnn:123/user/test.json", - hFile.getValue()); - // test file on local-fs - hFile = new FileName("file:///user/test.json"); - assertEquals("Filename error!", "file:///user/test.json", - hFile.getValue()); - // test dir on hdfs - hFile = new FileName("hdfs://testnn:123/user/"); - assertEquals("Filename error!", "hdfs://testnn:123/user/", - hFile.getValue()); - // test dir on local-fs - hFile = new FileName("file:///user/"); - assertEquals("Filename error!", "file:///user/", hFile.getValue()); - // test absolute file - hFile = new FileName("/user/test/test.json"); - assertEquals("Filename error!", "/user/test/test.json", hFile.getValue()); - // test absolute directory - hFile = new FileName("/user/test/"); - assertEquals("Filename error!", "/user/test/", hFile.getValue()); - // test relative file - hFile = new FileName("user/test/test2.json"); - assertEquals("Filename error!", "user/test/test2.json", hFile.getValue()); - // test relative directory - hFile = new FileName("user/test/"); - assertEquals("Filename error!", "user/test/", hFile.getValue()); - // test absolute file - hFile = new FileName("user"); - assertEquals("Filename error!", "user", hFile.getValue()); - // test absolute directory - hFile = new FileName("user/"); - assertEquals("Filename error!", "user/", hFile.getValue()); - hFile = new FileName("./tmp"); - assertEquals("Filename error!","./tmp", hFile.getValue()); - hFile = new FileName("./tmp/"); - assertEquals("Filename error!","./tmp/", hFile.getValue()); - hFile = new FileName("../tmp"); - assertEquals("Filename error!","../tmp", hFile.getValue()); - hFile = new FileName("../tmp/"); - assertEquals("Filename error!","../tmp/", hFile.getValue()); - - // test comma separated filenames - // test hdfs filenames, absolute and local-fs filenames - hFile = new FileName("hdfs://testnn:123/user/test1," - + "file:///user/test2,/user/test3"); - assertEquals("Filename error!", - "hdfs://testnn:123/user/test1,file:///user/test2,/user/test3", - hFile.getValue()); - } - - /** - * Test {@link FileName} serialization. - */ - @Test - public void testFileNameSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - // test filename serialization - // test with no anonymization - // test a file on hdfs - testSerializer(new FileName("hdfs://mynn:123/home/user/test.json"), - "hdfs://mynn:123/home/user/test.json", defaultSerializer); - // test a file on local-fs - testSerializer(new FileName("file:///home/user/test.json"), - "file:///home/user/test.json", defaultSerializer); - // test directory on hdfs - testSerializer(new FileName("hdfs://mynn:123/home/user/"), - "hdfs://mynn:123/home/user/", defaultSerializer); - // test directory on local fs - testSerializer(new FileName("file:///home/user/"), - "file:///home/user/", defaultSerializer); - // test absolute file - testSerializer(new FileName("/home/user/test.json"), - "/home/user/test.json", defaultSerializer); - // test relative file - testSerializer(new FileName("home/user/test.json"), - "home/user/test.json", defaultSerializer); - // test absolute folder - testSerializer(new FileName("/home/user/"), "/home/user/", - defaultSerializer); - // test relative folder - testSerializer(new FileName("home/user/"), "home/user/", - defaultSerializer); - // relative file - testSerializer(new FileName("home"), "home", defaultSerializer); - // relative folder - testSerializer(new FileName("home/"), "home/", defaultSerializer); - // absolute file - testSerializer(new FileName("/home"), "/home", defaultSerializer); - // absolute folder - testSerializer(new FileName("/home/"), "/home/", defaultSerializer); - // relative folder - testSerializer(new FileName("./tmp"), "./tmp", defaultSerializer); - testSerializer(new FileName("./tmp/"), "./tmp/", defaultSerializer); - testSerializer(new FileName("../tmp"), "../tmp", defaultSerializer); - - // test comma separated filenames - // test hdfs filenames, absolute and local-fs filenames - FileName fileName = - new FileName("hdfs://testnn:123/user/test1,file:///user/test2," - + "/user/test3"); - testSerializer(fileName, - "hdfs://testnn:123/user/test1,file:///user/test2,/user/test3", - defaultSerializer); - } - - /** - * Test {@link FileName} anonymization. - */ - @Test - public void testFileNameAnonymization() throws IOException { - JsonSerializer anonymizingSerializer = - new DefaultAnonymizingRumenSerializer(new StatePool(), - new Configuration()); - - // test filename serialization - // test with no anonymization - // test hdfs file - testSerializer(new FileName("hdfs://mynn:123/home/user/bob/test.json"), - "hdfs://host0/home/user/dir0/test.json", anonymizingSerializer); - // test local-fs file - testSerializer(new FileName("file:///home/user/alice/test.jar"), - "file:///home/user/dir1/test.jar", anonymizingSerializer); - // test hdfs dir - testSerializer(new FileName("hdfs://mynn:123/home/user/"), - "hdfs://host0/home/user/", anonymizingSerializer); - // test local-fs dir - testSerializer(new FileName("file:///home/user/secret/more-secret/"), - "file:///home/user/dir2/dir3/", anonymizingSerializer); - // test absolute filenames - testSerializer(new FileName("/home/user/top-secret.txt"), - "/home/user/file0.txt", anonymizingSerializer); - // test relative filenames - testSerializer(new FileName("home/user/top-top-secret.zip"), - "home/user/file1.zip", anonymizingSerializer); - // test absolute dirnames - testSerializer(new FileName("/home/user/project1/"), - "/home/user/dir4/", anonymizingSerializer); - // test relative filenames - testSerializer(new FileName("home/user/project1"), - "home/user/file2", anonymizingSerializer); - // test absolute dirnames (re-use) - testSerializer(new FileName("more-secret/"), - "dir3/", anonymizingSerializer); - // test relative filenames (re-use) - testSerializer(new FileName("secret/project1"), - "dir2/file2", anonymizingSerializer); - // test absolute filenames (re-use) - testSerializer(new FileName("/top-secret.txt"), - "/file0.txt", anonymizingSerializer); - // test relative filenames (re-use) - testSerializer(new FileName("top-top-secret.tar"), - "file1.tar", anonymizingSerializer); - // test absolute dirname - testSerializer(new FileName("sensitive-projectname/"), - "dir5/", anonymizingSerializer); - // test relative filenames - testSerializer(new FileName("/real-sensitive-projectname/"), - "/dir6/", anonymizingSerializer); - // test absolute filenames - testSerializer(new FileName("/usernames.xml"), - "/file3.xml", anonymizingSerializer); - // test relative filenames - testSerializer(new FileName("passwords.zip"), - "file4.zip", anonymizingSerializer); - // test relative filenames - testSerializer(new FileName("./tmp"), - "./tmp", anonymizingSerializer); - testSerializer(new FileName("./tmp/"), - "./tmp/", anonymizingSerializer); - testSerializer(new FileName("../tmp"), - "../tmp", anonymizingSerializer); - testSerializer(new FileName("../tmp/"), - "../tmp/", anonymizingSerializer); - - // test comma separated filenames - // test hdfs filenames, absolute and local-fs filenames - FileName fileName = - new FileName("hdfs://mynn:123/home/user/bob/test.json," - + "file:///home/user/bob/test.json,/user/alice/test.json"); - testSerializer(fileName, - "hdfs://host0/home/user/dir0/test.json,file:///home/user/dir0/test.json" - + ",/user/dir1/test.json", - anonymizingSerializer); - } - - - /** - * Test {@link DefaultDataType} serialization. - */ - @Test - public void testDefaultDataTypeSerialization() throws IOException { - JsonSerializer defaultSerializer = new DefaultRumenSerializer(); - - // test default data-type - DefaultDataType dt = new DefaultDataType("test"); - assertEquals("DefaultDataType error!", "test", dt.getValue()); - - // test default data-type - // test with no anonymization - // test data - testSerializer(new DefaultDataType("test"), "test", defaultSerializer); - } - - // A faked OutputStream which stores the stream content into a StringBuffer. - private static class MyOutputStream extends OutputStream { - private StringBuffer data = new StringBuffer(); - - @Override - public void write(int b) throws IOException { - data.append((char)b); - } - - @Override - public void write(byte[] b) throws IOException { - data.append(b); - } - - @Override - public String toString() { - // remove all the '"' for ease of testing - return data.toString().trim().replaceAll("\"", ""); - } - } - - // tests the object serializing using the class of the specified object - @SuppressWarnings("unchecked") - private static void testSerializer(Object toBeSerialized, String expData, - JsonSerializer serializer) - throws IOException { - // define a module - SimpleModule module = new SimpleModule("Test Anonymization Serializer", - new Version(0, 0, 0, "TEST")); - // add various serializers to the module - module.addSerializer(toBeSerialized.getClass(), serializer); - testSerializer(toBeSerialized, expData, module); - } - - // tests the object serializing using the specified class - private static void testSerializer(Object toBeSerialized, String expData, - SimpleModule module) - throws IOException { - // define a custom generator - ObjectMapper outMapper = new ObjectMapper(); - - // register the module - outMapper.registerModule(module); - - // get the json factory - JsonFactory outFactory = outMapper.getJsonFactory(); - // define a fake output stream which will cache the data - MyOutputStream output = new MyOutputStream(); - // define the json output generator - JsonGenerator outGen = - outFactory.createJsonGenerator(output, JsonEncoding.UTF8); - - // serialize the object - outGen.writeObject(toBeSerialized); - //serializer.serialize(toBeSerialized, outGen, null); - - // close the json generator so that it flushes out the data to the output - // stream - outGen.close(); - - assertEquals("Serialization failed!", expData, output.toString()); - } - - /** - * Test {@link DefaultRumenSerializer}. - */ - @Test - public void testDefaultDataSerializers() throws Exception { - JsonSerializer defaultSer = new DefaultRumenSerializer(); - // test default data-type - // test with no anonymization - // test data - testSerializer(new DefaultDataType("test"), "test", defaultSer); - } - - @Test - public void testBlockingDataSerializers() throws Exception { - JsonSerializer blockingSerializer = new BlockingSerializer(); - - // test string serializer - testSerializer("username:password", "null", blockingSerializer); - } - - @Test - public void testObjectStringDataSerializers() throws Exception { - JsonSerializer objectStringSerializer = new ObjectStringSerializer(); - // test job/task/attempt id serializer - // test job-id - JobID jid = JobID.forName("job_1_1"); - testSerializer(jid, jid.toString(), objectStringSerializer); - // test task-id - TaskID tid = new TaskID(jid, TaskType.MAP, 1); - testSerializer(tid, tid.toString(), objectStringSerializer); - // test attempt-id - TaskAttemptID aid = new TaskAttemptID(tid, 0); - testSerializer(aid, aid.toString(), objectStringSerializer); - } - - // test anonymizer - @Test - public void testRumenAnonymization() throws Exception { - Configuration conf = new Configuration(); - - // Run a MR job - // create a MR cluster - conf.setInt(TTConfig.TT_MAP_SLOTS, 1); - conf.setInt(TTConfig.TT_REDUCE_SLOTS, 1); - - MiniDFSCluster dfsCluster = null; - MiniMRCluster mrCluster = null; - - // local filesystem for running TraceBuilder - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testRumenAnonymization"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - try { - dfsCluster = new MiniDFSCluster(conf, 1, true, null); - String[] racks = new String[] {"/rack123.myorg.com", - "/rack456.myorg.com"}; - String[] hosts = new String[] {"host1230.myorg.com", - "host4560.myorg.com"}; - mrCluster = - new MiniMRCluster(2, dfsCluster.getFileSystem().getUri().toString(), - 1, racks, hosts, new JobConf(conf)); - - // run a job - Path inDir = new Path("secret-input"); - Path outDir = new Path("secret-output"); - - JobConf jConf = mrCluster.createJobConf(); - // add some usr sensitive data in the job conf - jConf.set("user-secret-code", "abracadabra"); - - jConf.setJobName("top-secret"); - // construct a job with 1 map and 1 reduce task. - Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 2, 2); - // wait for the job to complete - job.waitForCompletion(false); - - assertTrue("Job failed", job.isSuccessful()); - - JobID id = job.getJobID(); - Cluster cluster = new Cluster(jConf); - String user = cluster.getAllJobStatuses()[0].getUsername(); - - // get the jobhistory filepath - Path jhPath = - new Path(mrCluster.getJobTrackerRunner().getJobTracker() - .getJobHistoryDir()); - Path inputLogPath = JobHistory.getJobHistoryFile(jhPath, id, user); - Path inputConfPath = JobHistory.getConfFile(jhPath, id); - // wait for 10 secs for the jobhistory file to move into the done folder - FileSystem fs = inputLogPath.getFileSystem(jConf); - for (int i = 0; i < 100; ++i) { - if (fs.exists(inputLogPath)) { - break; - } - TimeUnit.MILLISECONDS.wait(100); - } - - assertTrue("Missing job history file", fs.exists(inputLogPath)); - - // run trace builder on the job history logs - Path goldTraceFilename = new Path(tempDir, "trace.json"); - Path goldTopologyFilename = new Path(tempDir, "topology.json"); - - // build the trace-builder command line args - String[] args = new String[] {goldTraceFilename.toString(), - goldTopologyFilename.toString(), - inputLogPath.toString(), - inputConfPath.toString()}; - Tool analyzer = new TraceBuilder(); - int result = ToolRunner.run(analyzer, args); - assertEquals("Non-zero exit", 0, result); - - // anonymize the job trace - Path anonymizedTraceFilename = new Path(tempDir, "trace-anonymized.json"); - Path anonymizedClusterTopologyFilename = - new Path(tempDir, "topology-anonymized.json"); - args = new String[] {"-trace", goldTraceFilename.toString(), - anonymizedTraceFilename.toString(), - "-topology", goldTopologyFilename.toString(), - anonymizedClusterTopologyFilename.toString()}; - Tool anonymizer = new Anonymizer(); - result = ToolRunner.run(anonymizer, args); - assertEquals("Non-zero exit", 0, result); - - JobTraceReader reader = new JobTraceReader(anonymizedTraceFilename, conf); - LoggedJob anonymizedJob = reader.getNext(); - reader.close(); // close the reader as we need only 1 job - // test - // user-name - String currentUser = UserGroupInformation.getCurrentUser().getUserName(); - assertFalse("Username not anonymized!", - currentUser.equals(anonymizedJob.getUser().getValue())); - // jobid - assertEquals("JobID mismatch!", - id.toString(), anonymizedJob.getJobID().toString()); - // queue-name - assertFalse("Queuename mismatch!", - "default".equals(anonymizedJob.getQueue().getValue())); - // job-name - assertFalse("Jobname mismatch!", - "top-secret".equals(anonymizedJob.getJobName().getValue())); - - // job properties - for (Map.Entry entry : - anonymizedJob.getJobProperties().getValue().entrySet()) { - assertFalse("User sensitive configuration key not anonymized", - entry.getKey().toString().equals("user-secret-code")); - assertFalse("User sensitive data not anonymized", - entry.getValue().toString().contains(currentUser)); - assertFalse("User sensitive data not anonymized", - entry.getValue().toString().contains("secret")); - } - - // test map tasks - testTasks(anonymizedJob.getMapTasks(), id, TaskType.MAP); - - // test reduce tasks - testTasks(anonymizedJob.getReduceTasks(), id, TaskType.REDUCE); - - // test other tasks - testTasks(anonymizedJob.getOtherTasks(), id, null); - - // test the anonymized cluster topology file - ClusterTopologyReader cReader = - new ClusterTopologyReader(anonymizedClusterTopologyFilename, conf); - LoggedNetworkTopology loggedNetworkTopology = cReader.get(); - // test the cluster topology - testClusterTopology(loggedNetworkTopology, 0, "myorg"); - } finally { - // shutdown and cleanup - if (mrCluster != null) { - mrCluster.shutdown(); - } - - if (dfsCluster != null) { - dfsCluster.formatDataNodeDirs(); - dfsCluster.shutdown(); - } - lfs.delete(tempDir, true); - } - } - - // test task level details lije - // - taskid - // - locality info - // - attempt details - // - attempt execution hostname - private static void testTasks(List tasks, JobID id, - TaskType type) { - int index = 0; - for (LoggedTask task : tasks) { - // generate the expected task id for this task - if (type != null) { - TaskID tid = new TaskID(id, type, index++); - assertEquals("TaskID mismatch!", - tid.toString(), task.getTaskID().toString()); - } - - // check locality information - if (task.getPreferredLocations() != null) { - for (LoggedLocation loc : task.getPreferredLocations()) { - for (NodeName name : loc.getLayers()) { - assertFalse("Hostname mismatch!", - name.getValue().contains("myorg")); - } - } - } - - // check execution host - for (LoggedTaskAttempt attempt : task.getAttempts()) { - // generate the expected task id for this task - TaskAttemptID aid = new TaskAttemptID(task.getTaskID(), 0); - assertEquals("TaskAttemptID mismatch!", - aid.toString(), attempt.getAttemptID().toString()); - - assertNotNull("Hostname null!", attempt.getHostName()); - assertFalse("Hostname mismatch!", - attempt.getHostName().getValue().contains("myorg")); - } - } - } - - // tests the logged network topology - private static void testClusterTopology(LoggedNetworkTopology topology, - int level, String bannedString) { - assertFalse("Cluster topology test failed!", - topology.getName().getValue().contains(bannedString)); - if (level == 0) { - assertEquals("Level-1 data mismatch!", - "", topology.getName().getValue()); - } else if (level == 1) { - assertTrue("Level-2 data mismatch!", - topology.getName().getValue().contains("rack")); - assertFalse("Level-2 data mismatch!", - topology.getName().getValue().contains("host")); - } else { - assertTrue("Level-2 data mismatch!", - topology.getName().getValue().contains("host")); - assertFalse("Level-2 data mismatch!", - topology.getName().getValue().contains("rack")); - } - - // if the current node is a rack, then test the nodes under it - if (topology.getChildren() != null) { - for (LoggedNetworkTopology child : topology.getChildren()) { - testClusterTopology(child, level + 1, bannedString); - } - } - } - - @Test - public void testCLI() throws Exception { - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testCLI"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // test no args - String[] args = new String[] {}; - testAnonymizerCLI(args, -1); - - // test with wrong args - args = new String[] {"-test"}; - testAnonymizerCLI(args, -1); - - args = new String[] {"-trace"}; - testAnonymizerCLI(args, -1); - - args = new String[] {"-topology"}; - testAnonymizerCLI(args, -1); - - args = new String[] {"-trace -topology"}; - testAnonymizerCLI(args, -1); - - Path testTraceInputFilename = new Path(tempDir, "trace-in.json"); - args = new String[] {"-trace", testTraceInputFilename.toString()}; - testAnonymizerCLI(args, -1); - - Path testTraceOutputFilename = new Path(tempDir, "trace-out.json"); - args = new String[] {"-trace", testTraceInputFilename.toString(), - testTraceOutputFilename.toString()}; - testAnonymizerCLI(args, -1); - - OutputStream out = lfs.create(testTraceInputFilename); - out.write("{\n}".getBytes()); - out.close(); - args = new String[] {"-trace", testTraceInputFilename.toString(), - testTraceOutputFilename.toString()}; - testAnonymizerCLI(args, 0); - - Path testToplogyInputFilename = new Path(tempDir, "topology-in.json"); - args = new String[] {"-topology", testToplogyInputFilename.toString()}; - testAnonymizerCLI(args, -1); - - Path testTopologyOutputFilename = new Path(tempDir, "topology-out.json"); - args = new String[] {"-topology", testToplogyInputFilename.toString(), - testTopologyOutputFilename.toString()}; - testAnonymizerCLI(args, -1); - - out = lfs.create(testToplogyInputFilename); - out.write("{\n}".getBytes()); - out.close(); - args = new String[] {"-topology", testToplogyInputFilename.toString(), - testTopologyOutputFilename.toString()}; - testAnonymizerCLI(args, 0); - - args = new String[] {"-trace", testTraceInputFilename.toString(), - "-topology", testToplogyInputFilename.toString()}; - testAnonymizerCLI(args, -1); - - args = new String[] {"-trace", testTraceInputFilename.toString(), - testTraceOutputFilename.toString(), - "-topology", testToplogyInputFilename.toString(), - testTopologyOutputFilename.toString()}; - testAnonymizerCLI(args, 0); - } - - // tests the Anonymizer CLI via the Tools interface - private static void testAnonymizerCLI(String[] args, int eExitCode) - throws Exception { - Anonymizer anonymizer = new Anonymizer(); - - int exitCode = ToolRunner.run(anonymizer, args); - assertEquals("Exit code mismatch", eExitCode, exitCode); - } - - /** - * Test {@link StatePool}'s reload and persistence feature. - */ - @Test - public void testStatePool() throws Exception { - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testStatePool"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the state dir - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - - StatePool pool = new StatePool(); - // test reload, persist and dir config - // test with no reload and persist - pool.initialize(conf); - - // test with reload and/or persist enabled with no dir - assertNull("Default state pool error", - pool.getState(MyState.class)); - - // try persisting - pool.persist(); - assertFalse("State pool persisted when disabled", lfs.exists(tempDir)); - - // test wrongly configured state-pool - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - conf.unset(StatePool.DIR_CONFIG); - pool = new StatePool(); - boolean success = true; - try { - pool.initialize(conf); - } catch (Exception e) { - success = false; - } - assertFalse("State pool bad configuration succeeded", success); - - // test wrongly configured state-pool - conf.setBoolean(StatePool.RELOAD_CONFIG, false); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - pool = new StatePool(); - success = true; - try { - pool.initialize(conf); - } catch (Exception e) { - success = false; - } - assertFalse("State manager bad configuration succeeded", success); - - - // test persistence - conf.setBoolean(StatePool.RELOAD_CONFIG, false); - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - pool = new StatePool(); - pool.initialize(conf); - - // add states to the state pool - MyState myState = new MyState(); - pool.addState(MyState.class, myState); - myState.setState("test-1"); - // try persisting - pool.persist(); - assertTrue("State pool persisted when enabled", lfs.exists(tempDir)); - assertEquals("State pool persisted when enabled", - 1, lfs.listStatus(tempDir).length); - - // reload - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - pool = new StatePool(); - pool.initialize(conf); - MyState pState = - (MyState) pool.getState(MyState.class); - assertEquals("State pool persistence/reload failed", "test-1", - pState.getState()); - - // try persisting with no state change - pool.persist(); - assertEquals("State pool persisted when disabled", - 1, lfs.listStatus(tempDir).length); - - // modify the state of the pool and check persistence - pState.setUpdated(true); - pool.persist(); - assertEquals("State pool persisted when disabled", - 2, lfs.listStatus(tempDir).length); - - // delete the temp directory if everything goes fine - lfs.delete(tempDir, true); - } - - /** - * Test state. - */ - static class MyState implements State { - private boolean updated = false; - private String state = "null"; - - @Override - @JsonIgnore - public String getName() { - return "test"; - } - - @Override - public void setName(String name) { - // for now, simply assert since this class has a hardcoded name - if (!getName().equals(name)) { - throw new RuntimeException("State name mismatch! Expected '" - + getName() + "' but found '" + name + "'."); - } - } - - public void setState(String state) { - this.state = state; - } - - public String getState() { - return state; - } - - void setUpdated(boolean up) { - this.updated = up; - } - - @Override - @JsonIgnore - public boolean isUpdated() { - return updated; - } - } - - @SuppressWarnings("unchecked") - private static String getValueFromDataType(Object object) { - DataType dt = (DataType) object; - return dt.getValue(); - } - - @Test - public void testJobPropertiesParser() { - // test default parser - Properties properties = new Properties(); - Configuration conf = new Configuration(); - JobProperties jp = new JobProperties(properties); - assertEquals("Job Properties (default filter) store error", - 0, jp.getAnonymizedValue(null, conf).size()); - - // define key-value pairs for job configuration - String key1 = "test-key"; - String value1 = "test-value"; - properties.put(key1, value1); // user config - String key2 = MRJobConfig.USER_NAME; - String value2 = "bob"; - properties.put(key2, value2); // job config - String key3 = JobConf.MAPRED_MAP_TASK_JAVA_OPTS; - String value3 = "-Xmx1G"; - properties.put(key3, value3); // deprecated - String key4 = MRJobConfig.REDUCE_JAVA_OPTS; - String value4 = "-Xms100m"; - properties.put(key4, value4); - - jp = new JobProperties(properties); - - // Configure the default parser - conf.set(JobProperties.PARSERS_CONFIG_KEY, - DefaultJobPropertiesParser.class.getName()); - // anonymize - Properties defaultProp = jp.getAnonymizedValue(null, conf); - assertEquals("Job Properties (all-pass filter) store error", - 4, defaultProp.size()); - assertEquals("Job Properties (default filter) key#1 error", value1, - getValueFromDataType(defaultProp.get(key1))); - assertEquals("Job Properties (default filter) key#2 error", value2, - getValueFromDataType(defaultProp.get(key2))); - assertEquals("Job Properties (default filter) key#3 error", value3, - getValueFromDataType(defaultProp.get(key3))); - assertEquals("Job Properties (default filter) key#4 error", value4, - getValueFromDataType(defaultProp.get(key4))); - - // test MR parser - conf.set(JobProperties.PARSERS_CONFIG_KEY, - MapReduceJobPropertiesParser.class.getName()); - // anonymize - Properties filteredProp = jp.getAnonymizedValue(null, conf); - assertEquals("Job Properties (MR filter) store error", - 3, filteredProp.size()); - assertNull("Job Properties (MR filter) key#1 error", - filteredProp.get(key1)); - assertEquals("Job Properties (MR filter) key#2 error", value2, - getValueFromDataType(filteredProp.get(key2))); - assertEquals("Job Properties (MR filter) key#3 error", value3, - getValueFromDataType(filteredProp.get(key3))); - assertEquals("Job Properties (MR filter) key#4 error", value4, - getValueFromDataType(filteredProp.get(key4))); - } - - /** - * Test {@link WordListAnonymizerUtility}. Test various features like - * - test known words - * - test known suffix - */ - @Test - public void testWordListBasedAnonymizer() { - String[] knownSuffixes = new String[] {".1", ".2", ".3", ".4"}; - - // test with valid suffix - assertTrue("suffix test#0 failed!", - WordListAnonymizerUtility.hasSuffix("a.1", knownSuffixes)); - String split[] = - WordListAnonymizerUtility.extractSuffix("a.1", knownSuffixes); - assertEquals("suffix test#1 failed!", 2, split.length); - assertEquals("suffix test#2 failed!", "a", split[0]); - assertEquals("suffix test#3 failed!", ".1", split[1]); - - // test with valid suffix - assertTrue("suffix test#0 failed!", - WordListAnonymizerUtility.hasSuffix("a.1", knownSuffixes)); - split = - WordListAnonymizerUtility.extractSuffix("/a/b.2", knownSuffixes); - assertEquals("suffix test#0 failed!", 2, split.length); - assertEquals("suffix test#1 failed!", "/a/b", split[0]); - assertEquals("suffix test#2 failed!", ".2", split[1]); - - // test with invalid suffix - assertFalse("suffix test#0 failed!", - WordListAnonymizerUtility.hasSuffix("a.b", knownSuffixes)); - - boolean failed = false; - try { - split = WordListAnonymizerUtility.extractSuffix("a.b", knownSuffixes); - } catch (Exception e) { - failed = true; - } - assertTrue("Exception expected!", failed); - - String[] knownWords = new String[] {"a", "b"}; - - // test with valid data - assertTrue("data test#0 failed!", - WordListAnonymizerUtility.isKnownData("a", knownWords)); - // test with valid data - assertTrue("data test#1 failed!", - WordListAnonymizerUtility.isKnownData("b", knownWords)); - // test with invalid data - assertFalse("data test#2 failed!", - WordListAnonymizerUtility.isKnownData("c", knownWords)); - - // test with valid known word - assertTrue("data test#3 failed!", - WordListAnonymizerUtility.isKnownData("job")); - // test with invalid known word - assertFalse("data test#4 failed!", - WordListAnonymizerUtility.isKnownData("bob")); - - // test numeric data - assertFalse("Numeric test failed!", - WordListAnonymizerUtility.needsAnonymization("123")); - // test numeric data (unsupported) - assertTrue("Numeric test failed!", - WordListAnonymizerUtility.needsAnonymization("123.456")); - // test text data - assertTrue("Text test failed!", - WordListAnonymizerUtility.needsAnonymization("123abc")); - } - - /** - * Test {@link WordList} features like - * - add words - * - index - * - contains - */ - @Test - public void testWordList() throws Exception { - // test features with fresh state - WordList wordList = new WordList(); - assertFalse("Word list state incorrect", wordList.isUpdated()); - - // add some special word - String test = "abbracadabra"; - wordList.add(test); - assertTrue("Word list failed to store", wordList.contains(test)); - assertEquals("Word list index failed", 0, wordList.indexOf(test)); - assertEquals("Word list size failed", 1, wordList.getSize()); - assertTrue("Word list state incorrect", wordList.isUpdated()); - - // add already added word - wordList.add(test); - assertEquals("Word list index failed", 0, wordList.indexOf(test)); - assertEquals("Word list size failed", 1, wordList.getSize()); - assertTrue("Word list state incorrect", wordList.isUpdated()); - - String test2 = "hakuna-matata"; - wordList.add(test2); - assertTrue("Word list failed to store", wordList.contains(test2)); - assertEquals("Word list index failed", 1, wordList.indexOf(test2)); - assertEquals("Word list size failed", 2, wordList.getSize()); - assertTrue("Word list state incorrect", wordList.isUpdated()); - - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testWordList"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // define a state pool to help persist the wordlist - StatePool pool = new StatePool(); - - try { - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), wordList); - - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", WordList.class, state.getClass()); - WordList pList = (WordList) state; - - // check size - assertEquals("Word list size on reload failed", 2, pList.getSize()); - assertFalse("Word list state incorrect", pList.isUpdated()); - - // add already added word - pList.add(test); - assertEquals("Word list index on reload failed", 0, pList.indexOf(test)); - assertEquals("Word list size on reload failed", 2, pList.getSize()); - assertFalse("Word list state on reload incorrect", pList.isUpdated()); - - String test3 = "disco-dancer"; - assertFalse("Word list failed to after reload", pList.contains(test3)); - pList.add(test3); - assertTrue("Word list failed to store on reload", pList.contains(test3)); - assertEquals("Word list index on reload failed", 2, pList.indexOf(test3)); - assertEquals("Word list size on reload failed", 3, pList.getSize()); - assertTrue("Word list state on reload incorrect", pList.isUpdated()); - - // test previously added (persisted) word - assertTrue("Word list failed to store on reload", pList.contains(test2)); - assertEquals("Word list index on reload failed", 1, pList.indexOf(test2)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link FileName#FileNameState} persistence with directories only. - */ - @Test - public void testFileNameStateWithDir() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testFileNameStateWithDir"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the dirs - StatePool pool = new StatePool(); - - FileNameState fState = new FileNameState(); - - // define the directory names - String test1 = "test"; - String test2 = "home"; - - // test dir only - WordList dirState = new WordList("dir"); - dirState.add(test1); - dirState.add(test2); - - // set the directory state - fState.setDirectoryState(dirState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), fState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - FileNameState.class, state.getClass()); - FileNameState newFState = (FileNameState) state; - - // check the state contents - WordList newStateWordList = newFState.getDirectoryState(); - assertTrue("File state failed to store on reload", - newStateWordList.contains(test1)); - assertEquals("File state index on reload failed", - 0, newStateWordList.indexOf(test1)); - - assertTrue("File state failed to store on reload", - newStateWordList.contains(test2)); - assertEquals("File state index on reload failed", - 1, newStateWordList.indexOf(test2)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link FileName#FileNameState} persistence with files only. - */ - @Test - public void testFileNameStateWithFiles() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testFileNameStateWithFiles"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the filename parts - StatePool pool = new StatePool(); - - FileNameState fState = new FileNameState(); - - // define the file names - String test1 = "part-00.bzip"; - String test2 = "file1.txt"; - - // test filenames only - WordList fileNameState = new WordList("files"); - fileNameState.add(test1); - fileNameState.add(test2); - - // set the filename state - fState.setDirectoryState(fileNameState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), fState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - FileNameState.class, state.getClass()); - FileNameState newFState = (FileNameState) state; - - // check the state contents - WordList newFileWordList = newFState.getDirectoryState(); - assertTrue("File state failed on reload", - newFileWordList.contains(test1)); - assertEquals("File state indexing on reload failed", - 0, newFileWordList.indexOf(test1)); - - assertTrue("File state failed on reload", - newFileWordList.contains(test2)); - assertEquals("File state indexing on reload failed", - 1, newFileWordList.indexOf(test2)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link FileName#FileNameState} persistence with files and directories. - */ - @Test - public void testFileNameState() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testFileNameState"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the filename parts - StatePool pool = new StatePool(); - - FileNameState fState = new FileNameState(); - - // define the directory names - String testD1 = "test"; - String testD2 = "home"; - String testD3 = "tmp"; - - // test dir only - WordList dirState = new WordList("dir"); - dirState.add(testD1); - dirState.add(testD2); - dirState.add(testD3); - - // define the file names - String testF1 = "part-00.bzip"; - String testF2 = "file1.txt"; - String testF3 = "tmp"; - - // test filenames only - WordList fileNameState = new WordList("files"); - fileNameState.add(testF1); - fileNameState.add(testF2); - fileNameState.add(testF3); - - // set the filename state - fState.setFileNameState(fileNameState); - // set the directory state - fState.setDirectoryState(dirState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), fState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - FileNameState.class, state.getClass()); - FileNameState newFState = (FileNameState) state; - - // test filenames - WordList newStateWordList = newFState.getFileNameState(); - assertTrue("File state failed on reload", - newStateWordList.contains(testF1)); - assertEquals("File state indexing on reload failed", - 0, newStateWordList.indexOf(testF1)); - - assertTrue("File state failed on reload", - newStateWordList.contains(testF2)); - assertEquals("File state indexing on reload failed", - 1, newStateWordList.indexOf(testF2)); - - assertTrue("File state failed on reload", - newStateWordList.contains(testF3)); - assertEquals("File state indexing on reload failed", - 2, newStateWordList.indexOf(testF3)); - - // test dirs - WordList newDirWordList = newFState.getDirectoryState(); - assertTrue("File state failed on reload", - newDirWordList.contains(testD1)); - assertEquals("File state indexing on reload failed", - 0, newDirWordList.indexOf(testD1)); - - assertTrue("File state failed on reload", - newDirWordList.contains(testD2)); - assertEquals("File state indexing on reload failed", - 1, newDirWordList.indexOf(testD2)); - assertTrue("File state failed on reload", - newDirWordList.contains(testD3)); - assertEquals("File state indexing on reload failed", - 2, newDirWordList.indexOf(testD3)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link NodeName#NodeName} persistence with hostnames only. - */ - @Test - public void testNodeNameStateWithHostNameOnly() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testNodeNameStateWithHostNameOnly"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the hostnames - StatePool pool = new StatePool(); - - NodeNameState nState = new NodeNameState(); - - // define the host names - String test1 = "abc123"; - String test2 = "xyz789"; - - // test hostname only - WordList hostNameState = new WordList("hostname"); - hostNameState.add(test1); - hostNameState.add(test2); - - // set the directory state - nState.setHostNameState(hostNameState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), nState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - NodeNameState.class, state.getClass()); - NodeNameState newNState = (NodeNameState) state; - - // check the state contents - WordList newStateWordList = newNState.getHostNameState(); - assertTrue("Node state failed to store on reload", - newStateWordList.contains(test1)); - assertEquals("Node state index on reload failed", - 0, newStateWordList.indexOf(test1)); - - assertTrue("Node state failed to store on reload", - newStateWordList.contains(test2)); - assertEquals("Node state index on reload failed", - 1, newStateWordList.indexOf(test2)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link NodeName#NodeNameState} persistence with racknames only. - */ - @Test - public void testNodeNameWithRackNamesOnly() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testNodeNameWithRackNamesOnly"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the rack names - StatePool pool = new StatePool(); - - NodeNameState nState = new NodeNameState(); - - // define the rack names - String test1 = "rack1"; - String test2 = "rack2"; - - // test filenames only - WordList rackNameState = new WordList("racknames"); - rackNameState.add(test1); - rackNameState.add(test2); - - // set the rackname state - nState.setRackNameState(rackNameState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), nState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - NodeNameState.class, state.getClass()); - NodeNameState newNState = (NodeNameState) state; - - // check the state contents - WordList newFileWordList = newNState.getRackNameState(); - assertTrue("File state failed on reload", - newFileWordList.contains(test1)); - assertEquals("File state indexing on reload failed", - 0, newFileWordList.indexOf(test1)); - - assertTrue("File state failed on reload", - newFileWordList.contains(test2)); - assertEquals("File state indexing on reload failed", - 1, newFileWordList.indexOf(test2)); - } finally { - lfs.delete(tempDir, true); - } - } - - /** - * Test {@link NodeName#NodeNameState} persistence with hosts and racks. - */ - @Test - public void testNodeNameState() throws Exception { - // test persistence - Configuration conf = new Configuration(); - FileSystem lfs = FileSystem.getLocal(conf); - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - Path tempDir = new Path(rootTempDir, "testNodeNameState"); - tempDir = lfs.makeQualified(tempDir); - lfs.delete(tempDir, true); - - // set the persistence directory - conf.set(StatePool.DIR_CONFIG, tempDir.toString()); - conf.setBoolean(StatePool.PERSIST_CONFIG, true); - - // define a state pool to help persist the node names. - StatePool pool = new StatePool(); - - NodeNameState nState = new NodeNameState(); - - // define the rack names - String testR1 = "rack1"; - String testR2 = "rack2"; - String testR3 = "rack3"; - - WordList rackState = new WordList("rack"); - rackState.add(testR1); - rackState.add(testR2); - rackState.add(testR3); - - String testH1 = "host1"; - String testH2 = "host2"; - String testH3 = "host3"; - - WordList hostNameState = new WordList("host"); - hostNameState.add(testH1); - hostNameState.add(testH2); - hostNameState.add(testH3); - - // set the filename state - nState.setHostNameState(hostNameState); - nState.setRackNameState(rackState); - - try { - // initialize the state-pool - pool.initialize(conf); - - // add the wordlist to the pool - pool.addState(getClass(), nState); - - // persist the state - pool.persist(); - - // now clear the pool state - pool = new StatePool(); - - // set reload to true - conf.setBoolean(StatePool.RELOAD_CONFIG, true); - - // initialize the state-pool - pool.initialize(conf); - - State state = pool.getState(getClass()); - assertNotNull("Missing state!", state); - assertEquals("Incorrect state class!", - NodeNameState.class, state.getClass()); - NodeNameState newNState = (NodeNameState) state; - - // test nodenames - WordList newHostWordList = newNState.getHostNameState(); - assertTrue("File state failed on reload", - newHostWordList.contains(testH1)); - assertEquals("File state indexing on reload failed", - 0, newHostWordList.indexOf(testH1)); - - assertTrue("File state failed on reload", - newHostWordList.contains(testH2)); - assertEquals("File state indexing on reload failed", - 1, newHostWordList.indexOf(testH2)); - - assertTrue("File state failed on reload", - newHostWordList.contains(testH3)); - assertEquals("File state indexing on reload failed", - 2, newHostWordList.indexOf(testH3)); - - // test racknames - WordList newRackWordList = newNState.getRackNameState(); - assertTrue("File state failed on reload", - newRackWordList.contains(testR1)); - assertEquals("File state indexing on reload failed", - 0, newRackWordList.indexOf(testR1)); - - assertTrue("File state failed on reload", - newRackWordList.contains(testR2)); - assertEquals("File state indexing on reload failed", - 1, newRackWordList.indexOf(testR2)); - assertTrue("File state failed on reload", - newRackWordList.contains(testR3)); - assertEquals("File state indexing on reload failed", - 2, newRackWordList.indexOf(testR3)); - } finally { - lfs.delete(tempDir, true); - } - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenFolder.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenFolder.java deleted file mode 100644 index 2fe0d7194a..0000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenFolder.java +++ /dev/null @@ -1,196 +0,0 @@ -package org.apache.hadoop.tools.rumen; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.InputStream; -import java.util.LinkedList; -import java.util.List; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.ToolRunner; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestRumenFolder { - @Test - public void testFoldingSmallTrace() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - @SuppressWarnings("deprecation") - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")) - .makeQualified(lfs); - @SuppressWarnings("deprecation") - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(lfs); - - final Path rootInputFile = new Path(rootInputDir, "rumen/small-trace-test"); - final Path tempDir = new Path(rootTempDir, "TestRumenJobTraces"); - lfs.delete(tempDir, true); - - final Path foldedTracePath = new Path(tempDir, "folded-trace.json"); - - final Path inputFile = - new Path(rootInputFile, "folder-input-trace.json.gz"); - - System.out.println("folded trace result path = " + foldedTracePath); - - String[] args = - { "-input-cycle", "100S", "-output-duration", "300S", - "-skew-buffer-length", "1", "-seed", "100", "-concentration", "2", - inputFile.toString(), foldedTracePath.toString() }; - - final Path foldedGoldFile = - new Path(rootInputFile, "goldFoldedTrace.json.gz"); - - Folder folder = new Folder(); - int result = ToolRunner.run(folder, args); - assertEquals("Non-zero exit", 0, result); - - TestRumenFolder. jsonFileMatchesGold(conf, lfs, foldedTracePath, - foldedGoldFile, LoggedJob.class, "trace"); - } - - @Test - public void testStartsAfterOption() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - @SuppressWarnings("deprecation") - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")) - .makeQualified(lfs); - @SuppressWarnings("deprecation") - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(lfs); - - final Path rootInputFile = new Path(rootInputDir, "rumen/small-trace-test"); - final Path tempDir = new Path(rootTempDir, "TestRumenJobTraces"); - lfs.delete(tempDir, true); - - final Path inputFile = - new Path(rootInputFile, "goldFoldedTrace.json.gz"); - - final Path foldedTracePath = new Path(tempDir, - "folded-skippedjob-trace.json"); - String[] args = - { "-input-cycle", "300S", "-output-duration", "300S", - "-starts-after", "30S", - inputFile.toString(), foldedTracePath.toString() }; - - Folder folder = new Folder(); - int result = ToolRunner.run(folder, args); - assertEquals("Non-zero exit", 0, result); - - TestRumenFolder. checkValidityAfterSkippingJobs(conf, lfs, foldedTracePath, - inputFile, LoggedJob.class, "trace", 30000, 300000); - } - - static private void - checkValidityAfterSkippingJobs(Configuration conf, - FileSystem lfs, Path result, Path inputFile, - Class clazz, String fileDescription, - long startsAfter, long duration) throws IOException { - - JsonObjectMapperParser inputFileParser = - new JsonObjectMapperParser(inputFile, clazz, conf); - InputStream resultStream = lfs.open(result); - JsonObjectMapperParser resultParser = - new JsonObjectMapperParser(resultStream, clazz); - List gpSubmitTimes = new LinkedList(); - List rpSubmitTimes = new LinkedList(); - try { - //Get submitTime of first job - LoggedJob firstJob = (LoggedJob)inputFileParser.getNext(); - gpSubmitTimes.add(firstJob.getSubmitTime()); - long absoluteStartsAfterTime = firstJob.getSubmitTime() + startsAfter; - - //total duration - long endTime = firstJob.getSubmitTime() + duration; - - //read original trace - LoggedJob oriJob = null; - while((oriJob = (LoggedJob)inputFileParser.getNext()) != null) { - gpSubmitTimes.add(oriJob.getSubmitTime()); - } - - //check if retained jobs have submittime > starts-after - LoggedJob job = null; - while((job = (LoggedJob) resultParser.getNext()) != null) { - assertTrue("job's submit time in the output trace is less " + - "than the specified value of starts-after", - (job.getSubmitTime() >= absoluteStartsAfterTime)); - rpSubmitTimes.add(job.getSubmitTime()); - } - - List skippedJobs = new LinkedList(); - skippedJobs.addAll(gpSubmitTimes); - skippedJobs.removeAll(rpSubmitTimes); - - //check if the skipped job submittime < starts-after - for(Long submitTime : skippedJobs) { - assertTrue("skipped job submit time " + submitTime + - " in the trace is greater " + - "than the specified value of starts-after " - + absoluteStartsAfterTime, - (submitTime < absoluteStartsAfterTime)); - } - } finally { - IOUtils.cleanup(null, inputFileParser, resultParser); - } - } - - static private void jsonFileMatchesGold( - Configuration conf, FileSystem lfs, Path result, Path gold, - Class clazz, String fileDescription) throws IOException { - JsonObjectMapperParser goldParser = - new JsonObjectMapperParser(gold, clazz, conf); - InputStream resultStream = lfs.open(result); - JsonObjectMapperParser resultParser = - new JsonObjectMapperParser(resultStream, clazz); - try { - while (true) { - DeepCompare goldJob = goldParser.getNext(); - DeepCompare resultJob = resultParser.getNext(); - if ((goldJob == null) || (resultJob == null)) { - assertTrue(goldJob == resultJob); - break; - } - - try { - resultJob.deepCompare(goldJob, new TreePath(null, "")); - } catch (DeepInequalityException e) { - String error = e.path.toString(); - - assertFalse(fileDescription + " mismatches: " + error, true); - } - } - } finally { - IOUtils.cleanup(null, goldParser, resultParser); - } - } -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java deleted file mode 100644 index bb92426f5f..0000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java +++ /dev/null @@ -1,1259 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.tools.rumen; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.MapReduceTestUtil; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.TaskCounter; -import org.apache.hadoop.mapreduce.TaskID; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.TestNoJobSetupCleanup.MyOutputFormat; -import org.apache.hadoop.mapreduce.jobhistory.HistoryEvent; -import org.apache.hadoop.mapreduce.jobhistory.JobHistory; -import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptFinishedEvent; -import org.apache.hadoop.mapreduce.jobhistory.TaskAttemptUnsuccessfulCompletionEvent; -import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent; -import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; -import org.apache.hadoop.mapreduce.v2.api.records.JobId; -import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils; -import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo; -import org.apache.hadoop.tools.rumen.TraceBuilder.MyOptions; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestRumenJobTraces { - private static final Log LOG = LogFactory.getLog(TestRumenJobTraces.class); - - @Test - public void testSmallTrace() throws Exception { - performSingleTest("sample-job-tracker-logs.gz", - "job-tracker-logs-topology-output", "job-tracker-logs-trace-output.gz"); - } - - @Test - public void testTruncatedTask() throws Exception { - performSingleTest("truncated-job-tracker-log", "truncated-topology-output", - "truncated-trace-output"); - } - - private void performSingleTest(String jtLogName, String goldTopology, - String goldTrace) throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path rootInputFile = new Path(rootInputDir, "rumen/small-trace-test"); - final Path tempDir = new Path(rootTempDir, "TestRumenJobTraces"); - lfs.delete(tempDir, true); - - final Path topologyFile = new Path(tempDir, jtLogName + "-topology.json"); - final Path traceFile = new Path(tempDir, jtLogName + "-trace.json"); - - final Path inputFile = new Path(rootInputFile, jtLogName); - - System.out.println("topology result file = " + topologyFile); - System.out.println("trace result file = " + traceFile); - - String[] args = new String[6]; - - args[0] = "-v1"; - - args[1] = "-write-topology"; - args[2] = topologyFile.toString(); - - args[3] = "-write-job-trace"; - args[4] = traceFile.toString(); - - args[5] = inputFile.toString(); - - final Path topologyGoldFile = new Path(rootInputFile, goldTopology); - final Path traceGoldFile = new Path(rootInputFile, goldTrace); - - @SuppressWarnings("deprecation") - HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer(); - int result = ToolRunner.run(analyzer, args); - assertEquals("Non-zero exit", 0, result); - - TestRumenJobTraces - . jsonFileMatchesGold(conf, topologyFile, - topologyGoldFile, LoggedNetworkTopology.class, "topology"); - TestRumenJobTraces. jsonFileMatchesGold(conf, traceFile, - traceGoldFile, LoggedJob.class, "trace"); - } - - @Test - public void testRumenViaDispatch() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); - final Path tempDir = new Path(rootTempDir, "TestRumenViaDispatch"); - lfs.delete(tempDir, true); - - final Path topologyPath = new Path(tempDir, "dispatch-topology.json"); - final Path tracePath = new Path(tempDir, "dispatch-trace.json"); - - final Path inputPath = - new Path(rootInputPath, "dispatch-sample-v20-jt-log.gz"); - - System.out.println("topology result file = " + topologyPath); - System.out.println("testRumenViaDispatch() trace result file = " + tracePath); - - String demuxerClassName = ConcatenatedInputFilesDemuxer.class.getName(); - - String[] args = - { "-demuxer", demuxerClassName, tracePath.toString(), - topologyPath.toString(), inputPath.toString() }; - - final Path topologyGoldFile = - new Path(rootInputPath, "dispatch-topology-output.json.gz"); - final Path traceGoldFile = - new Path(rootInputPath, "dispatch-trace-output.json.gz"); - - Tool analyzer = new TraceBuilder(); - int result = ToolRunner.run(analyzer, args); - assertEquals("Non-zero exit", 0, result); - - TestRumenJobTraces - . jsonFileMatchesGold(conf, topologyPath, - topologyGoldFile, LoggedNetworkTopology.class, "topology"); - TestRumenJobTraces. jsonFileMatchesGold(conf, tracePath, - traceGoldFile, LoggedJob.class, "trace"); - } - - @Test - public void testBracketedCounters() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); - final Path tempDir = new Path(rootTempDir, "TestBracketedCounters"); - lfs.delete(tempDir, true); - - final Path topologyPath = new Path(tempDir, "dispatch-topology.json"); - final Path tracePath = new Path(tempDir, "dispatch-trace.json"); - - final Path inputPath = new Path(rootInputPath, "counters-format-test-logs"); - - System.out.println("topology result file = " + topologyPath); - System.out.println("testBracketedCounters() trace result file = " + tracePath); - - final Path goldPath = - new Path(rootInputPath, "counters-test-trace.json.gz"); - - String[] args = - { tracePath.toString(), topologyPath.toString(), inputPath.toString() }; - - Tool analyzer = new TraceBuilder(); - int result = ToolRunner.run(analyzer, args); - assertEquals("Non-zero exit", 0, result); - - TestRumenJobTraces. jsonFileMatchesGold(conf, tracePath, - goldPath, LoggedJob.class, "trace"); - } - - @Test - public void testHadoop20JHParser() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); - - // history file to be parsed to get events - final Path inputPath = new Path(rootInputPath, "v20-single-input-log.gz"); - - RewindableInputStream ris = getRewindableInputStream(inputPath, conf); - assertNotNull(ris); - - Hadoop20JHParser parser = null; - - try { - assertEquals("Hadoop20JHParser can't parse the test file " + - inputPath, true, Hadoop20JHParser.canParse(ris)); - - ris.rewind(); - parser = new Hadoop20JHParser(ris); - ArrayList seenEvents = new ArrayList(150); - - // this is same as the one in input history file - String jobId = "job_200904211745_0002"; - JobBuilder builder = new JobBuilder(jobId); - - // get events into seenEvents - getHistoryEvents(parser, seenEvents, builder); - - // Validate the events seen by history parser from - // history file v20-single-input-log.gz - validateSeenHistoryEvents(seenEvents, goldLines); - - ParsedJob parsedJob = builder.build(); - // validate the obtainXXX api of ParsedJob, ParsedTask and - // ParsedTaskAttempt - validateParsedJob(parsedJob, 20, 1, true); - } finally { - if (parser != null) { - parser.close(); - } - ris.close(); - } - } - - /** - * Validate the parsing of given history file name. - * - * TODO: Also validate the history file name suffixed with old/stale file - * suffix. - * @param jhFileName job history file path - * @param jid JobID - */ - private void validateHistoryFileNameParsing(Path jhFileName, - org.apache.hadoop.mapred.JobID jid) { - JobID extractedJID = - JobID.forName(JobHistoryUtils.extractJobID(jhFileName.getName())); - assertEquals("TraceBuilder failed to parse the current JH filename" - + jhFileName, jid, extractedJID); - //TODO test jobhistory filename with old/stale file suffix - } - - /** - * Validate the parsing of given history conf file name. Also validate the - * history conf file name suffixed with old/stale file suffix. - * @param jhConfFileName job history conf file path - * @param jid JobID - */ - private void validateJHConfFileNameParsing(Path jhConfFileName, - org.apache.hadoop.mapred.JobID jid) { - assertTrue("TraceBuilder failed to parse the JH conf filename:" - + jhConfFileName, - JobHistoryUtils.isJobConfXml(jhConfFileName.getName())); - JobID extractedJID = - JobID.forName(JobHistoryUtils.extractJobID(jhConfFileName.getName())); - assertEquals("TraceBuilder failed to parse the current JH conf filename:" - + jhConfFileName, jid, extractedJID); - // Test jobhistory conf filename with old/stale file suffix - jhConfFileName = jhConfFileName.suffix(JobHistory.getOldFileSuffix("123")); - assertTrue("TraceBuilder failed to parse the current JH conf filename" - + " (old suffix):" + jhConfFileName, - JobHistoryUtils.isJobConfXml(jhConfFileName.getName())); - extractedJID = - JobID.forName(JobHistoryUtils.extractJobID(jhConfFileName.getName())); - assertEquals("TraceBuilder failed to parse the JH conf filename" - + "(old-suffix):" + jhConfFileName, - jid, extractedJID); - } - - /** - * Tests if {@link TraceBuilder} can correctly identify and parse different - * versions of jobhistory filenames. The testcase checks if - * {@link TraceBuilder} - * - correctly identifies a jobhistory filename without suffix - * - correctly parses a jobhistory filename without suffix to extract out - * the jobid - * - correctly identifies a jobhistory filename with suffix - * - correctly parses a jobhistory filename with suffix to extract out the - * jobid - * - correctly identifies a job-configuration filename stored along with the - * jobhistory files - */ - @Test - public void testJobHistoryFilenameParsing() throws IOException { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - org.apache.hadoop.mapred.JobID jid = - new org.apache.hadoop.mapred.JobID("12345", 1); - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")) - .makeQualified(lfs.getUri(), lfs.getWorkingDirectory()); - - // Check if current jobhistory filenames are detected properly - JobId jobId = TypeConverter.toYarn(jid); - JobIndexInfo info = new JobIndexInfo(0L, 0L, "", "", jobId, 0, 0, ""); - Path jhFilename = new Path(FileNameIndexUtils.getDoneFileName(info)); - validateHistoryFileNameParsing(jhFilename, jid); - - // Check if Pre21 V1 jophistory file names are detected properly - jhFilename = new Path("jt-identifier_" + jid + "_user-name_job-name"); - validateHistoryFileNameParsing(jhFilename, jid); - - // Check if Pre21 V2 jobhistory file names are detected properly - jhFilename = new Path(jid + "_user-name_job-name"); - validateHistoryFileNameParsing(jhFilename, jid); - - // Check if the current jobhistory conf filenames are detected properly - Path jhConfFilename = JobHistory.getConfFile(rootInputDir, jid); - validateJHConfFileNameParsing(jhConfFilename, jid); - - // Check if Pre21 V1 jobhistory conf file names are detected properly - jhConfFilename = new Path("jt-identifier_" + jid + "_conf.xml"); - validateJHConfFileNameParsing(jhConfFilename, jid); - - // Check if Pre21 V2 jobhistory conf file names are detected properly - jhConfFilename = new Path(jid + "_conf.xml"); - validateJHConfFileNameParsing(jhConfFilename, jid); - } - - /** - * Check if processing of input arguments is as expected by passing globbed - * input path - *
  • without -recursive option and - *
  • with -recursive option. - */ - @Test - public void testProcessInputArgument() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - // define the test's root temporary directory - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")) - .makeQualified(lfs.getUri(), lfs.getWorkingDirectory()); - // define the test's root input directory - Path testRootInputDir = new Path(rootTempDir, "TestProcessInputArgument"); - // define the nested input directory - Path nestedInputDir = new Path(testRootInputDir, "1/2/3/4"); - // define the globbed version of the nested input directory - Path globbedInputNestedDir = - lfs.makeQualified(new Path(testRootInputDir, "*/*/*/*/*")); - try { - lfs.delete(nestedInputDir, true); - - List recursiveInputPaths = new ArrayList(); - List nonRecursiveInputPaths = new ArrayList(); - // Create input files under the given path with multiple levels of - // sub directories - createHistoryLogsHierarchy(nestedInputDir, lfs, recursiveInputPaths, - nonRecursiveInputPaths); - - // Check the case of globbed input path and without -recursive option - List inputs = MyOptions.processInputArgument( - globbedInputNestedDir.toString(), conf, false); - validateHistoryLogPaths(inputs, nonRecursiveInputPaths); - - // Check the case of globbed input path and with -recursive option - inputs = MyOptions.processInputArgument( - globbedInputNestedDir.toString(), conf, true); - validateHistoryLogPaths(inputs, recursiveInputPaths); - - } finally { - lfs.delete(testRootInputDir, true); - } - } - - /** - * Validate if the input history log paths are as expected. - * @param inputs the resultant input paths to be validated - * @param expectedHistoryFileNames the expected input history logs - * @throws IOException - */ - private void validateHistoryLogPaths(List inputs, - List expectedHistoryFileNames) throws IOException { - - System.out.println("\nExpected history files are:"); - for (String historyFile : expectedHistoryFileNames) { - System.out.println(historyFile); - } - System.out.println("\nResultant history files are:"); - List historyLogs = new ArrayList(); - for (Path p : inputs) { - historyLogs.add(p.toUri().getPath()); - System.out.println(p.toUri().getPath()); - } - - assertEquals("Number of history logs found is different from the expected.", - expectedHistoryFileNames.size(), inputs.size()); - - // Verify if all the history logs are expected ones and they are in the - // expected order - assertTrue("Some of the history log files do not match the expected.", - historyLogs.equals(expectedHistoryFileNames)); - } - - /** - * Create history logs under the given path with multiple levels of - * sub directories as shown below. - *
    - * Create a file, an empty subdirectory and a nonempty subdirectory - * <historyDir> under the given input path. - *
    - * The subdirectory <historyDir> contains the following dir structure: - *
    - *
    <historyDir>/historyFile1.txt - *
    <historyDir>/historyFile1.gz - *
    <historyDir>/subDir1/historyFile2.txt - *
    <historyDir>/subDir1/historyFile2.gz - *
    <historyDir>/subDir2/historyFile3.txt - *
    <historyDir>/subDir2/historyFile3.gz - *
    <historyDir>/subDir1/subDir11/historyFile4.txt - *
    <historyDir>/subDir1/subDir11/historyFile4.gz - *
    <historyDir>/subDir2/subDir21/ - *
    - * Create the lists of input paths that should be processed by TraceBuilder - * for recursive case and non-recursive case. - * @param nestedInputDir the input history logs directory where history files - * with nested subdirectories are created - * @param fs FileSystem of the input paths - * @param recursiveInputPaths input paths for recursive case - * @param nonRecursiveInputPaths input paths for non-recursive case - * @throws IOException - */ - private void createHistoryLogsHierarchy(Path nestedInputDir, FileSystem fs, - List recursiveInputPaths, List nonRecursiveInputPaths) - throws IOException { - List dirs = new ArrayList(); - // define a file in the nested test input directory - Path inputPath1 = new Path(nestedInputDir, "historyFile.txt"); - // define an empty sub-folder in the nested test input directory - Path emptyDir = new Path(nestedInputDir, "emptyDir"); - // define a nonempty sub-folder in the nested test input directory - Path historyDir = new Path(nestedInputDir, "historyDir"); - - fs.mkdirs(nestedInputDir); - // Create an empty input file - fs.createNewFile(inputPath1); - // Create empty subdir - fs.mkdirs(emptyDir);// let us not create any files under this dir - - fs.mkdirs(historyDir); - dirs.add(historyDir); - - Path subDir1 = new Path(historyDir, "subDir1"); - fs.mkdirs(subDir1); - dirs.add(subDir1); - Path subDir2 = new Path(historyDir, "subDir2"); - fs.mkdirs(subDir2); - dirs.add(subDir2); - - Path subDir11 = new Path(subDir1, "subDir11"); - fs.mkdirs(subDir11); - dirs.add(subDir11); - Path subDir21 = new Path(subDir2, "subDir21"); - fs.mkdirs(subDir21);// let us not create any files under this dir - - int i = 0; - for (Path dir : dirs) { - i++; - Path gzPath = new Path(dir, "historyFile" + i + ".gz"); - Path txtPath = new Path(dir, "historyFile" + i + ".txt"); - fs.createNewFile(txtPath); - fs.createNewFile(gzPath); - recursiveInputPaths.add(gzPath.toUri().getPath()); - recursiveInputPaths.add(txtPath.toUri().getPath()); - if (i == 1) { - nonRecursiveInputPaths.add(gzPath.toUri().getPath()); - nonRecursiveInputPaths.add(txtPath.toUri().getPath()); - } - } - recursiveInputPaths.add(inputPath1.toUri().getPath()); - nonRecursiveInputPaths.add(inputPath1.toUri().getPath()); - } - - /** - * Test if {@link CurrentJHParser} can read events from current JH files. - */ - @Test - public void testCurrentJHParser() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser"); - lfs.delete(tempDir, true); - - // Run a MR job - // create a MR cluster - conf.setInt(TTConfig.TT_MAP_SLOTS, 1); - conf.setInt(TTConfig.TT_REDUCE_SLOTS, 1); - MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, - new JobConf(conf)); - - // run a job - Path inDir = new Path(tempDir, "input"); - Path outDir = new Path(tempDir, "output"); - JobHistoryParser parser = null; - RewindableInputStream ris = null; - ArrayList seenEvents = new ArrayList(15); - - try { - JobConf jConf = mrCluster.createJobConf(); - // construct a job with 1 map and 1 reduce task. - Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 1, 1); - // disable setup/cleanup - job.setJobSetupCleanupNeeded(false); - // set the output format to take care of the _temporary folder - job.setOutputFormatClass(MyOutputFormat.class); - // wait for the job to complete - job.waitForCompletion(false); - - assertTrue("Job failed", job.isSuccessful()); - - JobID id = job.getJobID(); - JobClient jc = new JobClient(jConf); - String user = jc.getAllJobs()[0].getUsername(); - - // get the jobhistory filepath - Path jhPath = - new Path(mrCluster.getJobTrackerRunner().getJobTracker() - .getJobHistoryDir()); - Path inputPath = JobHistory.getJobHistoryFile(jhPath, id, user); - // wait for 10 secs for the jobhistory file to move into the done folder - for (int i = 0; i < 100; ++i) { - if (lfs.exists(inputPath)) { - break; - } - TimeUnit.MILLISECONDS.wait(100); - } - - assertTrue("Missing job history file", lfs.exists(inputPath)); - - ris = getRewindableInputStream(inputPath, conf); - - // Test if the JobHistoryParserFactory can detect the parser correctly - parser = JobHistoryParserFactory.getParser(ris); - - // create a job builder - JobBuilder builder = new JobBuilder(id.toString()); - - // get events into seenEvents and also process them using builder - getHistoryEvents(parser, seenEvents, builder); - - // Check against the gold standard - System.out.println("testCurrentJHParser validating using gold std "); - // The list of history events expected when parsing the above job's - // history log file - String[] goldLinesExpected = new String[] { - JSE, JPCE, JIE, JSCE, TSE, ASE, MFE, TFE, TSE, ASE, RFE, TFE, JFE - }; - - validateSeenHistoryEvents(seenEvents, goldLinesExpected); - - // validate resource usage metrics - // get the job counters - Counters counters = job.getTaskReports(TaskType.MAP)[0].getTaskCounters(); - - // get the parsed job - ParsedJob parsedJob = builder.build(); - // get the logged job - LoggedJob loggedJob = parsedJob; - // get the logged attempts - LoggedTaskAttempt attempt = - loggedJob.getMapTasks().get(0).getAttempts().get(0); - // get the resource usage metrics - ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics(); - - // check with the actual values - testResourceUsageMetricViaDeepCompare(metrics, - counters.findCounter(TaskCounter.CPU_MILLISECONDS).getValue(), - counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).getValue(), - counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).getValue(), - counters.findCounter(TaskCounter.COMMITTED_HEAP_BYTES).getValue(), - true); - - // validate the obtainXXX api of ParsedJob, ParsedTask and - // ParsedTaskAttempt - validateParsedJob(parsedJob, 1, 1, false); - } finally { - // stop the MR cluster - mrCluster.shutdown(); - - if (ris != null) { - ris.close(); - } - if (parser != null) { - parser.close(); - } - - // cleanup the filesystem - lfs.delete(tempDir, true); - } - } - - /** - * Verify if the obtainXXX methods of {@link ParsedJob}, {@link ParsedTask} - * and {@link ParsedTaskAttempt} give valid info - */ - private void validateParsedJob(ParsedJob parsedJob, int numMaps, - int numReduces, boolean pre21JobHistory) { - validateParsedJobAPI(parsedJob, numMaps, numReduces, pre21JobHistory); - - List maps = parsedJob.obtainMapTasks(); - for (ParsedTask task : maps) { - validateParsedTask(task); - } - List reduces = parsedJob.obtainReduceTasks(); - for (ParsedTask task : reduces) { - validateParsedTask(task); - } - List others = parsedJob.obtainOtherTasks(); - for (ParsedTask task : others) { - validateParsedTask(task); - } - } - - /** Verify if the obtainXXX methods of {@link ParsedJob} give valid info */ - private void validateParsedJobAPI(ParsedJob parsedJob, int numMaps, - int numReduces, boolean pre21JobHistory) { - LOG.info("Validating ParsedJob.obtainXXX api... for " - + parsedJob.getJobID()); - assertNotNull("Job acls in ParsedJob is null", - parsedJob.obtainJobAcls()); - assertNotNull("Job conf path in ParsedJob is null", - parsedJob.obtainJobConfpath()); - - assertNotNull("Map Counters in ParsedJob is null", - parsedJob.obtainMapCounters()); - assertNotNull("Reduce Counters in ParsedJob is null", - parsedJob.obtainReduceCounters()); - assertNotNull("Total Counters in ParsedJob is null", - parsedJob.obtainTotalCounters()); - - assertNotNull("Map Tasks List in ParsedJob is null", - parsedJob.obtainMapTasks()); - assertNotNull("Reduce Tasks List in ParsedJob is null", - parsedJob.obtainReduceTasks()); - assertNotNull("Other Tasks List in ParsedJob is null", - parsedJob.obtainOtherTasks()); - - // 1 map and 1 reduce task should be there - assertEquals("Number of map tasks in ParsedJob is wrong", - numMaps, parsedJob.obtainMapTasks().size()); - assertEquals("Number of reduce tasks in ParsedJob is wrong", - numReduces, parsedJob.obtainReduceTasks().size(), 1); - - // old hadoop20 version history files don't have job-level-map-counters and - // job-level-reduce-counters. Only total counters exist there. - assertTrue("Total Counters in ParsedJob is empty", - parsedJob.obtainTotalCounters().size() > 0); - if (!pre21JobHistory) { - assertTrue("Map Counters in ParsedJob is empty", - parsedJob.obtainMapCounters().size() > 0); - assertTrue("Reduce Counters in ParsedJob is empty", - parsedJob.obtainReduceCounters().size() > 0); - } - } - - /** - * Verify if the obtainXXX methods of {@link ParsedTask} and - * {@link ParsedTaskAttempt} give valid info - */ - private void validateParsedTask(ParsedTask parsedTask) { - validateParsedTaskAPI(parsedTask); - - List attempts = parsedTask.obtainTaskAttempts(); - for (ParsedTaskAttempt attempt : attempts) { - validateParsedTaskAttemptAPI(attempt); - } - } - - /** Verify if the obtainXXX methods of {@link ParsedTask} give valid info */ - private void validateParsedTaskAPI(ParsedTask parsedTask) { - LOG.info("Validating ParsedTask.obtainXXX api... for " - + parsedTask.getTaskID()); - assertNotNull("Task counters in ParsedTask is null", - parsedTask.obtainCounters()); - - if (parsedTask.getTaskStatus() - == Pre21JobHistoryConstants.Values.SUCCESS) { - // task counters should not be empty - assertTrue("Task counters in ParsedTask is empty", - parsedTask.obtainCounters().size() > 0); - assertNull("Diagnostic-info is non-null for a succeeded task", - parsedTask.obtainDiagnosticInfo()); - assertNull("Failed-due-to-attemptId is non-null for a succeeded task", - parsedTask.obtainFailedDueToAttemptId()); - } else { - assertNotNull("Diagnostic-info is non-null for a succeeded task", - parsedTask.obtainDiagnosticInfo()); - assertNotNull("Failed-due-to-attemptId is non-null for a succeeded task", - parsedTask.obtainFailedDueToAttemptId()); - } - - List attempts = parsedTask.obtainTaskAttempts(); - assertNotNull("TaskAttempts list in ParsedTask is null", attempts); - assertTrue("TaskAttempts list in ParsedTask is empty", - attempts.size() > 0); - } - - /** - * Verify if the obtainXXX methods of {@link ParsedTaskAttempt} give - * valid info - */ - private void validateParsedTaskAttemptAPI( - ParsedTaskAttempt parsedTaskAttempt) { - LOG.info("Validating ParsedTaskAttempt.obtainXXX api... for " - + parsedTaskAttempt.getAttemptID()); - assertNotNull("Counters in ParsedTaskAttempt is null", - parsedTaskAttempt.obtainCounters()); - - if (parsedTaskAttempt.getResult() - == Pre21JobHistoryConstants.Values.SUCCESS) { - assertTrue("Counters in ParsedTaskAttempt is empty", - parsedTaskAttempt.obtainCounters().size() > 0); - assertNull("Diagnostic-info is non-null for a succeeded taskAttempt", - parsedTaskAttempt.obtainDiagnosticInfo()); - } else { - assertNotNull("Diagnostic-info is non-null for a succeeded taskAttempt", - parsedTaskAttempt.obtainDiagnosticInfo()); - } - assertNotNull("TrackerName in ParsedTaskAttempt is null", - parsedTaskAttempt.obtainTrackerName()); - - assertNotNull("http-port info in ParsedTaskAttempt is null", - parsedTaskAttempt.obtainHttpPort()); - assertNotNull("Shuffle-port info in ParsedTaskAttempt is null", - parsedTaskAttempt.obtainShufflePort()); - } - - @Test - public void testJobConfigurationParser() throws Exception { - - // Validate parser with old mapred config properties from - // sample-conf-file.xml - String[] oldProps1 = { "mapred.job.queue.name", "mapred.job.name", - "mapred.child.java.opts" }; - - validateJobConfParser("sample-conf.file.xml", false); - validateJobConfParser("sample-conf.file.new.xml", true); - } - - private void validateJobConfParser(String confFile, boolean newConfig) - throws Exception { - - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - @SuppressWarnings("deprecation") - final Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")) - .makeQualified(lfs); - - final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); - - final Path inputPath = new Path(rootInputPath, confFile); - - InputStream inputConfStream = - new PossiblyDecompressedInputStream(inputPath, conf); - - try { - Properties props = JobConfigurationParser.parse(inputConfStream); - inputConfStream.close(); - - String oldOrNew = newConfig ? "New" : "Old"; - assertEquals(oldOrNew + " config property for job queue name is not " - + " extracted properly.", "TheQueue", - JobBuilder.extract(props, JobConfPropertyNames.QUEUE_NAMES - .getCandidates(), null)); - assertEquals(oldOrNew + " config property for job name is not " - + " extracted properly.", "MyMRJob", - JobBuilder.extract(props, JobConfPropertyNames.JOB_NAMES - .getCandidates(), null)); - - validateChildJavaOpts(newConfig, props); - - } finally { - inputConfStream.close(); - } - } - - // Validate child java opts in properties. - // newConfigProperties: boolean that specifies if the config properties to be - // validated are new OR old. - private void validateChildJavaOpts(boolean newConfigProperties, - Properties props) { - if (newConfigProperties) { - assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS - + " is not extracted properly.", - "-server -Xmx640m -Djava.net.preferIPv4Stack=true", - JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S - .getCandidates(), null)); - assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS - + " is not extracted properly.", - "-server -Xmx650m -Djava.net.preferIPv4Stack=true", - JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S - .getCandidates(), null)); - } - else { - // if old property mapred.child.java.opts is set, then extraction of all - // the following 3 properties should give that value. - assertEquals("mapred.child.java.opts is not extracted properly.", - "-server -Xmx640m -Djava.net.preferIPv4Stack=true", - JobBuilder.extract(props, JobConfPropertyNames.TASK_JAVA_OPTS_S - .getCandidates(), null)); - assertEquals("New config property " + MRJobConfig.MAP_JAVA_OPTS - + " is not extracted properly when the old config property " - + "mapred.child.java.opts is set.", - "-server -Xmx640m -Djava.net.preferIPv4Stack=true", - JobBuilder.extract(props, JobConfPropertyNames.MAP_JAVA_OPTS_S - .getCandidates(), null)); - assertEquals("New config property " + MRJobConfig.REDUCE_JAVA_OPTS - + " is not extracted properly when the old config property " - + "mapred.child.java.opts is set.", - "-server -Xmx640m -Djava.net.preferIPv4Stack=true", - JobBuilder.extract(props, JobConfPropertyNames.REDUCE_JAVA_OPTS_S - .getCandidates(), null)); - } - } - - /** - * Test if the {@link JobConfigurationParser} can correctly extract out - * key-value pairs from the job configuration. - */ - @Test - public void testJobConfigurationParsing() throws Exception { - final FileSystem lfs = FileSystem.getLocal(new Configuration()); - - final Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")).makeQualified( - lfs.getUri(), lfs.getWorkingDirectory()); - - final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser"); - lfs.delete(tempDir, true); - - // Add some configuration parameters to the conf - JobConf jConf = new JobConf(false); - String key = "test.data"; - String value = "hello world"; - jConf.set(key, value); - - // create the job conf file - Path jobConfPath = new Path(tempDir.toString(), "job.xml"); - lfs.delete(jobConfPath, false); - DataOutputStream jobConfStream = lfs.create(jobConfPath); - jConf.writeXml(jobConfStream); - jobConfStream.close(); - - // now read the job conf file using the job configuration parser - Properties properties = - JobConfigurationParser.parse(lfs.open(jobConfPath)); - - // check if the required parameter is loaded - assertEquals("Total number of extracted properties (" + properties.size() - + ") doesn't match the expected size of 1 [" - + "JobConfigurationParser]", - 1, properties.size()); - // check if the key is present in the extracted configuration - assertTrue("Key " + key + " is missing in the configuration extracted " - + "[JobConfigurationParser]", - properties.keySet().contains(key)); - // check if the desired property has the correct value - assertEquals("JobConfigurationParser couldn't recover the parameters" - + " correctly", - value, properties.get(key)); - - // Test ZombieJob - LoggedJob job = new LoggedJob(); - job.setJobProperties(properties); - - ZombieJob zjob = new ZombieJob(job, null); - Configuration zconf = zjob.getJobConf(); - // check if the required parameter is loaded - assertEquals("ZombieJob couldn't recover the parameters correctly", - value, zconf.get(key)); - } - - - /** - * Test {@link ResourceUsageMetrics}. - */ - @Test - public void testResourceUsageMetrics() throws Exception { - final long cpuUsage = 100; - final long pMemUsage = 200; - final long vMemUsage = 300; - final long heapUsage = 400; - - // test ResourceUsageMetrics's setters - ResourceUsageMetrics metrics = new ResourceUsageMetrics(); - metrics.setCumulativeCpuUsage(cpuUsage); - metrics.setPhysicalMemoryUsage(pMemUsage); - metrics.setVirtualMemoryUsage(vMemUsage); - metrics.setHeapUsage(heapUsage); - // test cpu usage value - assertEquals("Cpu usage values mismatch via set", cpuUsage, - metrics.getCumulativeCpuUsage()); - // test pMem usage value - assertEquals("Physical memory usage values mismatch via set", pMemUsage, - metrics.getPhysicalMemoryUsage()); - // test vMem usage value - assertEquals("Virtual memory usage values mismatch via set", vMemUsage, - metrics.getVirtualMemoryUsage()); - // test heap usage value - assertEquals("Heap usage values mismatch via set", heapUsage, - metrics.getHeapUsage()); - - // test deepCompare() (pass case) - testResourceUsageMetricViaDeepCompare(metrics, cpuUsage, vMemUsage, - pMemUsage, heapUsage, true); - - // test deepCompare (fail case) - // test cpu usage mismatch - testResourceUsageMetricViaDeepCompare(metrics, 0, vMemUsage, pMemUsage, - heapUsage, false); - // test pMem usage mismatch - testResourceUsageMetricViaDeepCompare(metrics, cpuUsage, vMemUsage, 0, - heapUsage, false); - // test vMem usage mismatch - testResourceUsageMetricViaDeepCompare(metrics, cpuUsage, 0, pMemUsage, - heapUsage, false); - // test heap usage mismatch - testResourceUsageMetricViaDeepCompare(metrics, cpuUsage, vMemUsage, - pMemUsage, 0, false); - - // define a metric with a fixed value of size() - ResourceUsageMetrics metrics2 = new ResourceUsageMetrics() { - @Override - public int size() { - return -1; - } - }; - metrics2.setCumulativeCpuUsage(cpuUsage); - metrics2.setPhysicalMemoryUsage(pMemUsage); - metrics2.setVirtualMemoryUsage(vMemUsage); - metrics2.setHeapUsage(heapUsage); - - // test with size mismatch - testResourceUsageMetricViaDeepCompare(metrics2, cpuUsage, vMemUsage, - pMemUsage, heapUsage, false); - } - - // test ResourceUsageMetric's deepCompare() method - private static void testResourceUsageMetricViaDeepCompare( - ResourceUsageMetrics metrics, long cpuUsage, - long vMemUsage, long pMemUsage, long heapUsage, - boolean shouldPass) { - ResourceUsageMetrics testMetrics = new ResourceUsageMetrics(); - testMetrics.setCumulativeCpuUsage(cpuUsage); - testMetrics.setPhysicalMemoryUsage(pMemUsage); - testMetrics.setVirtualMemoryUsage(vMemUsage); - testMetrics.setHeapUsage(heapUsage); - - Boolean passed = null; - try { - metrics.deepCompare(testMetrics, new TreePath(null, "")); - passed = true; - } catch (DeepInequalityException die) { - passed = false; - } - - assertEquals("ResourceUsageMetrics deepCompare() failed!", - shouldPass, passed); - } - - /** - * Testing {@link ResourceUsageMetrics} using {@link HadoopLogsAnalyzer}. - */ - @Test - @SuppressWarnings("deprecation") - public void testResourceUsageMetricsWithHadoopLogsAnalyzer() - throws IOException { - Configuration conf = new Configuration(); - // get the input trace file - Path rootInputDir = - new Path(System.getProperty("test.tools.input.dir", "")); - Path rootInputSubFolder = new Path(rootInputDir, "rumen/small-trace-test"); - Path traceFile = new Path(rootInputSubFolder, "v20-resource-usage-log.gz"); - - FileSystem lfs = FileSystem.getLocal(conf); - - // define the root test directory - Path rootTempDir = - new Path(System.getProperty("test.build.data", "/tmp")); - - // define output directory - Path outputDir = - new Path(rootTempDir, "testResourceUsageMetricsWithHadoopLogsAnalyzer"); - lfs.delete(outputDir, true); - lfs.deleteOnExit(outputDir); - - // run HadoopLogsAnalyzer - HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer(); - analyzer.setConf(conf); - Path traceOutput = new Path(outputDir, "trace.json"); - analyzer.run(new String[] {"-write-job-trace", traceOutput.toString(), - "-v1", traceFile.toString()}); - - // test HadoopLogsAnalyzer's output w.r.t ResourceUsageMetrics - // get the logged job - JsonObjectMapperParser traceParser = - new JsonObjectMapperParser(traceOutput, LoggedJob.class, - conf); - - // get the logged job from the output trace file - LoggedJob job = traceParser.getNext(); - LoggedTaskAttempt attempt = job.getMapTasks().get(0).getAttempts().get(0); - ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics(); - - // test via deepCompare() - testResourceUsageMetricViaDeepCompare(metrics, 200, 100, 75, 50, true); - } - - @Test - public void testTopologyBuilder() throws Exception { - final TopologyBuilder subject = new TopologyBuilder(); - - // This 4 comes from - // TaskInProgress.ProgressibleSplitsBlock.burst().size , which - // is invisible here. - - int[][] splits = new int[4][]; - - splits[0] = new int[12]; - splits[1] = new int[12]; - splits[2] = new int[12]; - splits[3] = new int[12]; - - for (int j = 0; j < 4; ++j) { - for (int i = 0; i < 12; ++i) { - splits[j][i] = -1; - } - } - - for (int i = 0; i < 6; ++i) { - splits[0][i] = 500000 * i; - splits[1][i] = 300000 * i; - splits[2][i] = 500000; - splits[3][i] = 700000; - } - - // currently we extract no host names from the Properties - subject.process(new Properties()); - - subject.process(new TaskAttemptFinishedEvent(TaskAttemptID - .forName("attempt_200904211745_0003_m_000004_0"), TaskType - .valueOf("MAP"), "STATUS", 1234567890L, - "/194\\.6\\.134\\.64", "cluster50261\\.secondleveldomain\\.com", - "SUCCESS", null)); - subject.process(new TaskAttemptUnsuccessfulCompletionEvent - (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_1"), - TaskType.valueOf("MAP"), "STATUS", 1234567890L, - "cluster50262\\.secondleveldomain\\.com", - -1, "/194\\.6\\.134\\.80", "MACHINE_EXPLODED", splits)); - subject.process(new TaskAttemptUnsuccessfulCompletionEvent - (TaskAttemptID.forName("attempt_200904211745_0003_m_000004_2"), - TaskType.valueOf("MAP"), "STATUS", 1234567890L, - "cluster50263\\.secondleveldomain\\.com", - -1, "/194\\.6\\.134\\.80", "MACHINE_EXPLODED", splits)); - subject.process(new TaskStartedEvent(TaskID - .forName("task_200904211745_0003_m_000004"), 1234567890L, TaskType - .valueOf("MAP"), - "/194\\.6\\.134\\.80/cluster50263\\.secondleveldomain\\.com")); - - final LoggedNetworkTopology topology = subject.build(); - - List racks = topology.getChildren(); - - assertEquals("Wrong number of racks", 2, racks.size()); - - boolean sawSingleton = false; - boolean sawDoubleton = false; - - for (LoggedNetworkTopology rack : racks) { - List nodes = rack.getChildren(); - if (rack.getName().getValue().endsWith(".64")) { - assertEquals("The singleton rack has the wrong number of elements", 1, - nodes.size()); - sawSingleton = true; - } else if (rack.getName().getValue().endsWith(".80")) { - assertEquals("The doubleton rack has the wrong number of elements", 2, - nodes.size()); - sawDoubleton = true; - } else { - assertTrue("Unrecognized rack name", false); - } - } - - assertTrue("Did not see singleton rack", sawSingleton); - assertTrue("Did not see doubleton rack", sawDoubleton); - } - - static private void jsonFileMatchesGold( - Configuration conf, Path result, Path gold, Class clazz, - String fileDescription) throws IOException { - JsonObjectMapperParser goldParser = - new JsonObjectMapperParser(gold, clazz, conf); - JsonObjectMapperParser resultParser = - new JsonObjectMapperParser(result, clazz, conf); - try { - while (true) { - DeepCompare goldJob = goldParser.getNext(); - DeepCompare resultJob = resultParser.getNext(); - if ((goldJob == null) || (resultJob == null)) { - assertTrue(goldJob == resultJob); - break; - } - - try { - resultJob.deepCompare(goldJob, new TreePath(null, "")); - } catch (DeepInequalityException e) { - String error = e.path.toString(); - - assertFalse(fileDescription + " mismatches: " + error, true); - } - } - } finally { - IOUtils.cleanup(null, goldParser, resultParser); - } - } - - /** - * Creates {@link RewindableInputStream} for the given file path. - * @param inputPath the input file path - * @param conf configuration - * @return {@link RewindableInputStream} - * @throws IOException - */ - private RewindableInputStream getRewindableInputStream(Path inputPath, - Configuration conf) throws IOException { - - PossiblyDecompressedInputStream in = - new PossiblyDecompressedInputStream(inputPath, conf); - - return new RewindableInputStream(in, BUFSIZE); - } - - /** - * Allows given history parser to parse the history events and places in - * the given list - * @param parser the job history parser - * @param events the job history events seen while parsing - * @throws IOException - */ - private void getHistoryEvents(JobHistoryParser parser, - ArrayList events, JobBuilder builder) throws IOException { - HistoryEvent e; - while ((e = parser.nextEvent()) != null) { - String eventString = e.getClass().getSimpleName(); - System.out.println(eventString); - events.add(eventString); - if (builder != null) { - builder.process(e); - } - } - } - - /** - * Validate if history events seen are as expected - * @param seenEvents the list of history events seen - * @param goldLinesExpected the expected history events - */ - private void validateSeenHistoryEvents(ArrayList seenEvents, - String[] goldLinesExpected) { - - // Check the output with gold std - assertEquals("Number of events expected is different from the events seen" - + " by the history parser.", - goldLinesExpected.length, seenEvents.size()); - - int index = 0; - for (String goldLine : goldLinesExpected) { - assertEquals("History Event mismatch at line " + (index + 1), - goldLine, seenEvents.get(index)); - index++; - } - } - - final static int BUFSIZE = 8192; // 8K - - // Any Map Reduce Job History Event should be 1 of the following 16 - final static String JSE = "JobSubmittedEvent"; - final static String JPCE = "JobPriorityChangeEvent"; - final static String JSCE = "JobStatusChangedEvent"; - final static String JIE = "JobInitedEvent"; - final static String JICE = "JobInfoChangeEvent"; - static String TSE = "TaskStartedEvent"; - static String ASE = "TaskAttemptStartedEvent"; - static String AFE = "TaskAttemptFinishedEvent"; - static String MFE = "MapAttemptFinishedEvent"; - static String TUE = "TaskUpdatedEvent"; - static String TFE = "TaskFinishedEvent"; - static String JUCE = "JobUnsuccessfulCompletionEvent"; - static String RFE = "ReduceAttemptFinishedEvent"; - static String AUCE = "TaskAttemptUnsuccessfulCompletionEvent"; - static String TFLE = "TaskFailedEvent"; - static String JFE = "JobFinishedEvent"; - - // The expected job history events(in order) when parsing - // the job history file v20-single-input-log.gz - final static String[] goldLines = new String[] { - JSE, JPCE, JSCE, JIE, JICE, TSE, ASE, AFE, MFE, TUE, TFE, JSCE, TSE, - TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, TSE, - TSE, TSE, TSE, TSE, TSE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, - TFE, ASE, AFE, MFE, TUE, TFE, TSE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, - MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, - AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, - ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AUCE, ASE, AFE, - MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, - AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, - ASE, AFE, MFE, TUE, TFE, ASE, AFE, MFE, TUE, TFE, ASE, AFE, RFE, TUE, - TFE, TSE, ASE, AFE, MFE, TUE, TFE, JSCE, JFE - }; - -} diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestZombieJob.java b/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestZombieJob.java deleted file mode 100644 index 306d1ba486..0000000000 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestZombieJob.java +++ /dev/null @@ -1,338 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.tools.rumen; - -import java.util.List; -import java.util.ArrayList; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.TaskStatus.State; -import org.apache.hadoop.mapreduce.TaskType; - -import org.junit.Before; -import org.junit.Test; -import static org.junit.Assert.*; - -public class TestZombieJob { - final double epsilon = 0.01; - private final int[] attemptTimesPercentiles = new int[] { 10, 50, 90 }; - private long[] succeededCDF = new long[] { 5268, 5268, 5268, 5268, 5268 }; - private long[] failedCDF = new long[] { 18592, 18592, 18592, 18592, 18592 }; - private double[] expectedPs = new double[] { 0.000001, 0.18707660239708182, - 0.0013027618551328818, 2.605523710265763E-4 }; - - private final long[] mapTaskCounts = new long[] { 7838525L, 342277L, 100228L, - 1564L, 1234L }; - private final long[] reduceTaskCounts = new long[] { 4405338L, 139391L, - 1514383L, 139391, 1234L }; - - List loggedJobs = new ArrayList(); - List jobStories = new ArrayList(); - - @Before - public void setUp() throws Exception { - final Configuration conf = new Configuration(); - final FileSystem lfs = FileSystem.getLocal(conf); - - final Path rootInputDir = new Path( - System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); - final Path rootInputFile = new Path(rootInputDir, "rumen/zombie"); - - ZombieJobProducer parser = new ZombieJobProducer(new Path(rootInputFile, - "input-trace.json"), new ZombieCluster(new Path(rootInputFile, - "input-topology.json"), null, conf), conf); - - JobStory job = null; - for (int i = 0; i < 4; i++) { - job = parser.getNextJob(); - ZombieJob zJob = (ZombieJob) job; - LoggedJob loggedJob = zJob.getLoggedJob(); - System.out.println(i + ":" + job.getNumberMaps() + "m, " - + job.getNumberReduces() + "r"); - System.out - .println(loggedJob.getOutcome() + ", " + loggedJob.getJobtype()); - - System.out.println("Input Splits -- " + job.getInputSplits().length - + ", " + job.getNumberMaps()); - - System.out.println("Successful Map CDF -------"); - for (LoggedDiscreteCDF cdf : loggedJob.getSuccessfulMapAttemptCDFs()) { - System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() - + "--" + cdf.getMaximum()); - for (LoggedSingleRelativeRanking ranking : cdf.getRankings()) { - System.out.println(" " + ranking.getRelativeRanking() + ":" - + ranking.getDatum()); - } - } - System.out.println("Failed Map CDF -----------"); - for (LoggedDiscreteCDF cdf : loggedJob.getFailedMapAttemptCDFs()) { - System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() - + "--" + cdf.getMaximum()); - for (LoggedSingleRelativeRanking ranking : cdf.getRankings()) { - System.out.println(" " + ranking.getRelativeRanking() + ":" - + ranking.getDatum()); - } - } - System.out.println("Successful Reduce CDF ----"); - LoggedDiscreteCDF cdf = loggedJob.getSuccessfulReduceAttemptCDF(); - System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() + "--" - + cdf.getMaximum()); - for (LoggedSingleRelativeRanking ranking : cdf.getRankings()) { - System.out.println(" " + ranking.getRelativeRanking() + ":" - + ranking.getDatum()); - } - System.out.println("Failed Reduce CDF --------"); - cdf = loggedJob.getFailedReduceAttemptCDF(); - System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() + "--" - + cdf.getMaximum()); - for (LoggedSingleRelativeRanking ranking : cdf.getRankings()) { - System.out.println(" " + ranking.getRelativeRanking() + ":" - + ranking.getDatum()); - } - System.out.print("map attempts to success -- "); - for (double p : loggedJob.getMapperTriesToSucceed()) { - System.out.print(p + ", "); - } - System.out.println(); - System.out.println("==============="); - - loggedJobs.add(loggedJob); - jobStories.add(job); - } - } - - @Test - public void testFirstJob() { - // 20th job seems reasonable: "totalMaps":329,"totalReduces":101 - // successful map: 80 node-local, 196 rack-local, 53 rack-remote, 2 unknown - // failed map: 0-0-0-1 - // successful reduce: 99 failed reduce: 13 - // map attempts to success -- 0.9969879518072289, 0.0030120481927710845, - JobStory job = jobStories.get(0); - assertEquals(1, job.getNumberMaps()); - assertEquals(1, job.getNumberReduces()); - - // get splits - - TaskAttemptInfo taInfo = null; - long expectedRuntime = 2423; - // get a succeeded map task attempt, expect the exact same task attempt - taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 1); - assertEquals(expectedRuntime, taInfo.getRuntime()); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a succeeded map attempt, but reschedule with different locality. - taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 2); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 0); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - expectedRuntime = 97502; - // get a succeeded reduce task attempt, expect the exact same task attempt - taInfo = job.getTaskAttemptInfo(TaskType.REDUCE, 14, 0); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a failed reduce task attempt, expect the exact same task attempt - taInfo = job.getTaskAttemptInfo(TaskType.REDUCE, 14, 0); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a non-exist reduce task attempt, expect a made-up task attempt - // TODO fill in test case - } - - @Test - public void testSecondJob() { - // 7th job has many failed tasks. - // 3204 m, 0 r - // successful maps 497-586-23-1, failed maps 0-0-0-2714 - // map attempts to success -- 0.8113600833767587, 0.18707660239708182, - // 0.0013027618551328818, 2.605523710265763E-4, - JobStory job = jobStories.get(1); - assertEquals(20, job.getNumberMaps()); - assertEquals(1, job.getNumberReduces()); - - TaskAttemptInfo taInfo = null; - // get a succeeded map task attempt - taInfo = job.getMapTaskAttemptInfoAdjusted(17, 1, 1); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a succeeded map task attempt, with different locality - taInfo = job.getMapTaskAttemptInfoAdjusted(17, 1, 2); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - taInfo = job.getMapTaskAttemptInfoAdjusted(17, 1, 0); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a failed map task attempt - taInfo = job.getMapTaskAttemptInfoAdjusted(14, 0, 1); - assertEquals(1927, taInfo.getRuntime()); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - - // get a failed map task attempt, with different locality - // TODO: this test does not make sense here, because I don't have - // available data set. - } - - @Test - public void testFourthJob() { - // 7th job has many failed tasks. - // 3204 m, 0 r - // successful maps 497-586-23-1, failed maps 0-0-0-2714 - // map attempts to success -- 0.8113600833767587, 0.18707660239708182, - // 0.0013027618551328818, 2.605523710265763E-4, - JobStory job = jobStories.get(3); - assertEquals(131, job.getNumberMaps()); - assertEquals(47, job.getNumberReduces()); - - TaskAttemptInfo taInfo = null; - // get a succeeded map task attempt - long runtime = 5268; - taInfo = job.getMapTaskAttemptInfoAdjusted(113, 1, 1); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - assertEquals(runtime, taInfo.getRuntime()); - - // get a succeeded map task attempt, with different locality - taInfo = job.getMapTaskAttemptInfoAdjusted(113, 1, 2); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - assertEquals(runtime, taInfo.getRuntime() / 2); - taInfo = job.getMapTaskAttemptInfoAdjusted(113, 1, 0); - assertEquals(State.SUCCEEDED, taInfo.getRunState()); - assertEquals((long) (runtime / 1.5), taInfo.getRuntime()); - - // get a failed map task attempt - taInfo = job.getMapTaskAttemptInfoAdjusted(113, 0, 1); - assertEquals(18592, taInfo.getRuntime()); - assertEquals(State.FAILED, taInfo.getRunState()); - } - - @Test - public void testRecordIOInfo() { - JobStory job = jobStories.get(3); - - TaskInfo mapTask = job.getTaskInfo(TaskType.MAP, 113); - - TaskInfo reduceTask = job.getTaskInfo(TaskType.REDUCE, 0); - - assertEquals(mapTaskCounts[0], mapTask.getInputBytes()); - assertEquals(mapTaskCounts[1], mapTask.getInputRecords()); - assertEquals(mapTaskCounts[2], mapTask.getOutputBytes()); - assertEquals(mapTaskCounts[3], mapTask.getOutputRecords()); - assertEquals(mapTaskCounts[4], mapTask.getTaskMemory()); - - assertEquals(reduceTaskCounts[0], reduceTask.getInputBytes()); - assertEquals(reduceTaskCounts[1], reduceTask.getInputRecords()); - assertEquals(reduceTaskCounts[2], reduceTask.getOutputBytes()); - assertEquals(reduceTaskCounts[3], reduceTask.getOutputRecords()); - assertEquals(reduceTaskCounts[4], reduceTask.getTaskMemory()); - } - - @Test - public void testMakeUpInfo() { - // get many non-exist tasks - // total 3204 map tasks, 3300 is a non-exist task. - checkMakeUpTask(jobStories.get(3), 113, 1); - } - - private void checkMakeUpTask(JobStory job, int taskNumber, int locality) { - TaskAttemptInfo taInfo = null; - - Histogram sampleSucceeded = new Histogram(); - Histogram sampleFailed = new Histogram(); - List sampleAttempts = new ArrayList(); - for (int i = 0; i < 100000; i++) { - int attemptId = 0; - while (true) { - taInfo = job.getMapTaskAttemptInfoAdjusted(taskNumber, attemptId, 1); - if (taInfo.getRunState() == State.SUCCEEDED) { - sampleSucceeded.enter(taInfo.getRuntime()); - break; - } - sampleFailed.enter(taInfo.getRuntime()); - attemptId++; - } - sampleAttempts.add(attemptId); - } - - // check state distribution - int[] countTries = new int[] { 0, 0, 0, 0 }; - for (int attempts : sampleAttempts) { - assertTrue(attempts < 4); - countTries[attempts]++; - } - /* - * System.out.print("Generated map attempts to success -- "); for (int - * count: countTries) { System.out.print((double)count/sampleAttempts.size() - * + ", "); } System.out.println(); System.out.println("==============="); - */ - for (int i = 0; i < 4; i++) { - int count = countTries[i]; - double p = (double) count / sampleAttempts.size(); - assertTrue(expectedPs[i] - p < epsilon); - } - - // check succeeded attempts runtime distribution - long[] expectedCDF = succeededCDF; - LoggedDiscreteCDF cdf = new LoggedDiscreteCDF(); - cdf.setCDF(sampleSucceeded, attemptTimesPercentiles, 100); - /* - * System.out.println("generated succeeded map runtime distribution"); - * System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() + "--" - * + cdf.getMaximum()); for (LoggedSingleRelativeRanking ranking: - * cdf.getRankings()) { System.out.println(" " + - * ranking.getRelativeRanking() + ":" + ranking.getDatum()); } - */ - assertRuntimeEqual(cdf.getMinimum(), expectedCDF[0]); - assertRuntimeEqual(cdf.getMaximum(), expectedCDF[4]); - for (int i = 0; i < 3; i++) { - LoggedSingleRelativeRanking ranking = cdf.getRankings().get(i); - assertRuntimeEqual(expectedCDF[i + 1], ranking.getDatum()); - } - - // check failed attempts runtime distribution - expectedCDF = failedCDF; - cdf = new LoggedDiscreteCDF(); - cdf.setCDF(sampleFailed, attemptTimesPercentiles, 100); - - System.out.println("generated failed map runtime distribution"); - System.out.println(cdf.getNumberValues() + ": " + cdf.getMinimum() + "--" - + cdf.getMaximum()); - for (LoggedSingleRelativeRanking ranking : cdf.getRankings()) { - System.out.println(" " + ranking.getRelativeRanking() + ":" - + ranking.getDatum()); - } - assertRuntimeEqual(cdf.getMinimum(), expectedCDF[0]); - assertRuntimeEqual(cdf.getMaximum(), expectedCDF[4]); - for (int i = 0; i < 3; i++) { - LoggedSingleRelativeRanking ranking = cdf.getRankings().get(i); - assertRuntimeEqual(expectedCDF[i + 1], ranking.getDatum()); - } - } - - private void assertRuntimeEqual(long expected, long generated) { - if (expected == 0) { - assertTrue(generated > -1000 && generated < 1000); - } else { - long epsilon = Math.max(expected / 10, 5000); - assertTrue(expected - generated > -epsilon); - assertTrue(expected - generated < epsilon); - } - } - -} From ece854211f117424913cdf7da48eea6ec6dd3b5c Mon Sep 17 00:00:00 2001 From: Harsh J Date: Sun, 22 Jan 2012 17:30:33 +0000 Subject: [PATCH 8/8] HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. (Devaraj K via harsh) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1234555 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp | 2 +- .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp | 2 +- .../hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 50b40c03dc..99bb920b95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -274,6 +274,8 @@ Release 0.23.1 - UNRELEASED HDFS-2817. Combine the two TestSafeMode test suites. (todd) + HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. (Devaraj K via harsh) + OPTIMIZATIONS HDFS-2130. Switch default checksum to CRC32C. (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp index a5edcd3ed9..a71f40f26e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/corrupt_files.jsp @@ -41,7 +41,7 @@ -Hadoop <%=namenodeRole%> <%=namenodeLabel%> +Hadoop <%=namenodeRole%> <%=namenodeLabel%>

    <%=namenodeRole%> '<%=namenodeLabel%>'

    <%=NamenodeJspHelper.getVersionTable(fsn)%> diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp index 648200ce8a..ecce30ae88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.jsp @@ -37,7 +37,7 @@ -Hadoop <%=namenodeRole%> <%=namenodeLabel%> +Hadoop <%=namenodeRole%> <%=namenodeLabel%>

    <%=namenodeRole%> '<%=namenodeLabel%>'

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp index ed1c5ef5fc..886fbeaa35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfsnodelist.jsp @@ -37,7 +37,7 @@ String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameN -Hadoop <%=namenodeRole%> <%=namenodeLabel%> +Hadoop <%=namenodeRole%> <%=namenodeLabel%>

    <%=namenodeRole%> '<%=namenodeLabel%>'