diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 70b75f092c..dc6220e80a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -523,6 +523,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3689. RM web UI doesn't handle newline in job name. (Thomas Graves via mahadev) + MAPREDUCE-3549. write api documentation for web service apis for RM, NM, + mapreduce app master, and job history server (Thomas Graves via mahadev) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java index 2026c76ddb..f61b930430 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java @@ -30,7 +30,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.util.MRApps; -@XmlRootElement(name = "JobTaskAttemptCounters") +@XmlRootElement(name = "jobTaskAttemptCounters") @XmlAccessorType(XmlAccessType.FIELD) public class JobTaskAttemptCounterInfo { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java index ee824ee10a..e33a50671c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java @@ -629,7 +629,7 @@ public void testTaskAttemptIdCounters() throws JSONException, Exception { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("JobTaskAttemptCounters"); + JSONObject info = json.getJSONObject("jobTaskAttemptCounters"); verifyAMJobTaskAttemptCounters(info, att); } } @@ -661,7 +661,7 @@ public void testTaskAttemptIdXMLCounters() throws JSONException, Exception { InputSource is = new InputSource(); is.setCharacterStream(new StringReader(xml)); Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("JobTaskAttemptCounters"); + NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters"); verifyAMTaskCountersXML(nodes, att); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java index 6fdb94d902..7ba200fcc5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java @@ -642,7 +642,7 @@ public void testTaskAttemptIdCounters() throws JSONException, Exception { assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); - JSONObject info = json.getJSONObject("JobTaskAttemptCounters"); + JSONObject info = json.getJSONObject("jobTaskAttemptCounters"); verifyHsJobTaskAttemptCounters(info, att); } } @@ -674,7 +674,7 @@ public void testTaskAttemptIdXMLCounters() throws JSONException, Exception { InputSource is = new InputSource(); is.setCharacterStream(new StringReader(xml)); Document dom = db.parse(is); - NodeList nodes = dom.getElementsByTagName("JobTaskAttemptCounters"); + NodeList nodes = dom.getElementsByTagName("jobTaskAttemptCounters"); verifyHsTaskCountersXML(nodes, att); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm new file mode 100644 index 0000000000..a7dda193df --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm @@ -0,0 +1,2733 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + History Server REST API's. + --- + --- + ${maven.build.timestamp} + +History Server REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=3} + +* Overview + + The history server REST API's allow the user to get status on finished applications. Currently it only supports MapReduce and provides information on finished jobs. + +* History Server Information API + + The history server information resource provides overall information about the history server. + +** URI + + Both of the following URI's give you the history server information, from an application id identified by the appid value. + +------ + * http:///ws/v1/history + * http:///ws/v1/history/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "historyInfo" : { + "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012", + "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7", + "hadoopVersion" : "0.23.1-SNAPSHOT" + } +} ++---+ + + <> + + HTTP Request: + +----- + GET http:///ws/v1/history/info + Accept: application/xml +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 330 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7 + Wed Jan 11 21:18:36 UTC 2012 + ++---+ + +* MapReduce API's + + The following list of resources apply to MapReduce. + +** Jobs API + + The jobs resource provides a list of the MapReduce jobs that have finished. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + + Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all jobs that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + +------ + * user - user name + * queue - queue name + * limit - total number of app objects to be returned + * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch + * startedTimeEnd - jobs with start time ending with this time, specified in ms since epoch + * finishedTimeBegin - jobs with finish time beginning with this time, specified in ms since epoch + * finishedTimeEnd - jobs with finish time ending with this time, specified in ms since epoch +------ + +*** Elements of the object + + When you make a request for the list of jobs, the information will be returned as an array of job objects. + See also {{Job API}} for syntax of the job object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| job | array of job objects(json)/zero or more job objects(XML) | The collection of job objects | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobs" : { + "job" : [ + { + "avgReduceTime" : 833, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381344489, + "id" : "job_1326381300833_1_1", + "avgMapTime" : 2671, + "successfulMapAttempts" : 1, + "name" : "word count", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2570, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381356010 + }, + { + "avgReduceTime" : 124961, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381446529, + "id" : "job_1326381300833_2_2", + "avgMapTime" : 2638, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2589, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381582106 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1922 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381344489 + 1326381356010 + job_1326381300833_1_1 + word count + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2671 + 833 + 2540 + 2570 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + + 1326381446529 + 1326381582106 + job_1326381300833_2_2 + Sleep job + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2638 + 124961 + 2540 + 2589 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + ++---+ + +** {Job API} + + A Job resource contains information about a particular job identified by {jobid}. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id| +*---------------+--------------+-------------------------------+ +| name | string | The job name | +*---------------+--------------+-------------------------------+ +| queue | string | The queue the job was submitted to| +*---------------+--------------+-------------------------------+ +| user | string | The user name | +*---------------+--------------+-------------------------------+ +| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR| +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message | +*---------------+--------------+-------------------------------+ +| startTime | long | The time the job started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| finishTime | long | The time the job finished (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| mapsTotal | int | The total number of maps | +*---------------+--------------+-------------------------------+ +| mapsCompleted | int | The number of completed maps | +*---------------+--------------+-------------------------------+ +| reducesTotal | int | The total number of reduces | +*---------------+--------------+-------------------------------+ +| reducesCompleted | int | The number of completed reduces| +*---------------+--------------+-------------------------------+ +| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master| +*---------------+--------------+-------------------------------+ +| avgMapTime | long | The average time of a map task (in ms)| +*---------------+--------------+-------------------------------+ +| avgReduceTime | long | The average time of the reduce (in ms)| +*---------------+--------------+-------------------------------+ +| avgShuffleTime | long | The average time of the shuffle (in ms)| +*---------------+--------------+-------------------------------+ +| avgMergeTime | long | The average time of the merge (in ms)| +*---------------+--------------+-------------------------------+ +| failedReduceAttempts | int | The number of failed reduce attempts | +*---------------+--------------+-------------------------------+ +| killedReduceAttempts | int | The number of killed reduce attempts | +*---------------+--------------+-------------------------------+ +| successfulReduceAttempts | int | The number of successful reduce attempts | +*---------------+--------------+-------------------------------+ +| failedMapAttempts | int | The number of failed map attempts | +*---------------+--------------+-------------------------------+ +| killedMapAttempts | int | The number of killed map attempts | +*---------------+--------------+-------------------------------+ +| successfulMapAttempts | int | The number of successful map attempts | +*---------------+--------------+-------------------------------+ +| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| value | string | The acl value| +*---------------+--------------+-------------------------------+ +| name | string | The acl name | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Server: Jetty(6.1.26) + Content-Length: 720 ++---+ + + Response Body: + ++---+ +{ + "job" : { + "avgReduceTime" : 124961, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326381446529, + "id" : "job_1326381300833_2_2", + "avgMapTime" : 2638, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2540, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2589, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "default", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326381582106 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 983 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381446529 + 1326381582106 + job_1326381300833_2_2 + Sleep job + default + user1 + SUCCEEDED + 1 + 1 + 1 + 1 + false + + 2638 + 124961 + 2540 + 2589 + 0 + 0 + 1 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + ++---+ + +** Job Attempts API + + With the job attempts API, you can obtain a collection of resources that represent a job attempt. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/jobattempts +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + + When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects. + + jobAttempts: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects | +*---------------+--------------+--------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job attempt id | +*---------------+--------------+--------------------------------+ +| nodeId | string | The node id of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The node http address of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| logsLink | string | The http link to the job attempt logs | +*---------------+--------------+--------------------------------+ +| containerId | string | The id of the container for the job attempt | +*---------------+--------------+--------------------------------+ +| startTime | long | The start time of the attempt (in ms since epoch)| +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobAttempts" : { + "jobAttempt" : [ + { + "nodeId" : "host.domain.com:45454", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381444693, + "id" : 1, + "logsLink" : "http://host.domain.com:19888/jobhistory/logs/host.domain.com:45454/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1", + "containerId" : "container_1326381300833_0002_01_000001" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattmpts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 575 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + host.domain.com:9999 + host.domain.com:45454 + 1 + 1326381444693 + container_1326381300833_0002_01_000001 + http://host.domain.com:19888/jobhistory/logs/host.domain.com:45454/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1 + + ++---+ + +** Job Counters API + + With the job counters API, you can object a collection of resources that represent al the counters for that job. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id | +*---------------+--------------+-------------------------------+ +| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the objecs + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| reduceCounterValue | long | The counter value of reduce tasks | +*---------------+--------------+-------------------------------+ +| mapCounterValue | long | The counter value of map tasks | +*---------------+--------------+-------------------------------+ +| totalCounterValue | long | The counter value of all tasks | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobCounters" : { + "id" : "job_1326381300833_2_2", + "counterGroup" : [ + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BAD_ID" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "CONNECTION" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "IO_ERROR" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_LENGTH" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_MAP" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2483, + "name" : "FILE_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 108525, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "HDFS_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "HDFS_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MAP_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "MAP_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 4800, + "name" : "MAP_OUTPUT_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "MAP_OUTPUT_MATERIALIZED_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "SPLIT_RAW_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2400, + "name" : "SPILLED_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 113, + "name" : "GC_TIME_MILLIS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1830, + "name" : "CPU_MILLISECONDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 478068736, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2159284224, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 378863616, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_READ" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 7030 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + job_1326381300833_2_2 + + Shuffle Errors + + BAD_ID + 0 + 0 + 0 + + + CONNECTION + 0 + 0 + 0 + + + IO_ERROR + 0 + 0 + 0 + + + WRONG_LENGTH + 0 + 0 + 0 + + + WRONG_MAP + 0 + 0 + 0 + + + WRONG_REDUCE + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2483 + 0 + 0 + + + FILE_BYTES_WRITTEN + 108525 + 0 + 0 + + + FILE_READ_OPS + 0 + 0 + 0 + + + FILE_LARGE_READ_OPS + 0 + 0 + 0 + + + FILE_WRITE_OPS + 0 + 0 + 0 + + + HDFS_BYTES_READ + 48 + 0 + 0 + + + HDFS_BYTES_WRITTEN + 0 + 0 + 0 + + + HDFS_READ_OPS + 1 + 0 + 0 + + + HDFS_LARGE_READ_OPS + 0 + 0 + 0 + + + HDFS_WRITE_OPS + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + MAP_INPUT_RECORDS + 1 + 0 + 0 + + + MAP_OUTPUT_RECORDS + 1200 + 0 + 0 + + + MAP_OUTPUT_BYTES + 4800 + 0 + 0 + + + MAP_OUTPUT_MATERIALIZED_BYTES + 2235 + 0 + 0 + + + SPLIT_RAW_BYTES + 48 + 0 + 0 + + + COMBINE_INPUT_RECORDS + 0 + 0 + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + 0 + 0 + + + REDUCE_INPUT_GROUPS + 1200 + 0 + 0 + + + REDUCE_SHUFFLE_BYTES + 2235 + 0 + 0 + + + REDUCE_INPUT_RECORDS + 1200 + 0 + 0 + + + REDUCE_OUTPUT_RECORDS + 0 + 0 + 0 + + + SPILLED_RECORDS + 2400 + 0 + 0 + + + SHUFFLED_MAPS + 1 + 0 + 0 + + + FAILED_SHUFFLE + 0 + 0 + 0 + + + MERGED_MAP_OUTPUTS + 1 + 0 + 0 + + + GC_TIME_MILLIS + 113 + 0 + 0 + + + CPU_MILLISECONDS + 1830 + 0 + 0 + + + PHYSICAL_MEMORY_BYTES + 478068736 + 0 + 0 + + + VIRTUAL_MEMORY_BYTES + 2159284224 + 0 + 0 + + + COMMITTED_HEAP_BYTES + 378863616 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter + + BYTES_READ + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + 0 + 0 + + + ++---+ + + +** Job Conf API + + A job configuration resource contains information about the job configuration for this job. + +*** URI + + Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/conf +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| path | string | The path to the job configuration file| +*---------------+--------------+-------------------------------+ +| property | array of the configuration properties(JSON)/zero or more configuration properties(XML) | Collection of configuration property objects| +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the configuration property | +*---------------+--------------+-------------------------------+ +| value | string | The value of the configuration property | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + + This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file. + ++---+ +{ + "conf" : { + "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml", + "property" : [ + { + "value" : "/home/hadoop/hdfs/data", + "name" : "dfs.datanode.data.dir" + }, + { + "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", + "name" : "hadoop.http.filter.initializers" + }, + { + "value" : "/home/hadoop/tmp", + "name" : "mapreduce.cluster.temp.dir" + }, + ... + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml + + dfs.datanode.data.dir + /home/hadoop/hdfs/data + + + hadoop.http.filter.initializers + org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + + + mapreduce.cluster.temp.dir + /home/hadoop/tmp + + ... + ++---+ + +** Tasks API + + With the tasks API, you can obtain a collection of resources that represent a task within a job. When you run a GET operation on this resource, you obtain a collection of Task Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + * type - type of task, valid values are m or r. m for map task or r for reduce task. +------ + +*** Elements of the object + + When you make a request for the list of tasks , the information will be returned as an array of task objects. + See also {{Task API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects. | +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 6777, + "state" : "SUCCEEDED", + "startTime" : 1326381446541, + "id" : "task_1326381300833_2_2_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0", + "finishTime" : 1326381453318 + }, + { + "progress" : 100, + "elapsedTime" : 135559, + "state" : "SUCCEEDED", + "startTime" : 1326381446544, + "id" : "task_1326381300833_2_2_r_0", + "type" : "REDUCE", + "successfulAttempt" : "attempt_1326381300833_2_2_r_0_0", + "finishTime" : 1326381582103 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 653 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381446541 + 1326381453318 + 6777 + 100.0 + task_1326381300833_2_2_m_0 + SUCCEEDED + MAP + attempt_1326381300833_2_2_m_0_0 + + + 1326381446544 + 1326381582103 + 135559 + 100.0 + task_1326381300833_2_2_r_0 + SUCCEEDED + REDUCE + attempt_1326381300833_2_2_r_0_0 + + ++---+ + +** {Task API} + + A Task resource contains information about a particular task within a job. + +*** URI + + Use the following URI to obtain an Task Object, from a task identified by the {taskid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED +*---------------+--------------+--------------------------------+ +| type | string | The task type - MAP or REDUCE| +*---------------+--------------+--------------------------------+ +| successfulAttempt | string | The id of the last successful attempt | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ + + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "task" : { + "progress" : 100, + "elapsedTime" : 6777, + "state" : "SUCCEEDED", + "startTime" : 1326381446541, + "id" : "task_1326381300833_2_2_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0", + "finishTime" : 1326381453318 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 299 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381446541 + 1326381453318 + 6777 + 100.0 + task_1326381300833_2_2_m_0 + SUCCEEDED + MAP + attempt_1326381300833_2_2_m_0_0 + ++---+ + +** Task Counters API + + With the task counters API, you can object a collection of resources that represent al the counters for that task. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+-------------------------------+ +| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskCounters" : { + "id" : "task_1326381300833_2_2_m_0", + "taskCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2660 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + task_1326381300833_2_2_m_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + +** Task Attempts API + + With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + + When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. + See also {{Task Attempt API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects | +*---------------+--------------+--------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "assignedContainerId" : "container_1326381300833_0002_01_000002", + "progress" : 100, + "elapsedTime" : 2638, + "state" : "SUCCEEDED", + "diagnostics" : "", + "rack" : "/98.139.92.0", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381450680, + "id" : "attempt_1326381300833_2_2_m_0_0", + "type" : "MAP", + "finishTime" : 1326381453318 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 537 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326381450680 + 1326381453318 + 2638 + 100.0 + attempt_1326381300833_2_2_m_0_0 + /98.139.92.0 + SUCCEEDED + host.domain.com:9999 + + MAP + container_1326381300833_0002_01_000002 + + ++---+ + +** {Task Attempt API} + + A Task Attempt resource contains information about a particular task attempt within a job. + +*** URI + + Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value. + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid} +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| rack | string | The rack | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED | +*---------------+--------------+--------------------------------+ +| type | string | The type of task | +*---------------+--------------+--------------------------------+ +| assignedContainerId | string | The container id this attempt is assigned to| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The http address of the node this task attempt ran on | +*---------------+--------------+--------------------------------+ +| diagnostics| string | A diagnostics message | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task attempt as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task attempt started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task attempt finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the task attempt started (in ms)| +*---------------+--------------+--------------------------------+ + + For reduce task attempts you also have the following fields: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| mergeFinishTime | long | The time at which merge finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)| +*---------------+--------------+--------------------------------+ +| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)| +*---------------+--------------+--------------------------------+ +| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)| +*---------------+--------------+--------------------------------+ + + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempt" : { + "assignedContainerId" : "container_1326381300833_0002_01_000002", + "progress" : 100, + "elapsedTime" : 2638, + "state" : "SUCCEEDED", + "diagnostics" : "", + "rack" : "/98.139.92.0", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326381450680, + "id" : "attempt_1326381300833_2_2_m_0_0", + "type" : "MAP", + "finishTime" : 1326381453318 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 691 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326381450680 + 1326381453318 + 2638 + 100.0 + attempt_1326381300833_2_2_m_0_0 + /98.139.92.0 + SUCCEEDED + host.domain.com:9999 + + MAP + container_1326381300833_0002_01_000002 + ++---+ + +** Task Attempt Counters API + + With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt. + +*** URI + +------ + * http:///ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters +------ + +*** HTTP Operations Supported + +------ + * GET +------ + +*** Query Parameters Supported + +------ + None +------ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task attempt id | +*---------------+--------------+-------------------------------+ +| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +*** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326381300833_2_2_m_0_0" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2735 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + attempt_1326381300833_2_2_m_0_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm new file mode 100644 index 0000000000..186f044c22 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm @@ -0,0 +1,2701 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + MapReduce Application Master REST API's. + --- + --- + ${maven.build.timestamp} + +MapReduce Application Master REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The MapReduce Application Master REST API's allow the user to get status on the running MapReduce application master. Currently this is the equivalent to a running MapReduce job. The information includes the jobs the app master is running and all the job particulars like tasks, counters, configuration, attempts, etc. The application master should be accessed via the proxy. This proxy is configurable to run either on the resource manager or on a separate host. The proxy URL usually looks like: http:///proxy/{appid}. + +* Mapreduce Application Master Info API + + The MapReduce application master information resource provides overall information about that mapreduce application master. This includes application id, time it was started, user, name, etc. + +** URI + + Both of the following URI's give you the MapReduce application master information, from an application id identified by the appid value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce + * http:///proxy/{appid}/ws/v1/mapreduce/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the mapreduce application master information, the information will be returned as an info object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| appId | long | The application id | +*---------------+--------------+-------------------------------+ +| startedOn | long | The time the application started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| name | string | The name of the application | +*---------------+--------------+-------------------------------+ +| user | string | The user name of the user who started the application | +*---------------+--------------+-------------------------------+ +| elapsedTime | long | The time since the application was started (in ms)| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0003/ws/v1/mapreduce/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "info" : { + "appId" : "application_1326232085508_0003", + "startedOn" : 1326238244047, + "user" : "user1", + "name" : "Sleep job", + "elapsedTime" : 32374 + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///proxy/application_1326232085508_0003/ws/v1/mapreduce/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 223 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326232085508_0003 + Sleep job + user1 + 1326238244047 + 32407 + ++---+ + +* Jobs API + + The jobs resource provides a list of the jobs running on this application master. See also {{Job API}} for syntax of the job object. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of jobs, the information will be returned as a collection of job objects. See also {{Job API}} for syntax of the job object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| job | array of job objects(JSON)/Zero or more job objects(XML) | The collection of job objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobs" : { + "job" : [ + { + "runningReduceAttempts" : 1, + "reduceProgress" : 100, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326238769379, + "id" : "job_1326232085508_4_4", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 59377, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } + ] + } + } ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1214 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238769379 + 0 + 59416 + job_1326232085508_4_4 + Sleep job + user1 + RUNNING + 1 + 1 + 1 + 0 + 100.0 + 100.0 + 0 + 0 + 0 + 1 + false + + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + + + ++---+ + +* {Job API} + + A job resource contains information about a particular job that was started by this application master. Certain fields are only accessible if user has permissions - depends on acl settings. + +** URI + + Use the following URI to obtain a job object, for a job identified by the jobid value. + +------ + * http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id| +*---------------+--------------+-------------------------------+ +| name | string | The job name | +*---------------+--------------+-------------------------------+ +| user | string | The user name | +*---------------+--------------+-------------------------------+ +| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR| +*---------------+--------------+-------------------------------+ +| startTime | long | The time the job started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| finishTime | long | The time the job finished (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| elapsedTime | long | The elapsed time since job started (in ms)| +*---------------+--------------+-------------------------------+ +| mapsTotal | int | The total number of maps | +*---------------+--------------+-------------------------------+ +| mapsCompleted | int | The number of completed maps | +*---------------+--------------+-------------------------------+ +| reducesTotal | int | The total number of reduces | +*---------------+--------------+-------------------------------+ +| reducesCompleted | int | The number of completed reduces| +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message | +*---------------+--------------+-------------------------------+ +| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master| +*---------------+--------------+-------------------------------+ +| mapsPending | int | The number of maps still to be run| +*---------------+--------------+-------------------------------+ +| mapsRunning | int | The number of running maps | +*---------------+--------------+-------------------------------+ +| reducesPending | int | The number of reduces still to be run | +*---------------+--------------+-------------------------------+ +| reducesRunning | int | The number of running reduces| +*---------------+--------------+-------------------------------+ +| newReduceAttempts | int | The number of new reduce attempts | +*---------------+--------------+-------------------------------+ +| runningReduceAttempts | int | The number of running reduce attempts | +*---------------+--------------+-------------------------------+ +| failedReduceAttempts | int | The number of failed reduce attempts | +*---------------+--------------+-------------------------------+ +| killedReduceAttempts | int | The number of killed reduce attempts | +*---------------+--------------+-------------------------------+ +| successfulReduceAttempts | int | The number of successful reduce attempts | +*---------------+--------------+-------------------------------+ +| newMapAttempts | int | The number of new map attempts | +*---------------+--------------+-------------------------------+ +| runningMapAttempts | int | The number of running map attempts | +*---------------+--------------+-------------------------------+ +| failedMapAttempts | int | The number of failed map attempts | +*---------------+--------------+-------------------------------+ +| killedMapAttempts | int | The number of killed map attempts | +*---------------+--------------+-------------------------------+ +| successfulMapAttempts | int | The number of successful map attempts | +*---------------+--------------+-------------------------------+ +| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| value | string | The acl value| +*---------------+--------------+-------------------------------+ +| name | string | The acl name | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Server: Jetty(6.1.26) + Content-Length: 720 ++---+ + + Response Body: + ++---+ +{ + "job" : { + "runningReduceAttempts" : 1, + "reduceProgress" : 100, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326238769379, + "id" : "job_1326232085508_4_4", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 59437, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1201 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238769379 + 0 + 59474 + job_1326232085508_4_4 + Sleep job + user1 + RUNNING + 1 + 1 + 1 + 0 + 100.0 + 100.0 + 0 + 0 + 0 + 1 + false + + 0 + 1 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 1 + + mapreduce.job.acl-modify-job + + + + mapreduce.job.acl-view-job + + ++---+ + +* Job Attempts API + + With the job attempts API, you can obtain a collection of resources that represent the job attempts. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects. + +** URI + +------ + * http:///ws/v1/history/jobs/{jobid}/jobattempts +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects | +*---------------+--------------+--------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job attempt id | +*---------------+--------------+--------------------------------+ +| nodeId | string | The node id of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The node http address of the node the attempt ran on| +*---------------+--------------+--------------------------------+ +| logsLink | string | The http link to the job attempt logs | +*---------------+--------------+--------------------------------+ +| containerId | string | The id of the container for the job attempt | +*---------------+--------------+--------------------------------+ +| startTime | long | The start time of the attempt (in ms since epoch)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobAttempts" : { + "jobAttempt" : [ + { + "nodeId" : "host.domain.com:45454", + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326238773493, + "id" : 1, + "logsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326232085508_0004_01_000001", + "containerId" : "container_1326232085508_0004_01_000001" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 498 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + host.domain.com:9999 + host.domain.com:45454 + 1 + 1326238773493 + container_1326232085508_0004_01_000001 + http://host.domain.com:9999/node/containerlogs/container_1326232085508_0004_01_000001 + + ++---+ + +* Job Counters API + + With the job counters API, you can object a collection of resources that represent all the counters for that job. + +** URI + +------ + * http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The job id | +*---------------+--------------+-------------------------------+ +| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| reduceCounterValue | long | The counter value of reduce tasks | +*---------------+--------------+-------------------------------+ +| mapCounterValue | long | The counter value of map tasks | +*---------------+--------------+-------------------------------+ +| totalCounterValue | long | The counter value of all tasks | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobCounters" : { + "id" : "job_1326232085508_4_4", + "counterGroup" : [ + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BAD_ID" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "CONNECTION" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "IO_ERROR" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_LENGTH" + }, { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_MAP" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2483, + "name" : "FILE_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 108763, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "HDFS_BYTES_READ" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "HDFS_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MAP_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "MAP_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 4800, + "name" : "MAP_OUTPUT_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "MAP_OUTPUT_MATERIALIZED_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 48, + "name" : "SPLIT_RAW_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1200, + "name" : "SPILLED_RECORDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 58, + "name" : "GC_TIME_MILLIS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 1580, + "name" : "CPU_MILLISECONDS" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 462643200, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 2149728256, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 357957632, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_READ" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "reduceCounterValue" : 0, + "mapCounterValue" : 0, + "totalCounterValue" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 7027 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + job_1326232085508_4_4 + + Shuffle Errors + + BAD_ID + 0 + 0 + 0 + + + CONNECTION + 0 + 0 + 0 + + + IO_ERROR + 0 + 0 + 0 + + + WRONG_LENGTH + 0 + 0 + 0 + + + WRONG_MAP + 0 + 0 + 0 + + + WRONG_REDUCE + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2483 + 0 + 0 + + + FILE_BYTES_WRITTEN + 108763 + 0 + 0 + + + FILE_READ_OPS + 0 + 0 + 0 + + + FILE_LARGE_READ_OPS + 0 + 0 + 0 + + + FILE_WRITE_OPS + 0 + 0 + 0 + + + HDFS_BYTES_READ + 48 + 0 + 0 + + + HDFS_BYTES_WRITTEN + 0 + 0 + 0 + + + HDFS_READ_OPS + 1 + 0 + 0 + + + HDFS_LARGE_READ_OPS + 0 + 0 + 0 + + + HDFS_WRITE_OPS + 0 + 0 + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + MAP_INPUT_RECORDS + 1 + 0 + 0 + + + MAP_OUTPUT_RECORDS + 1200 + 0 + 0 + + + MAP_OUTPUT_BYTES + 4800 + 0 + 0 + + + MAP_OUTPUT_MATERIALIZED_BYTES + 2235 + 0 + 0 + + + SPLIT_RAW_BYTES + 48 + 0 + 0 + + + COMBINE_INPUT_RECORDS + 0 + 0 + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + 0 + 0 + + + REDUCE_INPUT_GROUPS + 460 + 0 + 0 + + + REDUCE_SHUFFLE_BYTES + 2235 + 0 + 0 + + + REDUCE_INPUT_RECORDS + 460 + 0 + 0 + + + REDUCE_OUTPUT_RECORDS + 0 + 0 + 0 + + + SPILLED_RECORDS + 1200 + 0 + 0 + + + SHUFFLED_MAPS + 1 + 0 + 0 + + + FAILED_SHUFFLE + 0 + 0 + 0 + + + MERGED_MAP_OUTPUTS + 1 + 0 + 0 + + + GC_TIME_MILLIS + 58 + 0 + 0 + + + CPU_MILLISECONDS + 1580 + 0 + 0 + + + PHYSICAL_MEMORY_BYTES + 462643200 + 0 + 0 + + + VIRTUAL_MEMORY_BYTES + 2149728256 + 0 + 0 + + + COMMITTED_HEAP_BYTES + 357957632 + 0 + 0 + + + + org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter + + BYTES_READ + 0 + 0 + 0 + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + BYTES_WRITTEN + 0 + 0 + 0 + + + ++---+ + +* Job Conf API + + A job configuration resource contains information about the job configuration for this job. + +** URI + + Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/conf +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| path | string | The path to the job configuration file| +*---------------+--------------+-------------------------------+ +| property | array of the configuration properties(JSON)/zero or more property objects(XML) | Collection of property objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the configuration property | +*---------------+--------------+-------------------------------+ +| value | string | The value of the configuration property | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + + This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file. + ++---+ +{ + "conf" : { + "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml", + "property" : [ + { + "value" : "/home/hadoop/hdfs/data", + "name" : "dfs.datanode.data.dir" + }, + { + "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer", + "name" : "hadoop.http.filter.initializers" + }, + { + "value" : "/home/hadoop/tmp", + "name" : "mapreduce.cluster.temp.dir" + }, + ... + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml + + dfs.datanode.data.dir + /home/hadoop/hdfs/data + + + hadoop.http.filter.initializers + org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer + + + mapreduce.cluster.temp.dir + /home/hadoop/tmp + + ... + ++---+ + +* Tasks API + + With the tasks API, you can obtain a collection of resources that represent all the tasks for a job. When you run a GET operation on this resource, you obtain a collection of Task Objects. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + * type - type of task, valid values are m or r. m for map task or r for reduce task. +------ + +** Elements of the object + + When you make a request for the list of tasks , the information will be returned as an array of task objects. + See also {{Task API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 2768, + "state" : "SUCCEEDED", + "startTime" : 1326238773493, + "id" : "task_1326232085508_4_4_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326232085508_4_4_m_0_0", + "finishTime" : 1326238776261 + }, + { + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326238777460, + "id" : "task_1326232085508_4_4_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 603 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238773493 + 1326238776261 + 2768 + 100.0 + task_1326232085508_4_4_m_0 + SUCCEEDED + MAP + attempt_1326232085508_4_4_m_0_0 + + + 1326238777460 + 0 + 0 + 100.0 + task_1326232085508_4_4_r_0 + RUNNING + REDUCE + + + ++---+ + +* {Task API} + + A Task resource contains information about a particular task within a job. + +** URI + + Use the following URI to obtain an Task Object, from a task identified by the {taskid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED | +*---------------+--------------+--------------------------------+ +| type | string | The task type - MAP or REDUCE| +*---------------+--------------+--------------------------------+ +| successfulAttempt | string | The the id of the last successful attempt | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "task" : { + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326238777460, + "id" : "task_1326232085508_4_4_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 299 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238777460 + 0 + 0 + 100.0 + task_1326232085508_4_4_r_0 + RUNNING + REDUCE + + ++---+ + +* Task Counters API + + With the task counters API, you can object a collection of resources that represent all the counters for that task. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+-------------------------------+ +| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskCounters" : { + "id" : "task_1326232085508_4_4_r_0", + "taskCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2660 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + task_1326232085508_4_4_r_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + +* Task Attempts API + + With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. + See also {{Task Attempt API}} for syntax of the task object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "elapsedMergeTime" : 47, + "shuffleFinishTime" : 1326238780052, + "assignedContainerId" : "container_1326232085508_0004_01_000003", + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2592, + "mergeFinishTime" : 1326238780099, + "rack" : "/98.139.92.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "type" : "REDUCE", + "startTime" : 1326238777460, + "id" : "attempt_1326232085508_4_4_r_0_0", + "finishTime" : 0 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 807 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1326238777460 + 0 + 0 + 100.0 + attempt_1326232085508_4_4_r_0_0 + /98.139.92.0 + RUNNING + host.domain.com:9999 + REDUCE + container_1326232085508_0004_01_000003 + 1326238780052 + 1326238780099 + 2592 + 47 + 0 + + ++---+ + +* {Task Attempt API} + + A Task Attempt resource contains information about a particular task attempt within a job. + +** URI + + Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value. + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task id | +*---------------+--------------+--------------------------------+ +| rack | string | The rack | +*---------------+--------------+--------------------------------+ +| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED| +*---------------+--------------+--------------------------------+ +| type | string | The type of task | +*---------------+--------------+--------------------------------+ +| assignedContainerId | string | The container id this attempt is assigned to| +*---------------+--------------+--------------------------------+ +| nodeHttpAddress | string | The http address of the node this task attempt ran on | +*---------------+--------------+--------------------------------+ +| diagnostics| string | The diagnostics message | +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the task attempt as a percent| +*---------------+--------------+--------------------------------+ +| startTime | long | The time in which the task attempt started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishTime | long | The time in which the task attempt finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the task attempt started (in ms)| +*---------------+--------------+--------------------------------+ + + For reduce task attempts you also have the following fields: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| mergeFinishTime | long | The time at which merge finished (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)| +*---------------+--------------+--------------------------------+ +| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)| +*---------------+--------------+--------------------------------+ +| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "taskAttempt" : { + "elapsedMergeTime" : 47, + "shuffleFinishTime" : 1326238780052, + "assignedContainerId" : "container_1326232085508_0004_01_000003", + "progress" : 100, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2592, + "mergeFinishTime" : 1326238780099, + "rack" : "/98.139.92.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "startTime" : 1326238777460, + "id" : "attempt_1326232085508_4_4_r_0_0", + "type" : "REDUCE", + "finishTime" : 0 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 691 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1326238777460 + 0 + 0 + 100.0 + attempt_1326232085508_4_4_r_0_0 + /98.139.92.0 + RUNNING + host.domain.com:9999 + REDUCE + container_1326232085508_0004_01_000003 + 1326238780052 + 1326238780099 + 2592 + 47 + 0 + ++---+ + +* Task Attempt Counters API + + With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt. + +** URI + +------ + * http:///proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The task attempt id | +*---------------+--------------+-------------------------------+ +| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| counterGroupName | string | The name of the counter group | +*---------------+--------------+-------------------------------+ +| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects | +*---------------+--------------+-------------------------------+ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| name | string | The name of the counter | +*---------------+--------------+-------------------------------+ +| value | long | The value of the counter | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "jobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 2363, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 54372, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 2235, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 460, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 26, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 860, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 107839488, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1123147776, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 57475072, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326232085508_4_4_r_0_0" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2735 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + attempt_1326232085508_4_4_r_0_0 + + org.apache.hadoop.mapreduce.FileSystemCounter + + FILE_BYTES_READ + 2363 + + + FILE_BYTES_WRITTEN + 54372 + + + FILE_READ_OPS + 0 + + + FILE_LARGE_READ_OPS + 0 + + + FILE_WRITE_OPS + 0 + + + HDFS_BYTES_READ + 0 + + + HDFS_BYTES_WRITTEN + 0 + + + HDFS_READ_OPS + 0 + + + HDFS_LARGE_READ_OPS + 0 + + + HDFS_WRITE_OPS + 0 + + + + org.apache.hadoop.mapreduce.TaskCounter + + COMBINE_INPUT_RECORDS + 0 + + + COMBINE_OUTPUT_RECORDS + 0 + + + REDUCE_INPUT_GROUPS + 460 + + + REDUCE_SHUFFLE_BYTES + 2235 + + + REDUCE_INPUT_RECORDS + 460 + + + REDUCE_OUTPUT_RECORDS + 0 + + + SPILLED_RECORDS + 0 + + + SHUFFLED_MAPS + 1 + + + FAILED_SHUFFLE + 0 + + + MERGED_MAP_OUTPUTS + 1 + + + GC_TIME_MILLIS + 26 + + + CPU_MILLISECONDS + 860 + + + PHYSICAL_MEMORY_BYTES + 107839488 + + + VIRTUAL_MEMORY_BYTES + 1123147776 + + + COMMITTED_HEAP_BYTES + 57475072 + + + + Shuffle Errors + + BAD_ID + 0 + + + CONNECTION + 0 + + + IO_ERROR + 0 + + + WRONG_LENGTH + 0 + + + WRONG_MAP + 0 + + + WRONG_REDUCE + 0 + + + + org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter + + BYTES_WRITTEN + 0 + + + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm new file mode 100644 index 0000000000..a733d71378 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm @@ -0,0 +1,635 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + NodeManager REST API's. + --- + --- + ${maven.build.timestamp} + +NodeManager REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The NodeManager REST API's allow the user to get status on the node and information about applications and containers running on that node. + +* NodeManager Information API + + The node information resource provides overall information about that particular node. + +** URI + + Both of the following URI's give you the cluster information. + +------ + * http:///ws/v1/node + * http:///ws/v1/node/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | long | The NodeManager id | +*---------------+--------------+-------------------------------+ +| nodeHostName | string | The host name of the NodeManager | +*---------------+--------------+-------------------------------+ +| totalPmemAllocatedContainersMB | long | The amount of physical memory allocated for use by containers in MB | +*---------------+--------------+-------------------------------+ +| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB | +*---------------+--------------+-------------------------------+ +| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| healthReport | string | The diagnostic health report of the node | +*---------------+--------------+-------------------------------+ +| nodeHealthy | boolean | true/false indicator of if the node is healthy| +*---------------+--------------+-------------------------------+ +| nodeManagerVersion | string | Version of the NodeManager | +*---------------+--------------+-------------------------------+ +| nodeManagerBuildVersion | string | NodeManager build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| nodeManagerVersionBuiltOn | string | Timestamp when NodeManager was built(in ms since epoch) | +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built(in ms since epoch) | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "nodeInfo" : { + "hadoopVersionBuiltOn" : "Mon Jan 9 14:58:42 UTC 2012", + "nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c", + "lastNodeUpdateTime" : 1326222266126, + "totalVmemAllocatedContainersMB" : 17203, + "nodeHealthy" : true, + "healthReport" : "", + "totalPmemAllocatedContainersMB" : 8192, + "nodeManagerVersionBuiltOn" : "Mon Jan 9 15:01:59 UTC 2012", + "nodeManagerVersion" : "0.23.1-SNAPSHOT", + "id" : "host.domain.com:45454", + "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1228292 by user1 source checksum 3eba233f2248a089e9b28841a784dd00", + "nodeHostName" : "host.domain.com", + "hadoopVersion" : "0.23.1-SNAPSHOT" + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/node/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 983 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 17203 + 8192 + 1326222386134 + true + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c + Mon Jan 9 15:01:59 UTC 2012 + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1228292 by user1 source checksum 3eba233f2248a089e9b28841a784dd00 + Mon Jan 9 14:58:42 UTC 2012 + host.domain.com:45454 + host.domain.com + ++---+ + +* Applications API + + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. See also {{Application API}} for syntax of the application object. + +** URI + +------ + * http:///ws/v1/node/apps +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + + Multiple paramters can be specified. + +------ + * state - application state + * user - user name +------ + +** Elements of the (Applications) object + + When you make a request for the list of applications, the information will be returned as a collection of app objects. + See also {{Application API}} for syntax of the app object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| app | array of app objects(JSON)/zero or more app objects(XML) | A collection of application objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "apps" : { + "app" : [ + { + "containerids" : [ + "container_1326121700862_0003_01_000001", + "container_1326121700862_0003_01_000002" + ], + "user" : "user1", + "id" : "application_1326121700862_0003", + "state" : "RUNNING" + }, + { + "user" : "user1", + "id" : "application_1326121700862_0002", + "state" : "FINISHED" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 400 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + application_1326121700862_0002 + FINISHED + user1 + + + application_1326121700862_0003 + RUNNING + user1 + container_1326121700862_0003_01_000002 + container_1326121700862_0003_01_000001 + + + ++---+ + +* {Application API} + + An application resource contains information about a particular application that was run or is running on this NodeManager. + +** URI + + Use the following URI to obtain an app Object, for a application identified by the {appid} value. + +------ + * http:///ws/v1/node/apps/{appid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the (Application) object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The application id | +*---------------+--------------+--------------------------------+ +| user | string | The user who started the application | +*---------------+--------------+--------------------------------+ +| state | string | The state of the application - valid states are: NEW, INITING, RUNNING, FINISHING_CONTAINERS_WAIT, APPLICATION_RESOURCES_CLEANINGUP, FINISHED | +*---------------+--------------+--------------------------------+ +| containerids | array of containerids(JSON)/zero or more containerids(XML) | The list of containerids currently being used by the application on this node. If not present then no containers are currently running for this application.| +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps/application_1326121700862_0005 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "app" : { + "containerids" : [ + "container_1326121700862_0005_01_000003", + "container_1326121700862_0005_01_000001" + ], + "user" : "user1", + "id" : "application_1326121700862_0005", + "state" : "RUNNING" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/apps/application_1326121700862_0005 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 281 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326121700862_0005 + RUNNING + user1 + container_1326121700862_0005_01_000003 + container_1326121700862_0005_01_000001 + ++---+ + + +* Containers API + + With the containers API, you can obtain a collection of resources, each of which represents a container. When you run a GET operation on this resource, you obtain a collection of Container Objects. See also {{Container API}} for syntax of the container object. + +** URI + +------ + * http:///ws/v1/node/containers +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + + When you make a request for the list of containers, the information will be returned as collection of container objects. + See also {{Container API}} for syntax of the container object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| containers | array of container objects(JSON)/zero or more container objects(XML) | A collection of container objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "containers" : { + "container" : [ + { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000001/user1", + "user" : "user1", + "id" : "container_1326121700862_0006_01_000001", + "exitCode" : -1000 + }, + { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000003/user1", + "user" : "user1", + "id" : "container_1326121700862_0006_01_000003", + "exitCode" : -1000 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 988 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + container_1326121700862_0006_01_000001 + RUNNING + -1000 + + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000001/user1 + host.domain.com:45454 + + + container_1326121700862_0006_01_000003 + DONE + 0 + Container killed by the ApplicationMaster. + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0006_01_000003/user1 + host.domain.com:45454 + + ++---+ + + +* {Container API} + + A container resource contains information about a particular container that is running on this NodeManager. + +** URI + + Use the following URI to obtain a Container Object, from a container identified by the {containerid} value. + +------ + * http:///ws/v1/node/containers/{containerid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The container id | +*---------------+--------------+-------------------------------+ +| state | string | State of the container - valid states are: NEW, LOCALIZING, LOCALIZATION_FAILED, LOCALIZED, RUNNING, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING, CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE| +*---------------+--------------+-------------------------------+ +| nodeId | string | The id of the node the container is on| +*---------------+--------------+-------------------------------+ +| containerLogsLink | string | The http link to the container logs | +*---------------+--------------+-------------------------------+ +| user | string | The user name of the user which started the container| +*---------------+--------------+-------------------------------+ +| exitCode | int | Exit code of the container | +*---------------+--------------+-------------------------------+ +| diagnostics | string | A diagnostic message for failed containers | +*---------------+--------------+-------------------------------+ +| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/nodes/containers/container_1326121700862_0007_01_000001 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "container" : { + "nodeId" : "host.domain.com:45454", + "totalMemoryNeededMB" : 2048, + "state" : "RUNNING", + "diagnostics" : "", + "containerLogsLink" : "http://host.domain.com:9999/node/containerlogs/container_1326121700862_0007_01_000001/user1", + "user" : "user1", + "id" : "container_1326121700862_0007_01_000001", + "exitCode" : -1000 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/node/containers/container_1326121700862_0007_01_000001 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 491 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + container_1326121700862_0007_01_000001 + RUNNING + -1000 + + user1 + 2048 + http://host.domain.com:9999/node/containerlogs/container_1326121700862_0007_01_000001/user1 + host.domain.com:45454 + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm new file mode 100644 index 0000000000..e762594af8 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -0,0 +1,1469 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + ResourceManager REST API's. + --- + --- + ${maven.build.timestamp} + +ResourceManager REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0|toDepth=2} + +* Overview + + The ResourceManager REST API's allow the user to get information about the cluster - status on the cluster, metrics on the cluster, scheduler information, information about nodes in the cluster, and information about applications on the cluster. + +* Cluster Information API + + The cluster information resource provides overall information about the cluster. + +** URI + + Both of the following URI's give you the cluster information. + +------ + * http:///ws/v1/cluster + * http:///ws/v1/cluster/info +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | long | The cluster id | +*---------------+--------------+-------------------------------+ +| startedOn | long | The time the cluster started (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| state | string | The ResourceManager state - valid values are: NOTINITED, INITED, STARTED, STOPPED| +*---------------+--------------+-------------------------------+ +| resourceManagerVersion | string | Version of the ResourceManager | +*---------------+--------------+-------------------------------+ +| resourceManagerBuildVersion | string | ResourceManager build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| resourceManagerVersionBuiltOn | string | Timestamp when ResourceManager was built (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| hadoopVersion | string | Version of hadoop common | +*---------------+--------------+-------------------------------+ +| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum | +*---------------+--------------+-------------------------------+ +| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built(in ms since epoch)| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/info +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "clusterInfo": + { + "id":1324053971963, + "startedOn":1324053971963, + "state":"STARTED", + "resourceManagerVersion":"0.23.1-SNAPSHOT", + "resourceManagerBuildVersion":"0.23.1-SNAPSHOT from 1214049 by user1 source checksum 050cd664439d931c8743a6428fd6a693", + "resourceManagerVersionBuiltOn":"Tue Dec 13 22:12:48 CST 2011", + "hadoopVersion":"0.23.1-SNAPSHOT", + "hadoopBuildVersion":"0.23.1-SNAPSHOT from 1214049 by user1 source checksum 11458df3bb77342dca5f917198fad328", + "hadoopVersionBuiltOn":"Tue Dec 13 22:12:26 CST 2011" + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/cluster/info +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 712 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 1324053971963 + 1324053971963 + STARTED + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1214049 by user1 source checksum 050cd664439d931c8743a6428fd6a693 + Tue Dec 13 22:12:48 CST 2011 + 0.23.1-SNAPSHOT + 0.23.1-SNAPSHOT from 1214049 by user1 source checksum 11458df3bb77342dca5f917198fad328 + Tue Dec 13 22:12:48 CST 2011 + ++---+ + +* Cluster Metrics API + + The cluster metrics resource provides some overall metrics about the cluster. More detailed metrics should be retrieved from the jmx interface. + +** URI + +------ + * http:///ws/v1/cluster/metrics +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| appsSubmitted | int | The number of applications submitted | +*---------------+--------------+-------------------------------+ +| reservedMB | long | The amount of memory reserved in MB | +*---------------+--------------+-------------------------------+ +| availableMB | long | The amount of memory available in MB | +*---------------+--------------+-------------------------------+ +| allocatedMB | long | The amount of memory allocated in MB | +*---------------+--------------+-------------------------------+ +| totalMB | long | The amount of total memory in MB | +*---------------+--------------+-------------------------------+ +| containersAllocated | int | The number of containers allocated | +*---------------+--------------+-------------------------------+ +| totalNodes | int | The total number of nodes | +*---------------+--------------+-------------------------------+ +| activeNodes | int | The number of active nodes | +*---------------+--------------+-------------------------------+ +| lostNodes | int | The number of lost nodes | +*---------------+--------------+-------------------------------+ +| unhealthyNodes | int | The number of unhealthy nodes | +*---------------+--------------+-------------------------------+ +| decommissionedNodes | int | The number of nodes decommissioned | +*---------------+--------------+-------------------------------+ +| rebootedNodes | int | The number of nodes rebooted | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/metrics +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + { + "clusterMetrics": + { + "appsSubmitted":4, + "reservedMB":0, + "availableMB":8192, + "allocatedMB":0, + "totalMB":8192, + "containersAllocated":0, + "totalNodes":1, + "activeNodes":1, + "lostNodes":0, + "unhealthyNodes":0, + "decommissionedNodes":0, + "rebootedNodes":0 + } + } ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/metrics + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 432 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + 4 + 0 + 8192 + 0 + 0 + 8192 + 1 + 1 + 0 + 0 + 0 + 0 + ++---+ + +* Cluster Scheduler API + + A scheduler resource contains information about the current scheduler configured in a cluster. It currently supports both the Fifo and Capacity Scheduler. You will get different information depending on which scheduler is configured so be sure to look at the type information. + +** URI + +------ + * http:///ws/v1/cluster/scheduler +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Capacity Scheduler API + + The capacity scheduler supports hierarchical queues. This one request will print information about all the queues and any subqueues they have. + Queues that can actually have jobs submitted to them are referred to as leaf queues. These queues have additional data associated with them. + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | string | Scheduler type - capacityScheduler| +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +*---------------+--------------+-------------------------------+ +| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| queueName | string | Name of the queue | +*---------------+--------------+-------------------------------+ +| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of queue resources| +*---------------+--------------+-------------------------------+ + +** Elements of the queues/subQueues object for a Parent queue + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage relative its to parent queue | +*---------------+--------------+-------------------------------+ +| maxCapacity | float | Maximum queue capacity in percentage relative to its parent queue | +*---------------+--------------+-------------------------------+ +| absoluteCapacity | float | Absolute capacity percentage this queue can use of entire cluster | +*---------------+--------------+-------------------------------+ +| absoluteMaxCapacity | float | Absolute maximum capacity percentage this queue can use of the entire cluster | +*---------------+--------------+-------------------------------+ +| utilization | float | Queue utilization percentage relative to the entire cluster | +*---------------+--------------+-------------------------------+ +| numApplications | int | The number of applications currently in the queue | +*---------------+--------------+-------------------------------+ +| usedResources | string | A string describing the current resources used by the queue | +*---------------+--------------+-------------------------------+ +| queueName | string | The name of the queue | +*---------------+--------------+-------------------------------+ +| state | string of QueueState | The state of the queue | +*---------------+--------------+-------------------------------+ +| subQueues | array of queues(JSON)/zero or more queue objects(XML) | A collection of sub-queue information| +*---------------+--------------+-------------------------------+ + +** Elements of the queues/subQueues object for a Leaf queue - contains all elements in parent plus the following: + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | String | type of the queue - capacitySchedulerLeafQueueInfo | +*---------------+--------------+-------------------------------+ +| numActiveApplications | int | The number of active applications in this queue | +*---------------+--------------+-------------------------------+ +| numPendingApplications | int | The number of pending applications in this queue | +*---------------+--------------+-------------------------------+ +| numContainers | int | The number of containers being used | +*---------------+--------------+-------------------------------+ +| maxApplications | int | The maximum number of applications this queue can have | +*---------------+--------------+-------------------------------+ +| maxApplicationsPerUser | int | The maximum number of applications per user this queue can have | +*---------------+--------------+-------------------------------+ +| maxActiveApplications | int | The maximum number of active applications this queue can have | +*---------------+--------------+-------------------------------+ +| maxActiveApplicationsPerUser | int | The maximum number of active applications per user this queue can have| +*---------------+--------------+-------------------------------+ +| userLimit | int | The minimum user limit percent set in the configuration | +*---------------+--------------+-------------------------------+ +| userLimitFactor | float | The user limit factor set in the configuration | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "scheduler" : { + "schedulerInfo" : { + "queueName" : "root", + "maxCapacity" : 100, + "type" : "capacityScheduler", + "queues" : [ + { + "numPendingApplications" : 0, + "queueName" : "default", + "userLimitFactor" : 1, + "maxApplications" : 7000, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 90, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 90, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 70, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 70, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 7000 + }, + { + "queueName" : "test", + "utilization" : 0, + "absoluteCapacity" : 20, + "usedCapacity" : 0, + "capacity" : 20, + "subQueues" : [ + { + "numPendingApplications" : 0, + "queueName" : "a1", + "userLimitFactor" : 1, + "maxApplications" : 1200, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 80, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 16.000002, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 12, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 60.000004, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 1200 + }, + { + "numPendingApplications" : 0, + "queueName" : "a2", + "userLimitFactor" : 1, + "maxApplications" : 800, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 8.000001, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 40, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 800 + } + ], + "state" : "RUNNING", + "maxCapacity" : 80, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 80 + }, + { + "queueName" : "test2", + "utilization" : 0, + "absoluteCapacity" : 10, + "usedCapacity" : 0, + "capacity" : 10, + "subQueues" : [ + { + "numPendingApplications" : 0, + "queueName" : "a5", + "userLimitFactor" : 1, + "maxApplications" : 500, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 5, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 50, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 500 + }, + { + "numPendingApplications" : 0, + "queueName" : "a3", + "userLimitFactor" : 1, + "maxApplications" : 400, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 4.0000005, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 40, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 400 + }, + { + "numPendingApplications" : 0, + "queueName" : "a4", + "userLimitFactor" : 1, + "maxApplications" : 100, + "usedCapacity" : 0, + "numContainers" : 0, + "state" : "RUNNING", + "maxCapacity" : 100, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 100, + "maxActiveApplications" : 1, + "numActiveApplications" : 0, + "utilization" : 0, + "userLimit" : 100, + "absoluteCapacity" : 1.0000001, + "maxActiveApplicationsPerUser" : 1, + "capacity" : 10, + "type" : "capacitySchedulerLeafQueueInfo", + "maxApplicationsPerUser" : 100 + } + ], + "state" : "RUNNING", + "maxCapacity" : 15.000001, + "numApplications" : 0, + "usedResources" : "memory: 0", + "absoluteMaxCapacity" : 15.000001 + } + ], + "usedCapacity" : 0, + "capacity" : 100 + } + } +} ++---+ + + <> + + HTTP Request: + +----- + Accept: application/xml + GET http:///ws/v1/cluster/scheduler +----- + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 5778 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 100.0 + 0.0 + 100.0 + root + + 70.0 + 0.0 + 90.0 + 70.0 + 90.0 + 0.0 + 0 + memory: 0 + default + RUNNING + 0 + 0 + 0 + 7000 + 7000 + 1 + 1 + 100 + 1.0 + + + 20.0 + 0.0 + 80.0 + 20.0 + 80.0 + 0.0 + 0 + memory: 0 + test + RUNNING + + 60.000004 + 0.0 + 80.0 + 12.0 + 16.000002 + 0.0 + 0 + memory: 0 + a1 + RUNNING + 0 + 0 + 0 + 1200 + 1200 + 1 + 1 + 100 + 1.0 + + + 40.0 + 0.0 + 100.0 + 8.000001 + 100.0 + 0.0 + 0 + memory: 0 + a2 + RUNNING + 0 + 0 + 0 + 800 + 800 + 1 + 1 + 100 + 1.0 + + + + 10.0 + 0.0 + 15.000001 + 10.0 + 15.000001 + 0.0 + 0 + memory: 0 + test2 + RUNNING + + 50.0 + 0.0 + 100.0 + 5.0 + 100.0 + 0.0 + 0 + memory: 0 + A4 + RUNNING + 0 + 0 + 0 + 500 + 500 + 1 + 1 + 100 + 1.0 + + + 40.0 + 0.0 + 100.0 + 4.0000005 + 100.0 + 0.0 + 0 + memory: 0 + a3 + RUNNING + 0 + 0 + 0 + 400 + 400 + 1 + 1 + 100 + 1.0 + + + 10.0 + 0.0 + 100.0 + 1.0000001 + 100.0 + 0.0 + 0 + memory: 0 + a4 + RUNNING + 0 + 0 + 0 + 100 + 100 + 1 + 1 + 100 + 1.0 + + + + ++---+ + +** Fifo Scheduler API + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| type | string | Scheduler type - fifoScheduler | +*---------------+--------------+-------------------------------+ +| capacity | float | Queue capacity in percentage | +*---------------+--------------+-------------------------------+ +| usedCapacity | float | Used queue capacity in percentage | +*---------------+--------------+-------------------------------+ +| qstate | string | State of the queue - valid values are: STOPPED, RUNNING| +*---------------+--------------+-------------------------------+ +| minQueueMemoryCapacity | int | Minimum queue memory capacity | +*---------------+--------------+-------------------------------+ +| maxQueueMemoryCapacity | int | Maximum queue memory capacity | +*---------------+--------------+-------------------------------+ +| numNodes | int | The total number of nodes | +*---------------+--------------+-------------------------------+ +| usedNodeCapacity | int | The used node capacity | +*---------------+--------------+-------------------------------+ +| availNodeCapacity | int | The available node capacity | +*---------------+--------------+-------------------------------+ +| totalNodeCapacity | int | The total node capacity | +*---------------+--------------+-------------------------------+ +| numContainers | int | The number of containers | +*---------------+--------------+-------------------------------+ + +*** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "scheduler": + { + "schedulerInfo": + { + "type":"fifoScheduler", + "capacity":1, + "usedCapacity":"NaN", + "qstate":"RUNNING", + "minQueueMemoryCapacity":1024, + "maxQueueMemoryCapacity":10240, + "numNodes":0, + "usedNodeCapacity":0, + "availNodeCapacity":0, + "totalNodeCapacity":0, + "numContainers":0 + } + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/scheduler + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 432 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + 1.0 + NaN + RUNNING + 1024 + 10240 + 0 + 0 + 0 + 0 + 0 + + ++---+ + +* Cluster Applications API + + With the Applications API, you can obtain a collection of resources, each of which represents an application. When you run a GET operation on this resource, you obtain a collection of Application Objects. + +** URI + +------ + * http:///ws/v1/cluster/apps +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + + Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all applications that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity. + +------ + * state - state of the application + * user - user name + * queue - queue name + * limit - total number of app objects to be returned + * startedTimeBegin - applications with start time beginning with this time, specified in ms since epoch + * startedTimeEnd - applications with start time ending with this time, specified in ms since epoch + * finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch + * finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch +------ + +** Elements of the (Applications) object + + When you make a request for the list of applications, the information will be returned as a collection of app objects. + See also {{Application API}} for syntax of the app object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| app | array of app objects(JSON)/zero or more application objects(XML) | The collection of application objects | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "apps": + { + "app": + [ + { + "finishedTime" : 1326815598530, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326815542473_0001_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326815542473_0001", + "clusterId" : 1326815542473, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "word count", + "startedTime" : 1326815573334, + "elapsedTime" : 25196, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job/job_1326815542473_1_1", + "queue" : "default" + }, + { + "finishedTime" : 1326815789546, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326815542473_0002_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326815542473_0002", + "clusterId" : 1326815542473, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326815641380, + "elapsedTime" : 148166, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2", + "queue" : "default" + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 2459 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + application_1326815542473_0001 + user1 + word count + default + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326815542473_0001/jobhistory/job +/job_1326815542473_1_1 + + 1326815542473 + 1326815573334 + 1326815598530 + 25196 + http://host.domain.com:9999/node/containerlogs/container_1326815542473_0001 +_01_000001 + host.domain.com:9999 + + + application_1326815542473_0002 + user1 + Sleep job + default + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326815542473_0002/jobhistory/job/job_1326815542473_2_2 + + 1326815542473 + 1326815641380 + 1326815789546 + 148166 + http://host.domain.com:9999/node/containerlogs/container_1326815542473_0002_01_000001 + host.domain.com:9999 + + + ++---+ + +* Cluster {Application API} + + An application resource contains information about a particular application that was submitted to a cluster. + +** URI + + Use the following URI to obtain an app object, from a application identified by the {appid} value. + +------ + * http:///ws/v1/cluster/apps/{appid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the (Application) object + + Note that depending on security settings a user might not be able to see all the fields. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| id | string | The application id | +*---------------+--------------+--------------------------------+ +| user | string | The user who started the application | +*---------------+--------------+--------------------------------+ +| name | string | The application name | +*---------------+--------------+--------------------------------+ +| queue | string | The queue the application was submitted to| +*---------------+--------------+--------------------------------+ +| state | string | The application state according to the ResourceManager - valid values are: NEW, SUBMITTED, ACCEPTED, RUNNING, FINISHED, FAILED, KILLED| +*---------------+--------------+--------------------------------+ +| finalStatus | string | The final status of the application if finished - reported by the application itself - valid values are: UNDEFINED, SUCCEEDED, FAILED, KILLED| +*---------------+--------------+--------------------------------+ +| progress | float | The progress of the application as a percent | +*---------------+--------------+--------------------------------+ +| trackingUI | string | Where the tracking url is currently pointing - History (for history server) or ApplicationMaster | +*---------------+--------------+--------------------------------+ +| trackingUrl | string | The web URL that can be used to track the application | +*---------------+--------------+--------------------------------+ +| diagnostics | string | Detailed diagnostics information | +*---------------+--------------+--------------------------------+ +| clusterId | long | The cluster id | +*---------------+--------------+--------------------------------+ +| startedTime | long | The time in which application started (in ms since epoch)| +*---------------+--------------+--------------------------------+ +| finishedTime | long | The time in which the application finished (in ms since epoch) | +*---------------+--------------+--------------------------------+ +| elapsedTime | long | The elapsed time since the application started (in ms)| +*---------------+--------------+--------------------------------+ +| amContainerLogs | string | The URL of the application master container logs| +*---------------+--------------+--------------------------------+ +| amHostHttpAddress | string | The nodes http address of the application master | +*---------------+--------------+--------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "app" : { + "finishedTime" : 1326824991300, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0005_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326821518301_0005", + "clusterId" : 1326821518301, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326824544552, + "elapsedTime" : 446748, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5", + "queue" : "a1" + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/apps/application_1326821518301_0005 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 847 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + application_1326821518301_0005 + user1 + Sleep job + a1 + FINISHED + SUCCEEDED + 100.0 + History + http://host.domain.com:8088/proxy/application_1326821518301_0005/jobhistory/job/job_1326821518301_5_5 + + 1326821518301 + 1326824544552 + 1326824991300 + 446748 + http://host.domain.com:9999/node/containerlogs/container_1326821518301_0005_01_000001 + host.domain.com:9999 + ++---+ + +* Cluster Nodes API + + With the Nodes API, you can obtain a collection of resources, each of which represents a node. When you run a GET operation on this resource, you obtain a collection of Node Objects. + +** URI + +------ + * http:///ws/v1/cluster/nodes +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + * state - the state of the node + * healthy - true or false +------ + +** Elements of the object + + When you make a request for the list of nodes, the information will be returned as a collection of node objects. + See also {{Node API}} for syntax of the node object. + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| node | array of node objects(JSON)/zero or more node objects(XML) | A collection of node objects | +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "nodes": + { + "node": + [ + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h2:1235", + "nodeHostName":"h2", + "nodeHTTPAddress":"h2:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895432, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0 + "availMemoryMB":8192 + }, + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h1:1234", + "nodeHostName":"h1", + "nodeHTTPAddress":"h1:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895092, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0, + "availMemoryMB":8192 + } + ] + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 1104 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + + /default-rack + RUNNING + h2:1234 + h2 + h2:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + + + /default-rack + RUNNING + h1:1234 + h1 + h1:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + + ++---+ + + +* Cluster {Node API} + + A node resource contains information about a node in the cluster. + +** URI + + Use the following URI to obtain a Node Object, from a node identified by the {nodeid} value. + +------ + * http:///ws/v1/cluster/nodes/{nodeid} +------ + +** HTTP Operations Supported + +------ + * GET +------ + +** Query Parameters Supported + +------ + None +------ + +** Elements of the object + +*---------------+--------------+-------------------------------+ +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| rack | string | The rack location of this node | +*---------------+--------------+-------------------------------+ +| state | string | State of the node - valid values are: NEW, RUNNING, UNHEALTHY, DECOMMISSIONED, LOST, REBOOTED | +*---------------+--------------+-------------------------------+ +| id | string | The node id | +*---------------+--------------+-------------------------------+ +| nodeHostName | string | The host name of the node| +*---------------+--------------+-------------------------------+ +| nodeHTTPAddress | string | The nodes HTTP address| +*---------------+--------------+-------------------------------+ +| healthStatus | string | The health status of the node - Healthy or Unhealthy | +*---------------+--------------+-------------------------------+ +| healthReport | string | A detailed health report | +*---------------+--------------+-------------------------------+ +| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)| +*---------------+--------------+-------------------------------+ +| usedMemoryMB | long | The total about of memory currently used on the node (in MB)| +*---------------+--------------+-------------------------------+ +| availMemoryMB | long | The total amount of memory currently available on the node (in MB)| +*---------------+--------------+-------------------------------+ +| numContainers | int | The total number of containers currently running on the node| +*---------------+--------------+-------------------------------+ + +** Response Examples + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/nodes/h2:1235 +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "node": + { + "rack":"\/default-rack", + "state":"NEW", + "id":"h2:1235", + "nodeHostName":"h2", + "nodeHTTPAddress":"h2:2", + "healthStatus":"Healthy", + "lastHealthUpdate":1324056895432, + "healthReport":"Healthy", + "numContainers":0, + "usedMemoryMB":0, + "availMemoryMB":5120 + } +} ++---+ + + <> + + HTTP Request: + +------ + GET http:///ws/v1/cluster/node/h2:1235 + Accept: application/xml +------ + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/xml + Content-Length: 552 + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ + + + /default-rack + NEW + h2:1235 + h2 + h2:2 + Healthy + 1324333268447 + Healthy + 0 + 0 + 5120 + ++---+ + diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm new file mode 100644 index 0000000000..2cdbfbe472 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm @@ -0,0 +1,595 @@ +~~ Licensed under the Apache License, Version 2.0 (the "License"); +~~ you may not use this file except in compliance with the License. +~~ You may obtain a copy of the License at +~~ +~~ http://www.apache.org/licenses/LICENSE-2.0 +~~ +~~ Unless required by applicable law or agreed to in writing, software +~~ distributed under the License is distributed on an "AS IS" BASIS, +~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~~ See the License for the specific language governing permissions and +~~ limitations under the License. See accompanying LICENSE file. + + --- + Hadoop YARN - Introduction to the web services REST API's. + --- + --- + ${maven.build.timestamp} + +Hadoop YARN - Introduction to the web services REST API's. + + \[ {{{./index.html}Go Back}} \] + +%{toc|section=1|fromDepth=0} + +* Overview + + The Hadoop YARN web service REST APIs are a set of URI resources that give access to the cluster, nodes, applications, and application historical information. The URI resources are grouped into APIs based on the type of information returned. Some URI resources return collections while others return singletons. + +* URI's + + The URIs for the REST-based Web services have the following syntax: + +------ + http://{http address of service}/ws/{version}/{resourcepath} +------ + + The elements in this syntax are as follows: + +------ + {http address of service} - The http address of the service to get information about. + Currently supported are the ResourceManager, NodeManager, + MapReduce application master, and history server. + {version} - The version of the APIs. In this release, the version is v1. + {resourcepath} - A path that defines a singleton resource or a collection of resources. +------ + +* HTTP Requests + + To invoke a REST API, your application calls an HTTP operation on the URI associated with a resource. + +** Summary of HTTP operations + + Currently only GET is supported. It retrieves information about the resource specified. + +** Security + + The web service REST API's go through the same security as the web ui. If your cluster adminstrators have filters enabled you must authenticate via the mechanism they specified. + +** Headers Supported + +----- + * Accept + * Accept-Encoding +----- + + Currently the only fields used in the header is Accept and Accept-Encoding. Accept currently supports XML and JSON for the response type you accept. Accept-Encoding currently only supports gzip format and will return gzip compressed output if this is specified, otherwise output is uncompressed. All other header fields are ignored. + +* HTTP Responses + + The next few sections describe some of the syntax and other details of the HTTP Responses of the web service REST APIs. + +** Compression + + This release supports gzip compression if you specify gzip in the Accept-Encoding header of the HTTP request (Accept-Encoding: gzip). + +** Response Formats + + This release of the web service REST APIs supports responses in JSON and XML formats. JSON is the default. To set the response format, you can specify the format in the Accept header of the HTTP request. + + As specified in HTTP Response Codes, the response body can contain the data that represents the resource or an error message. In the case of success, the response body is in the selected format, either JSON or XML. In the case of error, the resonse body is in either JSON or XML based on the format requested. The Content-Type header of the response contains the format requested. If the application requests an unsupported format, the response status code is 500. +Note that the order of the fields within response body is not specified and might change. Also, additional fields might be added to a response body. Therefore, your applications should use parsing routines that can extract data from a response body in any order. + +** Response Errors + + After calling an HTTP request, an application should check the response status code to verify success or detect an error. If the response status code indicates an error, the response body contains an error message. The first field is the exception type, currently only RemoteException is returned. The following table lists the items within the RemoteException error message: + +*---------------*--------------*-------------------------------* +|| Item || Data Type || Description | +*---------------+--------------+-------------------------------+ +| exception | String | Exception type | +*---------------+--------------+-------------------------------+ +| javaClassName | String | Java class name of exception | +*---------------+--------------+-------------------------------+ +| message | String | Detailed message of exception | +*---------------+--------------+-------------------------------+ + +** Response Examples + +*** JSON response with single resource + + HTTP Request: + GET http://rmhost.domain:8088/ws/v1/cluster/app/application_1324057493980_0001 + + Response Status Line: + HTTP/1.1 200 OK + + Response Header: + ++---+ + HTTP/1.1 200 OK + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + app": + { + "id":"application_1324057493980_0001", + "user":"user1", + "name":"", + "queue":"default", + "state":"ACCEPTED", + "finalStatus":"UNDEFINED", + "progress":0, + "trackingUI":"UNASSIGNED", + "diagnostics":"", + "clusterId":1324057493980, + "startedTime":1324057495921, + "finishedTime":0, + "elapsedTime":2063, + "amContainerLogs":"http:\/\/amNM:2\/node\/containerlogs\/container_1324057493980_0001_01_000001", + "amHostHttpAddress":"amNM:2" + } +} ++---+ + +*** JSON response with Error response + + Here we request information about an application that doesn't exist yet. + + HTTP Request: + GET http://rmhost.domain:8088/ws/v1/cluster/app/application_1324057493980_9999 + + Response Status Line: + HTTP/1.1 404 Not Found + + Response Header: + ++---+ + HTTP/1.1 404 Not Found + Content-Type: application/json + Transfer-Encoding: chunked + Server: Jetty(6.1.26) ++---+ + + Response Body: + ++---+ +{ + "RemoteException" : { + "javaClassName" : "org.apache.hadoop.yarn.webapp.NotFoundException", + "exception" : "NotFoundException", + "message" : "java.lang.Exception: app with id: application_1324057493980_9999 not found" + } +} ++---+ + +* Example usage + + You can use any number of ways/languages to use the web services REST API's. This example uses the curl command line interface to do the REST GET calls. + + In this example, a user submits a MapReduce application to the ResourceManager using a command like: + ++---+ + hadoop jar hadoop-mapreduce-test.jar sleep -Dmapred.job.queue.name=a1 -m 1 -r 1 -rt 1200000 -mt 20 ++---+ + + The client prints information about the job submitted along with the application id, similar to: + ++---+ +12/01/18 04:25:15 INFO mapred.ResourceMgrDelegate: Submitted application application_1326821518301_0010 to ResourceManager at host.domain.com/10.10.10.10:8040 +12/01/18 04:25:15 INFO mapreduce.Job: Running job: job_1326821518301_0010 +12/01/18 04:25:21 INFO mapred.ClientServiceDelegate: The url to track the job: host.domain.com:8088/proxy/application_1326821518301_0010/ +12/01/18 04:25:22 INFO mapreduce.Job: Job job_1326821518301_0010 running in uber mode : false +12/01/18 04:25:22 INFO mapreduce.Job: map 0% reduce 0% ++---+ + + The user then wishes to track the application. The users starts by getting the information about the application from the ResourceManager. Use the --comopressed option to request output compressed. curl handles uncompressing on client side. + ++---+ +curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/ws/v1/cluster/apps/application_1326821518301_0010" ++---+ + + Output: + ++---+ +{ + "app" : { + "finishedTime" : 0, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0010_01_000001", + "trackingUI" : "ApplicationMaster", + "state" : "RUNNING", + "user" : "user1", + "id" : "application_1326821518301_0010", + "clusterId" : 1326821518301, + "finalStatus" : "UNDEFINED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 82.44703, + "name" : "Sleep job", + "startedTime" : 1326860715335, + "elapsedTime" : 31814, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0010/", + "queue" : "a1" + } +} ++---+ + + The user then wishes to get more details about the running application and goes directly to the MapReduce application master for this application. The ResourceManager lists the trackingUrl that can be used for this application: http://host.domain.com:8088/proxy/application_1326821518301_0010. This could either go to the web browser or use the web service REST API's. The user uses the web services REST API's to get the list of jobs this MapReduce application master is running: + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs" ++---+ + + Output: + ++---+ +{ + "jobs" : { + "job" : [ + { + "runningReduceAttempts" : 1, + "reduceProgress" : 72.104515, + "failedReduceAttempts" : 0, + "newMapAttempts" : 0, + "mapsRunning" : 0, + "state" : "RUNNING", + "successfulReduceAttempts" : 0, + "reducesRunning" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "reducesPending" : 0, + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326860720902, + "id" : "job_1326821518301_10_10", + "successfulMapAttempts" : 1, + "runningMapAttempts" : 0, + "newReduceAttempts" : 0, + "name" : "Sleep job", + "mapsPending" : 0, + "elapsedTime" : 64432, + "reducesCompleted" : 0, + "mapProgress" : 100, + "diagnostics" : "", + "failedMapAttempts" : 0, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 0 + } + ] + } +} ++---+ + + The user then wishes to get the task details about the job with job id job_1326821518301_10_10 that was listed above. + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks" ++---+ + + Output: + ++---+ +{ + "tasks" : { + "task" : [ + { + "progress" : 100, + "elapsedTime" : 5059, + "state" : "SUCCEEDED", + "startTime" : 1326860725014, + "id" : "task_1326821518301_10_10_m_0", + "type" : "MAP", + "successfulAttempt" : "attempt_1326821518301_10_10_m_0_0", + "finishTime" : 1326860730073 + }, + { + "progress" : 72.104515, + "elapsedTime" : 0, + "state" : "RUNNING", + "startTime" : 1326860732984, + "id" : "task_1326821518301_10_10_r_0", + "type" : "REDUCE", + "successfulAttempt" : "", + "finishTime" : 0 + } + ] + } +} ++---+ + + The map task has finished but the reduce task is still running. The users wishes to get the task attempt information for the reduce task task_1326821518301_10_10_r_0, note that the Accept header isn't really required here since JSON is the default output format: + ++---+ + curl --compressed -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks/task_1326821518301_10_10_r_0/attempts" ++---+ + + Output: + ++---+ +{ + "taskAttempts" : { + "taskAttempt" : [ + { + "elapsedMergeTime" : 158, + "shuffleFinishTime" : 1326860735378, + "assignedContainerId" : "container_1326821518301_0010_01_000003", + "progress" : 72.104515, + "elapsedTime" : 0, + "state" : "RUNNING", + "elapsedShuffleTime" : 2394, + "mergeFinishTime" : 1326860735536, + "rack" : "/10.10.10.0", + "elapsedReduceTime" : 0, + "nodeHttpAddress" : "host.domain.com:9999", + "type" : "REDUCE", + "startTime" : 1326860732984, + "id" : "attempt_1326821518301_10_10_r_0_0", + "finishTime" : 0 + } + ] + } +} ++---+ + + The reduce attempt is still running and the user wishes to see the current counter values for that attempt: + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/proxy/application_1326821518301_0010/ws/v1/mapreduce/jobs/job_1326821518301_10_10/tasks/task_1326821518301_10_10_r_0/attempts/attempt_1326821518301_10_10_r_0_0/counters" ++---+ + + Output: + ++---+ +{ + "JobTaskAttemptCounters" : { + "taskAttemptCounterGroup" : [ + { + "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter", + "counter" : [ + { + "value" : 4216, + "name" : "FILE_BYTES_READ" + }, + { + "value" : 77151, + "name" : "FILE_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "FILE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "FILE_WRITE_OPS" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_READ" + }, + { + "value" : 0, + "name" : "HDFS_BYTES_WRITTEN" + }, + { + "value" : 0, + "name" : "HDFS_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_LARGE_READ_OPS" + }, + { + "value" : 0, + "name" : "HDFS_WRITE_OPS" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter", + "counter" : [ + { + "value" : 0, + "name" : "COMBINE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "COMBINE_OUTPUT_RECORDS" + }, + { + "value" : 1767, + "name" : "REDUCE_INPUT_GROUPS" + }, + { + "value" : 25104, + "name" : "REDUCE_SHUFFLE_BYTES" + }, + { + "value" : 1767, + "name" : "REDUCE_INPUT_RECORDS" + }, + { + "value" : 0, + "name" : "REDUCE_OUTPUT_RECORDS" + }, + { + "value" : 0, + "name" : "SPILLED_RECORDS" + }, + { + "value" : 1, + "name" : "SHUFFLED_MAPS" + }, + { + "value" : 0, + "name" : "FAILED_SHUFFLE" + }, + { + "value" : 1, + "name" : "MERGED_MAP_OUTPUTS" + }, + { + "value" : 50, + "name" : "GC_TIME_MILLIS" + }, + { + "value" : 1580, + "name" : "CPU_MILLISECONDS" + }, + { + "value" : 141320192, + "name" : "PHYSICAL_MEMORY_BYTES" + }, + { + "value" : 1118552064, + "name" : "VIRTUAL_MEMORY_BYTES" + }, + { + "value" : 73728000, + "name" : "COMMITTED_HEAP_BYTES" + } + ] + }, + { + "counterGroupName" : "Shuffle Errors", + "counter" : [ + { + "value" : 0, + "name" : "BAD_ID" + }, + { + "value" : 0, + "name" : "CONNECTION" + }, + { + "value" : 0, + "name" : "IO_ERROR" + }, + { + "value" : 0, + "name" : "WRONG_LENGTH" + }, + { + "value" : 0, + "name" : "WRONG_MAP" + }, + { + "value" : 0, + "name" : "WRONG_REDUCE" + } + ] + }, + { + "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter", + "counter" : [ + { + "value" : 0, + "name" : "BYTES_WRITTEN" + } + ] + } + ], + "id" : "attempt_1326821518301_10_10_r_0_0" + } +} ++---+ + + The job finishes and the user wishes to get the final job information from the history server for this job. + ++---+ + curl --compressed -X GET "http://host.domain.com:19888/ws/v1/history/mapreduce/jobs/job_1326821518301_10_10" ++---+ + + Output: + ++---+ +{ + "job" : { + "avgReduceTime" : 1250784, + "failedReduceAttempts" : 0, + "state" : "SUCCEEDED", + "successfulReduceAttempts" : 1, + "acls" : [ + { + "value" : " ", + "name" : "mapreduce.job.acl-modify-job" + }, + { + "value" : " ", + "name" : "mapreduce.job.acl-view-job" + } + ], + "user" : "user1", + "reducesTotal" : 1, + "mapsCompleted" : 1, + "startTime" : 1326860720902, + "id" : "job_1326821518301_10_10", + "avgMapTime" : 5059, + "successfulMapAttempts" : 1, + "name" : "Sleep job", + "avgShuffleTime" : 2394, + "reducesCompleted" : 1, + "diagnostics" : "", + "failedMapAttempts" : 0, + "avgMergeTime" : 2552, + "killedReduceAttempts" : 0, + "mapsTotal" : 1, + "queue" : "a1", + "uberized" : false, + "killedMapAttempts" : 0, + "finishTime" : 1326861986164 + } +} ++---+ + + The user also gets the final applications information from the ResourceManager. + ++---+ + curl --compressed -H "Accept: application/json" -X GET "http://host.domain.com:8088/ws/v1/cluster/apps/application_1326821518301_0010" ++---+ + + Output: + ++---+ +{ + "app" : { + "finishedTime" : 1326861991282, + "amContainerLogs" : "http://host.domain.com:9999/node/containerlogs/container_1326821518301_0010_01_000001", + "trackingUI" : "History", + "state" : "FINISHED", + "user" : "user1", + "id" : "application_1326821518301_0010", + "clusterId" : 1326821518301, + "finalStatus" : "SUCCEEDED", + "amHostHttpAddress" : "host.domain.com:9999", + "progress" : 100, + "name" : "Sleep job", + "startedTime" : 1326860715335, + "elapsedTime" : 1275947, + "diagnostics" : "", + "trackingUrl" : "http://host.domain.com:8088/proxy/application_1326821518301_0010/jobhistory/job/job_1326821518301_10_10", + "queue" : "a1" + } +} ++---+ diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 6b966883e0..95c3325775 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -62,6 +62,14 @@ + + + + + + + +