big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-6787. Allow job_conf.xml to be downloadable on the job overview page in JHS (haibochen via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2016-12-01 17:29:16 -08:00
parent 2d77dc727d
commit c87b3a448a
6 changed files with 83 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
View File

@ -323,6 +323,40 @@ public void conf() {
render(confPage());
}
/**
* Handle requests to download the job configuration.
*/
public void downloadConf() {
try {
requireJob();
} catch (Exception e) {
renderText(e.getMessage());
return;
}
writeJobConf();
}
private void writeJobConf() {
String jobId = $(JOB_ID);
assert(!jobId.isEmpty());
JobId jobID = MRApps.toJobID($(JOB_ID));
Job job = app.context.getJob(jobID);
assert(job != null);
try {
Configuration jobConf = job.loadConfFile();
response().setContentType("text/xml");
response().setHeader("Content-Disposition",
"attachment; filename=" + jobId + ".xml");
jobConf.writeXml(writer());
} catch (IOException e) {
LOG.error("Error reading/writing job" +
" conf file for job: " + jobId, e);
renderText(e.getMessage());
}
}
/**
* Render a BAD_REQUEST error.
* @param s the error message to include.

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
View File

@ -70,7 +70,7 @@ public class ConfBlock extends HtmlBlock {
try {
ConfInfo info = new ConfInfo(job);
html.div().h3(confPath.toString())._();
html.div().a("/jobhistory/downloadconf/" + jid, confPath.toString());
TBODY<TABLE<Hamlet>> tbody = html.
// Tasks table
table("#conf").

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAppController.java
View File

@ -24,6 +24,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@ -59,6 +60,8 @@ public void setUp() throws IOException {
Task task = mock(Task.class);
when(job.getTask(any(TaskId.class))).thenReturn(task);
when(job.loadConfFile()).thenReturn(new Configuration());
when(job.getConfFile()).thenReturn(new Path("/"));
JobId jobID = MRApps.toJobID("job_01_01");
when(context.getJob(jobID)).thenReturn(job);
@ -265,6 +268,17 @@ public void testConfiguration() {
assertEquals(JobConfPage.class, appController.getClazz());
}
/**
* Test downloadConf request handling.
*/
@Test
public void testDownloadConfiguration() {
appController.downloadConf();
String jobConfXml = appController.getData();
assertTrue("Error downloading the job configuration file.",
!jobConfXml.contains("Error"));
}
/**
* Test method 'conf'. Should set AttemptsPage class for rendering or print information about error
*/

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebApp.java
View File

@ -50,6 +50,8 @@ public void setup() {
route("/app", HsController.class);
route(pajoin("/job", JOB_ID), HsController.class, "job");
route(pajoin("/conf", JOB_ID), HsController.class, "conf");
routeWithoutDefaultView(pajoin("/downloadconf", JOB_ID),
HsController.class, "downloadConf");
route(pajoin("/jobcounters", JOB_ID), HsController.class, "jobCounters");
route(pajoin("/singlejobcounter",JOB_ID, COUNTER_GROUP, COUNTER_NAME),
HsController.class, "singleJobCounter");

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Router.java
View File

@ -74,17 +74,32 @@ static class Dest {
final TreeMap<String, Dest> routes = Maps.newTreeMap(); // path->dest
synchronized Dest add(WebApp.HTTP httpMethod, String path,
Class<? extends Controller> cls,
String action, List<String> names){
return addWithOptionalDefaultView(
httpMethod, path, cls, action, names, true);
}
synchronized Dest addWithoutDefaultView(WebApp.HTTP httpMethod,
String path, Class<? extends Controller> cls, String action,
List<String> names){
return addWithOptionalDefaultView(httpMethod, path, cls, action,
names, false);
}
/**
* Add a route to the router.
* e.g., add(GET, "/foo/show", FooController.class, "show", [name...]);
* The name list is from /foo/show/:name/...
*/
synchronized Dest add(WebApp.HTTP httpMethod, String path,
Class<? extends Controller> cls,
String action, List<String> names) {
synchronized Dest addWithOptionalDefaultView(WebApp.HTTP httpMethod,
String path, Class<? extends Controller> cls,
String action, List<String> names, boolean defaultViewNeeded) {
LOG.debug("adding {}({})->{}#{}", new Object[]{path, names, cls, action});
Dest dest = addController(httpMethod, path, cls, action, names);
addDefaultView(dest);
if (defaultViewNeeded) {
addDefaultView(dest);
}
return dest;
}

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
View File

@ -210,6 +210,19 @@ public void route(HTTP method, String pathSpec,
res.subList(R_PARAMS, res.size()));
}
/**
* Setup of a webapp serving route without default views added to the page.
* @param pathSpec the path spec in the form of /controller/action/:args etc.
* @param cls the controller class
* @param action the controller method
*/
public void routeWithoutDefaultView(String pathSpec,
Class<? extends Controller> cls, String action) {
List<String> res = parseRoute(pathSpec);
router.addWithoutDefaultView(HTTP.GET, res.get(R_PATH), cls, action,
res.subList(R_PARAMS, res.size()));
}
public void route(String pathSpec, Class<? extends Controller> cls,
String action) {
route(HTTP.GET, pathSpec, cls, action);