big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17594. DistCp: Expose the JobId for applications executing through run method (#2786). Contributed by Ayush Saxena.

Browse Source 
Signed-off-by: Mingliang Liu <liuml07@apache.org>
Signed-off-by: Steve Loughran <stevel@apache.org>
This commit is contained in:
Ayush Saxena 2021-03-19 14:19:49 +05:30 committed by GitHub
parent f5f3fc6fe9
commit 4781761dc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
View File

@ -225,6 +225,8 @@ public Job createAndSubmitJob() throws Exception {
String jobID = job.getJobID().toString(); String jobID = job.getJobID().toString();
job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, job.getConfiguration().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID,
jobID); jobID);
// Set the jobId for the applications running through run method.
getConf().set(DistCpConstants.CONF_LABEL_DISTCP_JOB_ID, jobID);
LOG.info("DistCp job-id: " + jobID); LOG.info("DistCp job-id: " + jobID);
return job; return job;

13
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.tools.contract; package org.apache.hadoop.tools.contract;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_DISTCP_JOB_ID;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
@ -42,6 +43,7 @@
import org.apache.hadoop.tools.DistCpConstants; import org.apache.hadoop.tools.DistCpConstants;
import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.mapred.CopyMapper; import org.apache.hadoop.tools.mapred.CopyMapper;
import org.apache.hadoop.tools.util.DistCpTestUtils;
import org.junit.Before; import org.junit.Before;
import org.junit.Rule; import org.junit.Rule;
@ -464,6 +466,17 @@ public void testLargeFilesFromRemote() throws Exception {
largeFiles(remoteFS, remoteDir, localFS, localDir); largeFiles(remoteFS, remoteDir, localFS, localDir);
} }
@Test
public void testSetJobId() throws Exception {
describe("check jobId is set in the conf");
remoteFS.create(new Path(remoteDir, "file1")).close();
DistCpTestUtils
.assertRunDistCp(DistCpConstants.SUCCESS, remoteDir.toString(),
localDir.toString(), null, conf);
assertNotNull("DistCp job id isn't set",
conf.get(CONF_LABEL_DISTCP_JOB_ID));
}
/** /**
* Executes a DistCp using a file system sub-tree with multiple nesting * Executes a DistCp using a file system sub-tree with multiple nesting
* levels. * levels.