big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19283. Move all DistCp execution logic to execute() (#7060)

Browse Source 
Co-authored-by: Felix Nguyen <kokonguyen191@gmail.com>
This commit is contained in:
Felix Nguyen 2024-09-23 15:39:56 +08:00 committed by GitHub
parent f5345b881d
commit fccc268cde
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
View File

@ -140,9 +140,7 @@ public int run(String[] argv) {
try { try {
context = new DistCpContext(OptionsParser.parse(argv)); context = new DistCpContext(OptionsParser.parse(argv));
checkSplitLargeFile(); LOG.info("Input Options: {}", context);
setTargetPathExists();
LOG.info("Input Options: " + context);
} catch (Throwable e) { } catch (Throwable e) {
LOG.error("Invalid arguments: ", e); LOG.error("Invalid arguments: ", e);
System.err.println("Invalid arguments: " + e.getMessage()); System.err.println("Invalid arguments: " + e.getMessage());
@ -152,7 +150,7 @@ public int run(String[] argv) {
Job job = null; Job job = null;
try { try {
job = execute(); job = execute(true);
} catch (InvalidInputException e) { } catch (InvalidInputException e) {
LOG.error("Invalid input: ", e); LOG.error("Invalid input: ", e);
return DistCpConstants.INVALID_ARGUMENT; return DistCpConstants.INVALID_ARGUMENT;
@ -169,7 +167,7 @@ public int run(String[] argv) {
LOG.error("Exception encountered ", e); LOG.error("Exception encountered ", e);
return DistCpConstants.UNKNOWN_ERROR; return DistCpConstants.UNKNOWN_ERROR;
} finally { } finally {
//Blocking distcp so close the job after its done // Blocking distcp so close the job after it's done
if (job != null && context.shouldBlock()) { if (job != null && context.shouldBlock()) {
try { try {
job.close(); job.close();
@ -182,14 +180,30 @@ public int run(String[] argv) {
} }
/** /**
* Implements the core-execution. Creates the file-list for copy, * Original entrypoint of a distcp job. Calls {@link DistCp#execute(boolean)}
* and launches the Hadoop-job, to do the copy. * without doing extra context checks and setting some configs.
* @return Job handle * @return Job handle
* @throws Exception * @throws Exception when fails to submit distcp job or distcp job fails
*/ */
public Job execute() throws Exception { public Job execute() throws Exception {
return execute(false);
}
/**
* Implements the core-execution. Creates the file-list for copy,
* and launches the Hadoop-job, to do the copy.
* @param extraContextChecks if true, does extra context checks and sets some configs.
* @return Job handle
* @throws Exception when fails to submit distcp job or distcp job fails, or context checks fail
*/
public Job execute(boolean extraContextChecks) throws Exception {
Preconditions.checkState(context != null, Preconditions.checkState(context != null,
"The DistCpContext should have been created before running DistCp!"); "The DistCpContext should have been created before running DistCp!");
if (extraContextChecks) {
checkSplitLargeFile();
setTargetPathExists();
}
Job job = createAndSubmitJob(); Job job = createAndSubmitJob();
if (context.shouldBlock()) { if (context.shouldBlock()) {

5
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestExternalCall.java
View File

@ -156,7 +156,10 @@ public void testCleanupOfJob() throws Exception {
DistCp distcp = mock(DistCp.class); DistCp distcp = mock(DistCp.class);
Job job = spy(Job.class); Job job = spy(Job.class);
Mockito.when(distcp.getConf()).thenReturn(conf); Mockito.when(distcp.getConf()).thenReturn(conf);
Mockito.when(distcp.execute()).thenReturn(job); Mockito.when(distcp.createAndSubmitJob()).thenReturn(job);
Mockito.when(distcp.execute()).thenCallRealMethod();
Mockito.when(distcp.execute(Mockito.anyBoolean())).thenCallRealMethod();
Mockito.doReturn(true).when(job).waitForCompletion(Mockito.anyBoolean());
Mockito.when(distcp.run(Mockito.any())).thenCallRealMethod(); Mockito.when(distcp.run(Mockito.any())).thenCallRealMethod();
String[] arg = { soure.toString(), target.toString() }; String[] arg = { soure.toString(), target.toString() };