HDFS-10397. Distcp should ignore -delete option if -diff option is provided instead of exiting. Contributed by Mingliang Liu.
This commit is contained in:
parent
9478484845
commit
03788d3015
@ -160,7 +160,6 @@ public boolean shouldAtomicCommit() {
|
|||||||
* @param atomicCommit - boolean switch
|
* @param atomicCommit - boolean switch
|
||||||
*/
|
*/
|
||||||
public void setAtomicCommit(boolean atomicCommit) {
|
public void setAtomicCommit(boolean atomicCommit) {
|
||||||
validate(DistCpOptionSwitch.ATOMIC_COMMIT, atomicCommit);
|
|
||||||
this.atomicCommit = atomicCommit;
|
this.atomicCommit = atomicCommit;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -179,7 +178,6 @@ public boolean shouldSyncFolder() {
|
|||||||
* @param syncFolder - boolean switch
|
* @param syncFolder - boolean switch
|
||||||
*/
|
*/
|
||||||
public void setSyncFolder(boolean syncFolder) {
|
public void setSyncFolder(boolean syncFolder) {
|
||||||
validate(DistCpOptionSwitch.SYNC_FOLDERS, syncFolder);
|
|
||||||
this.syncFolder = syncFolder;
|
this.syncFolder = syncFolder;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -198,7 +196,6 @@ public boolean shouldDeleteMissing() {
|
|||||||
* @param deleteMissing - boolean switch
|
* @param deleteMissing - boolean switch
|
||||||
*/
|
*/
|
||||||
public void setDeleteMissing(boolean deleteMissing) {
|
public void setDeleteMissing(boolean deleteMissing) {
|
||||||
validate(DistCpOptionSwitch.DELETE_MISSING, deleteMissing);
|
|
||||||
this.deleteMissing = deleteMissing;
|
this.deleteMissing = deleteMissing;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -254,7 +251,6 @@ public boolean shouldOverwrite() {
|
|||||||
* @param overwrite - boolean switch
|
* @param overwrite - boolean switch
|
||||||
*/
|
*/
|
||||||
public void setOverwrite(boolean overwrite) {
|
public void setOverwrite(boolean overwrite) {
|
||||||
validate(DistCpOptionSwitch.OVERWRITE, overwrite);
|
|
||||||
this.overwrite = overwrite;
|
this.overwrite = overwrite;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -270,7 +266,6 @@ public boolean shouldAppend() {
|
|||||||
* update option and CRC is not skipped.
|
* update option and CRC is not skipped.
|
||||||
*/
|
*/
|
||||||
public void setAppend(boolean append) {
|
public void setAppend(boolean append) {
|
||||||
validate(DistCpOptionSwitch.APPEND, append);
|
|
||||||
this.append = append;
|
this.append = append;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -287,7 +282,6 @@ public String getToSnapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public void setUseDiff(boolean useDiff, String fromSnapshot, String toSnapshot) {
|
public void setUseDiff(boolean useDiff, String fromSnapshot, String toSnapshot) {
|
||||||
validate(DistCpOptionSwitch.DIFF, useDiff);
|
|
||||||
this.useDiff = useDiff;
|
this.useDiff = useDiff;
|
||||||
this.fromSnapshot = fromSnapshot;
|
this.fromSnapshot = fromSnapshot;
|
||||||
this.toSnapshot = toSnapshot;
|
this.toSnapshot = toSnapshot;
|
||||||
@ -314,7 +308,6 @@ public boolean shouldSkipCRC() {
|
|||||||
* @param skipCRC - boolean switch
|
* @param skipCRC - boolean switch
|
||||||
*/
|
*/
|
||||||
public void setSkipCRC(boolean skipCRC) {
|
public void setSkipCRC(boolean skipCRC) {
|
||||||
validate(DistCpOptionSwitch.SKIP_CRC, skipCRC);
|
|
||||||
this.skipCRC = skipCRC;
|
this.skipCRC = skipCRC;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -551,20 +544,15 @@ public final void setFiltersFile(String filtersFilename) {
|
|||||||
this.filtersFile = filtersFilename;
|
this.filtersFile = filtersFilename;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void validate(DistCpOptionSwitch option, boolean value) {
|
void validate() {
|
||||||
|
if (useDiff && deleteMissing) {
|
||||||
boolean syncFolder = (option == DistCpOptionSwitch.SYNC_FOLDERS ?
|
// -delete and -diff are mutually exclusive. For backward compatibility,
|
||||||
value : this.syncFolder);
|
// we ignore the -delete option here, instead of throwing an
|
||||||
boolean overwrite = (option == DistCpOptionSwitch.OVERWRITE ?
|
// IllegalArgumentException. See HDFS-10397 for more discussion.
|
||||||
value : this.overwrite);
|
OptionsParser.LOG.warn("-delete and -diff are mutually exclusive. " +
|
||||||
boolean deleteMissing = (option == DistCpOptionSwitch.DELETE_MISSING ?
|
"The -delete option will be ignored.");
|
||||||
value : this.deleteMissing);
|
setDeleteMissing(false);
|
||||||
boolean atomicCommit = (option == DistCpOptionSwitch.ATOMIC_COMMIT ?
|
}
|
||||||
value : this.atomicCommit);
|
|
||||||
boolean skipCRC = (option == DistCpOptionSwitch.SKIP_CRC ?
|
|
||||||
value : this.skipCRC);
|
|
||||||
boolean append = (option == DistCpOptionSwitch.APPEND ? value : this.append);
|
|
||||||
boolean useDiff = (option == DistCpOptionSwitch.DIFF ? value : this.useDiff);
|
|
||||||
|
|
||||||
if (syncFolder && atomicCommit) {
|
if (syncFolder && atomicCommit) {
|
||||||
throw new IllegalArgumentException("Atomic commit can't be used with " +
|
throw new IllegalArgumentException("Atomic commit can't be used with " +
|
||||||
@ -593,7 +581,7 @@ public void validate(DistCpOptionSwitch option, boolean value) {
|
|||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"Append is disallowed when skipping CRC");
|
"Append is disallowed when skipping CRC");
|
||||||
}
|
}
|
||||||
if ((!syncFolder || deleteMissing) && useDiff) {
|
if (!syncFolder && useDiff) {
|
||||||
throw new IllegalArgumentException(
|
throw new IllegalArgumentException(
|
||||||
"Diff is valid only with update options");
|
"Diff is valid only with update options");
|
||||||
}
|
}
|
||||||
|
@ -41,7 +41,7 @@
|
|||||||
*/
|
*/
|
||||||
public class OptionsParser {
|
public class OptionsParser {
|
||||||
|
|
||||||
private static final Log LOG = LogFactory.getLog(OptionsParser.class);
|
static final Log LOG = LogFactory.getLog(OptionsParser.class);
|
||||||
|
|
||||||
private static final Options cliOptions = new Options();
|
private static final Options cliOptions = new Options();
|
||||||
|
|
||||||
@ -88,14 +88,26 @@ public static DistCpOptions parse(String args[]) throws IllegalArgumentException
|
|||||||
|
|
||||||
DistCpOptions option = parseSourceAndTargetPaths(command);
|
DistCpOptions option = parseSourceAndTargetPaths(command);
|
||||||
|
|
||||||
//Process all the other option switches and set options appropriately
|
option.setIgnoreFailures(
|
||||||
if (command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch())) {
|
command.hasOption(DistCpOptionSwitch.IGNORE_FAILURES.getSwitch()));
|
||||||
option.setIgnoreFailures(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch())) {
|
option.setAtomicCommit(
|
||||||
option.setAtomicCommit(true);
|
command.hasOption(DistCpOptionSwitch.ATOMIC_COMMIT.getSwitch()));
|
||||||
}
|
|
||||||
|
option.setSyncFolder(
|
||||||
|
command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch()));
|
||||||
|
|
||||||
|
option.setOverwrite(
|
||||||
|
command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch()));
|
||||||
|
|
||||||
|
option.setAppend(
|
||||||
|
command.hasOption(DistCpOptionSwitch.APPEND.getSwitch()));
|
||||||
|
|
||||||
|
option.setDeleteMissing(
|
||||||
|
command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch()));
|
||||||
|
|
||||||
|
option.setSkipCRC(
|
||||||
|
command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch()));
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) &&
|
if (command.hasOption(DistCpOptionSwitch.WORK_PATH.getSwitch()) &&
|
||||||
option.shouldAtomicCommit()) {
|
option.shouldAtomicCommit()) {
|
||||||
@ -111,25 +123,6 @@ public static DistCpOptions parse(String args[]) throws IllegalArgumentException
|
|||||||
option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch())));
|
option.setLogPath(new Path(getVal(command, DistCpOptionSwitch.LOG_PATH.getSwitch())));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.SYNC_FOLDERS.getSwitch())) {
|
|
||||||
option.setSyncFolder(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.OVERWRITE.getSwitch())) {
|
|
||||||
option.setOverwrite(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.APPEND.getSwitch())) {
|
|
||||||
option.setAppend(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.DELETE_MISSING.getSwitch())) {
|
|
||||||
option.setDeleteMissing(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.SKIP_CRC.getSwitch())) {
|
|
||||||
option.setSkipCRC(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) {
|
if (command.hasOption(DistCpOptionSwitch.BLOCKING.getSwitch())) {
|
||||||
option.setBlocking(false);
|
option.setBlocking(false);
|
||||||
@ -164,6 +157,8 @@ public static DistCpOptions parse(String args[]) throws IllegalArgumentException
|
|||||||
DistCpOptionSwitch.FILTERS.getSwitch()));
|
DistCpOptionSwitch.FILTERS.getSwitch()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
option.validate();
|
||||||
|
|
||||||
return option;
|
return option;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.tools;
|
package org.apache.hadoop.tools;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertFalse;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
@ -691,11 +692,25 @@ public void testDiffOption() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
OptionsParser.parse(new String[] { "-diff", "s1", "s2", "-update", "-delete",
|
options = OptionsParser.parse(new String[] {
|
||||||
|
"-diff", "s1", "s2", "-update", "-delete",
|
||||||
"hdfs://localhost:9820/source/first",
|
"hdfs://localhost:9820/source/first",
|
||||||
"hdfs://localhost:9820/target/" });
|
"hdfs://localhost:9820/target/" });
|
||||||
fail("-diff should fail if -delete option is specified");
|
assertFalse("-delete should be ignored when -diff is specified",
|
||||||
|
options.shouldDeleteMissing());
|
||||||
} catch (IllegalArgumentException e) {
|
} catch (IllegalArgumentException e) {
|
||||||
|
fail("Got unexpected IllegalArgumentException: " + e.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
options = OptionsParser.parse(new String[] {
|
||||||
|
"-diff", "s1", "s2", "-delete",
|
||||||
|
"hdfs://localhost:9820/source/first",
|
||||||
|
"hdfs://localhost:9820/target/" });
|
||||||
|
fail("-diff should fail if -update option is not specified");
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
assertFalse("-delete should be ignored when -diff is specified",
|
||||||
|
options.shouldDeleteMissing());
|
||||||
GenericTestUtils.assertExceptionContains(
|
GenericTestUtils.assertExceptionContains(
|
||||||
"Diff is valid only with update options", e);
|
"Diff is valid only with update options", e);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user