From 9faccd104672dfef123735ca8ada178fc3a6196f Mon Sep 17 00:00:00 2001 From: Ravi Prakash Date: Mon, 12 Sep 2016 08:26:08 -0700 Subject: [PATCH] HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic --- .../java/org/apache/hadoop/tools/DistCp.java | 4 ++ .../apache/hadoop/tools/DistCpOptions.java | 8 ++- .../hadoop/tools/TestOptionsParser.java | 64 ++++++++++++++++++- 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 36e5ee4cd6..be58f13164 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool { private static final String PREFIX = "_distcp"; private static final String WIP_PREFIX = "._WIP_"; private static final String DISTCP_DEFAULT_XML = "distcp-default.xml"; + private static final String DISTCP_SITE_XML = "distcp-site.xml"; static final Random rand = new Random(); private boolean submitted; @@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool { public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception { Configuration config = new Configuration(configuration); config.addResource(DISTCP_DEFAULT_XML); + config.addResource(DISTCP_SITE_XML); setConf(config); this.inputOptions = inputOptions; this.metaFolder = createMetaFolderPath(); @@ -393,10 +395,12 @@ public static void main(String argv[]) { * Loads properties from distcp-default.xml into configuration * object * @return Configuration which includes properties from distcp-default.xml + * and distcp-site.xml */ private static Configuration getDefaultConf() { Configuration config = new Configuration(); config.addResource(DISTCP_DEFAULT_XML); + config.addResource(DISTCP_SITE_XML); return config; } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index e6f53f5b60..4c5518f3ad 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -47,7 +47,7 @@ public class DistCpOptions { public static final int maxNumListstatusThreads = 40; private int numListstatusThreads = 0; // Indicates that flag is not set. private int maxMaps = DistCpConstants.DEFAULT_MAPS; - private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB; + private float mapBandwidth = 0; // Indicates that we should use the default. private String copyStrategy = DistCpConstants.UNIFORMSIZE; @@ -609,8 +609,10 @@ public void appendToConf(Configuration conf) { String.valueOf(useDiff)); DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC, String.valueOf(skipCRC)); - DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, - String.valueOf(mapBandwidth)); + if (mapBandwidth > 0) { + DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH, + String.valueOf(mapBandwidth)); + } DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); if (filtersFile != null) { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index c46bcd9cf5..218de4edfd 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -107,7 +107,7 @@ public void testParsebandwidth() { DistCpOptions options = OptionsParser.parse(new String[] { "hdfs://localhost:9820/source/first", "hdfs://localhost:9820/target/"}); - Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA); + Assert.assertEquals(options.getMapBandwidth(), 0, DELTA); options = OptionsParser.parse(new String[] { "-bandwidth", @@ -389,7 +389,7 @@ public void testToString() { + "deleteMissing=false, ignoreFailures=false, overwrite=false, " + "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, " + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, " - + "mapBandwidth=100.0, " + + "mapBandwidth=0.0, " + "copyStrategy='uniformsize', preserveStatus=[], " + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, " + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, " @@ -572,6 +572,8 @@ public void testOptionsAppendToConf() { Configuration conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); DistCpOptions options = OptionsParser.parse(new String[] { "-atomic", "-i", @@ -581,7 +583,7 @@ public void testOptionsAppendToConf() { Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), - DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA); + -1.0, DELTA); conf = new Configuration(); Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); @@ -602,6 +604,62 @@ public void testOptionsAppendToConf() { Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11.2, DELTA); } + @Test + public void testOptionsAppendToConfDoesntOverwriteBandwidth() { + Configuration conf = new Configuration(); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); + DistCpOptions options = OptionsParser.parse(new String[] { + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0, + DELTA); + + conf = new Configuration(); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null); + options = OptionsParser.parse(new String[] { + "-update", + "-delete", + "-pu", + "-bandwidth", + "77", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0, + DELTA); + + conf = new Configuration(); + conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88"); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88"); + options = OptionsParser.parse(new String[] { + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0, + DELTA); + + conf = new Configuration(); + conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0"); + Assert.assertEquals( + conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0"); + options = OptionsParser.parse(new String[] { + "-bandwidth", + "99", + "hdfs://localhost:8020/source/first", + "hdfs://localhost:8020/target/"}); + options.appendToConf(conf); + Assert.assertEquals( + conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0, + DELTA); + } + @Test public void testAppendOption() { Configuration conf = new Configuration();