big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic

Browse Source
This commit is contained in:
Ravi Prakash 2016-09-12 08:26:08 -07:00
parent cc01ed7026
commit 9faccd1046
3 changed files with 70 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
View File

@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = "_distcp";
private static final String WIP_PREFIX = "._WIP_";
private static final String DISTCP_DEFAULT_XML = "distcp-default.xml";
private static final String DISTCP_SITE_XML = "distcp-site.xml";
static final Random rand = new Random();
private boolean submitted;
@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool {
public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
Configuration config = new Configuration(configuration);
config.addResource(DISTCP_DEFAULT_XML);
config.addResource(DISTCP_SITE_XML);
setConf(config);
this.inputOptions = inputOptions;
this.metaFolder = createMetaFolderPath();
@ -393,10 +395,12 @@ public static void main(String argv[]) {
* Loads properties from distcp-default.xml into configuration
* object
* @return Configuration which includes properties from distcp-default.xml
* and distcp-site.xml
*/
private static Configuration getDefaultConf() {
Configuration config = new Configuration();
config.addResource(DISTCP_DEFAULT_XML);
config.addResource(DISTCP_SITE_XML);
return config;
}

8
hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
View File

@ -47,7 +47,7 @@ public class DistCpOptions {
public static final int maxNumListstatusThreads = 40;
private int numListstatusThreads = 0; // Indicates that flag is not set.
private int maxMaps = DistCpConstants.DEFAULT_MAPS;
private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB;
private float mapBandwidth = 0; // Indicates that we should use the default.
private String copyStrategy = DistCpConstants.UNIFORMSIZE;
@ -609,8 +609,10 @@ public void appendToConf(Configuration conf) {
String.valueOf(useDiff));
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC,
String.valueOf(skipCRC));
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
String.valueOf(mapBandwidth));
if (mapBandwidth > 0) {
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
String.valueOf(mapBandwidth));
}
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
DistCpUtils.packAttributes(preserveStatus));
if (filtersFile != null) {

64
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
View File

@ -107,7 +107,7 @@ public void testParsebandwidth() {
DistCpOptions options = OptionsParser.parse(new String[] {
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/"});
Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
Assert.assertEquals(options.getMapBandwidth(), 0, DELTA);
options = OptionsParser.parse(new String[] {
"-bandwidth",
@ -389,7 +389,7 @@ public void testToString() {
+ "deleteMissing=false, ignoreFailures=false, overwrite=false, "
+ "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, "
+ "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
+ "mapBandwidth=100.0, "
+ "mapBandwidth=0.0, "
+ "copyStrategy='uniformsize', preserveStatus=[], "
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
@ -572,6 +572,8 @@ public void testOptionsAppendToConf() {
Configuration conf = new Configuration();
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
Assert.assertEquals(
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
DistCpOptions options = OptionsParser.parse(new String[] {
"-atomic",
"-i",
@ -581,7 +583,7 @@ public void testOptionsAppendToConf() {
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
-1.0, DELTA);
conf = new Configuration();
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
@ -602,6 +604,62 @@ public void testOptionsAppendToConf() {
Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11.2, DELTA);
}
@Test
public void testOptionsAppendToConfDoesntOverwriteBandwidth() {
Configuration conf = new Configuration();
Assert.assertEquals(
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
DistCpOptions options = OptionsParser.parse(new String[] {
"hdfs://localhost:8020/source/first",
"hdfs://localhost:8020/target/"});
options.appendToConf(conf);
Assert.assertEquals(
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0,
DELTA);
conf = new Configuration();
Assert.assertEquals(
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
options = OptionsParser.parse(new String[] {
"-update",
"-delete",
"-pu",
"-bandwidth",
"77",
"hdfs://localhost:8020/source/first",
"hdfs://localhost:8020/target/"});
options.appendToConf(conf);
Assert.assertEquals(
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0,
DELTA);
conf = new Configuration();
conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88");
Assert.assertEquals(
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88");
options = OptionsParser.parse(new String[] {
"hdfs://localhost:8020/source/first",
"hdfs://localhost:8020/target/"});
options.appendToConf(conf);
Assert.assertEquals(
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0,
DELTA);
conf = new Configuration();
conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0");
Assert.assertEquals(
conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0");
options = OptionsParser.parse(new String[] {
"-bandwidth",
"99",
"hdfs://localhost:8020/source/first",
"hdfs://localhost:8020/target/"});
options.appendToConf(conf);
Assert.assertEquals(
conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0,
DELTA);
}
@Test
public void testAppendOption() {
Configuration conf = new Configuration();