HADOOP-13024. Distcp with -delete feature on raw data not implemented. Contributed by Mavin Martin.
This commit is contained in:
parent
8c721aa00a
commit
0a85d07983
@ -18,6 +18,8 @@
|
|||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility class to hold commonly used constants.
|
* Utility class to hold commonly used constants.
|
||||||
*/
|
*/
|
||||||
@ -124,10 +126,18 @@ public class DistCpConstants {
|
|||||||
public static final int MIN_RECORDS_PER_CHUNK_DEFAULT = 5;
|
public static final int MIN_RECORDS_PER_CHUNK_DEFAULT = 5;
|
||||||
public static final int SPLIT_RATIO_DEFAULT = 2;
|
public static final int SPLIT_RATIO_DEFAULT = 2;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constants for NONE file deletion
|
||||||
|
*/
|
||||||
|
public static final String NONE_PATH_NAME = "/NONE";
|
||||||
|
public static final Path NONE_PATH = new Path(NONE_PATH_NAME);
|
||||||
|
public static final Path RAW_NONE_PATH = new Path(
|
||||||
|
DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME + NONE_PATH_NAME);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Value of reserved raw HDFS directory when copying raw.* xattrs.
|
* Value of reserved raw HDFS directory when copying raw.* xattrs.
|
||||||
*/
|
*/
|
||||||
static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
|
public static final String HDFS_RESERVED_RAW_DIRECTORY_NAME = "/.reserved/raw";
|
||||||
|
|
||||||
static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp";
|
static final String HDFS_DISTCP_DIFF_DIRECTORY_NAME = ".distcp.diff.tmp";
|
||||||
}
|
}
|
||||||
|
@ -238,7 +238,10 @@ private void deleteMissing(Configuration conf) throws IOException {
|
|||||||
List<Path> targets = new ArrayList<Path>(1);
|
List<Path> targets = new ArrayList<Path>(1);
|
||||||
Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
|
Path targetFinalPath = new Path(conf.get(DistCpConstants.CONF_LABEL_TARGET_FINAL_PATH));
|
||||||
targets.add(targetFinalPath);
|
targets.add(targetFinalPath);
|
||||||
DistCpOptions options = new DistCpOptions(targets, new Path("/NONE"));
|
Path resultNonePath = Path.getPathWithoutSchemeAndAuthority(targetFinalPath)
|
||||||
|
.toString().startsWith(DistCpConstants.HDFS_RESERVED_RAW_DIRECTORY_NAME)
|
||||||
|
? DistCpConstants.RAW_NONE_PATH : DistCpConstants.NONE_PATH;
|
||||||
|
DistCpOptions options = new DistCpOptions(targets, resultNonePath);
|
||||||
//
|
//
|
||||||
// Set up options to be the same from the CopyListing.buildListing's perspective,
|
// Set up options to be the same from the CopyListing.buildListing's perspective,
|
||||||
// so to collect similar listings as when doing the copy
|
// so to collect similar listings as when doing the copy
|
||||||
|
@ -82,14 +82,7 @@ public void testPreserveRawXAttrs1() throws Exception {
|
|||||||
final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest";
|
final String relDst = "/./.reserved/../.reserved/raw/../raw/dest/../dest";
|
||||||
doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true,
|
doTestPreserveRawXAttrs(relSrc, relDst, "-px", true, true,
|
||||||
DistCpConstants.SUCCESS);
|
DistCpConstants.SUCCESS);
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-px",
|
doTestStandardPreserveRawXAttrs("-px", true);
|
||||||
false, true, DistCpConstants.SUCCESS);
|
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-px",
|
|
||||||
false, true, DistCpConstants.INVALID_ARGUMENT);
|
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-px",
|
|
||||||
false, true, DistCpConstants.INVALID_ARGUMENT);
|
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-px",
|
|
||||||
true, true, DistCpConstants.SUCCESS);
|
|
||||||
final Path savedWd = fs.getWorkingDirectory();
|
final Path savedWd = fs.getWorkingDirectory();
|
||||||
try {
|
try {
|
||||||
fs.setWorkingDirectory(new Path("/.reserved/raw"));
|
fs.setWorkingDirectory(new Path("/.reserved/raw"));
|
||||||
@ -103,27 +96,18 @@ public void testPreserveRawXAttrs1() throws Exception {
|
|||||||
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
|
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
|
||||||
@Test
|
@Test
|
||||||
public void testPreserveRawXAttrs2() throws Exception {
|
public void testPreserveRawXAttrs2() throws Exception {
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, "-p",
|
doTestStandardPreserveRawXAttrs("-p", false);
|
||||||
false, false, DistCpConstants.SUCCESS);
|
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, "-p",
|
|
||||||
false, false, DistCpConstants.INVALID_ARGUMENT);
|
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, "-p",
|
|
||||||
false, false, DistCpConstants.INVALID_ARGUMENT);
|
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rawDestName, "-p",
|
|
||||||
true, false, DistCpConstants.SUCCESS);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
|
/* Test that XAttrs are not preserved and raw.* are when appropriate. */
|
||||||
@Test
|
@Test
|
||||||
public void testPreserveRawXAttrs3() throws Exception {
|
public void testPreserveRawXAttrs3() throws Exception {
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, null,
|
doTestStandardPreserveRawXAttrs(null, false);
|
||||||
false, false, DistCpConstants.SUCCESS);
|
}
|
||||||
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, null,
|
|
||||||
false, false, DistCpConstants.INVALID_ARGUMENT);
|
@Test
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, null,
|
public void testPreserveRawXAttrs4() throws Exception {
|
||||||
false, false, DistCpConstants.INVALID_ARGUMENT);
|
doTestStandardPreserveRawXAttrs("-update -delete", false);
|
||||||
doTestPreserveRawXAttrs(rawSrcName, rawDestName, null,
|
|
||||||
true, false, DistCpConstants.SUCCESS);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Path[] pathnames = { new Path("dir1"),
|
private static Path[] pathnames = { new Path("dir1"),
|
||||||
@ -145,6 +129,19 @@ private void initXAttrs() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void doTestStandardPreserveRawXAttrs(String options,
|
||||||
|
boolean expectUser)
|
||||||
|
throws Exception {
|
||||||
|
doTestPreserveRawXAttrs(rootedSrcName, rootedDestName, options,
|
||||||
|
false, expectUser, DistCpConstants.SUCCESS);
|
||||||
|
doTestPreserveRawXAttrs(rootedSrcName, rawDestName, options,
|
||||||
|
false, expectUser, DistCpConstants.INVALID_ARGUMENT);
|
||||||
|
doTestPreserveRawXAttrs(rawSrcName, rootedDestName, options,
|
||||||
|
false, expectUser, DistCpConstants.INVALID_ARGUMENT);
|
||||||
|
doTestPreserveRawXAttrs(rawSrcName, rawDestName, options,
|
||||||
|
true, expectUser, DistCpConstants.SUCCESS);
|
||||||
|
}
|
||||||
|
|
||||||
private void doTestPreserveRawXAttrs(String src, String dest,
|
private void doTestPreserveRawXAttrs(String src, String dest,
|
||||||
String preserveOpts, boolean expectRaw, boolean expectUser,
|
String preserveOpts, boolean expectRaw, boolean expectUser,
|
||||||
int expectedExitCode) throws Exception {
|
int expectedExitCode) throws Exception {
|
||||||
|
@ -18,20 +18,19 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.tools.util;
|
package org.apache.hadoop.tools.util;
|
||||||
|
|
||||||
import static org.junit.Assert.assertArrayEquals;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import static org.junit.Assert.assertEquals;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import static org.junit.Assert.assertTrue;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.tools.DistCp;
|
||||||
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import static org.junit.Assert.assertArrayEquals;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import static org.junit.Assert.assertEquals;
|
||||||
import org.apache.hadoop.fs.Path;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import org.apache.hadoop.tools.DistCp;
|
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility class for DistCpTests
|
* Utility class for DistCpTests
|
||||||
@ -79,10 +78,19 @@ public static void assertXAttrs(Path path, FileSystem fs,
|
|||||||
public static void assertRunDistCp(int exitCode, String src, String dst,
|
public static void assertRunDistCp(int exitCode, String src, String dst,
|
||||||
String options, Configuration conf)
|
String options, Configuration conf)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
assertRunDistCp(exitCode, src, dst,
|
||||||
|
options == null ? new String[0] : options.trim().split(" "), conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void assertRunDistCp(int exitCode, String src, String dst,
|
||||||
|
String[] options, Configuration conf)
|
||||||
|
throws Exception {
|
||||||
DistCp distCp = new DistCp(conf, null);
|
DistCp distCp = new DistCp(conf, null);
|
||||||
String[] optsArr = options == null ?
|
String[] optsArr = new String[options.length + 2];
|
||||||
new String[] { src, dst } :
|
System.arraycopy(options, 0, optsArr, 0, options.length);
|
||||||
new String[] { options, src, dst };
|
optsArr[optsArr.length - 2] = src;
|
||||||
|
optsArr[optsArr.length - 1] = dst;
|
||||||
|
|
||||||
assertEquals(exitCode,
|
assertEquals(exitCode,
|
||||||
ToolRunner.run(conf, distCp, optsArr));
|
ToolRunner.run(conf, distCp, optsArr));
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user