HADOOP-8327. distcpv2 and distcpv1 jars should not coexist (Dave Thompson via bobby)
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1335075 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
e7d03636a0
commit
8069771a2d
@ -536,6 +536,9 @@ Release 0.23.3 - UNRELEASED
|
||||
HADOOP-8335. Improve Configuration's address handling (Daryn Sharp via
|
||||
bobby)
|
||||
|
||||
HADOOP-8327. distcpv2 and distcpv1 jars should not coexist (Dave Thompson
|
||||
via bobby)
|
||||
|
||||
Release 0.23.2 - UNRELEASED
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
@ -78,8 +78,8 @@
|
||||
* A Map-reduce program to recursively copy directories between
|
||||
* different file-systems.
|
||||
*/
|
||||
public class DistCp implements Tool {
|
||||
public static final Log LOG = LogFactory.getLog(DistCp.class);
|
||||
public class DistCpV1 implements Tool {
|
||||
public static final Log LOG = LogFactory.getLog(DistCpV1.class);
|
||||
|
||||
private static final String NAME = "distcp";
|
||||
|
||||
@ -221,7 +221,7 @@ public Configuration getConf() {
|
||||
return conf;
|
||||
}
|
||||
|
||||
public DistCp(Configuration conf) {
|
||||
public DistCpV1(Configuration conf) {
|
||||
setConf(conf);
|
||||
}
|
||||
|
||||
@ -565,7 +565,7 @@ private void rename(Path tmp, Path dst) throws IOException {
|
||||
private void updateDestStatus(FileStatus src, FileStatus dst
|
||||
) throws IOException {
|
||||
if (preserve_status) {
|
||||
DistCp.updateDestStatus(src, dst, preseved, destFileSys);
|
||||
DistCpV1.updateDestStatus(src, dst, preseved, destFileSys);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1049,8 +1049,8 @@ public int run(String[] args) {
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
JobConf job = new JobConf(DistCp.class);
|
||||
DistCp distcp = new DistCp(job);
|
||||
JobConf job = new JobConf(DistCpV1.class);
|
||||
DistCpV1 distcp = new DistCpV1(job);
|
||||
int res = ToolRunner.run(distcp, args);
|
||||
System.exit(res);
|
||||
}
|
||||
@ -1117,7 +1117,7 @@ static void fullyDelete(String dir, Configuration conf) throws IOException {
|
||||
|
||||
//Job configuration
|
||||
private static JobConf createJobConf(Configuration conf) {
|
||||
JobConf jobconf = new JobConf(conf, DistCp.class);
|
||||
JobConf jobconf = new JobConf(conf, DistCpV1.class);
|
||||
jobconf.setJobName(conf.get("mapred.job.name", NAME));
|
||||
|
||||
// turn off speculative execution, because DFS doesn't handle
|
@ -194,7 +194,7 @@ public int compare(byte[] b1, int s1, int l1,
|
||||
throws IOException
|
||||
{
|
||||
String destURL = FileSystem.getDefaultUri(fsConfig) + archiveDirectory;
|
||||
DistCp.copy(new JobConf(fsConfig), logListURI, destURL, null, true, false);
|
||||
DistCpV1.copy(new JobConf(fsConfig), logListURI, destURL, null, true, false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -48,7 +48,7 @@
|
||||
import org.apache.hadoop.mapred.JobConf;
|
||||
import org.apache.hadoop.mapred.MiniMRCluster;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.tools.DistCp;
|
||||
import org.apache.hadoop.tools.DistCpV1;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.apache.log4j.Level;
|
||||
import org.junit.Ignore;
|
||||
@ -64,7 +64,7 @@ public class TestCopyFiles extends TestCase {
|
||||
).getLogger().setLevel(Level.OFF);
|
||||
((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.OFF);
|
||||
((Log4JLogger)LogFactory.getLog(FSNamesystem.class)).getLogger().setLevel(Level.OFF);
|
||||
((Log4JLogger)DistCp.LOG).getLogger().setLevel(Level.ALL);
|
||||
((Log4JLogger)DistCpV1.LOG).getLogger().setLevel(Level.ALL);
|
||||
}
|
||||
|
||||
static final URI LOCAL_FS = URI.create("file:///");
|
||||
@ -267,7 +267,7 @@ public void testCopyFromLocalToLocal() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
|
||||
MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
|
||||
"file:///"+TEST_ROOT_DIR+"/destdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
@ -287,7 +287,7 @@ public void testCopyFromDfsToDfs() throws Exception {
|
||||
namenode = FileSystem.getDefaultUri(conf).toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
namenode+"/srcdat",
|
||||
@ -320,7 +320,7 @@ public void testEmptyDir() throws Exception {
|
||||
FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
|
||||
fs.mkdirs(new Path("/empty"));
|
||||
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
namenode+"/empty",
|
||||
@ -347,7 +347,7 @@ public void testCopyFromLocalToDfs() throws Exception {
|
||||
final String namenode = hdfs.getUri().toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(LOCAL_FS, TEST_ROOT_DIR+"/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
"file:///"+TEST_ROOT_DIR+"/srcdat",
|
||||
@ -376,7 +376,7 @@ public void testCopyFromDfsToLocal() throws Exception {
|
||||
final String namenode = FileSystem.getDefaultUri(conf).toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-log",
|
||||
"/logs",
|
||||
namenode+"/srcdat",
|
||||
@ -403,7 +403,7 @@ public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
|
||||
final String namenode = hdfs.getUri().toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-p",
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
@ -420,7 +420,7 @@ public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
|
||||
updateFiles(cluster.getFileSystem(), "/srcdat", files, nupdate);
|
||||
deldir(hdfs, "/logs");
|
||||
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-prbugp", // no t to avoid preserving mod. times
|
||||
"-update",
|
||||
"-log",
|
||||
@ -433,7 +433,7 @@ public void testCopyDfsToDfsUpdateOverwrite() throws Exception {
|
||||
checkUpdate(hdfs, dchkpoint, "/destdat", files, nupdate));
|
||||
|
||||
deldir(hdfs, "/logs");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-prbugp", // no t to avoid preserving mod. times
|
||||
"-overwrite",
|
||||
"-log",
|
||||
@ -483,7 +483,7 @@ public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
|
||||
out.close();
|
||||
|
||||
// Run with -skipcrccheck option
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-p",
|
||||
"-update",
|
||||
"-skipcrccheck",
|
||||
@ -503,7 +503,7 @@ public void testCopyDfsToDfsUpdateWithSkipCRC() throws Exception {
|
||||
deldir(hdfs, "/logs");
|
||||
|
||||
// Run without the option
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-p",
|
||||
"-update",
|
||||
"-log",
|
||||
@ -533,14 +533,14 @@ public void testCopyDuplication() throws Exception {
|
||||
final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
|
||||
try {
|
||||
MyFile[] files = createFiles(localfs, TEST_ROOT_DIR+"/srcdat");
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
|
||||
"file:///"+TEST_ROOT_DIR+"/src2/srcdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
checkFiles(localfs, TEST_ROOT_DIR+"/src2/srcdat", files));
|
||||
|
||||
assertEquals(DistCp.DuplicationException.ERROR_CODE,
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
assertEquals(DistCpV1.DuplicationException.ERROR_CODE,
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
|
||||
"file:///"+TEST_ROOT_DIR+"/src2/srcdat",
|
||||
"file:///"+TEST_ROOT_DIR+"/destdat",}));
|
||||
@ -558,7 +558,7 @@ public void testCopySingleFile() throws Exception {
|
||||
try {
|
||||
MyFile[] files = {createFile(root, fs)};
|
||||
//copy a dir with a single file
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat",
|
||||
"file:///"+TEST_ROOT_DIR+"/destdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
@ -568,7 +568,7 @@ public void testCopySingleFile() throws Exception {
|
||||
String fname = files[0].getName();
|
||||
Path p = new Path(root, fname);
|
||||
FileSystem.LOG.info("fname=" + fname + ", exists? " + fs.exists(p));
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
|
||||
"file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
@ -578,17 +578,17 @@ public void testCopySingleFile() throws Exception {
|
||||
String[] args = {"-update", "file:///"+TEST_ROOT_DIR+"/srcdat/"+fname,
|
||||
"file:///"+TEST_ROOT_DIR+"/dest2/"+fname};
|
||||
Configuration conf = new Configuration();
|
||||
JobConf job = new JobConf(conf, DistCp.class);
|
||||
DistCp.Arguments distcpArgs = DistCp.Arguments.valueOf(args, conf);
|
||||
JobConf job = new JobConf(conf, DistCpV1.class);
|
||||
DistCpV1.Arguments distcpArgs = DistCpV1.Arguments.valueOf(args, conf);
|
||||
assertFalse("Single file update failed to skip copying even though the "
|
||||
+ "file exists at destination.", DistCp.setup(conf, job, distcpArgs));
|
||||
+ "file exists at destination.", DistCpV1.setup(conf, job, distcpArgs));
|
||||
|
||||
//copy single file to existing dir
|
||||
deldir(fs, TEST_ROOT_DIR+"/dest2");
|
||||
fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
|
||||
MyFile[] files2 = {createFile(root, fs, 0)};
|
||||
String sname = files2[0].getName();
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"-update",
|
||||
"file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
|
||||
"file:///"+TEST_ROOT_DIR+"/dest2/"});
|
||||
@ -596,7 +596,7 @@ public void testCopySingleFile() throws Exception {
|
||||
checkFiles(fs, TEST_ROOT_DIR+"/dest2", files2));
|
||||
updateFiles(fs, TEST_ROOT_DIR+"/srcdat", files2, 1);
|
||||
//copy single file to existing dir w/ dst name conflict
|
||||
ToolRunner.run(new DistCp(new Configuration()),
|
||||
ToolRunner.run(new DistCpV1(new Configuration()),
|
||||
new String[] {"-update",
|
||||
"file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
|
||||
"file:///"+TEST_ROOT_DIR+"/dest2/"});
|
||||
@ -621,7 +621,7 @@ public void testBasedir() throws Exception {
|
||||
namenode = FileSystem.getDefaultUri(conf).toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-basedir",
|
||||
"/basedir",
|
||||
namenode+"/basedir/middle/srcdat",
|
||||
@ -651,7 +651,7 @@ public void testPreserveOption() throws Exception {
|
||||
for(int i = 0; i < srcstat.length; i++) {
|
||||
fs.setOwner(srcstat[i].getPath(), "u" + i, null);
|
||||
}
|
||||
ToolRunner.run(new DistCp(conf),
|
||||
ToolRunner.run(new DistCpV1(conf),
|
||||
new String[]{"-pu", nnUri+"/srcdat", nnUri+"/destdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
checkFiles(fs, "/destdat", files));
|
||||
@ -670,7 +670,7 @@ public void testPreserveOption() throws Exception {
|
||||
for(int i = 0; i < srcstat.length; i++) {
|
||||
fs.setOwner(srcstat[i].getPath(), null, "g" + i);
|
||||
}
|
||||
ToolRunner.run(new DistCp(conf),
|
||||
ToolRunner.run(new DistCpV1(conf),
|
||||
new String[]{"-pg", nnUri+"/srcdat", nnUri+"/destdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
checkFiles(fs, "/destdat", files));
|
||||
@ -692,7 +692,7 @@ public void testPreserveOption() throws Exception {
|
||||
fs.setPermission(srcstat[i].getPath(), permissions[i]);
|
||||
}
|
||||
|
||||
ToolRunner.run(new DistCp(conf),
|
||||
ToolRunner.run(new DistCpV1(conf),
|
||||
new String[]{"-pp", nnUri+"/srcdat", nnUri+"/destdat"});
|
||||
assertTrue("Source and destination directories do not match.",
|
||||
checkFiles(fs, "/destdat", files));
|
||||
@ -715,7 +715,7 @@ public void testPreserveOption() throws Exception {
|
||||
fs.setTimes(srcstat[i].getPath(), 40, 50);
|
||||
}
|
||||
|
||||
ToolRunner.run(new DistCp(conf),
|
||||
ToolRunner.run(new DistCpV1(conf),
|
||||
new String[]{"-pt", nnUri+"/srcdat", nnUri+"/destdat"});
|
||||
|
||||
FileStatus[] dststat = getFileStatus(fs, "/destdat", files);
|
||||
@ -753,7 +753,7 @@ public void testMapCount() throws Exception {
|
||||
}
|
||||
Configuration job = mr.createJobConf();
|
||||
job.setLong("distcp.bytes.per.map", totsize / 3);
|
||||
ToolRunner.run(new DistCp(job),
|
||||
ToolRunner.run(new DistCpV1(job),
|
||||
new String[] {"-m", "100",
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
@ -771,7 +771,7 @@ public void testMapCount() throws Exception {
|
||||
|
||||
deldir(fs, "/destdat");
|
||||
deldir(fs, "/logs");
|
||||
ToolRunner.run(new DistCp(job),
|
||||
ToolRunner.run(new DistCpV1(job),
|
||||
new String[] {"-m", "1",
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
@ -795,7 +795,7 @@ public void testLimits() throws Exception {
|
||||
cluster = new MiniDFSCluster(conf, 2, true, null);
|
||||
final String nnUri = FileSystem.getDefaultUri(conf).toString();
|
||||
final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
|
||||
final DistCp distcp = new DistCp(conf);
|
||||
final DistCpV1 distcp = new DistCpV1(conf);
|
||||
final FsShell shell = new FsShell(conf);
|
||||
|
||||
final String srcrootdir = "/src_root";
|
||||
@ -927,9 +927,9 @@ public FileSystem run() throws IOException {
|
||||
final String srcrootdir = srcrootpath.toString();
|
||||
final Path dstrootpath = new Path(home, "dst_root");
|
||||
final String dstrootdir = dstrootpath.toString();
|
||||
final DistCp distcp = USER_UGI.doAs(new PrivilegedExceptionAction<DistCp>() {
|
||||
public DistCp run() {
|
||||
return new DistCp(userConf);
|
||||
final DistCpV1 distcp = USER_UGI.doAs(new PrivilegedExceptionAction<DistCpV1>() {
|
||||
public DistCpV1 run() {
|
||||
return new DistCpV1(userConf);
|
||||
}
|
||||
});
|
||||
|
||||
@ -961,7 +961,7 @@ public void testDelete() throws Exception {
|
||||
final String nnUri = nnURI.toString();
|
||||
final FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
|
||||
|
||||
final DistCp distcp = new DistCp(conf);
|
||||
final DistCpV1 distcp = new DistCpV1(conf);
|
||||
final FsShell shell = new FsShell(conf);
|
||||
|
||||
final String srcrootdir = "/src_root";
|
||||
@ -1035,7 +1035,7 @@ public void testDeleteLocal() throws Exception {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
|
||||
String destdir = TEST_ROOT_DIR + "/destdat";
|
||||
MyFile[] localFiles = createFiles(localfs, destdir);
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-delete",
|
||||
"-update",
|
||||
"-log",
|
||||
@ -1066,7 +1066,7 @@ public void testGlobbing() throws Exception {
|
||||
namenode = FileSystem.getDefaultUri(conf).toString();
|
||||
if (namenode.startsWith("hdfs://")) {
|
||||
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
|
||||
ToolRunner.run(new DistCp(conf), new String[] {
|
||||
ToolRunner.run(new DistCpV1(conf), new String[] {
|
||||
"-log",
|
||||
namenode+"/logs",
|
||||
namenode+"/srcdat/*",
|
||||
|
Loading…
Reference in New Issue
Block a user