big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-10300. TestDistCpSystem should share MiniDFSCluster. Contributed by John Zhuge.

Browse Source
This commit is contained in:
Andrew Wang 2016-07-11 18:06:24 -07:00
parent 7bd5d4272c
commit f292624bd8
1 changed files with 100 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpSystem.java
View File

@ -18,7 +18,10 @@
package org.apache.hadoop.tools; package org.apache.hadoop.tools;
import static org.apache.hadoop.test.GenericTestUtils.getMethodName;
import static org.hamcrest.core.Is.is; import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
@ -26,26 +29,33 @@
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.util.ToolRunner;
import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
import org.junit.rules.Timeout;
/** /**
* A JUnit test for copying files recursively. * A JUnit test for copying files recursively.
*/ */
public class TestDistCpSystem extends TestCase { public class TestDistCpSystem {
@Rule
public Timeout globalTimeout = new Timeout(30000);
private static final String SRCDAT = "srcdat"; private static final String SRCDAT = "srcdat";
private static final String DSTDAT = "dstdat"; private static final String DSTDAT = "dstdat";
private static MiniDFSCluster cluster;
private static Configuration conf;
private class FileEntry { private class FileEntry {
String path; String path;
boolean isDir; boolean isDir;
@ -93,59 +103,65 @@ private static void deldir(FileSystem fs, String topdir) throws IOException {
fs.delete(new Path(topdir), true); fs.delete(new Path(topdir), true);
} }
private void testPreserveUserHelper( private void testPreserveUserHelper(String testRoot,
FileEntry[] srcEntries, FileEntry[] srcEntries,
FileEntry[] dstEntries, FileEntry[] dstEntries,
boolean createSrcDir, boolean createSrcDir,
boolean createTgtDir, boolean createTgtDir,
boolean update) throws Exception { boolean update) throws Exception {
Configuration conf = null; final String testSrcRel = SRCDAT;
MiniDFSCluster cluster = null; final String testSrc = testRoot + "/" + testSrcRel;
try { final String testDstRel = DSTDAT;
final String testRoot = "/testdir"; final String testDst = testRoot + "/" + testDstRel;
final String testSrcRel = SRCDAT;
final String testSrc = testRoot + "/" + testSrcRel;
final String testDstRel = DSTDAT;
final String testDst = testRoot + "/" + testDstRel;
conf = new Configuration(); String nnUri = FileSystem.getDefaultUri(conf).toString();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); FileSystem fs = FileSystem.get(URI.create(nnUri), conf);
fs.mkdirs(new Path(testRoot));
if (createSrcDir) {
fs.mkdirs(new Path(testSrc));
}
if (createTgtDir) {
fs.mkdirs(new Path(testDst));
}
String nnUri = FileSystem.getDefaultUri(conf).toString(); createFiles(fs, testRoot, srcEntries);
FileSystem fs = FileSystem.get(URI.create(nnUri), conf); FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries);
fs.mkdirs(new Path(testRoot)); for(int i = 0; i < srcEntries.length; i++) {
if (createSrcDir) { fs.setOwner(srcstats[i].getPath(), "u" + i, null);
fs.mkdirs(new Path(testSrc)); }
} String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
if (createTgtDir) { nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
fs.mkdirs(new Path(testDst));
}
createFiles(fs, testRoot, srcEntries); ToolRunner.run(conf, new DistCp(), args);
FileStatus[] srcstats = getFileStatus(fs, testRoot, srcEntries);
for(int i = 0; i < srcEntries.length; i++) {
fs.setOwner(srcstats[i].getPath(), "u" + i, null);
}
String[] args = update? new String[]{"-pu", "-update", nnUri+testSrc,
nnUri+testDst} : new String[]{"-pu", nnUri+testSrc, nnUri+testDst};
ToolRunner.run(conf, new DistCp(), args); String realTgtPath = testDst;
if (!createTgtDir) {
realTgtPath = testRoot;
}
FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries);
for(int i = 0; i < dststat.length; i++) {
assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
}
deldir(fs, testRoot);
}
String realTgtPath = testDst; @BeforeClass
if (!createTgtDir) { public static void beforeClass() throws IOException {
realTgtPath = testRoot; conf = new Configuration();
} cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
FileStatus[] dststat = getFileStatus(fs, realTgtPath, dstEntries); cluster.waitActive();
for(int i = 0; i < dststat.length; i++) { }
assertEquals("i=" + i, "u" + i, dststat[i].getOwner());
} @AfterClass
deldir(fs, testRoot); public static void afterClass() throws IOException {
} finally { if (cluster != null) {
if (cluster != null) { cluster.shutdown(); } cluster.shutdown();
} }
} }
@Test
public void testPreserveUseNonEmptyDir() throws Exception { public void testPreserveUseNonEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
FileEntry[] srcfiles = { FileEntry[] srcfiles = {
new FileEntry(SRCDAT, true), new FileEntry(SRCDAT, true),
new FileEntry(SRCDAT + "/a", false), new FileEntry(SRCDAT + "/a", false),
@ -160,12 +176,14 @@ public void testPreserveUseNonEmptyDir() throws Exception {
new FileEntry(DSTDAT + "/b/c", false) new FileEntry(DSTDAT + "/b/c", false)
}; };
testPreserveUserHelper(srcfiles, srcfiles, false, true, false); testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
testPreserveUserHelper(srcfiles, dstfiles, false, false, false); testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
} }
@Test
public void testPreserveUserEmptyDir() throws Exception { public void testPreserveUserEmptyDir() throws Exception {
String testRoot = "/testdir." + getMethodName();
FileEntry[] srcfiles = { FileEntry[] srcfiles = {
new FileEntry(SRCDAT, true) new FileEntry(SRCDAT, true)
}; };
@ -174,22 +192,26 @@ public void testPreserveUserEmptyDir() throws Exception {
new FileEntry(DSTDAT, true) new FileEntry(DSTDAT, true)
}; };
testPreserveUserHelper(srcfiles, srcfiles, false, true, false); testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
testPreserveUserHelper(srcfiles, dstfiles, false, false, false); testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
} }
@Test
public void testPreserveUserSingleFile() throws Exception { public void testPreserveUserSingleFile() throws Exception {
String testRoot = "/testdir." + getMethodName();
FileEntry[] srcfiles = { FileEntry[] srcfiles = {
new FileEntry(SRCDAT, false) new FileEntry(SRCDAT, false)
}; };
FileEntry[] dstfiles = { FileEntry[] dstfiles = {
new FileEntry(DSTDAT, false) new FileEntry(DSTDAT, false)
}; };
testPreserveUserHelper(srcfiles, srcfiles, false, true, false); testPreserveUserHelper(testRoot, srcfiles, srcfiles, false, true, false);
testPreserveUserHelper(srcfiles, dstfiles, false, false, false); testPreserveUserHelper(testRoot, srcfiles, dstfiles, false, false, false);
} }
@Test
public void testPreserveUserNonEmptyDirWithUpdate() throws Exception { public void testPreserveUserNonEmptyDirWithUpdate() throws Exception {
String testRoot = "/testdir." + getMethodName();
FileEntry[] srcfiles = { FileEntry[] srcfiles = {
new FileEntry(SRCDAT + "/a", false), new FileEntry(SRCDAT + "/a", false),
new FileEntry(SRCDAT + "/b", true), new FileEntry(SRCDAT + "/b", true),
@ -202,37 +224,30 @@ public void testPreserveUserNonEmptyDirWithUpdate() throws Exception {
new FileEntry("b/c", false) new FileEntry("b/c", false)
}; };
testPreserveUserHelper(srcfiles, dstfiles, true, true, true); testPreserveUserHelper(testRoot, srcfiles, dstfiles, true, true, true);
} }
@Test @Test
public void testSourceRoot() throws Exception { public void testSourceRoot() throws Exception {
MiniDFSCluster cluster = null; FileSystem fs = cluster.getFileSystem();
Configuration conf = new Configuration();
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
FileSystem fs = cluster.getFileSystem();
String rootStr = fs.makeQualified(new Path("/")).toString(); String rootStr = fs.makeQualified(new Path("/")).toString();
// Case 1. The target does not exist. String testRoot = "/testdir." + getMethodName();
String tgtStr = fs.makeQualified(new Path("/nodir")).toString(); // Case 1. The target does not exist.
String[] args = new String[]{ rootStr, tgtStr };
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0));
// Case 2. The target exists. Path tgtPath = new Path(testRoot + "/nodir");
String tgtStr = fs.makeQualified(tgtPath).toString();
String[] args = new String[]{rootStr, tgtStr};
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args), is(0));
Path tgtPath2 = new Path("/dir"); // Case 2. The target exists.
assertTrue(fs.mkdirs(tgtPath2));
String tgtStr2 = fs.makeQualified(tgtPath2).toString(); Path tgtPath2 = new Path(testRoot + "/dir");
String[] args2 = new String[]{ rootStr, tgtStr2 }; assertTrue(fs.mkdirs(tgtPath2));
Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0)); String tgtStr2 = fs.makeQualified(tgtPath2).toString();
} finally { String[] args2 = new String[]{rootStr, tgtStr2};
if (cluster != null) { Assert.assertThat(ToolRunner.run(conf, new DistCp(), args2), is(0));
cluster.shutdown();
}
}
} }
} }