diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 836a86dc54..37b297bd7e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -339,6 +339,9 @@ Release 2.3.0 - UNRELEASED HADOOP-10006. Compilation failure in trunk for o.a.h.fs.swift.util.JSONUtil (Junping Du via stevel) + HADOOP-9063. enhance unit-test coverage of class + org.apache.hadoop.fs.FileUtil (Ivan A. Veselovsky via jlowe) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index f37bf4f0c0..3eaa97f247 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -24,6 +24,8 @@ import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; @@ -32,15 +34,20 @@ import java.util.jar.Attributes; import java.util.jar.JarFile; import java.util.jar.Manifest; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import org.apache.tools.tar.TarEntry; +import org.apache.tools.tar.TarOutputStream; import org.junit.After; import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.*; public class TestFileUtil { private static final Log LOG = LogFactory.getLog(TestFileUtil.class); @@ -48,14 +55,14 @@ public class TestFileUtil { private static final String TEST_ROOT_DIR = System.getProperty( "test.build.data", "/tmp") + "/fu"; private static final File TEST_DIR = new File(TEST_ROOT_DIR); - private static String FILE = "x"; - private static String LINK = "y"; - private static String DIR = "dir"; - private File del = new File(TEST_DIR, "del"); - private File tmp = new File(TEST_DIR, "tmp"); - private File dir1 = new File(del, DIR + "1"); - private File dir2 = new File(del, DIR + "2"); - private File partitioned = new File(TEST_DIR, "partitioned"); + private static final String FILE = "x"; + private static final String LINK = "y"; + private static final String DIR = "dir"; + private final File del = new File(TEST_DIR, "del"); + private final File tmp = new File(TEST_DIR, "tmp"); + private final File dir1 = new File(del, DIR + "1"); + private final File dir2 = new File(del, DIR + "2"); + private final File partitioned = new File(TEST_DIR, "partitioned"); /** * Creates multiple directories for testing. @@ -116,17 +123,17 @@ private void setupDirs() throws IOException { * @param contents String non-null file contents. * @throws IOException if an I/O error occurs. */ - private void createFile(File directory, String name, String contents) + private File createFile(File directory, String name, String contents) throws IOException { File newFile = new File(directory, name); PrintWriter pw = new PrintWriter(newFile); - try { pw.println(contents); } finally { pw.close(); } + return newFile; } @Test (timeout = 30000) @@ -553,14 +560,283 @@ private boolean copyMerge(String src, String dst) * @throws IOException */ @Test (timeout = 30000) - public void testGetDU() throws IOException { + public void testGetDU() throws Exception { setupDirs(); long du = FileUtil.getDU(TEST_DIR); // Only two files (in partitioned). Each has 3 characters + system-specific // line separator. - long expected = 2 * (3 + System.getProperty("line.separator").length()); + final long expected = 2 * (3 + System.getProperty("line.separator").length()); Assert.assertEquals(expected, du); + + // target file does not exist: + final File doesNotExist = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); + long duDoesNotExist = FileUtil.getDU(doesNotExist); + assertEquals(0, duDoesNotExist); + + // target file is not a directory: + File notADirectory = new File(partitioned, "part-r-00000"); + long duNotADirectoryActual = FileUtil.getDU(notADirectory); + long duNotADirectoryExpected = 3 + System.getProperty("line.separator").length(); + assertEquals(duNotADirectoryExpected, duNotADirectoryActual); + + try { + // one of target files is not accessible, but the containing directory + // is accessible: + try { + FileUtil.chmod(notADirectory.getAbsolutePath(), "0000"); + } catch (InterruptedException ie) { + // should never happen since that method never throws InterruptedException. + assertNull(ie); + } + assertFalse(notADirectory.canRead()); + final long du3 = FileUtil.getDU(partitioned); + assertEquals(expected, du3); + + // some target files and containing directory are not accessible: + try { + FileUtil.chmod(partitioned.getAbsolutePath(), "0000"); + } catch (InterruptedException ie) { + // should never happen since that method never throws InterruptedException. + assertNull(ie); + } + assertFalse(partitioned.canRead()); + final long du4 = FileUtil.getDU(partitioned); + assertEquals(0, du4); + } finally { + // Restore the permissions so that we can delete the folder + // in @After method: + FileUtil.chmod(partitioned.getAbsolutePath(), "0777", true/*recursive*/); + } + } + + @Test (timeout = 30000) + public void testUnTar() throws IOException { + setupDirs(); + + // make a simple tar: + final File simpleTar = new File(del, FILE); + OutputStream os = new FileOutputStream(simpleTar); + TarOutputStream tos = new TarOutputStream(os); + try { + TarEntry te = new TarEntry("foo"); + byte[] data = "some-content".getBytes("UTF-8"); + te.setSize(data.length); + tos.putNextEntry(te); + tos.write(data); + tos.closeEntry(); + tos.flush(); + tos.finish(); + } finally { + tos.close(); + } + + // successfully untar it into an existing dir: + FileUtil.unTar(simpleTar, tmp); + // check result: + assertTrue(new File(tmp, "foo").exists()); + assertEquals(12, new File(tmp, "foo").length()); + + final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); + regularFile.createNewFile(); + assertTrue(regularFile.exists()); + try { + FileUtil.unTar(simpleTar, regularFile); + assertTrue("An IOException expected.", false); + } catch (IOException ioe) { + // okay + } + } + + @Test (timeout = 30000) + public void testReplaceFile() throws IOException { + setupDirs(); + final File srcFile = new File(tmp, "src"); + + // src exists, and target does not exist: + srcFile.createNewFile(); + assertTrue(srcFile.exists()); + final File targetFile = new File(tmp, "target"); + assertTrue(!targetFile.exists()); + FileUtil.replaceFile(srcFile, targetFile); + assertTrue(!srcFile.exists()); + assertTrue(targetFile.exists()); + + // src exists and target is a regular file: + srcFile.createNewFile(); + assertTrue(srcFile.exists()); + FileUtil.replaceFile(srcFile, targetFile); + assertTrue(!srcFile.exists()); + assertTrue(targetFile.exists()); + + // src exists, and target is a non-empty directory: + srcFile.createNewFile(); + assertTrue(srcFile.exists()); + targetFile.delete(); + targetFile.mkdirs(); + File obstacle = new File(targetFile, "obstacle"); + obstacle.createNewFile(); + assertTrue(obstacle.exists()); + assertTrue(targetFile.exists() && targetFile.isDirectory()); + try { + FileUtil.replaceFile(srcFile, targetFile); + assertTrue(false); + } catch (IOException ioe) { + // okay + } + // check up the post-condition: nothing is deleted: + assertTrue(srcFile.exists()); + assertTrue(targetFile.exists() && targetFile.isDirectory()); + assertTrue(obstacle.exists()); + } + + @Test (timeout = 30000) + public void testCreateLocalTempFile() throws IOException { + setupDirs(); + final File baseFile = new File(tmp, "base"); + File tmp1 = FileUtil.createLocalTempFile(baseFile, "foo", false); + File tmp2 = FileUtil.createLocalTempFile(baseFile, "foo", true); + assertFalse(tmp1.getAbsolutePath().equals(baseFile.getAbsolutePath())); + assertFalse(tmp2.getAbsolutePath().equals(baseFile.getAbsolutePath())); + assertTrue(tmp1.exists() && tmp2.exists()); + assertTrue(tmp1.canWrite() && tmp2.canWrite()); + assertTrue(tmp1.canRead() && tmp2.canRead()); + tmp1.delete(); + tmp2.delete(); + assertTrue(!tmp1.exists() && !tmp2.exists()); + } + + @Test (timeout = 30000) + public void testUnZip() throws IOException { + // make sa simple zip + setupDirs(); + + // make a simple tar: + final File simpleZip = new File(del, FILE); + OutputStream os = new FileOutputStream(simpleZip); + ZipOutputStream tos = new ZipOutputStream(os); + try { + ZipEntry ze = new ZipEntry("foo"); + byte[] data = "some-content".getBytes("UTF-8"); + ze.setSize(data.length); + tos.putNextEntry(ze); + tos.write(data); + tos.closeEntry(); + tos.flush(); + tos.finish(); + } finally { + tos.close(); + } + + // successfully untar it into an existing dir: + FileUtil.unZip(simpleZip, tmp); + // check result: + assertTrue(new File(tmp, "foo").exists()); + assertEquals(12, new File(tmp, "foo").length()); + + final File regularFile = new File(tmp, "QuickBrownFoxJumpsOverTheLazyDog"); + regularFile.createNewFile(); + assertTrue(regularFile.exists()); + try { + FileUtil.unZip(simpleZip, regularFile); + assertTrue("An IOException expected.", false); + } catch (IOException ioe) { + // okay + } + } + + @Test (timeout = 30000) + /* + * Test method copy(FileSystem srcFS, Path src, File dst, boolean deleteSource, Configuration conf) + */ + public void testCopy5() throws IOException { + setupDirs(); + + URI uri = tmp.toURI(); + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.newInstance(uri, conf); + final String content = "some-content"; + File srcFile = createFile(tmp, "src", content); + Path srcPath = new Path(srcFile.toURI()); + + // copy regular file: + final File dest = new File(del, "dest"); + boolean result = FileUtil.copy(fs, srcPath, dest, false, conf); + assertTrue(result); + assertTrue(dest.exists()); + assertEquals(content.getBytes().length + + System.getProperty("line.separator").getBytes().length, dest.length()); + assertTrue(srcFile.exists()); // should not be deleted + + // copy regular file, delete src: + dest.delete(); + assertTrue(!dest.exists()); + result = FileUtil.copy(fs, srcPath, dest, true, conf); + assertTrue(result); + assertTrue(dest.exists()); + assertEquals(content.getBytes().length + + System.getProperty("line.separator").getBytes().length, dest.length()); + assertTrue(!srcFile.exists()); // should be deleted + + // copy a dir: + dest.delete(); + assertTrue(!dest.exists()); + srcPath = new Path(partitioned.toURI()); + result = FileUtil.copy(fs, srcPath, dest, true, conf); + assertTrue(result); + assertTrue(dest.exists() && dest.isDirectory()); + File[] files = dest.listFiles(); + assertTrue(files != null); + assertEquals(2, files.length); + for (File f: files) { + assertEquals(3 + + System.getProperty("line.separator").getBytes().length, f.length()); + } + assertTrue(!partitioned.exists()); // should be deleted + } + + @Test (timeout = 30000) + public void testStat2Paths1() { + assertNull(FileUtil.stat2Paths(null)); + + FileStatus[] fileStatuses = new FileStatus[0]; + Path[] paths = FileUtil.stat2Paths(fileStatuses); + assertEquals(0, paths.length); + + Path path1 = new Path("file://foo"); + Path path2 = new Path("file://moo"); + fileStatuses = new FileStatus[] { + new FileStatus(3, false, 0, 0, 0, path1), + new FileStatus(3, false, 0, 0, 0, path2) + }; + paths = FileUtil.stat2Paths(fileStatuses); + assertEquals(2, paths.length); + assertEquals(paths[0], path1); + assertEquals(paths[1], path2); + } + + @Test (timeout = 30000) + public void testStat2Paths2() { + Path defaultPath = new Path("file://default"); + Path[] paths = FileUtil.stat2Paths(null, defaultPath); + assertEquals(1, paths.length); + assertEquals(defaultPath, paths[0]); + + paths = FileUtil.stat2Paths(null, null); + assertTrue(paths != null); + assertEquals(1, paths.length); + assertEquals(null, paths[0]); + + Path path1 = new Path("file://foo"); + Path path2 = new Path("file://moo"); + FileStatus[] fileStatuses = new FileStatus[] { + new FileStatus(3, false, 0, 0, 0, path1), + new FileStatus(3, false, 0, 0, 0, path2) + }; + paths = FileUtil.stat2Paths(fileStatuses, defaultPath); + assertEquals(2, paths.length); + assertEquals(paths[0], path1); + assertEquals(paths[1], path2); } @Test (timeout = 30000)