From 6be30a7799fadb75bfe58ebbfba1ecffd0c95462 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 4 Oct 2013 05:59:07 +0000 Subject: [PATCH] YARN-1219. FSDownload changes file suffix making FileUtil.unTar() throw exception. Contributed by Shanyu Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1529084 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../apache/hadoop/yarn/util/FSDownload.java | 2 +- .../hadoop/yarn/util/TestFSDownload.java | 191 +++++++----------- 3 files changed, 78 insertions(+), 118 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1ac823a55c..9b8007b9fb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -156,6 +156,9 @@ Release 2.1.2 - UNRELEASED YARN-1131. $yarn logs command should return an appropriate error message if YARN application is still running. (Siddharth Seth via hitesh) + YARN-1219. FSDownload changes file suffix making FileUtil.unTar() throw + exception. (Shanyu Zhao via cnauroth) + Release 2.1.1-beta - 2013-09-23 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 028276b7fe..36dfc6bc8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -170,7 +170,7 @@ private static boolean checkPermissionOfOther(FileSystem fs, Path path, private Path copy(Path sCopy, Path dstdir) throws IOException { FileSystem sourceFs = sCopy.getFileSystem(conf); - Path dCopy = new Path(dstdir, sCopy.getName() + ".tmp"); + Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName()); FileStatus sStat = sourceFs.getFileStatus(sCopy); if (sStat.getModificationTime() != resource.getTimestamp()) { throw new IOException("Resource " + sCopy + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java index f3f9f7d0c3..86909e7322 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java @@ -41,6 +41,7 @@ import java.util.jar.Manifest; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import java.util.zip.GZIPOutputStream; import junit.framework.Assert; @@ -72,6 +73,9 @@ public class TestFSDownload { private static final Log LOG = LogFactory.getLog(TestFSDownload.class); private static AtomicLong uniqueNumberGenerator = new AtomicLong(System.currentTimeMillis()); + private enum TEST_FILE_TYPE { + TAR, JAR, ZIP, TGZ + }; @AfterClass public static void deleteTestDir() throws IOException { @@ -121,7 +125,7 @@ static LocalResource createJar(FileContext files, Path p, ret.setPattern("classes/.*"); return ret; } - + static LocalResource createTarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { @@ -149,7 +153,35 @@ static LocalResource createTarFile(FileContext files, Path p, int len, .getModificationTime()); return ret; } - + + static LocalResource createTgzFile(FileContext files, Path p, int len, + Random r, LocalResourceVisibility vis) throws IOException, + URISyntaxException { + byte[] bytes = new byte[len]; + r.nextBytes(bytes); + + File gzipFile = new File(p.toUri().getPath() + ".tar.gz"); + gzipFile.createNewFile(); + TarArchiveOutputStream out = new TarArchiveOutputStream( + new GZIPOutputStream(new FileOutputStream(gzipFile))); + TarArchiveEntry entry = new TarArchiveEntry(p.getName()); + entry.setSize(bytes.length); + out.putArchiveEntry(entry); + out.write(bytes); + out.closeArchiveEntry(); + out.close(); + + LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); + ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + + ".tar.gz"))); + ret.setSize(len); + ret.setType(LocalResourceType.ARCHIVE); + ret.setVisibility(vis); + ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz")) + .getModificationTime()); + return ret; + } + static LocalResource createJarFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { @@ -175,7 +207,7 @@ static LocalResource createJarFile(FileContext files, Path p, int len, .getModificationTime()); return ret; } - + static LocalResource createZipFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) throws IOException, URISyntaxException { @@ -201,7 +233,7 @@ static LocalResource createZipFile(FileContext files, Path p, int len, .getModificationTime()); return ret; } - + @Test (timeout=10000) public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException { @@ -252,7 +284,7 @@ public void testDownloadBadPublic() throws IOException, URISyntaxException, Assert.assertTrue(e.getCause() instanceof IOException); } } - + @Test (timeout=10000) public void testDownload() throws IOException, URISyntaxException, InterruptedException { @@ -326,10 +358,9 @@ public void testDownload() throws IOException, URISyntaxException, throw new IOException("Failed exec", e); } } - - @Test (timeout=10000) - public void testDownloadArchive() throws IOException, URISyntaxException, - InterruptedException { + + private void downloadWithFileType(TEST_FILE_TYPE fileType) throws IOException, + URISyntaxException, InterruptedException{ Configuration conf = new Configuration(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); FileContext files = FileContext.getLocalFSFileContext(conf); @@ -352,7 +383,22 @@ public void testDownloadArchive() throws IOException, URISyntaxException, LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; Path p = new Path(basedir, "" + 1); - LocalResource rsrc = createTarFile(files, p, size, rand, vis); + LocalResource rsrc = null; + switch (fileType) { + case TAR: + rsrc = createTarFile(files, p, size, rand, vis); + break; + case JAR: + rsrc = createJarFile(files, p, size, rand, vis); + rsrc.setType(LocalResourceType.PATTERN); + break; + case ZIP: + rsrc = createZipFile(files, p, size, rand, vis); + break; + case TGZ: + rsrc = createTgzFile(files, p, size, rand, vis); + break; + } Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf); destPath = new Path (destPath, Long.toString(uniqueNumberGenerator.incrementAndGet())); @@ -371,7 +417,7 @@ public void testDownloadArchive() throws IOException, URISyntaxException, FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( filestatus.getPath()); for (FileStatus childfile : childFiles) { - if (childfile.getPath().getName().equalsIgnoreCase("1.tar.tmp")) { + if (childfile.getPath().getName().startsWith("tmp")) { Assert.fail("Tmp File should not have been there " + childfile.getPath()); } @@ -384,118 +430,29 @@ public void testDownloadArchive() throws IOException, URISyntaxException, } @Test (timeout=10000) + public void testDownloadArchive() throws IOException, URISyntaxException, + InterruptedException { + downloadWithFileType(TEST_FILE_TYPE.TAR); + } + + @Test (timeout=10000) public void testDownloadPatternJar() throws IOException, URISyntaxException, InterruptedException { - Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); - FileContext files = FileContext.getLocalFSFileContext(conf); - final Path basedir = files.makeQualified(new Path("target", - TestFSDownload.class.getSimpleName())); - files.mkdir(basedir, null, true); - conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); - - Random rand = new Random(); - long sharedSeed = rand.nextLong(); - rand.setSeed(sharedSeed); - System.out.println("SEED: " + sharedSeed); - - Map> pending = new HashMap>(); - ExecutorService exec = Executors.newSingleThreadExecutor(); - LocalDirAllocator dirs = new LocalDirAllocator( - TestFSDownload.class.getName()); - - int size = rand.nextInt(512) + 512; - LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; - - Path p = new Path(basedir, "" + 1); - LocalResource rsrcjar = createJarFile(files, p, size, rand, vis); - rsrcjar.setType(LocalResourceType.PATTERN); - Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf); - destPathjar = new Path (destPathjar, - Long.toString(uniqueNumberGenerator.incrementAndGet())); - FSDownload fsdjar = new FSDownload(files, - UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrcjar); - pending.put(rsrcjar, exec.submit(fsdjar)); - exec.shutdown(); - while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)); - Assert.assertTrue(pending.get(rsrcjar).isDone()); - - try { - FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus( - basedir); - for (FileStatus filestatus : filesstatus) { - if (filestatus.isDirectory()) { - FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( - filestatus.getPath()); - for (FileStatus childfile : childFiles) { - if (childfile.getPath().getName().equalsIgnoreCase("1.jar.tmp")) { - Assert.fail("Tmp File should not have been there " - + childfile.getPath()); - } - } - } - } - }catch (Exception e) { - throw new IOException("Failed exec", e); - } + downloadWithFileType(TEST_FILE_TYPE.JAR); } - + @Test (timeout=10000) public void testDownloadArchiveZip() throws IOException, URISyntaxException, InterruptedException { - Configuration conf = new Configuration(); - conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); - FileContext files = FileContext.getLocalFSFileContext(conf); - final Path basedir = files.makeQualified(new Path("target", - TestFSDownload.class.getSimpleName())); - files.mkdir(basedir, null, true); - conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); - - Random rand = new Random(); - long sharedSeed = rand.nextLong(); - rand.setSeed(sharedSeed); - System.out.println("SEED: " + sharedSeed); - - Map> pending = new HashMap>(); - ExecutorService exec = Executors.newSingleThreadExecutor(); - LocalDirAllocator dirs = new LocalDirAllocator( - TestFSDownload.class.getName()); - - int size = rand.nextInt(512) + 512; - LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; - - Path p = new Path(basedir, "" + 1); - LocalResource rsrczip = createZipFile(files, p, size, rand, vis); - Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf); - destPathjar = new Path (destPathjar, - Long.toString(uniqueNumberGenerator.incrementAndGet())); - FSDownload fsdzip = new FSDownload(files, - UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrczip); - pending.put(rsrczip, exec.submit(fsdzip)); - exec.shutdown(); - while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS)); - Assert.assertTrue(pending.get(rsrczip).isDone()); - - try { - FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus( - basedir); - for (FileStatus filestatus : filesstatus) { - if (filestatus.isDirectory()) { - FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( - filestatus.getPath()); - for (FileStatus childfile : childFiles) { - if (childfile.getPath().getName().equalsIgnoreCase("1.gz.tmp")) { - Assert.fail("Tmp File should not have been there " - + childfile.getPath()); - } - } - } - } - }catch (Exception e) { - throw new IOException("Failed exec", e); - } + downloadWithFileType(TEST_FILE_TYPE.ZIP); } - + + @Test (timeout=10000) + public void testDownloadArchiveTgz() throws IOException, URISyntaxException, + InterruptedException { + downloadWithFileType(TEST_FILE_TYPE.TGZ); + } + private void verifyPermsRecursively(FileSystem fs, FileContext files, Path p, LocalResourceVisibility vis) throws IOException { @@ -527,7 +484,7 @@ private void verifyPermsRecursively(FileSystem fs, } } } - + @Test (timeout=10000) public void testDirDownload() throws IOException, InterruptedException { Configuration conf = new Configuration();