From 71a57ded39a605166d616fe68f36017cdb0abe3e Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 31 Jan 2013 20:13:01 +0000 Subject: [PATCH] HADOOP-9264. Port change to use Java untar API on Windows from branch-1-win to trunk. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1441172 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-common/pom.xml | 23 ++++++ .../java/org/apache/hadoop/fs/FileUtil.java | 76 +++++++++++++++++- .../org/apache/hadoop/fs/TestFileUtil.java | 40 +++++++++ .../java/org/apache/hadoop/fs/test-untar.tar | Bin 0 -> 20480 bytes .../java/org/apache/hadoop/fs/test-untar.tgz | Bin 0 -> 2024 bytes 6 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 855a77fbd6..43a8e48334 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -325,6 +325,9 @@ Trunk (Unreleased) HADOOP-9249. hadoop-maven-plugins version-info goal causes build failure when running with Clover. (Chris Nauroth via suresh) + HADOOP-9264. Port change to use Java untar API on Windows from + branch-1-win to trunk. (Chris Nauroth via suresh) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index e154d4a820..545c0cb5da 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -241,6 +241,11 @@ test-jar test + + org.apache.commons + commons-compress + 1.4 + @@ -381,6 +386,23 @@ + + copy-test-tarballs + process-test-resources + + run + + + + + + + + + + + + pre-site @@ -485,6 +507,7 @@ src/test/all-tests src/test/resources/kdc/ldif/users.ldif src/main/native/src/org/apache/hadoop/io/compress/lz4/lz4.c + src/test/java/org/apache/hadoop/fs/test-untar.tgz diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 4593eedb9f..19c19cd2b6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -21,9 +21,12 @@ import java.io.*; import java.util.Arrays; import java.util.Enumeration; +import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -624,14 +627,28 @@ public static void unZip(File inFile, File unzipDir) throws IOException { * @throws IOException */ public static void unTar(File inFile, File untarDir) throws IOException { - if (!untarDir.mkdirs()) { + if (!untarDir.mkdirs()) { if (!untarDir.isDirectory()) { throw new IOException("Mkdirs failed to create " + untarDir); } } - StringBuilder untarCommand = new StringBuilder(); boolean gzipped = inFile.toString().endsWith("gz"); + if(Shell.WINDOWS) { + // Tar is not native to Windows. Use simple Java based implementation for + // tests and simple tar archives + unTarUsingJava(inFile, untarDir, gzipped); + } + else { + // spawn tar utility to untar archive for full fledged unix behavior such + // as resolving symlinks in tar archives + unTarUsingTar(inFile, untarDir, gzipped); + } + } + + private static void unTarUsingTar(File inFile, File untarDir, + boolean gzipped) throws IOException { + StringBuffer untarCommand = new StringBuffer(); if (gzipped) { untarCommand.append(" gzip -dc '"); untarCommand.append(FileUtil.makeShellPath(inFile)); @@ -656,7 +673,62 @@ public static void unTar(File inFile, File untarDir) throws IOException { ". Tar process exited with exit code " + exitcode); } } + + private static void unTarUsingJava(File inFile, File untarDir, + boolean gzipped) throws IOException { + InputStream inputStream = null; + if (gzipped) { + inputStream = new BufferedInputStream(new GZIPInputStream( + new FileInputStream(inFile))); + } else { + inputStream = new BufferedInputStream(new FileInputStream(inFile)); + } + TarArchiveInputStream tis = new TarArchiveInputStream(inputStream); + + for (TarArchiveEntry entry = tis.getNextTarEntry(); entry != null;) { + unpackEntries(tis, entry, untarDir); + entry = tis.getNextTarEntry(); + } + } + + private static void unpackEntries(TarArchiveInputStream tis, + TarArchiveEntry entry, File outputDir) throws IOException { + if (entry.isDirectory()) { + File subDir = new File(outputDir, entry.getName()); + if (!subDir.mkdir() && !subDir.isDirectory()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + + for (TarArchiveEntry e : entry.getDirectoryEntries()) { + unpackEntries(tis, e, subDir); + } + + return; + } + + File outputFile = new File(outputDir, entry.getName()); + if (!outputDir.exists()) { + if (!outputDir.mkdirs()) { + throw new IOException("Mkdirs failed to create tar internal dir " + + outputDir); + } + } + + int count; + byte data[] = new byte[2048]; + BufferedOutputStream outputStream = new BufferedOutputStream( + new FileOutputStream(outputFile)); + + while ((count = tis.read(data)) != -1) { + outputStream.write(data, 0, count); + } + + outputStream.flush(); + outputStream.close(); + } + /** * Class for creating hardlinks. * Supports Unix, Cygwin, WindXP. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index a64b45d80f..e73c644fb0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -546,4 +546,44 @@ public void testGetDU() throws IOException { long expected = 2 * (3 + System.getProperty("line.separator").length()); Assert.assertEquals(expected, du); } + + private void doUntarAndVerify(File tarFile, File untarDir) + throws IOException { + if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { + throw new IOException("Could not delete directory '" + untarDir + "'"); + } + FileUtil.unTar(tarFile, untarDir); + + String parentDir = untarDir.getCanonicalPath() + Path.SEPARATOR + "name"; + File testFile = new File(parentDir + Path.SEPARATOR + "version"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 0); + String imageDir = parentDir + Path.SEPARATOR + "image"; + testFile = new File(imageDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 157); + String currentDir = parentDir + Path.SEPARATOR + "current"; + testFile = new File(currentDir + Path.SEPARATOR + "fsimage"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 4331); + testFile = new File(currentDir + Path.SEPARATOR + "edits"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 1033); + testFile = new File(currentDir + Path.SEPARATOR + "fstime"); + Assert.assertTrue(testFile.exists()); + Assert.assertTrue(testFile.length() == 8); + } + + @Test + public void testUntar() throws IOException { + String tarGzFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tgz"; + String tarFileName = System.getProperty("test.cache.data", + "build/test/cache") + "/test-untar.tar"; + String dataDir = System.getProperty("test.build.data", "build/test/data"); + File untarDir = new File(dataDir, "untarDir"); + + doUntarAndVerify(new File(tarGzFileName), untarDir); + doUntarAndVerify(new File(tarFileName), untarDir); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/test-untar.tar new file mode 100644 index 0000000000000000000000000000000000000000..949e985c731aa2e4da7d1871fd1e7c114d656021 GIT binary patch literal 20480 zcmeHNe^gXe9)AP)BO+H=@&~N()U%4gzY35g)Yh}v;$W*C?!!@ObU4TwoC zT9PhmX-4L;W10stHhyU2kG9paG%~3Z*lxHUPUVkX^PCFXM6=m9@6MZf-uHNb2kPv0 z=a0E>KJVVo_ukL<-tWEdJI`S%u_j}~`ytqmUQb}YlKpU{UZ>@Ma~P>2R631DrO~P| zf+PtIf#KTW?G|A7xEYfR$1xX8GjjII%=Aikp+y|Iaa@Zt=ghfMVQ3Z?zx)tJw99`PV zV*6^fFtn?JOx6+vW1~H8lcPY=xIt$CJZgC!-(Z#H6k}6bXqU_5WRw<{<<`*~2tw^2 zNk5$mk~wHuo~iPkT4i<7q^J|T-=tjNYhkxK`fM{XXpj|NORDa*$xG%r2QLUnJm{KN$;}r zxi8uK@GaXy92pVvD^TE;{lCynm6(dGLp^Ny{huV%T6JLmSL@a6{?A5k9oqkg+7D8E zzWx8wjrafIzc&Gtp_f3Zm{>dQ@nvv3wvE?^J>k3Xe@2+8Vv{@E?QvRNMK0RoR9qR@ zi6Rtc_y3;5mSZP%YuvdTyEY3j?();JXByg{Qho;G`*j~byUu&04#r0-mwj<~F+CT? zcajrp=6611gz@IkGqpJ{HY#BJdSTd!{a3VMFdjP&zdUXI?maNRFP(|#nsNLbjPH+6 zoH_dYm;MRkkClA9I=6nsAsBbWpPBA8{m1VeY>xjv_QdPmN5{uKPQvxes@r<%@0-Ox z&;{zBem85)v9EI?;rb14w2gWD6}tz<#a?a2qJsMof7J6egJxxwG3Kp~{!ucatukEGJGPSJ6cHFL-NZ}-leUb-GU>;sIm{=(_Tsz0^eoX!0Z z;9ElMadgJFAZ`lqFGpq0zfc?o){g)uybakD^`TToE|VNm{{xT#4h3o!S?k@#M=Ky+A^y=inFZ)WQxJ@0^kzv-z)!=E1nm^-aRb zM*AKZZ;MQ;%-^;fbYH;#R^5g*vDXY>@(J)$yY{wpUHaW_nCJ9gYT6S=;cH=h%$_wV%Rxoe{IRjU%&R{Ol*kpZ|PJt^>vkGn(S=EZ_1k7;m2Z=Dqht z9M}%y*toUZYqE`7U_9x)d+h&SJvs!&UAD5XdUu#F!+69!sj=0y@ntYhJaZ<&=+Li* z@eFg;>n$n9128_XNBQ}8s?=35{#@_Ys?z9-nJ`|J*LyzhX3bO>-#oH&an*wl#KCxd zN798G%gQIgxb1wz0rQy;eh=ektvNf-_PzrGw!rUOGpbiar^c*+>!159@&(I`Sqov@ zmQlc1HrSTH_(L%jA&qZd`~!?{TN>T8ru(IRFn%EKyUvYY6^Z6x7RC4f1(ASdNt15X8zT`I(VJYVKVpAmettC#T!k7O5IR&0lTnfqg zYbhgAahZR33l-C17$28+#r}5tw6z!K!*#?)TCsjo5Td+@q+x%k9+7}uE$w*rEpSE= zxbDagvznroM1coP0sh>G!#DQUH5PFl72rjYA0B)Dy%SHt_?7auE<9|>pTWSga8OE5 zf!j*dNREz|^{QfyqNw#jFu}2S-nirh>_=A7lfSIts+)~H4;1Y6`%~WVfKoxS*3Rkw zo|j(SF)lDZ|Ao}-5vOPZ89!MY-&|XI>ecA|G)EmgI7mv_cj8Fi#0j@&dea*VK<|S# z1~!a-jwXO^Yk%y>g7F)dABO9j3!a`y#TR@J zv?pP_rMF?--K8g&b2t_a3iexSY=VQAKUEh9QQR{F7YMfoDEt?EToRUI zb}xnp1EHUN$itx6{{sqwvD;S`<=wwd6yn9BXvJi+bOQ^4e2}>L#~mwI{)flc{1Av@&3g{NJKhoYA~k`E51{lvuqWHRN4IQ!Am zvdYQe(9CPgmQr37+(wr&3PAZkhAOej)V{r2^7%jZE=fT7Keb+^NBO_oQ+u$j6o&3? zdxzfXgY5>Y|BEQVt$!}-pFC{#`(6L+zfb)8KWf64|5uTu7QO#5Yz`3+3;aJ!|Nda$ zeJ`2!zf@{1?|%~f|D)FEG${TH+%#;L>>0ls!@~1ylpFV*{BX9+$jY0Sm6ws7H8(SJ zDV~;{n`>N@H}8JjP2-Hs#NcICmz$y;?r^#gPd63Nv=eu_DB6WHG)^RuYCOK&Mp#%nV2bAy$-* z)$TIOjx0ON>@0{$g_fkDmS$ugJEnOcqd+wCqYo`hBa=FUePA9=B`4)-rBbd` z$uW|mNF|Bksy@Ciw;Xm}&*3=6u`Dn3f5irNp54?Rchs-4y7F&ya2##r`)VHaJpXc) zT#oYJAG#|4bRNUodp|}u`IigN`{n#oO1WCdzd|KU1Dx#rTk+QaJpWFov+V}q8i)@G45~Bh!M3TH!H?@G=q2v({DleFYKE@)ag= zUM7C$SeVF)OOuk4^5R#9m5E9}Zm7Hbc6u2lR&-P<)vnX=Ms3zbCr##8v97D zh>g>Aebm{d0S2Z(Zx6ORY&4h8u@0N;jDI_dKp!`Dr|>g$WSi2SwXAi;70j)}#ZT8Z zJwbd9`BwF(&n$NCu7-Rtk^9ZAne1f9*HI%XlUwf7LB1vMNLA(wbu!4$n}YUlJ);VO zJZczzdhC)78zJAE!iThu-*?RGK4ao04F2`SuOWZ9@Y4lZHFI}DZjCuI&Z++`0`f;+ zZ`(6G`Vk89;)=%hn!6?@Kz```^o4tW%nXIR_Kn6N@4RerK<=H@?O2%imcjL}?y2JS z1UW_I?bL}4g)3F3%ca-qQgTZwKI>%7dxYQDefUntgmFbnOjj_so_K-3%MmDAvYfn*=9KM(QS|)r8CzZ?Rf7PCf>e=D-3)^1Y4EeU~Us{%Z zm+zIAeYNMm9avF*9P*q$rhRh0K;~5v0$OLhGJJV74|(!w!=A>PG3AiYSf~4=AwHl5 z@`jpEDt9g~t%rQ&j$PyB2eyYnUPSM(#ZF#u4)SU5m67cao*NDM&l65m#RhKP33=kt z>UQVo>Lw4*>EvS7W~AgBVz&d=!cv9-`Ma=;7V%FP+W1nUGYWLw^B_`h%q8pc`ly@M zj9qjx8NMzyQawEvkt)>#VzIMD&Ck3Q0aLg8_lfmkv%*NopB=F4!lvrFe8}@dKi>P? z2m7CZ{7gw>D;_lKDNidMm+IIx9n?q;>si^Ivi$A$dL}p)^LK{T>z%)3;PQ)g?f2wu zO7KcLV3&&aSB0GUrFwjg8AgD@`HMD#)82-aT#jvN^jTH{?Azfr-ic z74nxu<6ix9$CSq*Z|JC9d~?x(Ii6OyWZ$AB&W_vz(BRoy% zI?=O(N$KS#ZfAJzgE>}z!uoNNDc!|M48g%~GKay$NR&VP4q$B+&dZtO-xi=dr8}w{Z;1;XI3z@st9ODKRrfGp^?ZcUB`?Xw&nI z!9oYSc9nt)Aw=iK54Y$`SqDEJFJUYe+@xnLE-$W}@bSU_^x*stBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxBqSsxDg6aIL-i*B GPyhg8PZ7ib literal 0 HcmV?d00001