From 460d77bd646d03d3eb7670f9017aeeb5410c4a95 Mon Sep 17 00:00:00 2001 From: Jason Lowe Date: Fri, 2 Feb 2018 11:31:39 -0600 Subject: [PATCH] HADOOP-15170. Add symlink support to FileUtil#unTarUsingJava. Contributed by Ajay Kumar --- .../java/org/apache/hadoop/fs/FileUtil.java | 12 ++- .../org/apache/hadoop/fs/TestFileUtil.java | 86 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index bf9b146378..8743be529d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -34,6 +34,8 @@ import java.net.UnknownHostException; import java.nio.charset.Charset; import java.nio.file.AccessDeniedException; +import java.nio.file.FileSystems; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; @@ -894,7 +896,7 @@ private static void unTarUsingTar(File inFile, File untarDir, } } - private static void unTarUsingJava(File inFile, File untarDir, + static void unTarUsingJava(File inFile, File untarDir, boolean gzipped) throws IOException { InputStream inputStream = null; TarArchiveInputStream tis = null; @@ -956,6 +958,14 @@ private static void unpackEntries(TarArchiveInputStream tis, return; } + if (entry.isSymbolicLink()) { + // Create symbolic link relative to tar parent dir + Files.createSymbolicLink(FileSystems.getDefault() + .getPath(outputDir.getPath(), entry.getName()), + FileSystems.getDefault().getPath(entry.getLinkName())); + return; + } + File outputFile = new File(outputDir, entry.getName()); if (!outputFile.getParentFile().exists()) { if (!outputFile.getParentFile().mkdirs()) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 0ad03fcbac..39f2f6bc1e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -37,6 +38,8 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.UnknownHostException; +import java.nio.file.FileSystems; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -47,6 +50,9 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; @@ -1173,4 +1179,84 @@ public void testCompareFsDirectories() throws Exception { assertEquals(FileUtil.compareFs(fs3,fs4),true); assertEquals(FileUtil.compareFs(fs1,fs6),false); } + + @Test(timeout = 8000) + public void testCreateSymbolicLinkUsingJava() throws IOException { + setupDirs(); + final File simpleTar = new File(del, FILE); + OutputStream os = new FileOutputStream(simpleTar); + TarArchiveOutputStream tos = new TarArchiveOutputStream(os); + File untarFile = null; + try { + // Files to tar + final String tmpDir = "tmp/test"; + File tmpDir1 = new File(tmpDir, "dir1/"); + File tmpDir2 = new File(tmpDir, "dir2/"); + // Delete the directories if they already exist + tmpDir1.mkdirs(); + tmpDir2.mkdirs(); + + java.nio.file.Path symLink = FileSystems + .getDefault().getPath(tmpDir1.getPath() + "/sl"); + + // Create Symbolic Link + Files.createSymbolicLink(symLink, + FileSystems.getDefault().getPath(tmpDir2.getPath())).toString(); + assertTrue(Files.isSymbolicLink(symLink.toAbsolutePath())); + // put entries in tar file + putEntriesInTar(tos, tmpDir1.getParentFile()); + tos.close(); + + untarFile = new File(tmpDir, "2"); + // Untar using java + FileUtil.unTarUsingJava(simpleTar, untarFile, false); + + // Check symbolic link and other directories are there in untar file + assertTrue(Files.exists(untarFile.toPath())); + assertTrue(Files.exists(FileSystems.getDefault().getPath(untarFile + .getPath(), tmpDir))); + assertTrue(Files.isSymbolicLink(FileSystems.getDefault().getPath(untarFile + .getPath().toString(), symLink.toString()))); + + } finally { + FileUtils.deleteDirectory(new File("tmp")); + tos.close(); + } + + } + + private void putEntriesInTar(TarArchiveOutputStream tos, File f) + throws IOException { + if (Files.isSymbolicLink(f.toPath())) { + TarArchiveEntry tarEntry = new TarArchiveEntry(f.getPath(), + TarArchiveEntry.LF_SYMLINK); + tarEntry.setLinkName(Files.readSymbolicLink(f.toPath()).toString()); + tos.putArchiveEntry(tarEntry); + tos.closeArchiveEntry(); + return; + } + + if (f.isDirectory()) { + tos.putArchiveEntry(new TarArchiveEntry(f)); + tos.closeArchiveEntry(); + for (File child : f.listFiles()) { + putEntriesInTar(tos, child); + } + } + + if (f.isFile()) { + tos.putArchiveEntry(new TarArchiveEntry(f)); + BufferedInputStream origin = new BufferedInputStream( + new FileInputStream(f)); + int count; + byte[] data = new byte[2048]; + while ((count = origin.read(data)) != -1) { + tos.write(data, 0, count); + } + tos.flush(); + tos.closeArchiveEntry(); + origin.close(); + } + } + }