big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-19052.Hadoop use Shell command to get the count of the hard link which takes a lot of time (#6587) Contributed by liangyu.

Browse Source 
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
yu liang 2024-03-24 10:14:27 +08:00 committed by GitHub
parent a60b5e2de3
commit 55dca911cc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 38 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java
View File

@ -23,6 +23,8 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.FileStore;
import java.nio.file.Files;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Shell;
@ -30,6 +32,8 @@
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.classification.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.nio.file.Files.createLink;
@ -50,6 +54,11 @@ public class HardLink {
private static HardLinkCommandGetter getHardLinkCommand;
public final LinkStats linkStats; //not static
static final Logger LOG = LoggerFactory.getLogger(HardLink.class);
private static final String FILE_ATTRIBUTE_VIEW = "unix";
private static final String FILE_ATTRIBUTE = "unix:nlink";
//initialize the command "getters" statically, so can use their
//methods without instantiating the HardLink object
@ -204,6 +213,21 @@ public static void createHardLinkMult(File parentDir, String[] fileBaseNames,
}
}
/**
* Determines whether the system supports hardlinks.
* @param f - file to examine
* @return true if hardlinks are supported, false otherwise
*/
public static boolean supportsHardLink(File f) {
try {
FileStore store = Files.getFileStore(f.toPath());
return store.supportsFileAttributeView(FILE_ATTRIBUTE_VIEW);
} catch (IOException e) {
LOG.warn("Failed to determine if hardlink is supported", e);
return false;
}
}
/**
* Retrieves the number of links to the specified file.
*
@ -220,6 +244,10 @@ public static int getLinkCount(File fileName) throws IOException {
throw new FileNotFoundException(fileName + " not found.");
}
if (supportsHardLink(fileName)) {
return (int) Files.getAttribute(fileName.toPath(), FILE_ATTRIBUTE);
}
// construct and execute shell command
String[] cmd = getHardLinkCommand.linkCount(fileName);
String inpMsg = null;

10
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java
View File

@ -219,6 +219,16 @@ public void testGetLinkCount() throws IOException {
assertEquals(1, getLinkCount(x3));
}
@Test
public void testGetLinkCountFromFileAttribute() throws IOException {
assertTrue(supportsHardLink(x1));
assertEquals(1, getLinkCount(x1));
assertTrue(supportsHardLink(x2));
assertEquals(1, getLinkCount(x2));
assertTrue(supportsHardLink(x3));
assertEquals(1, getLinkCount(x3));
}
/**
* Test the single-file method HardLink.createHardLink().
* Also tests getLinkCount() with values greater than one.