big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9018. Add functionality to AuxiliaryLocalPathHandler to return all locations to read for a given path. Contributed by Kuhu Shukla (kshukla)

Browse Source
This commit is contained in:
Eric E Payne 2020-01-09 17:18:44 +00:00
parent a40dc9ee31
commit 93233a7d6e
4 changed files with 25 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
View File

@ -173,6 +173,14 @@ public Path getLocalPathForWrite(String path, long size)
throws IOException {
return new Path(ABS_LOG_DIR.getAbsolutePath());
}
@Override
public Iterable<Path> getAllLocalPathsForRead(String path)
throws IOException {
ArrayList<Path> paths = new ArrayList<>();
paths.add(new Path(ABS_LOG_DIR.getAbsolutePath()));
return paths;
}
}
private static class MockShuffleHandler2 extends

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryLocalPathHandler.java
View File

@ -55,4 +55,12 @@ public interface AuxiliaryLocalPathHandler {
* @throws IOException if the path creations fails
*/
Path getLocalPathForWrite(String path, long size) throws IOException;
/**
* Get all paths from the local FS for reading for a given Auxiliary Service.
* @param path the requested path
* @return the complete path list to the file on a local disk as an Iterable
* @throws IOException if the file read encounters a problem
*/
Iterable<Path> getAllLocalPathsForRead(String path) throws IOException;
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
View File

@ -644,6 +644,10 @@ public Path getLocalPathForRead(String pathStr) throws IOException {
return getPathToRead(pathStr, getLocalDirsForRead());
}
public Iterable<Path> getAllLocalPathsForRead(String pathStr) throws IOException {
return localDirsAllocator.getAllLocalPathsToRead(pathStr, getConfig());
}
public Path getLogPathForWrite(String pathStr, boolean checkWrite)
throws IOException {
return logDirsAllocator.getLocalPathForWrite(pathStr,

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -1650,6 +1650,11 @@ public Path getLocalPathForWrite(String path, long size)
throws IOException {
return dirhandlerService.getLocalPathForWrite(path, size, false);
}
@Override
public Iterable<Path> getAllLocalPathsForRead(String path) throws IOException {
return dirhandlerService.getAllLocalPathsForRead(path);
}
}
@SuppressWarnings("unchecked")