big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-13453. RBF: getMountPointDates should fetch latest subdir time/date when parent dir is not present but /parent/child dirs are present in mount table. Contributed by Dibyendu Karmakar.

Browse Source
This commit is contained in:
Inigo Goiri 2018-04-19 14:56:36 -07:00
parent 7d06806dfd
commit 1134af9ad1
2 changed files with 65 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
View File

@ -2329,18 +2329,11 @@ private boolean isPathReadOnly(final String path) {
private Map<String, Long> getMountPointDates(String path) {
Map<String, Long> ret = new TreeMap<>();
if (subclusterResolver instanceof MountTableResolver) {
MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
String srcPath;
try {
final List<String> children = subclusterResolver.getMountPoints(path);
for (String child : children) {
if (path.equals(Path.SEPARATOR)) {
srcPath = Path.SEPARATOR + child;
} else {
srcPath = path + Path.SEPARATOR + child;
}
MountTable entry = mountTable.getMountPoint(srcPath);
ret.put(child, entry.getDateModified());
Long modTime = getModifiedTime(ret, path, child);
ret.put(child, modTime);
}
} catch (IOException e) {
LOG.error("Cannot get mount point", e);
@ -2349,6 +2342,50 @@ private Map<String, Long> getMountPointDates(String path) {
return ret;
}
/**
* Get modified time for child. If the child is present in mount table it
* will return the modified time. If the child is not present but subdirs of
* this child are present then it will return latest modified subdir's time
* as modified time of the requested child.
* @param ret contains children and modified times.
* @param mountTable.
* @param path Name of the path to start checking dates from.
* @param child child of the requested path.
* @return modified time.
*/
private long getModifiedTime(Map<String, Long> ret, String path,
String child) {
MountTableResolver mountTable = (MountTableResolver)subclusterResolver;
String srcPath;
if (path.equals(Path.SEPARATOR)) {
srcPath = Path.SEPARATOR + child;
} else {
srcPath = path + Path.SEPARATOR + child;
}
Long modTime = 0L;
try {
// Get mount table entry for the srcPath
MountTable entry = mountTable.getMountPoint(srcPath);
// if srcPath is not in mount table but its subdirs are in mount
// table we will display latest modified subdir date/time.
if (entry == null) {
List<MountTable> entries = mountTable.getMounts(srcPath);
for (MountTable eachEntry : entries) {
// Get the latest date
if (ret.get(child) == null ||
ret.get(child) < eachEntry.getDateModified()) {
modTime = eachEntry.getDateModified();
}
}
} else {
modTime = entry.getDateModified();
}
} catch (IOException e) {
LOG.error("Cannot get mount point", e);
}
return modTime;
}
/**
* Create a new file status for a mount point.
*

21
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterMountTable.java
View File

@ -24,6 +24,7 @@
import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@ -167,6 +168,12 @@ public void testListFilesTime() throws Exception {
addEntry = MountTable.newInstance(
"/testdir/subdir", Collections.singletonMap("ns0", "/testdir/subdir"));
assertTrue(addMountTable(addEntry));
addEntry = MountTable.newInstance(
"/testdir3/subdir1", Collections.singletonMap("ns0", "/testdir3"));
assertTrue(addMountTable(addEntry));
addEntry = MountTable.newInstance(
"/testA/testB/testC/testD", Collections.singletonMap("ns0", "/test"));
assertTrue(addMountTable(addEntry));
// Create test dir in NN
final FileSystem nnFs = nnContext.getFileSystem();
@ -174,8 +181,18 @@ public void testListFilesTime() throws Exception {
Map<String, Long> pathModTime = new TreeMap<>();
for (String mount : mountTable.getMountPoints("/")) {
pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
.getDateModified());
if (mountTable.getMountPoint("/"+mount) != null) {
pathModTime.put(mount, mountTable.getMountPoint("/"+mount)
.getDateModified());
} else {
List<MountTable> entries = mountTable.getMounts("/"+mount);
for (MountTable entry : entries) {
if (pathModTime.get(mount) == null ||
pathModTime.get(mount) < entry.getDateModified()) {
pathModTime.put(mount, entry.getDateModified());
}
}
}
}
FileStatus[] iterator = nnFs.listStatus(new Path("/"));
for (FileStatus file : iterator) {