big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14224. RBF: NPE in getContentSummary() for getEcPolicy() in case of multiple destinations. Contributed by Ayush Saxena.

Browse Source
This commit is contained in:
Brahma Reddy Battula 2019-01-28 09:03:32 +05:30
parent 8b9b58b58a
commit acdf911c01
2 changed files with 23 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientProtocol.java
View File

@ -1635,6 +1635,7 @@ private ContentSummary aggregateContentSummary(
long quota = 0; long quota = 0;
long spaceConsumed = 0; long spaceConsumed = 0;
long spaceQuota = 0; long spaceQuota = 0;
String ecPolicy = "";
for (ContentSummary summary : summaries) { for (ContentSummary summary : summaries) {
length += summary.getLength(); length += summary.getLength();
@ -1643,6 +1644,11 @@ private ContentSummary aggregateContentSummary(
quota += summary.getQuota(); quota += summary.getQuota();
spaceConsumed += summary.getSpaceConsumed(); spaceConsumed += summary.getSpaceConsumed();
spaceQuota += summary.getSpaceQuota(); spaceQuota += summary.getSpaceQuota();
// We return from the first response as we assume that the EC policy
// of each sub-cluster is same.
if (ecPolicy.isEmpty()) {
ecPolicy = summary.getErasureCodingPolicy();
}
} }
ContentSummary ret = new ContentSummary.Builder() ContentSummary ret = new ContentSummary.Builder()
@ -1652,6 +1658,7 @@ private ContentSummary aggregateContentSummary(
.quota(quota) .quota(quota)
.spaceConsumed(spaceConsumed) .spaceConsumed(spaceConsumed)
.spaceQuota(spaceQuota) .spaceQuota(spaceQuota)
.erasureCodingPolicy(ecPolicy)
.build(); .build();
return ret; return ret;
} }

16
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
View File

@ -41,6 +41,7 @@
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@ -229,6 +230,21 @@ public void testProxyRenameFiles() throws IOException, InterruptedException {
testRename2(getRouterContext(), filename1, renamedFile, false); testRename2(getRouterContext(), filename1, renamedFile, false);
} }
@Test
public void testGetContentSummaryEc() throws Exception {
DistributedFileSystem routerDFS =
(DistributedFileSystem) getRouterFileSystem();
Path dir = new Path("/");
String expectedECPolicy = "RS-6-3-1024k";
try {
routerDFS.setErasureCodingPolicy(dir, expectedECPolicy);
assertEquals(expectedECPolicy,
routerDFS.getContentSummary(dir).getErasureCodingPolicy());
} finally {
routerDFS.unsetErasureCodingPolicy(dir);
}
}
@Test @Test
public void testSubclusterDown() throws Exception { public void testSubclusterDown() throws Exception {
final int totalFiles = 6; final int totalFiles = 6;