big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14132. Add BlockLocation.isStriped() to determine if block is replicated or Striped

Browse Source 
(Contributed by Shweta Yakkali via Daniel Templeton)

Change-Id: I0ed8996a0bae2ad2c7d3513143195533f7191af8
This commit is contained in:
shwetayakkali@cloudera.com 2019-01-08 17:04:10 -08:00 committed by Daniel Templeton
parent 6a923464af
commit 4ab5260b7e
3 changed files with 35 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
View File

@ -239,6 +239,13 @@ public boolean isCorrupt() {
return corrupt; return corrupt;
} }
/**
* Return true if the block is striped (erasure coded).
*/
public boolean isStriped() {
return false;
}
/** /**
* Set the start offset of file associated with this block * Set the start offset of file associated with this block
*/ */

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
View File

@ -54,4 +54,8 @@ private void readObject(ObjectInputStream ois)
block = null; block = null;
} }
@Override
public boolean isStriped() {
return block.isStriped();
}
} }

24
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
View File

@ -17,6 +17,7 @@
*/ */
package org.apache.hadoop.hdfs.server.namenode; package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSStripedOutputStream; import org.apache.hadoop.hdfs.DFSStripedOutputStream;
@ -65,6 +66,8 @@
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestAddStripedBlocks { public class TestAddStripedBlocks {
private final ErasureCodingPolicy ecPolicy = private final ErasureCodingPolicy ecPolicy =
@ -476,4 +479,25 @@ public void testCheckStripedReplicaCorrupt() throws Exception {
Assert.assertEquals(3, bm.getCorruptReplicas(stored).size()); Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
} }
@Test
public void testStripedFlagInBlockLocation() throws IOException {
Path replicated = new Path("/blockLocation/replicated");
try (FSDataOutputStream out =
dfs.createFile(replicated).replicate().recursive().build()) {
out.write("this is a replicated file".getBytes());
}
BlockLocation[] locations = dfs.getFileBlockLocations(replicated, 0, 100);
assertEquals("There should be exactly one Block present",
1, locations.length);
assertFalse("The file is Striped", locations[0].isStriped());
Path striped = new Path("/blockLocation/striped");
try (FSDataOutputStream out = dfs.createFile(striped).recursive().build()) {
out.write("this is a striped file".getBytes());
}
locations = dfs.getFileBlockLocations(striped, 0, 100);
assertEquals("There should be exactly one Block present",
1, locations.length);
assertTrue("The file is not Striped", locations[0].isStriped());
}
} }