big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-16075. Use empty array constants present in StorageType and DatanodeInfo to avoid creating redundant objects (#3115)

Browse Source 
Reviewed-by: Hui Fei <ferhui@apache.org>
(cherry picked from commit c488abbc79)
This commit is contained in:
Viraj Jasani 2021-06-21 06:55:12 +05:30 committed by Takanobu Asanuma
parent 7d202e8365
commit b8a98e4f82
19 changed files with 29 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
View File

@ -74,7 +74,7 @@ public class BlockLocation implements Serializable {
private static final String[] EMPTY_STR_ARRAY = new String[0]; private static final String[] EMPTY_STR_ARRAY = new String[0];
private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY = private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY =
new StorageType[0]; StorageType.EMPTY_ARRAY;
/** /**
* Default Constructor. * Default Constructor.

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
View File

@ -27,7 +27,7 @@ public class TestBlockLocation {
private static final String[] EMPTY_STR_ARRAY = new String[0]; private static final String[] EMPTY_STR_ARRAY = new String[0];
private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY = private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY =
new StorageType[0]; StorageType.EMPTY_ARRAY;
private static void checkBlockLocation(final BlockLocation loc) private static void checkBlockLocation(final BlockLocation loc)
throws Exception { throws Exception {

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
View File

@ -1675,7 +1675,7 @@ public void updatePipeline(long newGS) throws IOException {
DatanodeInfo[] getExcludedNodes() { DatanodeInfo[] getExcludedNodes() {
return excludedNodes.getAllPresent(excludedNodes.asMap().keySet()) return excludedNodes.getAllPresent(excludedNodes.asMap().keySet())
.keySet().toArray(new DatanodeInfo[0]); .keySet().toArray(DatanodeInfo.EMPTY_ARRAY);
} }
/** /**

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
View File

@ -131,7 +131,7 @@ public ErasureCodingPolicy getErasureCodingPolicy() {
public int findBlock(long offset) { public int findBlock(long offset) {
// create fake block of size 0 as a key // create fake block of size 0 as a key
LocatedBlock key = new LocatedBlock( LocatedBlock key = new LocatedBlock(
new ExtendedBlock(), new DatanodeInfo[0]); new ExtendedBlock(), DatanodeInfo.EMPTY_ARRAY);
key.setStartOffset(offset); key.setStartOffset(offset);
key.getBlock().setNumBytes(1); key.getBlock().setNumBytes(1);
Comparator<LocatedBlock> comp = Comparator<LocatedBlock> comp =

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/MockNamenode.java
View File

@ -535,7 +535,7 @@ private static HdfsFileStatus getMockHdfsFileStatus(
*/ */
private static LocatedBlock getMockLocatedBlock(final String nsId) { private static LocatedBlock getMockLocatedBlock(final String nsId) {
LocatedBlock lb = mock(LocatedBlock.class); LocatedBlock lb = mock(LocatedBlock.class);
when(lb.getCachedLocations()).thenReturn(new DatanodeInfo[0]); when(lb.getCachedLocations()).thenReturn(DatanodeInfo.EMPTY_ARRAY);
DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0", DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0",
1111, 1112, 1113, 1114); 1111, 1112, 1113, 1114);
DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId); DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId);

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
View File

@ -1210,7 +1210,7 @@ public void testProxyGetAdditionalDatanode()
newRouterFile, clientName, null, null, newRouterFile, clientName, null, null,
status.getFileId(), null, null); status.getFileId(), null, null);
DatanodeInfo[] exclusions = new DatanodeInfo[0]; DatanodeInfo[] exclusions = DatanodeInfo.EMPTY_ARRAY;
LocatedBlock newBlock = routerProtocol.getAdditionalDatanode( LocatedBlock newBlock = routerProtocol.getAdditionalDatanode(
newRouterFile, status.getFileId(), block.getBlock(), newRouterFile, status.getFileId(), block.getBlock(),
block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName); block.getLocations(), block.getStorageIDs(), exclusions, 1, clientName);

2
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java
View File

@ -312,7 +312,7 @@ testPath, new FsPermission("777"), clientName,
assertEquals(1, proxyNumAddBlock2 - proxyNumAddBlock); assertEquals(1, proxyNumAddBlock2 - proxyNumAddBlock);
// Get additionalDatanode via router and block is not null. // Get additionalDatanode via router and block is not null.
DatanodeInfo[] exclusions = new DatanodeInfo[0]; DatanodeInfo[] exclusions = DatanodeInfo.EMPTY_ARRAY;
LocatedBlock newBlock = clientProtocol.getAdditionalDatanode( LocatedBlock newBlock = clientProtocol.getAdditionalDatanode(
testPath, status.getFileId(), blockTwo.getBlock(), testPath, status.getFileId(), blockTwo.getBlock(),
blockTwo.getLocations(), blockTwo.getStorageIDs(), exclusions, blockTwo.getLocations(), blockTwo.getStorageIDs(), exclusions,

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
View File

@ -209,7 +209,7 @@ static LocatedBlock prepareFileForAppend(final FSNamesystem fsn,
BlockInfo lastBlock = file.getLastBlock(); BlockInfo lastBlock = file.getLastBlock();
if (lastBlock != null) { if (lastBlock != null) {
ExtendedBlock blk = new ExtendedBlock(fsn.getBlockPoolId(), lastBlock); ExtendedBlock blk = new ExtendedBlock(fsn.getBlockPoolId(), lastBlock);
ret = new LocatedBlock(blk, new DatanodeInfo[0]); ret = new LocatedBlock(blk, DatanodeInfo.EMPTY_ARRAY);
} }
} }

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
View File

@ -883,7 +883,7 @@ public void testClientDNProtocolTimeout() throws IOException {
DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort()); DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L)); ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); LocatedBlock fakeBlock = new LocatedBlock(b, DatanodeInfo.EMPTY_ARRAY);
ClientDatanodeProtocol proxy = null; ClientDatanodeProtocol proxy = null;

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInputStream.java
View File

@ -242,7 +242,7 @@ public void testReadWithPreferredCachingReplica() throws IOException {
DFSInputStream dfsInputStream = DFSInputStream dfsInputStream =
(DFSInputStream) fs.open(filePath).getWrappedStream(); (DFSInputStream) fs.open(filePath).getWrappedStream();
LocatedBlock lb = mock(LocatedBlock.class); LocatedBlock lb = mock(LocatedBlock.class);
when(lb.getCachedLocations()).thenReturn(new DatanodeInfo[0]); when(lb.getCachedLocations()).thenReturn(DatanodeInfo.EMPTY_ARRAY);
DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0", 1111, DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0", 1111,
1112, 1113, 1114); 1112, 1113, 1114);
DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId); DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId);
@ -271,7 +271,7 @@ public void testReadWithoutPreferredCachingReplica() throws IOException {
DFSInputStream dfsInputStream = DFSInputStream dfsInputStream =
(DFSInputStream) fs.open(filePath).getWrappedStream(); (DFSInputStream) fs.open(filePath).getWrappedStream();
LocatedBlock lb = mock(LocatedBlock.class); LocatedBlock lb = mock(LocatedBlock.class);
when(lb.getCachedLocations()).thenReturn(new DatanodeInfo[0]); when(lb.getCachedLocations()).thenReturn(DatanodeInfo.EMPTY_ARRAY);
DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0", 1111, DatanodeID nodeId = new DatanodeID("localhost", "localhost", "dn0", 1111,
1112, 1113, 1114); 1112, 1113, 1114);
DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId); DatanodeInfo dnInfo = new DatanodeDescriptor(nodeId);

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
View File

@ -461,8 +461,8 @@ public void testAbortedRecovery() throws Exception {
// Add a block to the file // Add a block to the file
ExtendedBlock block = client.getNamenode().addBlock( ExtendedBlock block = client.getNamenode().addBlock(
file, client.clientName, null, new DatanodeInfo[0], stat.getFileId(), file, client.clientName, null, DatanodeInfo.EMPTY_ARRAY,
new String[0], null).getBlock(); stat.getFileId(), new String[0], null).getBlock();
// update the pipeline to get a new genstamp. // update the pipeline to get a new genstamp.
ExtendedBlock updatedBlock = client.getNamenode() ExtendedBlock updatedBlock = client.getNamenode()
@ -578,7 +578,7 @@ private void createCommittedNotCompleteFile(DFSClient client, String file,
// Add a block to the file // Add a block to the file
LocatedBlock blk = client.getNamenode() LocatedBlock blk = client.getNamenode()
.addBlock(file, client.clientName, null, .addBlock(file, client.clientName, null,
new DatanodeInfo[0], stat.getFileId(), new String[0], null); DatanodeInfo.EMPTY_ARRAY, stat.getFileId(), new String[0], null);
ExtendedBlock finalBlock = blk.getBlock(); ExtendedBlock finalBlock = blk.getBlock();
if (bytesToWrite != null) { if (bytesToWrite != null) {
// Here we create a output stream and then abort it so the block gets // Here we create a output stream and then abort it so the block gets

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java
View File

@ -65,7 +65,7 @@ public void testDefaultPolicy() throws Exception {
final DatanodeInfo[] infos = new DatanodeInfo[5]; final DatanodeInfo[] infos = new DatanodeInfo[5];
final DatanodeInfo[][] datanodes = new DatanodeInfo[infos.length + 1][]; final DatanodeInfo[][] datanodes = new DatanodeInfo[infos.length + 1][];
datanodes[0] = new DatanodeInfo[0]; datanodes[0] = DatanodeInfo.EMPTY_ARRAY;
for(int i = 0; i < infos.length; ) { for(int i = 0; i < infos.length; ) {
infos[i] = DFSTestUtil.getLocalDatanodeInfo(9867 + i); infos[i] = DFSTestUtil.getLocalDatanodeInfo(9867 + i);
i++; i++;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLocatedBlock.java
View File

@ -31,7 +31,7 @@ public class TestLocatedBlock {
@Test(timeout = 10000) @Test(timeout = 10000)
public void testAddCachedLocWhenEmpty() { public void testAddCachedLocWhenEmpty() {
DatanodeInfo[] ds = new DatanodeInfo[0]; DatanodeInfo[] ds = DatanodeInfo.EMPTY_ARRAY;
ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
LocatedBlock l1 = new LocatedBlock(b1, ds); LocatedBlock l1 = new LocatedBlock(b1, ds);
DatanodeDescriptor dn = new DatanodeDescriptor( DatanodeDescriptor dn = new DatanodeDescriptor(

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/token/block/TestBlockToken.java
View File

@ -391,7 +391,7 @@ private void testBlockTokenRpcLeak(boolean enableProtobuf) throws Exception {
DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort()); DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L)); ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); LocatedBlock fakeBlock = new LocatedBlock(b, DatanodeInfo.EMPTY_ARRAY);
fakeBlock.setBlockToken(token); fakeBlock.setBlockToken(token);
// Create another RPC proxy with the same configuration - this will never // Create another RPC proxy with the same configuration - this will never

9
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
View File

@ -391,7 +391,8 @@ public void HelperFunction(String scriptFileName, int providedStorages)
storageTypesList.add(StorageType.PROVIDED); storageTypesList.add(StorageType.PROVIDED);
} }
StorageType[] storageTypes= storageTypesList.toArray(new StorageType[0]); StorageType[] storageTypes = storageTypesList.toArray(
StorageType.EMPTY_ARRAY);
for (int i = 0; i < totalDNs; i++) { for (int i = 0; i < totalDNs; i++) {
// register new datanode // register new datanode
@ -693,7 +694,8 @@ public void testGetBlockLocationConsiderStorageType()
List<StorageType> storageTypesList = List<StorageType> storageTypesList =
new ArrayList<>(Arrays.asList(StorageType.ARCHIVE, StorageType.DISK, new ArrayList<>(Arrays.asList(StorageType.ARCHIVE, StorageType.DISK,
StorageType.SSD, StorageType.DEFAULT, StorageType.SSD)); StorageType.SSD, StorageType.DEFAULT, StorageType.SSD));
StorageType[] storageTypes = storageTypesList.toArray(new StorageType[0]); StorageType[] storageTypes = storageTypesList.toArray(
StorageType.EMPTY_ARRAY);
for (int i = 0; i < totalDNs; i++) { for (int i = 0; i < totalDNs; i++) {
// Register new datanode. // Register new datanode.
@ -778,7 +780,8 @@ public void testGetBlockLocationConsiderStorageTypeAndLoad()
List<StorageType> storageTypesList = List<StorageType> storageTypesList =
new ArrayList<>(Arrays.asList(StorageType.DISK, StorageType.DISK, new ArrayList<>(Arrays.asList(StorageType.DISK, StorageType.DISK,
StorageType.DEFAULT, StorageType.SSD, StorageType.SSD)); StorageType.DEFAULT, StorageType.SSD, StorageType.SSD));
StorageType[] storageTypes = storageTypesList.toArray(new StorageType[0]); StorageType[] storageTypes = storageTypesList.toArray(
StorageType.EMPTY_ARRAY);
for (int i = 0; i < totalDNs; i++) { for (int i = 0; i < totalDNs; i++) {
// Register new datanode. // Register new datanode.

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java
View File

@ -127,8 +127,8 @@ private void issueWriteBlockCall(DataXceiver xceiver, boolean lazyPersist)
StorageType.RAM_DISK, StorageType.RAM_DISK,
null, null,
"Dummy-Client", "Dummy-Client",
new DatanodeInfo[0], DatanodeInfo.EMPTY_ARRAY,
new StorageType[0], StorageType.EMPTY_ARRAY,
mock(DatanodeInfo.class), mock(DatanodeInfo.class),
BlockConstructionStage.PIPELINE_SETUP_CREATE, BlockConstructionStage.PIPELINE_SETUP_CREATE,
0, 0, 0, 0, 0, 0, 0, 0,

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
View File

@ -165,7 +165,7 @@ public void testReplicationError() throws Exception {
DataChecksum.Type.CRC32, 512); DataChecksum.Type.CRC32, 512);
new Sender(out).writeBlock(block.getBlock(), StorageType.DEFAULT, new Sender(out).writeBlock(block.getBlock(), StorageType.DEFAULT,
BlockTokenSecretManager.DUMMY_TOKEN, "", BlockTokenSecretManager.DUMMY_TOKEN, "",
new DatanodeInfo[0], new StorageType[0], null, DatanodeInfo.EMPTY_ARRAY, StorageType.EMPTY_ARRAY, null,
BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L, BlockConstructionStage.PIPELINE_SETUP_CREATE, 1, 0L, 0L, 0L,
checksum, CachingStrategy.newDefaultStrategy(), false, false, checksum, CachingStrategy.newDefaultStrategy(), false, false,
null, null, new String[0]); null, null, new String[0]);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestBlockPlacementPolicyRackFaultTolerant.java
View File

@ -119,7 +119,7 @@ private void doTestChooseTargetNormalCase() throws Exception {
LocatedBlock additionalLocatedBlock = LocatedBlock additionalLocatedBlock =
nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(), nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(),
locatedBlock.getBlock(), locatedBlock.getLocations(), locatedBlock.getBlock(), locatedBlock.getLocations(),
locatedBlock.getStorageIDs(), new DatanodeInfo[0], locatedBlock.getStorageIDs(), DatanodeInfo.EMPTY_ARRAY,
additionalReplication, clientMachine); additionalReplication, clientMachine);
doTestLocatedBlock(replication + additionalReplication, additionalLocatedBlock); doTestLocatedBlock(replication + additionalReplication, additionalLocatedBlock);
} }
@ -159,7 +159,7 @@ private void doTestChooseTargetSpecialCase() throws Exception {
LocatedBlock additionalLocatedBlock = LocatedBlock additionalLocatedBlock =
nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(), nameNodeRpc.getAdditionalDatanode(src, fileStatus.getFileId(),
locatedBlock.getBlock(), partLocs, locatedBlock.getBlock(), partLocs,
partStorageIDs, new DatanodeInfo[0], partStorageIDs, DatanodeInfo.EMPTY_ARRAY,
j, clientMachine); j, clientMachine);
doTestLocatedBlock(i + j, additionalLocatedBlock); doTestLocatedBlock(i + j, additionalLocatedBlock);
} }

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
View File

@ -857,7 +857,7 @@ public void testOpenFileWhenNNAndClientCrashAfterAddBlock() throws Exception {
pathString, pathString,
client.getClientName(), client.getClientName(),
new ExtendedBlock(previousBlock), new ExtendedBlock(previousBlock),
new DatanodeInfo[0], DatanodeInfo.EMPTY_ARRAY,
DFSClientAdapter.getFileId((DFSOutputStream) create DFSClientAdapter.getFileId((DFSOutputStream) create
.getWrappedStream()), null, null); .getWrappedStream()), null, null);
cluster.restartNameNode(0, true); cluster.restartNameNode(0, true);