big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-3924. Multi-byte id in HdfsVolumeId. Contributed by Andrew Wang.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1384602 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Aaron Myers 2012-09-13 23:48:25 +00:00
parent cae6885b71
commit 86635168fe
4 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -240,6 +240,8 @@ Release 2.0.3-alpha - Unreleased
HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever.
(Andy Isaacson via eli)
HDFS-3924. Multi-byte id in HdfsVolumeId. (Andrew Wang via atm)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
@ -31,10 +32,10 @@
@InterfaceAudience.Public
public class HdfsVolumeId implements VolumeId {
private final byte id;
private final byte[] id;
private final boolean isValid;
public HdfsVolumeId(byte id, boolean isValid) {
public HdfsVolumeId(byte[] id, boolean isValid) {
this.id = id;
this.isValid = isValid;
}
@ -69,6 +70,6 @@ public boolean equals(Object obj) {
@Override
public String toString() {
return Byte.toString(id);
return Base64.encodeBase64String(id);
}
}

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@ -201,7 +202,7 @@ static Map<LocatedBlock, List<VolumeId>> associateVolumeIdsWithBlocks(
ArrayList<VolumeId> l = new ArrayList<VolumeId>(b.getLocations().length);
// Start off all IDs as invalid, fill it in later with results from RPCs
for (int i = 0; i < b.getLocations().length; i++) {
l.add(new HdfsVolumeId((byte)-1, false));
l.add(new HdfsVolumeId(null, false));
}
blockVolumeIds.put(b, l);
}
@ -234,8 +235,8 @@ static Map<LocatedBlock, List<VolumeId>> associateVolumeIdsWithBlocks(
}
// Get the VolumeId by indexing into the list of VolumeIds
// provided by the datanode
HdfsVolumeId id = new HdfsVolumeId(metaVolumeIds.get(volumeIndex)[0],
true);
byte[] volumeId = metaVolumeIds.get(volumeIndex);
HdfsVolumeId id = new HdfsVolumeId(volumeId, true);
// Find out which index we are in the LocatedBlock's replicas
LocatedBlock locBlock = extBlockToLocBlock.get(extBlock);
DatanodeInfo[] dnInfos = locBlock.getLocations();
@ -255,8 +256,8 @@ static Map<LocatedBlock, List<VolumeId>> associateVolumeIdsWithBlocks(
}
// Place VolumeId at the same index as the DN's index in the list of
// replicas
List<VolumeId> VolumeIds = blockVolumeIds.get(locBlock);
VolumeIds.set(index, id);
List<VolumeId> volumeIds = blockVolumeIds.get(locBlock);
volumeIds.set(index, id);
}
}
return blockVolumeIds;

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
View File

@ -24,6 +24,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collection;
@ -1676,10 +1677,10 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
List<byte[]> blocksVolumeIds = new ArrayList<byte[]>(volumes.volumes.size());
// List of indexes into the list of VolumeIds, pointing at the VolumeId of
// the volume that the block is on
List<Integer> blocksVolumendexes = new ArrayList<Integer>(blocks.size());
List<Integer> blocksVolumeIndexes = new ArrayList<Integer>(blocks.size());
// Initialize the list of VolumeIds simply by enumerating the volumes
for (int i = 0; i < volumes.volumes.size(); i++) {
blocksVolumeIds.add(new byte[] { (byte) i });
blocksVolumeIds.add(ByteBuffer.allocate(4).putInt(i).array());
}
// Determine the index of the VolumeId of each block's volume, by comparing
// the block's volume against the enumerated volumes
@ -1700,10 +1701,10 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(List<ExtendedBlock> blocks)
if (!isValid) {
volumeIndex = Integer.MAX_VALUE;
}
blocksVolumendexes.add(volumeIndex);
blocksVolumeIndexes.add(volumeIndex);
}
return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}),
blocksVolumeIds, blocksVolumendexes);
blocksVolumeIds, blocksVolumeIndexes);
}
@Override