From 86635168feaaf9a8f20d01b5fd6c1b6fbe98c1ea Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 13 Sep 2012 23:48:25 +0000 Subject: [PATCH] HDFS-3924. Multi-byte id in HdfsVolumeId. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1384602 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../main/java/org/apache/hadoop/fs/HdfsVolumeId.java | 7 ++++--- .../apache/hadoop/hdfs/BlockStorageLocationUtil.java | 11 ++++++----- .../server/datanode/fsdataset/impl/FsDatasetImpl.java | 9 +++++---- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3352247e4a..1240edddb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -240,6 +240,8 @@ Release 2.0.3-alpha - Unreleased HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever. (Andy Isaacson via eli) + HDFS-3924. Multi-byte id in HdfsVolumeId. (Andrew Wang via atm) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java index e283e52627..8e328051f7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.classification.InterfaceAudience; @@ -31,10 +32,10 @@ @InterfaceAudience.Public public class HdfsVolumeId implements VolumeId { - private final byte id; + private final byte[] id; private final boolean isValid; - public HdfsVolumeId(byte id, boolean isValid) { + public HdfsVolumeId(byte[] id, boolean isValid) { this.id = id; this.isValid = isValid; } @@ -69,6 +70,6 @@ public boolean equals(Object obj) { @Override public String toString() { - return Byte.toString(id); + return Base64.encodeBase64String(id); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java index e588864988..de74e02340 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -201,7 +202,7 @@ static Map> associateVolumeIdsWithBlocks( ArrayList l = new ArrayList(b.getLocations().length); // Start off all IDs as invalid, fill it in later with results from RPCs for (int i = 0; i < b.getLocations().length; i++) { - l.add(new HdfsVolumeId((byte)-1, false)); + l.add(new HdfsVolumeId(null, false)); } blockVolumeIds.put(b, l); } @@ -234,8 +235,8 @@ static Map> associateVolumeIdsWithBlocks( } // Get the VolumeId by indexing into the list of VolumeIds // provided by the datanode - HdfsVolumeId id = new HdfsVolumeId(metaVolumeIds.get(volumeIndex)[0], - true); + byte[] volumeId = metaVolumeIds.get(volumeIndex); + HdfsVolumeId id = new HdfsVolumeId(volumeId, true); // Find out which index we are in the LocatedBlock's replicas LocatedBlock locBlock = extBlockToLocBlock.get(extBlock); DatanodeInfo[] dnInfos = locBlock.getLocations(); @@ -255,8 +256,8 @@ static Map> associateVolumeIdsWithBlocks( } // Place VolumeId at the same index as the DN's index in the list of // replicas - List VolumeIds = blockVolumeIds.get(locBlock); - VolumeIds.set(index, id); + List volumeIds = blockVolumeIds.get(locBlock); + volumeIds.set(index, id); } } return blockVolumeIds; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index d3f23c756d..0f9172247b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; +import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collection; @@ -1676,10 +1677,10 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks) List blocksVolumeIds = new ArrayList(volumes.volumes.size()); // List of indexes into the list of VolumeIds, pointing at the VolumeId of // the volume that the block is on - List blocksVolumendexes = new ArrayList(blocks.size()); + List blocksVolumeIndexes = new ArrayList(blocks.size()); // Initialize the list of VolumeIds simply by enumerating the volumes for (int i = 0; i < volumes.volumes.size(); i++) { - blocksVolumeIds.add(new byte[] { (byte) i }); + blocksVolumeIds.add(ByteBuffer.allocate(4).putInt(i).array()); } // Determine the index of the VolumeId of each block's volume, by comparing // the block's volume against the enumerated volumes @@ -1700,10 +1701,10 @@ public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks) if (!isValid) { volumeIndex = Integer.MAX_VALUE; } - blocksVolumendexes.add(volumeIndex); + blocksVolumeIndexes.add(volumeIndex); } return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}), - blocksVolumeIds, blocksVolumendexes); + blocksVolumeIds, blocksVolumeIndexes); } @Override