From b60ca37914b22550e3630fa02742d40697decb31 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Mon, 15 Oct 2018 15:48:26 +0530 Subject: [PATCH] Fix potential FSImage corruption. Contributed by Daryn Sharp. --- .../hadoop/hdfs/util/LongBitFormat.java | 8 + .../server/namenode/AclEntryStatusFormat.java | 109 ++++++------- .../server/namenode/FSImageFormatPBINode.java | 101 +++--------- .../namenode/FSImageFormatProtobuf.java | 26 ++- .../namenode/INodeWithAdditionalFields.java | 36 ++++- .../namenode/NameNodeLayoutVersion.java | 3 +- .../server/namenode/SerialNumberManager.java | 152 ++++++++++++++++-- .../hdfs/server/namenode/SerialNumberMap.java | 43 ++++- .../hdfs/server/namenode/XAttrFormat.java | 107 ++++++------ .../hdfs/server/namenode/XAttrStorage.java | 11 -- .../offlineImageViewer/FSImageLoader.java | 18 ++- .../offlineImageViewer/PBImageTextWriter.java | 3 +- .../offlineImageViewer/PBImageXmlWriter.java | 22 +-- .../hadoop-hdfs/src/main/proto/fsimage.proto | 1 + .../TestCommitBlockSynchronization.java | 6 +- 15 files changed, 389 insertions(+), 257 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java index 51ad08fc95..195fd8c48b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/LongBitFormat.java @@ -26,6 +26,10 @@ public class LongBitFormat implements Serializable { private static final long serialVersionUID = 1L; + public interface Enum { + int getLength(); + } + private final String NAME; /** Bit offset */ private final int OFFSET; @@ -69,4 +73,8 @@ public long combine(long value, long record) { public long getMin() { return MIN; } + + public int getLength() { + return LENGTH; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java index 2c5b23b07c..e9e3e598fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclEntryStatusFormat.java @@ -31,25 +31,23 @@ /** * Class to pack an AclEntry into an integer.
* An ACL entry is represented by a 32-bit integer in Big Endian format.
- * The bits can be divided in four segments:
- * [0:1) || [1:3) || [3:6) || [6:7) || [7:32)
- *
- * [0:1) -- the scope of the entry (AclEntryScope)
- * [1:3) -- the type of the entry (AclEntryType)
- * [3:6) -- the permission of the entry (FsAction)
- * [6:7) -- A flag to indicate whether Named entry or not
- * [7:8) -- Reserved
- * [8:32) -- the name of the entry, which is an ID that points to a
- * string in the StringTableSection.
+ * + * Note: this format is used both in-memory and on-disk. Changes will be + * incompatible. + * */ -public enum AclEntryStatusFormat { +public enum AclEntryStatusFormat implements LongBitFormat.Enum { - SCOPE(null, 1), - TYPE(SCOPE.BITS, 2), - PERMISSION(TYPE.BITS, 3), - NAMED_ENTRY_CHECK(PERMISSION.BITS, 1), - RESERVED(NAMED_ENTRY_CHECK.BITS, 1), - NAME(RESERVED.BITS, 24); + PERMISSION(null, 3), + TYPE(PERMISSION.BITS, 2), + SCOPE(TYPE.BITS, 1), + NAME(SCOPE.BITS, 24); + + private static final FsAction[] FSACTION_VALUES = FsAction.values(); + private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = + AclEntryScope.values(); + private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = + AclEntryType.values(); private final LongBitFormat BITS; @@ -59,30 +57,29 @@ private AclEntryStatusFormat(LongBitFormat previous, int length) { static AclEntryScope getScope(int aclEntry) { int ordinal = (int) SCOPE.BITS.retrieve(aclEntry); - return AclEntryScope.values()[ordinal]; + return ACL_ENTRY_SCOPE_VALUES[ordinal]; } static AclEntryType getType(int aclEntry) { int ordinal = (int) TYPE.BITS.retrieve(aclEntry); - return AclEntryType.values()[ordinal]; + return ACL_ENTRY_TYPE_VALUES[ordinal]; } static FsAction getPermission(int aclEntry) { int ordinal = (int) PERMISSION.BITS.retrieve(aclEntry); - return FsAction.values()[ordinal]; + return FSACTION_VALUES[ordinal]; } static String getName(int aclEntry) { - int nameExists = (int) NAMED_ENTRY_CHECK.BITS.retrieve(aclEntry); - if (nameExists == 0) { - return null; - } - int id = (int) NAME.BITS.retrieve(aclEntry); - AclEntryType type = getType(aclEntry); - if (type == AclEntryType.USER) { - return SerialNumberManager.INSTANCE.getUser(id); - } else if (type == AclEntryType.GROUP) { - return SerialNumberManager.INSTANCE.getGroup(id); + return getName(aclEntry, null); + } + + static String getName(int aclEntry, + SerialNumberManager.StringTable stringTable) { + SerialNumberManager snm = getSerialNumberManager(getType(aclEntry)); + if (snm != null) { + int nid = (int)NAME.BITS.retrieve(aclEntry); + return snm.getString(nid, stringTable); } return null; } @@ -94,29 +91,26 @@ static int toInt(AclEntry aclEntry) { aclEntryInt = TYPE.BITS.combine(aclEntry.getType().ordinal(), aclEntryInt); aclEntryInt = PERMISSION.BITS.combine(aclEntry.getPermission().ordinal(), aclEntryInt); - if (aclEntry.getName() != null) { - aclEntryInt = NAMED_ENTRY_CHECK.BITS.combine(1, aclEntryInt); - if (aclEntry.getType() == AclEntryType.USER) { - int userId = SerialNumberManager.INSTANCE.getUserSerialNumber(aclEntry - .getName()); - aclEntryInt = NAME.BITS.combine(userId, aclEntryInt); - } else if (aclEntry.getType() == AclEntryType.GROUP) { - int groupId = SerialNumberManager.INSTANCE - .getGroupSerialNumber(aclEntry.getName()); - aclEntryInt = NAME.BITS.combine(groupId, aclEntryInt); - } + SerialNumberManager snm = getSerialNumberManager(aclEntry.getType()); + if (snm != null) { + int nid = snm.getSerialNumber(aclEntry.getName()); + aclEntryInt = NAME.BITS.combine(nid, aclEntryInt); } return (int) aclEntryInt; } static AclEntry toAclEntry(int aclEntry) { - AclEntry.Builder builder = new AclEntry.Builder(); - builder.setScope(getScope(aclEntry)).setType(getType(aclEntry)) - .setPermission(getPermission(aclEntry)); - if (getName(aclEntry) != null) { - builder.setName(getName(aclEntry)); - } - return builder.build(); + return toAclEntry(aclEntry, null); + } + + static AclEntry toAclEntry(int aclEntry, + SerialNumberManager.StringTable stringTable) { + return new AclEntry.Builder() + .setScope(getScope(aclEntry)) + .setType(getType(aclEntry)) + .setPermission(getPermission(aclEntry)) + .setName(getName(aclEntry, stringTable)) + .build(); } public static int[] toInt(List aclEntries) { @@ -127,12 +121,19 @@ public static int[] toInt(List aclEntries) { return entries; } - public static ImmutableList toAclEntries(int[] entries) { - ImmutableList.Builder b = new ImmutableList.Builder(); - for (int entry : entries) { - AclEntry aclEntry = toAclEntry(entry); - b.add(aclEntry); + private static SerialNumberManager getSerialNumberManager(AclEntryType type) { + switch (type) { + case USER: + return SerialNumberManager.USER; + case GROUP: + return SerialNumberManager.GROUP; + default: + return null; } - return b.build(); + } + + @Override + public int getLength() { + return BITS.getLength(); } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 1571d57af3..7c4f21efd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -31,10 +31,6 @@ import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.permission.AclEntry; -import org.apache.hadoop.fs.permission.AclEntryScope; -import org.apache.hadoop.fs.permission.AclEntryType; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.XAttr; @@ -60,6 +56,8 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto; +import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields.PermissionStatusFormat; +import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager.StringTable; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; @@ -74,22 +72,11 @@ @InterfaceAudience.Private public final class FSImageFormatPBINode { - private final static long USER_GROUP_STRID_MASK = (1 << 24) - 1; - private final static int USER_STRID_OFFSET = 40; - private final static int GROUP_STRID_OFFSET = 16; - public static final int ACL_ENTRY_NAME_MASK = (1 << 24) - 1; public static final int ACL_ENTRY_NAME_OFFSET = 6; public static final int ACL_ENTRY_TYPE_OFFSET = 3; public static final int ACL_ENTRY_SCOPE_OFFSET = 5; public static final int ACL_ENTRY_PERM_MASK = 7; - private static final int ACL_ENTRY_TYPE_MASK = 3; - private static final int ACL_ENTRY_SCOPE_MASK = 1; - private static final FsAction[] FSACTION_VALUES = FsAction.values(); - private static final AclEntryScope[] ACL_ENTRY_SCOPE_VALUES = AclEntryScope - .values(); - private static final AclEntryType[] ACL_ENTRY_TYPE_VALUES = AclEntryType - .values(); public static final int XATTR_NAMESPACE_MASK = 3; public static final int XATTR_NAMESPACE_OFFSET = 30; @@ -100,56 +87,36 @@ public final class FSImageFormatPBINode { public static final int XATTR_NAMESPACE_EXT_OFFSET = 5; public static final int XATTR_NAMESPACE_EXT_MASK = 1; - private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES = - XAttr.NameSpace.values(); - - private static final Logger LOG = LoggerFactory.getLogger(FSImageFormatPBINode.class); + // the loader must decode all fields referencing serial number based fields + // via to methods with the string table. public final static class Loader { public static PermissionStatus loadPermission(long id, - final String[] stringTable) { - short perm = (short) (id & ((1 << GROUP_STRID_OFFSET) - 1)); - int gsid = (int) ((id >> GROUP_STRID_OFFSET) & USER_GROUP_STRID_MASK); - int usid = (int) ((id >> USER_STRID_OFFSET) & USER_GROUP_STRID_MASK); - return new PermissionStatus(stringTable[usid], stringTable[gsid], - new FsPermission(perm)); + final StringTable stringTable) { + return PermissionStatusFormat.toPermissionStatus(id, stringTable); } public static ImmutableList loadAclEntries( - AclFeatureProto proto, final String[] stringTable) { + AclFeatureProto proto, final StringTable stringTable) { ImmutableList.Builder b = ImmutableList.builder(); for (int v : proto.getEntriesList()) { - int p = v & ACL_ENTRY_PERM_MASK; - int t = (v >> ACL_ENTRY_TYPE_OFFSET) & ACL_ENTRY_TYPE_MASK; - int s = (v >> ACL_ENTRY_SCOPE_OFFSET) & ACL_ENTRY_SCOPE_MASK; - int nid = (v >> ACL_ENTRY_NAME_OFFSET) & ACL_ENTRY_NAME_MASK; - String name = stringTable[nid]; - b.add(new AclEntry.Builder().setName(name) - .setPermission(FSACTION_VALUES[p]) - .setScope(ACL_ENTRY_SCOPE_VALUES[s]) - .setType(ACL_ENTRY_TYPE_VALUES[t]).build()); + b.add(AclEntryStatusFormat.toAclEntry(v, stringTable)); } return b.build(); } public static List loadXAttrs( - XAttrFeatureProto proto, final String[] stringTable) { + XAttrFeatureProto proto, final StringTable stringTable) { List b = new ArrayList<>(); for (XAttrCompactProto xAttrCompactProto : proto.getXAttrsList()) { int v = xAttrCompactProto.getName(); - int nid = (v >> XATTR_NAME_OFFSET) & XATTR_NAME_MASK; - int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK; - ns |= - ((v >> XATTR_NAMESPACE_EXT_OFFSET) & XATTR_NAMESPACE_EXT_MASK) << 2; - String name = stringTable[nid]; byte[] value = null; if (xAttrCompactProto.getValue() != null) { value = xAttrCompactProto.getValue().toByteArray(); } - b.add(new XAttr.Builder().setNameSpace(XATTR_NAMESPACE_VALUES[ns]) - .setName(name).setValue(value).build()); + b.add(XAttrFormat.toXAttr(v, value, stringTable)); } return b; @@ -439,46 +406,30 @@ private void loadRootINode(INodeSection.INode p) { } } + // the saver can directly write out fields referencing serial numbers. + // the serial number maps will be compacted when loading. public final static class Saver { private long numImageErrors; - private static long buildPermissionStatus(INodeAttributes n, - final SaverContext.DeduplicationMap stringMap) { - long userId = stringMap.getId(n.getUserName()); - long groupId = stringMap.getId(n.getGroupName()); - return ((userId & USER_GROUP_STRID_MASK) << USER_STRID_OFFSET) - | ((groupId & USER_GROUP_STRID_MASK) << GROUP_STRID_OFFSET) - | n.getFsPermissionShort(); + private static long buildPermissionStatus(INodeAttributes n) { + return n.getPermissionLong(); } - private static AclFeatureProto.Builder buildAclEntries(AclFeature f, - final SaverContext.DeduplicationMap map) { + private static AclFeatureProto.Builder buildAclEntries(AclFeature f) { AclFeatureProto.Builder b = AclFeatureProto.newBuilder(); for (int pos = 0, e; pos < f.getEntriesSize(); pos++) { e = f.getEntryAt(pos); - int nameId = map.getId(AclEntryStatusFormat.getName(e)); - int v = ((nameId & ACL_ENTRY_NAME_MASK) << ACL_ENTRY_NAME_OFFSET) - | (AclEntryStatusFormat.getType(e).ordinal() << ACL_ENTRY_TYPE_OFFSET) - | (AclEntryStatusFormat.getScope(e).ordinal() << ACL_ENTRY_SCOPE_OFFSET) - | (AclEntryStatusFormat.getPermission(e).ordinal()); - b.addEntries(v); + b.addEntries(e); } return b; } - - private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f, - final SaverContext.DeduplicationMap stringMap) { + + private static XAttrFeatureProto.Builder buildXAttrs(XAttrFeature f) { XAttrFeatureProto.Builder b = XAttrFeatureProto.newBuilder(); for (XAttr a : f.getXAttrs()) { XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto. newBuilder(); - int nsOrd = a.getNameSpace().ordinal(); - Preconditions.checkArgument(nsOrd < 8, "Too many namespaces."); - int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) - | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) << - XATTR_NAME_OFFSET); - v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) << - XATTR_NAMESPACE_EXT_OFFSET); + int v = XAttrFormat.toInt(a); xAttrCompactBuilder.setName(v); if (a.getValue() != null) { xAttrCompactBuilder.setValue(PBHelperClient.getByteString(a.getValue())); @@ -510,7 +461,7 @@ public static INodeSection.INodeFile.Builder buildINodeFile( INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder() .setAccessTime(file.getAccessTime()) .setModificationTime(file.getModificationTime()) - .setPermission(buildPermissionStatus(file, state.getStringMap())) + .setPermission(buildPermissionStatus(file)) .setPreferredBlockSize(file.getPreferredBlockSize()) .setStoragePolicyID(file.getLocalStoragePolicyID()) .setBlockType(PBHelperClient.convert(file.getBlockType())); @@ -523,11 +474,11 @@ public static INodeSection.INodeFile.Builder buildINodeFile( AclFeature f = file.getAclFeature(); if (f != null) { - b.setAcl(buildAclEntries(f, state.getStringMap())); + b.setAcl(buildAclEntries(f)); } XAttrFeature xAttrFeature = file.getXAttrFeature(); if (xAttrFeature != null) { - b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap())); + b.setXAttrs(buildXAttrs(xAttrFeature)); } return b; } @@ -539,7 +490,7 @@ public static INodeSection.INodeDirectory.Builder buildINodeDirectory( .newBuilder().setModificationTime(dir.getModificationTime()) .setNsQuota(quota.getNameSpace()) .setDsQuota(quota.getStorageSpace()) - .setPermission(buildPermissionStatus(dir, state.getStringMap())); + .setPermission(buildPermissionStatus(dir)); if (quota.getTypeSpaces().anyGreaterOrEqual(0)) { b.setTypeQuotas(buildQuotaByStorageTypeEntries(quota)); @@ -547,11 +498,11 @@ public static INodeSection.INodeDirectory.Builder buildINodeDirectory( AclFeature f = dir.getAclFeature(); if (f != null) { - b.setAcl(buildAclEntries(f, state.getStringMap())); + b.setAcl(buildAclEntries(f)); } XAttrFeature xAttrFeature = dir.getXAttrFeature(); if (xAttrFeature != null) { - b.setXAttrs(buildXAttrs(xAttrFeature, state.getStringMap())); + b.setXAttrs(buildXAttrs(xAttrFeature)); } return b; } @@ -712,7 +663,7 @@ private void save(OutputStream out, INodeSymlink n) throws IOException { SaverContext state = parent.getSaverContext(); INodeSection.INodeSymlink.Builder b = INodeSection.INodeSymlink .newBuilder() - .setPermission(buildPermissionStatus(n, state.getStringMap())) + .setPermission(buildPermissionStatus(n)) .setTarget(ByteString.copyFrom(n.getSymlink())) .setModificationTime(n.getModificationTime()) .setAccessTime(n.getAccessTime()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java index 9752733c1c..95eade921d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java @@ -85,10 +85,10 @@ public final class FSImageFormatProtobuf { .getLogger(FSImageFormatProtobuf.class); public static final class LoaderContext { - private String[] stringTable; + private SerialNumberManager.StringTable stringTable; private final ArrayList refList = Lists.newArrayList(); - public String[] getStringTable() { + public SerialNumberManager.StringTable getStringTable() { return stringTable; } @@ -129,13 +129,6 @@ Set> entrySet() { } private final ArrayList refList = Lists.newArrayList(); - private final DeduplicationMap stringMap = DeduplicationMap - .newMap(); - - public DeduplicationMap getStringMap() { - return stringMap; - } - public ArrayList getRefList() { return refList; } @@ -327,11 +320,12 @@ private void loadNameSystemSection(InputStream in) throws IOException { private void loadStringTableSection(InputStream in) throws IOException { StringTableSection s = StringTableSection.parseDelimitedFrom(in); - ctx.stringTable = new String[s.getNumEntry() + 1]; + ctx.stringTable = + SerialNumberManager.newStringTable(s.getNumEntry(), s.getMaskBits()); for (int i = 0; i < s.getNumEntry(); ++i) { StringTableSection.Entry e = StringTableSection.Entry .parseDelimitedFrom(in); - ctx.stringTable[e.getId()] = e.getStr(); + ctx.stringTable.put(e.getId(), e.getStr()); } } @@ -651,12 +645,16 @@ private void saveNameSystemSection(FileSummary.Builder summary) private void saveStringTableSection(FileSummary.Builder summary) throws IOException { OutputStream out = sectionOutputStream; + + SerialNumberManager.StringTable stringTable = + SerialNumberManager.getStringTable(); StringTableSection.Builder b = StringTableSection.newBuilder() - .setNumEntry(saverContext.stringMap.size()); + .setNumEntry(stringTable.size()) + .setMaskBits(stringTable.getMaskBits()); b.build().writeDelimitedTo(out); - for (Entry e : saverContext.stringMap.entrySet()) { + for (Entry e : stringTable) { StringTableSection.Entry.Builder eb = StringTableSection.Entry - .newBuilder().setId(e.getValue()).setStr(e.getKey()); + .newBuilder().setId(e.getKey()).setStr(e.getValue()); eb.build().writeDelimitedTo(out); } commitSection(summary, SectionName.STRING_TABLE); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java index 84d99e483e..b7d2f2c1e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeWithAdditionalFields.java @@ -33,7 +33,9 @@ @InterfaceAudience.Private public abstract class INodeWithAdditionalFields extends INode implements LinkedElement { - enum PermissionStatusFormat { + // Note: this format is used both in-memory and on-disk. Changes will be + // incompatible. + enum PermissionStatusFormat implements LongBitFormat.Enum { MODE(null, 16), GROUP(MODE.BITS, 24), USER(GROUP.BITS, 24); @@ -46,12 +48,14 @@ private PermissionStatusFormat(LongBitFormat previous, int length) { static String getUser(long permission) { final int n = (int)USER.BITS.retrieve(permission); - return SerialNumberManager.INSTANCE.getUser(n); + String s = SerialNumberManager.USER.getString(n); + assert s != null; + return s; } static String getGroup(long permission) { final int n = (int)GROUP.BITS.retrieve(permission); - return SerialNumberManager.INSTANCE.getGroup(n); + return SerialNumberManager.GROUP.getString(n); } static short getMode(long permission) { @@ -61,16 +65,34 @@ static short getMode(long permission) { /** Encode the {@link PermissionStatus} to a long. */ static long toLong(PermissionStatus ps) { long permission = 0L; - final int user = SerialNumberManager.INSTANCE.getUserSerialNumber( + final int user = SerialNumberManager.USER.getSerialNumber( ps.getUserName()); + assert user != 0; permission = USER.BITS.combine(user, permission); - final int group = SerialNumberManager.INSTANCE.getGroupSerialNumber( + // ideally should assert on group but inodes are created with null + // group and then updated only when added to a directory. + final int group = SerialNumberManager.GROUP.getSerialNumber( ps.getGroupName()); permission = GROUP.BITS.combine(group, permission); final int mode = ps.getPermission().toShort(); permission = MODE.BITS.combine(mode, permission); return permission; } + + static PermissionStatus toPermissionStatus(long id, + SerialNumberManager.StringTable stringTable) { + int uid = (int)USER.BITS.retrieve(id); + int gid = (int)GROUP.BITS.retrieve(id); + return new PermissionStatus( + SerialNumberManager.USER.getString(uid, stringTable), + SerialNumberManager.GROUP.getString(gid, stringTable), + new FsPermission(getMode(id))); + } + + @Override + public int getLength() { + return BITS.getLength(); + } } /** The inode id. */ @@ -175,7 +197,7 @@ final String getUserName(int snapshotId) { @Override final void setUser(String user) { - int n = SerialNumberManager.INSTANCE.getUserSerialNumber(user); + int n = SerialNumberManager.USER.getSerialNumber(user); updatePermissionStatus(PermissionStatusFormat.USER, n); } @@ -189,7 +211,7 @@ final String getGroupName(int snapshotId) { @Override final void setGroup(String group) { - int n = SerialNumberManager.INSTANCE.getGroupSerialNumber(group); + int n = SerialNumberManager.GROUP.getSerialNumber(group); updatePermissionStatus(PermissionStatusFormat.GROUP, n); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java index 182cc08b02..297ca74c5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java @@ -88,7 +88,8 @@ public enum Feature implements LayoutFeature { TRUNCATE(-61, -61, "Truncate"), APPEND_NEW_BLOCK(-62, -61, "Support appending to new block"), QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types"), - ERASURE_CODING(-64, -61, "Support erasure coding"); + ERASURE_CODING(-64, -61, "Support erasure coding"), + EXPANDED_STRING_TABLE(-65, -61, "Support expanded string table in fsimage"); private final FeatureInfo info; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java index eb497abe35..61fdd7d5b3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java @@ -17,23 +17,147 @@ */ package org.apache.hadoop.hdfs.server.namenode; -/** Manage name-to-serial-number maps for users and groups. */ -class SerialNumberManager { - /** This is the only instance of {@link SerialNumberManager}.*/ - static final SerialNumberManager INSTANCE = new SerialNumberManager(); +import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields.PermissionStatusFormat; +import org.apache.hadoop.hdfs.util.LongBitFormat; - private final SerialNumberMap usermap = new SerialNumberMap(); - private final SerialNumberMap groupmap = new SerialNumberMap(); +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; - private SerialNumberManager() {} +/** Manage name-to-serial-number maps for various string tables. */ +public enum SerialNumberManager { + GLOBAL(), + USER(PermissionStatusFormat.USER, AclEntryStatusFormat.NAME), + GROUP(PermissionStatusFormat.GROUP, AclEntryStatusFormat.NAME), + XATTR(XAttrFormat.NAME); - int getUserSerialNumber(String u) {return usermap.get(u);} - int getGroupSerialNumber(String g) {return groupmap.get(g);} - String getUser(int n) {return usermap.get(n);} - String getGroup(int n) {return groupmap.get(n);} + private static final SerialNumberManager[] values = values(); + private static final int maxEntryBits; + private static final int maxEntryNumber; + private static final int maskBits; - { - getUserSerialNumber(null); - getGroupSerialNumber(null); + private SerialNumberMap serialMap; + private int bitLength = Integer.SIZE; + + static { + maxEntryBits = Integer.numberOfLeadingZeros(values.length); + maxEntryNumber = (1 << maxEntryBits) - 1; + maskBits = Integer.SIZE - maxEntryBits; + for (SerialNumberManager snm : values) { + // account for string table mask bits. + snm.updateLength(maxEntryBits); + snm.serialMap = new SerialNumberMap(snm); + FSDirectory.LOG.info(snm + " serial map: bits=" + snm.getLength() + + " maxEntries=" + snm.serialMap.getMax()); + } + } + + SerialNumberManager(LongBitFormat.Enum... elements) { + // compute the smallest bit length registered with the serial manager. + for (LongBitFormat.Enum element : elements) { + updateLength(element.getLength()); + } + } + + int getLength() { + return bitLength; + } + + private void updateLength(int maxLength) { + bitLength = Math.min(bitLength, maxLength); + } + + public int getSerialNumber(String u) { + return serialMap.get(u); + } + + public String getString(int id) { + return serialMap.get(id); + } + + public String getString(int id, StringTable stringTable) { + return (stringTable != null) + ? stringTable.get(this, id) : getString(id); + } + + private int getMask(int bits) { + return ordinal() << (Integer.SIZE - bits); + } + + private static int getMaskBits() { + return maskBits; + } + + private int size() { + return serialMap.size(); + } + + private Iterable> entrySet() { + return serialMap.entrySet(); + } + + // returns snapshot of current values for a save. + public static StringTable getStringTable() { + // approximate size for capacity. + int size = 0; + for (final SerialNumberManager snm : values) { + size += snm.size(); + } + int tableMaskBits = getMaskBits(); + StringTable map = new StringTable(size, tableMaskBits); + for (final SerialNumberManager snm : values) { + final int mask = snm.getMask(tableMaskBits); + for (Entry entry : snm.entrySet()) { + map.put(entry.getKey() | mask, entry.getValue()); + } + } + return map; + } + + // returns an empty table for load. + public static StringTable newStringTable(int size, int bits) { + if (bits > maskBits) { + throw new IllegalArgumentException( + "String table bits " + bits + " > " + maskBits); + } + return new StringTable(size, bits); + } + + public static class StringTable implements Iterable> { + private final int tableMaskBits; + private final Map map; + + private StringTable(int size, int loadingMaskBits) { + this.tableMaskBits = loadingMaskBits; + this.map = new HashMap<>(size); + } + + private String get(SerialNumberManager snm, int id) { + if (tableMaskBits != 0) { + if (id > maxEntryNumber) { + throw new IllegalStateException( + "serial id " + id + " > " + maxEntryNumber); + } + id |= snm.getMask(tableMaskBits); + } + return map.get(id); + } + + public void put(int id, String str) { + map.put(id, str); + } + + public Iterator> iterator() { + return map.entrySet().iterator(); + } + + public int size() { + return map.size(); + } + + public int getMaskBits() { + return tableMaskBits; + } } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java index 2965837f2c..ee6b8eb80c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberMap.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @@ -35,21 +39,33 @@ */ @InterfaceAudience.Private public class SerialNumberMap { - private final AtomicInteger max = new AtomicInteger(1); + private String name; + private final int max; + private final AtomicInteger current = new AtomicInteger(1); private final ConcurrentMap t2i = new ConcurrentHashMap(); private final ConcurrentMap i2t = new ConcurrentHashMap(); + SerialNumberMap(SerialNumberManager snm) { + this(snm.name(), snm.getLength()); + } + + SerialNumberMap(String name, int bitLength) { + this.name = name; + this.max = (1 << bitLength) - 1; + } + public int get(T t) { if (t == null) { return 0; } Integer sn = t2i.get(t); if (sn == null) { - sn = max.getAndIncrement(); - if (sn < 0) { - throw new IllegalStateException("Too many elements!"); + sn = current.getAndIncrement(); + if (sn > max) { + current.getAndDecrement(); + throw new IllegalStateException(name + ": serial number map is full"); } Integer old = t2i.putIfAbsent(t, sn); if (old != null) { @@ -66,14 +82,27 @@ public T get(int i) { } T t = i2t.get(i); if (t == null) { - throw new IllegalStateException("!i2t.containsKey(" + i - + "), this=" + this); + throw new IllegalStateException( + name + ": serial number " + i + " does not exist"); } return t; } + int getMax() { + return max; + } + + Set> entrySet() { + return new HashSet<>(i2t.entrySet()); + } + + public int size() { + return i2t.size(); + } + @Override public String toString() { - return "max=" + max + ",\n t2i=" + t2i + ",\n i2t=" + i2t; + return "current=" + current + ",\n" + + "max=" + max + ",\n t2i=" + t2i + ",\n i2t=" + i2t; } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java index f9f06db8fa..5b5992c7a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrFormat.java @@ -31,52 +31,66 @@ /** * Class to pack XAttrs into byte[].
- * For each XAttr:
- * The first 4 bytes represents XAttr namespace and name
- * [0:3) - XAttr namespace
- * [3:8) - Reserved
- * [8:32) - The name of the entry, which is an ID that points to a - * string in map
- * The following two bytes represents the length of XAttr value
- * The remaining bytes is the XAttr value
+ * + * Note: this format is used both in-memory and on-disk. Changes will be + * incompatible. + * */ -class XAttrFormat { - private enum XAttrStatusFormat { - NAMESPACE(null, 3), - RESERVED(NAMESPACE.BITS, 5), - NAME(RESERVED.BITS, 24); +public enum XAttrFormat implements LongBitFormat.Enum { + RESERVED(null, 5), + NS_EXT(RESERVED.BITS, 1), + NAME(NS_EXT.BITS, 24), + NS(NAME.BITS, 2); - private final LongBitFormat BITS; - - XAttrStatusFormat(LongBitFormat previous, int length) { - BITS = new LongBitFormat(name(), previous, length, 0); - } - - static XAttr.NameSpace getNamespace(int xattrStatus) { - int ordinal = (int) NAMESPACE.BITS.retrieve(xattrStatus); - return XAttr.NameSpace.values()[ordinal]; - } - - static String getName(int xattrStatus) { - int id = (int) NAME.BITS.retrieve(xattrStatus); - return XAttrStorage.getName(id); - } - - static int toInt(XAttr.NameSpace namespace, String name) { - long xattrStatusInt = 0; - - xattrStatusInt = NAMESPACE.BITS - .combine(namespace.ordinal(), xattrStatusInt); - int nid = XAttrStorage.getNameSerialNumber(name); - xattrStatusInt = NAME.BITS - .combine(nid, xattrStatusInt); - - return (int) xattrStatusInt; - } - } + private static final int NS_EXT_SHIFT = NS.BITS.getLength(); + private static final int NS_MASK = (1 << NS_EXT_SHIFT) - 1; private static final int XATTR_VALUE_LEN_MAX = 1 << 16; + private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES = + XAttr.NameSpace.values(); + + private final LongBitFormat BITS; + + XAttrFormat(LongBitFormat previous, int length) { + BITS = new LongBitFormat(name(), previous, length, 0); + } + + @Override + public int getLength() { + return BITS.getLength(); + } + + static XAttr.NameSpace getNamespace(int record) { + long nid = NS.BITS.retrieve(record); + nid |= NS_EXT.BITS.retrieve(record) << NS_EXT_SHIFT; + return XATTR_NAMESPACE_VALUES[(int) nid]; + } + + public static String getName(int record) { + int nid = (int)NAME.BITS.retrieve(record); + return SerialNumberManager.XATTR.getString(nid); + } + + static int toInt(XAttr a) { + int nid = SerialNumberManager.XATTR.getSerialNumber(a.getName()); + int nsOrd = a.getNameSpace().ordinal(); + long value = NS.BITS.combine(nsOrd & NS_MASK, 0L); + value = NS_EXT.BITS.combine(nsOrd >>> NS_EXT_SHIFT, value); + value = NAME.BITS.combine(nid, value); + return (int)value; + } + + static XAttr toXAttr(int record, byte[] value, + SerialNumberManager.StringTable stringTable) { + int nid = (int)NAME.BITS.retrieve(record); + String name = SerialNumberManager.XATTR.getString(nid, stringTable); + return new XAttr.Builder() + .setNameSpace(getNamespace(record)) + .setName(name) + .setValue(value) + .build(); + } /** * Unpack byte[] to XAttrs. @@ -95,8 +109,8 @@ static List toXAttrs(byte[] attrs) { int v = Ints.fromBytes(attrs[i], attrs[i + 1], attrs[i + 2], attrs[i + 3]); i += 4; - builder.setNameSpace(XAttrStatusFormat.getNamespace(v)); - builder.setName(XAttrStatusFormat.getName(v)); + builder.setNameSpace(XAttrFormat.getNamespace(v)); + builder.setName(XAttrFormat.getName(v)); int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]); i += 2; if (vlen > 0) { @@ -129,8 +143,8 @@ static XAttr getXAttr(byte[] attrs, String prefixedName) { int v = Ints.fromBytes(attrs[i], attrs[i + 1], attrs[i + 2], attrs[i + 3]); i += 4; - XAttr.NameSpace namespace = XAttrStatusFormat.getNamespace(v); - String name = XAttrStatusFormat.getName(v); + XAttr.NameSpace namespace = XAttrFormat.getNamespace(v); + String name = XAttrFormat.getName(v); int vlen = ((0xff & attrs[i]) << 8) | (0xff & attrs[i + 1]); i += 2; if (xAttr.getNameSpace() == namespace && @@ -161,7 +175,8 @@ static byte[] toBytes(List xAttrs) { ByteArrayOutputStream out = new ByteArrayOutputStream(); try { for (XAttr a : xAttrs) { - int v = XAttrStatusFormat.toInt(a.getNameSpace(), a.getName()); + // big-endian + int v = XAttrFormat.toInt(a); out.write(Ints.toByteArray(v)); int vlen = a.getValue() == null ? 0 : a.getValue().length; Preconditions.checkArgument(vlen < XATTR_VALUE_LEN_MAX, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java index 3b3747b1d8..1dab69c4dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java @@ -31,17 +31,6 @@ @InterfaceAudience.Private public class XAttrStorage { - private static final SerialNumberMap NAME_MAP = - new SerialNumberMap<>(); - - public static int getNameSerialNumber(String name) { - return NAME_MAP.get(name); - } - - public static String getName(int n) { - return NAME_MAP.get(n); - } - /** * Reads the extended attribute of an inode by name with prefix. *

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index 380f35aadd..7152c88f5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.server.namenode.INodeId; +import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager; import org.apache.hadoop.hdfs.web.JsonUtil; import org.apache.hadoop.hdfs.web.resources.XAttrEncodingParam; import org.apache.hadoop.io.IOUtils; @@ -68,7 +69,7 @@ class FSImageLoader { public static final Logger LOG = LoggerFactory.getLogger(FSImageHandler.class); - private final String[] stringTable; + private final SerialNumberManager.StringTable stringTable; // byte representation of inodes, sorted by id private final byte[][] inodes; private final Map dirmap; @@ -94,8 +95,8 @@ public int compare(byte[] o1, byte[] o2) { } }; - private FSImageLoader(String[] stringTable, byte[][] inodes, - Map dirmap) { + private FSImageLoader(SerialNumberManager.StringTable stringTable, + byte[][] inodes, Map dirmap) { this.stringTable = stringTable; this.inodes = inodes; this.dirmap = dirmap; @@ -120,7 +121,7 @@ static FSImageLoader load(String inputFile) throws IOException { try (FileInputStream fin = new FileInputStream(file.getFD())) { // Map to record INodeReference to the referred id ImmutableList refIdList = null; - String[] stringTable = null; + SerialNumberManager.StringTable stringTable = null; byte[][] inodes = null; Map dirmap = null; @@ -243,16 +244,17 @@ private static byte[][] loadINodeSection(InputStream in) return inodes; } - static String[] loadStringTable(InputStream in) throws - IOException { + static SerialNumberManager.StringTable loadStringTable(InputStream in) + throws IOException { FsImageProto.StringTableSection s = FsImageProto.StringTableSection .parseDelimitedFrom(in); LOG.info("Loading " + s.getNumEntry() + " strings"); - String[] stringTable = new String[s.getNumEntry() + 1]; + SerialNumberManager.StringTable stringTable = + SerialNumberManager.newStringTable(s.getNumEntry(), s.getMaskBits()); for (int i = 0; i < s.getNumEntry(); ++i) { FsImageProto.StringTableSection.Entry e = FsImageProto .StringTableSection.Entry.parseDelimitedFrom(in); - stringTable[e.getId()] = e.getStr(); + stringTable.put(e.getId(), e.getStr()); } return stringTable; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java index ae1cd79d60..5a1644ca63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode; import org.apache.hadoop.hdfs.server.namenode.INodeId; +import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.LimitInputStream; import org.apache.hadoop.util.Time; @@ -391,7 +392,7 @@ public void sync() throws IOException { } } - private String[] stringTable; + private SerialNumberManager.StringTable stringTable; private PrintStream out; private MetadataMap metadataMap = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index fb32400099..7a5ef31c9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -63,7 +63,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; -import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; +import org.apache.hadoop.hdfs.server.namenode.SerialNumberManager; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.io.erasurecode.ECSchema; @@ -269,7 +269,7 @@ public final class PBImageXmlWriter { private final Configuration conf; private final PrintStream out; private final SimpleDateFormat isoDateFormat; - private String[] stringTable; + private SerialNumberManager.StringTable stringTable; public static SimpleDateFormat createSimpleDateFormat() { SimpleDateFormat format = @@ -430,8 +430,9 @@ private void dumpXattrs(INodeSection.XAttrFeatureProto xattrs) { ((XATTR_NAMESPACE_EXT_MASK & (encodedName >> XATTR_NAMESPACE_EXT_OFFSET)) << 2); o(INODE_SECTION_NS, XAttrProtos.XAttrProto. XAttrNamespaceProto.valueOf(ns).toString()); - o(SECTION_NAME, - stringTable[XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET)]); + o(SECTION_NAME, SerialNumberManager.XATTR.getString( + XATTR_NAME_MASK & (encodedName >> XATTR_NAME_OFFSET), + stringTable)); ByteString val = xattr.getValue(); if (val.isValidUtf8()) { o(INODE_SECTION_VAL, val.toStringUtf8()); @@ -784,10 +785,9 @@ private void dumpSnapshotDiffSection(InputStream in) throws IOException { .o(SNAPSHOT_DIFF_SECTION_CHILDREN_SIZE, d.getChildrenSize()) .o(SNAPSHOT_DIFF_SECTION_IS_SNAPSHOT_ROOT, d.getIsSnapshotRoot()) .o(SECTION_NAME, d.getName().toStringUtf8()); - INodeDirectory snapshotCopy = d.getSnapshotCopy(); - if (snapshotCopy != null) { + if (d.hasSnapshotCopy()) { out.print("<" + SNAPSHOT_DIFF_SECTION_SNAPSHOT_COPY + ">"); - dumpINodeDirectory(snapshotCopy); + dumpINodeDirectory(d.getSnapshotCopy()); out.print("\n"); } o(SNAPSHOT_DIFF_SECTION_CREATED_LIST_SIZE, d.getCreatedListSize()); @@ -851,13 +851,7 @@ private void dumpSnapshotSection(InputStream in) throws IOException { } private void loadStringTable(InputStream in) throws IOException { - StringTableSection s = StringTableSection.parseDelimitedFrom(in); - stringTable = new String[s.getNumEntry() + 1]; - for (int i = 0; i < s.getNumEntry(); ++i) { - StringTableSection.Entry e = StringTableSection.Entry - .parseDelimitedFrom(in); - stringTable[e.getId()] = e.getStr(); - } + stringTable = FSImageLoader.loadStringTable(in); } private PBImageXmlWriter o(final String e, final Object v) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto index 101a0605ac..d943dbccd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto @@ -311,6 +311,7 @@ message StringTableSection { optional string str = 2; } optional uint32 numEntry = 1; + optional uint32 maskBits = 2 [default = 0]; // repeated Entry } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java index af15a93f93..844dab5a1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java @@ -57,12 +57,8 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) // set file's parent as root and put the file to inodeMap, so // FSNamesystem's isFileDeleted() method will return false on this file if (file.getParent() == null) { - INodeDirectory mparent = mock(INodeDirectory.class); - INodeDirectory parent = new INodeDirectory(mparent.getId(), new byte[0], - mparent.getPermissionStatus(), mparent.getAccessTime()); - parent.setLocalName(new byte[0]); + INodeDirectory parent = namesystem.getFSDirectory().getRoot(); parent.addChild(file); - file.setParent(parent); } namesystem.dir.getINodeMap().put(file);