diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java index 2b5a19f486..d102375563 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; +import com.google.common.base.Preconditions; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET; import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET; @@ -34,7 +35,6 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.nio.file.Files; @@ -61,6 +61,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; @@ -69,6 +71,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; import org.apache.hadoop.hdfs.server.namenode.FsImageProto; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection; @@ -147,6 +150,8 @@ class OfflineImageReconstructor { this.events = factory.createXMLEventReader(reader); this.sections = new HashMap<>(); this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor()); + this.sections.put(ErasureCodingSectionProcessor.NAME, + new ErasureCodingSectionProcessor()); this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor()); this.sections.put(SecretManagerSectionProcessor.NAME, new SecretManagerSectionProcessor()); @@ -490,6 +495,76 @@ class OfflineImageReconstructor { } } + private class ErasureCodingSectionProcessor implements SectionProcessor { + static final String NAME = "ErasureCodingSection"; + + @Override + public void process() throws IOException { + Node node = new Node(); + loadNodeChildren(node, "ErasureCodingSection fields"); + ErasureCodingSection.Builder builder = ErasureCodingSection.newBuilder(); + while (true) { + ErasureCodingPolicyProto.Builder policyBuilder = + ErasureCodingPolicyProto.newBuilder(); + Node ec = node.removeChild(ERASURE_CODING_SECTION_POLICY); + if (ec == null) { + break; + } + int policyId = ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_ID); + policyBuilder.setId(policyId); + String name = ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_NAME); + policyBuilder.setName(name); + Integer cellSize = + ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_CELL_SIZE); + policyBuilder.setCellSize(cellSize); + String policyState = + ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_STATE); + if (policyState != null) { + policyBuilder.setState( + HdfsProtos.ErasureCodingPolicyState.valueOf(policyState)); + } + + Node schema = ec.removeChild(ERASURE_CODING_SECTION_SCHEMA); + Preconditions.checkNotNull(schema); + + ECSchemaProto.Builder schemaBuilder = ECSchemaProto.newBuilder(); + String codecName = + schema.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME); + schemaBuilder.setCodecName(codecName); + Integer dataUnits = + schema.removeChildInt(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS); + schemaBuilder.setDataUnits(dataUnits); + Integer parityUnits = schema. + removeChildInt(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS); + schemaBuilder.setParityUnits(parityUnits); + Node options = schema + .removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTIONS); + if (options != null) { + while (true) { + Node option = + options.removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTION); + if (option == null) { + break; + } + String key = option + .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY); + String value = option + .removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE); + schemaBuilder.addOptions(HdfsProtos.ECSchemaOptionEntryProto + .newBuilder().setKey(key).setValue(value).build()); + } + } + policyBuilder.setSchema(schemaBuilder.build()); + + builder.addPolicies(policyBuilder.build()); + } + ErasureCodingSection section = builder.build(); + section.writeDelimitedTo(out); + node.verifyNoRemainingKeys("ErasureCodingSection"); + recordSectionLength(SectionName.ERASURE_CODING.name()); + } + } + private class INodeSectionProcessor implements SectionProcessor { static final String NAME = "INodeSection"; @@ -548,7 +623,7 @@ class OfflineImageReconstructor { } switch (type) { case "FILE": - processFileXml(node, inodeBld ); + processFileXml(node, inodeBld); break; case "DIRECTORY": processDirectoryXml(node, inodeBld); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java index 681ebf6862..fb32400099 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.Map; import java.util.TimeZone; import com.google.protobuf.ByteString; @@ -36,15 +37,20 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.PermissionStatus; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode; import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName; import org.apache.hadoop.hdfs.server.namenode.FSImageUtil; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection; +import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection; @@ -60,6 +66,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection; import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection; import org.apache.hadoop.hdfs.server.namenode.INodeFile; import org.apache.hadoop.hdfs.util.XMLUtils; +import org.apache.hadoop.io.erasurecode.ECSchema; import org.apache.hadoop.util.LimitInputStream; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -79,6 +86,8 @@ import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_ @InterfaceAudience.Private public final class PBImageXmlWriter { public static final String NAME_SECTION_NAME = "NameSection"; + public static final String ERASURE_CODING_SECTION_NAME = + "ErasureCodingSection"; public static final String INODE_SECTION_NAME = "INodeSection"; public static final String SECRET_MANAGER_SECTION_NAME = "SecretManagerSection"; @@ -109,6 +118,33 @@ public final class PBImageXmlWriter { public static final String NAME_SECTION_LAST_ALLOCATED_STRIPED_BLOCK_ID = "lastAllocatedStripedBlockId"; + public static final String ERASURE_CODING_SECTION_POLICY = + "erasureCodingPolicy"; + public static final String ERASURE_CODING_SECTION_POLICY_ID = + "policyId"; + public static final String ERASURE_CODING_SECTION_POLICY_NAME = + "policyName"; + public static final String ERASURE_CODING_SECTION_POLICY_CELL_SIZE = + "cellSize"; + public static final String ERASURE_CODING_SECTION_POLICY_STATE = + "policyState"; + public static final String ERASURE_CODING_SECTION_SCHEMA = + "ecSchema"; + public static final String ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME = + "codecName"; + public static final String ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS = + "dataUnits"; + public static final String ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS = + "parityUnits"; + public static final String ERASURE_CODING_SECTION_SCHEMA_OPTIONS = + "extraOptions"; + public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION = + "option"; + public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY = + "key"; + public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE = + "value"; + public static final String INODE_SECTION_LAST_INODE_ID = "lastInodeId"; public static final String INODE_SECTION_NUM_INODES = "numInodes"; public static final String INODE_SECTION_TYPE = "type"; @@ -297,6 +333,9 @@ public final class PBImageXmlWriter { case STRING_TABLE: loadStringTable(is); break; + case ERASURE_CODING: + dumpErasureCodingSection(is); + break; case INODE: dumpINodeSection(is); break; @@ -527,6 +566,47 @@ public final class PBImageXmlWriter { } } + private void dumpErasureCodingSection(InputStream in) throws IOException { + ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in); + if (s.getPoliciesCount() > 0) { + out.println("<" + ERASURE_CODING_SECTION_NAME + ">"); + for (int i = 0; i < s.getPoliciesCount(); ++i) { + HdfsProtos.ErasureCodingPolicyProto policy = s.getPolicies(i); + dumpErasureCodingPolicy(PBHelperClient + .convertErasureCodingPolicyInfo(policy)); + } + out.println("\n"); + } + } + + private void dumpErasureCodingPolicy(ErasureCodingPolicyInfo ecPolicyInfo) { + ErasureCodingPolicy ecPolicy = ecPolicyInfo.getPolicy(); + out.println("<" + ERASURE_CODING_SECTION_POLICY + ">"); + o(ERASURE_CODING_SECTION_POLICY_ID, ecPolicy.getId()); + o(ERASURE_CODING_SECTION_POLICY_NAME, ecPolicy.getName()); + o(ERASURE_CODING_SECTION_POLICY_CELL_SIZE, ecPolicy.getCellSize()); + o(ERASURE_CODING_SECTION_POLICY_STATE, ecPolicyInfo.getState()); + out.println("<" + ERASURE_CODING_SECTION_SCHEMA + ">"); + ECSchema schema = ecPolicy.getSchema(); + o(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME, schema.getCodecName()); + o(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS, schema.getNumDataUnits()); + o(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS, + schema.getNumParityUnits()); + if (schema.getExtraOptions().size() > 0) { + out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTIONS + ">"); + for (Map.Entry option : + schema.getExtraOptions().entrySet()) { + out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTION + ">"); + o(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY, option.getKey()); + o(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE, option.getValue()); + out.println(""); + } + out.println(""); + } + out.println(""); + out.println("\n"); + } + private void dumpINodeSection(InputStream in) throws IOException { INodeSection s = INodeSection.parseDelimitedFrom(in); out.print("<" + INODE_SECTION_NAME + ">"); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index b32b308958..5111265840 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.tools.offlineImageViewer; +import com.google.common.collect.ImmutableMap; import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; import static org.apache.hadoop.fs.permission.AclEntryType.GROUP; import static org.apache.hadoop.fs.permission.AclEntryType.OTHER; @@ -24,7 +25,19 @@ import static org.apache.hadoop.fs.permission.AclEntryType.USER; import static org.apache.hadoop.fs.permission.FsAction.ALL; import static org.apache.hadoop.fs.permission.FsAction.EXECUTE; import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE; +import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; +import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState; import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_NAME; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_CELL_SIZE; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_NAME; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_STATE; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME; +import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_OPTION; +import org.apache.hadoop.io.erasurecode.ECSchema; +import org.apache.hadoop.io.erasurecode.ErasureCodeConstants; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -48,11 +61,14 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.Map; import java.util.Random; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.SAXParser; import javax.xml.parsers.SAXParserFactory; @@ -91,6 +107,10 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import org.xml.sax.Attributes; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -106,6 +126,7 @@ public class TestOfflineImageViewer { private static final String TEST_RENEWER = "JobTracker"; private static File originalFsimage = null; private static int filesECCount = 0; + private static String addedErasureCodingPolicyName = null; // namespace as written to dfs, to be compared with viewer's output final static HashMap writtenFiles = Maps.newHashMap(); @@ -142,6 +163,15 @@ public class TestOfflineImageViewer { DistributedFileSystem hdfs = cluster.getFileSystem(); hdfs.enableErasureCodingPolicy(ecPolicy.getName()); + Map options = ImmutableMap.of("k1", "v1", "k2", "v2"); + ECSchema schema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME, + 10, 4, options); + ErasureCodingPolicy policy = new ErasureCodingPolicy(schema, 1024); + AddErasureCodingPolicyResponse[] responses = + hdfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{policy}); + addedErasureCodingPolicyName = responses[0].getPolicy().getName(); + hdfs.enableErasureCodingPolicy(addedErasureCodingPolicyName); + // Create a reasonable namespace for (int i = 0; i < NUM_DIRS; i++, dirCount++) { Path dir = new Path("/dir" + i); @@ -272,8 +302,9 @@ public class TestOfflineImageViewer { } LOG.debug("original FS image file is " + originalFsimage); } finally { - if (cluster != null) + if (cluster != null) { cluster.shutdown(); + } } } @@ -585,6 +616,7 @@ public class TestOfflineImageViewer { IOUtils.closeStream(out); } } + private void testPBDelimitedWriter(String db) throws IOException, InterruptedException { final String DELIMITER = "\t"; @@ -808,4 +840,71 @@ public class TestOfflineImageViewer { IOUtils.closeStream(out); } } + + private static String getXmlString(Element element, String name) { + NodeList id = element.getElementsByTagName(name); + Element line = (Element) id.item(0); + if (line == null) { + return ""; + } + Node first = line.getFirstChild(); + // handle empty + if (first == null) { + return ""; + } + String val = first.getNodeValue(); + if (val == null) { + return ""; + } + return val; + } + + @Test + public void testOfflineImageViewerForECPolicies() throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + PrintStream o = new PrintStream(output); + PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o); + v.visit(new RandomAccessFile(originalFsimage, "r")); + final String xml = output.toString(); + + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList ecSection = dom.getElementsByTagName(ERASURE_CODING_SECTION_NAME); + assertEquals(1, ecSection.getLength()); + NodeList policies = + dom.getElementsByTagName(ERASURE_CODING_SECTION_POLICY); + assertEquals(1 + SystemErasureCodingPolicies.getPolicies().size(), + policies.getLength()); + for (int i = 0; i < policies.getLength(); i++) { + Element policy = (Element) policies.item(i); + String name = getXmlString(policy, ERASURE_CODING_SECTION_POLICY_NAME); + if (name.equals(addedErasureCodingPolicyName)) { + String cellSize = + getXmlString(policy, ERASURE_CODING_SECTION_POLICY_CELL_SIZE); + assertEquals("1024", cellSize); + String state = + getXmlString(policy, ERASURE_CODING_SECTION_POLICY_STATE); + assertEquals(ErasureCodingPolicyState.ENABLED.toString(), state); + + Element schema = (Element) policy + .getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA).item(0); + String codecName = + getXmlString(schema, ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME); + assertEquals(ErasureCodeConstants.RS_CODEC_NAME, codecName); + + NodeList options = + schema.getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA_OPTION); + assertEquals(2, options.getLength()); + Element option1 = (Element) options.item(0); + assertEquals("k1", getXmlString(option1, "key")); + assertEquals("v1", getXmlString(option1, "value")); + Element option2 = (Element) options.item(1); + assertEquals("k2", getXmlString(option2, "key")); + assertEquals("v2", getXmlString(option2, "value")); + } + } + } }