big-data/hadoop
big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

HDFS-11467. Support ErasureCoding section in OIV XML/ReverseXML. Contributed by Huafeng Wang.

Browse Source
This commit is contained in:
Xiao Chen 2017-11-03 12:04:25 -07:00
parent c41728486b
commit 299d38295d
3 changed files with 257 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import com.google.common.base.Preconditions;
import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_MASK;
import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_NAME_OFFSET;
import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.ACL_ENTRY_SCOPE_OFFSET;
@ -34,7 +35,6 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.file.Files;
@ -61,6 +61,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ErasureCodingPolicyProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
@ -69,6 +71,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
@ -147,6 +150,8 @@ class OfflineImageReconstructor {
this.events = factory.createXMLEventReader(reader);
this.sections = new HashMap<>();
this.sections.put(NameSectionProcessor.NAME, new NameSectionProcessor());
this.sections.put(ErasureCodingSectionProcessor.NAME,
new ErasureCodingSectionProcessor());
this.sections.put(INodeSectionProcessor.NAME, new INodeSectionProcessor());
this.sections.put(SecretManagerSectionProcessor.NAME,
new SecretManagerSectionProcessor());
@ -490,6 +495,76 @@ class OfflineImageReconstructor {
}
}
private class ErasureCodingSectionProcessor implements SectionProcessor {
static final String NAME = "ErasureCodingSection";
@Override
public void process() throws IOException {
Node node = new Node();
loadNodeChildren(node, "ErasureCodingSection fields");
ErasureCodingSection.Builder builder = ErasureCodingSection.newBuilder();
while (true) {
ErasureCodingPolicyProto.Builder policyBuilder =
ErasureCodingPolicyProto.newBuilder();
Node ec = node.removeChild(ERASURE_CODING_SECTION_POLICY);
if (ec == null) {
break;
}
int policyId = ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_ID);
policyBuilder.setId(policyId);
String name = ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_NAME);
policyBuilder.setName(name);
Integer cellSize =
ec.removeChildInt(ERASURE_CODING_SECTION_POLICY_CELL_SIZE);
policyBuilder.setCellSize(cellSize);
String policyState =
ec.removeChildStr(ERASURE_CODING_SECTION_POLICY_STATE);
if (policyState != null) {
policyBuilder.setState(
HdfsProtos.ErasureCodingPolicyState.valueOf(policyState));
}
Node schema = ec.removeChild(ERASURE_CODING_SECTION_SCHEMA);
Preconditions.checkNotNull(schema);
ECSchemaProto.Builder schemaBuilder = ECSchemaProto.newBuilder();
String codecName =
schema.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME);
schemaBuilder.setCodecName(codecName);
Integer dataUnits =
schema.removeChildInt(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS);
schemaBuilder.setDataUnits(dataUnits);
Integer parityUnits = schema.
removeChildInt(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS);
schemaBuilder.setParityUnits(parityUnits);
Node options = schema
.removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTIONS);
if (options != null) {
while (true) {
Node option =
options.removeChild(ERASURE_CODING_SECTION_SCHEMA_OPTION);
if (option == null) {
break;
}
String key = option
.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY);
String value = option
.removeChildStr(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE);
schemaBuilder.addOptions(HdfsProtos.ECSchemaOptionEntryProto
.newBuilder().setKey(key).setValue(value).build());
}
}
policyBuilder.setSchema(schemaBuilder.build());
builder.addPolicies(policyBuilder.build());
}
ErasureCodingSection section = builder.build();
section.writeDelimitedTo(out);
node.verifyNoRemainingKeys("ErasureCodingSection");
recordSectionLength(SectionName.ERASURE_CODING.name());
}
}
private class INodeSectionProcessor implements SectionProcessor {
static final String NAME = "INodeSection";
@ -548,7 +623,7 @@ class OfflineImageReconstructor {
}
switch (type) {
case "FILE":
processFileXml(node, inodeBld );
processFileXml(node, inodeBld);
break;
case "DIRECTORY":
processDirectoryXml(node, inodeBld);

80
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
View File

@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.Map;
import java.util.TimeZone;
import com.google.protobuf.ByteString;
@ -36,15 +37,20 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.CacheManagerSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.ErasureCodingSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FilesUnderConstructionSection.FileUnderConstructionEntry;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeDirectorySection;
@ -60,6 +66,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.LimitInputStream;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@ -79,6 +86,8 @@ import static org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode.XATTR_
@InterfaceAudience.Private
public final class PBImageXmlWriter {
public static final String NAME_SECTION_NAME = "NameSection";
public static final String ERASURE_CODING_SECTION_NAME =
"ErasureCodingSection";
public static final String INODE_SECTION_NAME = "INodeSection";
public static final String SECRET_MANAGER_SECTION_NAME =
"SecretManagerSection";
@ -109,6 +118,33 @@ public final class PBImageXmlWriter {
public static final String NAME_SECTION_LAST_ALLOCATED_STRIPED_BLOCK_ID =
"lastAllocatedStripedBlockId";
public static final String ERASURE_CODING_SECTION_POLICY =
"erasureCodingPolicy";
public static final String ERASURE_CODING_SECTION_POLICY_ID =
"policyId";
public static final String ERASURE_CODING_SECTION_POLICY_NAME =
"policyName";
public static final String ERASURE_CODING_SECTION_POLICY_CELL_SIZE =
"cellSize";
public static final String ERASURE_CODING_SECTION_POLICY_STATE =
"policyState";
public static final String ERASURE_CODING_SECTION_SCHEMA =
"ecSchema";
public static final String ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME =
"codecName";
public static final String ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS =
"dataUnits";
public static final String ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS =
"parityUnits";
public static final String ERASURE_CODING_SECTION_SCHEMA_OPTIONS =
"extraOptions";
public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION =
"option";
public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY =
"key";
public static final String ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE =
"value";
public static final String INODE_SECTION_LAST_INODE_ID = "lastInodeId";
public static final String INODE_SECTION_NUM_INODES = "numInodes";
public static final String INODE_SECTION_TYPE = "type";
@ -297,6 +333,9 @@ public final class PBImageXmlWriter {
case STRING_TABLE:
loadStringTable(is);
break;
case ERASURE_CODING:
dumpErasureCodingSection(is);
break;
case INODE:
dumpINodeSection(is);
break;
@ -527,6 +566,47 @@ public final class PBImageXmlWriter {
}
}
private void dumpErasureCodingSection(InputStream in) throws IOException {
ErasureCodingSection s = ErasureCodingSection.parseDelimitedFrom(in);
if (s.getPoliciesCount() > 0) {
out.println("<" + ERASURE_CODING_SECTION_NAME + ">");
for (int i = 0; i < s.getPoliciesCount(); ++i) {
HdfsProtos.ErasureCodingPolicyProto policy = s.getPolicies(i);
dumpErasureCodingPolicy(PBHelperClient
.convertErasureCodingPolicyInfo(policy));
}
out.println("</" + ERASURE_CODING_SECTION_NAME + ">\n");
}
}
private void dumpErasureCodingPolicy(ErasureCodingPolicyInfo ecPolicyInfo) {
ErasureCodingPolicy ecPolicy = ecPolicyInfo.getPolicy();
out.println("<" + ERASURE_CODING_SECTION_POLICY + ">");
o(ERASURE_CODING_SECTION_POLICY_ID, ecPolicy.getId());
o(ERASURE_CODING_SECTION_POLICY_NAME, ecPolicy.getName());
o(ERASURE_CODING_SECTION_POLICY_CELL_SIZE, ecPolicy.getCellSize());
o(ERASURE_CODING_SECTION_POLICY_STATE, ecPolicyInfo.getState());
out.println("<" + ERASURE_CODING_SECTION_SCHEMA + ">");
ECSchema schema = ecPolicy.getSchema();
o(ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME, schema.getCodecName());
o(ERASURE_CODING_SECTION_SCHEMA_DATA_UNITS, schema.getNumDataUnits());
o(ERASURE_CODING_SECTION_SCHEMA_PARITY_UNITS,
schema.getNumParityUnits());
if (schema.getExtraOptions().size() > 0) {
out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTIONS + ">");
for (Map.Entry<String, String> option :
schema.getExtraOptions().entrySet()) {
out.println("<" + ERASURE_CODING_SECTION_SCHEMA_OPTION + ">");
o(ERASURE_CODING_SECTION_SCHEMA_OPTION_KEY, option.getKey());
o(ERASURE_CODING_SECTION_SCHEMA_OPTION_VALUE, option.getValue());
out.println("</" + ERASURE_CODING_SECTION_SCHEMA_OPTION + ">");
}
out.println("</" + ERASURE_CODING_SECTION_SCHEMA_OPTIONS + ">");
}
out.println("</" + ERASURE_CODING_SECTION_SCHEMA + ">");
out.println("</" + ERASURE_CODING_SECTION_POLICY + ">\n");
}
private void dumpINodeSection(InputStream in) throws IOException {
INodeSection s = INodeSection.parseDelimitedFrom(in);
out.print("<" + INODE_SECTION_NAME + ">");

101
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
View File

@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import com.google.common.collect.ImmutableMap;
import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS;
import static org.apache.hadoop.fs.permission.AclEntryType.GROUP;
import static org.apache.hadoop.fs.permission.AclEntryType.OTHER;
@ -24,7 +25,19 @@ import static org.apache.hadoop.fs.permission.AclEntryType.USER;
import static org.apache.hadoop.fs.permission.FsAction.ALL;
import static org.apache.hadoop.fs.permission.FsAction.EXECUTE;
import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyState;
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_NAME;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_CELL_SIZE;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_NAME;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_POLICY_STATE;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME;
import static org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter.ERASURE_CODING_SECTION_SCHEMA_OPTION;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@ -48,11 +61,14 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
@ -91,6 +107,10 @@ import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
@ -106,6 +126,7 @@ public class TestOfflineImageViewer {
private static final String TEST_RENEWER = "JobTracker";
private static File originalFsimage = null;
private static int filesECCount = 0;
private static String addedErasureCodingPolicyName = null;
// namespace as written to dfs, to be compared with viewer's output
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
@ -142,6 +163,15 @@ public class TestOfflineImageViewer {
DistributedFileSystem hdfs = cluster.getFileSystem();
hdfs.enableErasureCodingPolicy(ecPolicy.getName());
Map<String, String> options = ImmutableMap.of("k1", "v1", "k2", "v2");
ECSchema schema = new ECSchema(ErasureCodeConstants.RS_CODEC_NAME,
10, 4, options);
ErasureCodingPolicy policy = new ErasureCodingPolicy(schema, 1024);
AddErasureCodingPolicyResponse[] responses =
hdfs.addErasureCodingPolicies(new ErasureCodingPolicy[]{policy});
addedErasureCodingPolicyName = responses[0].getPolicy().getName();
hdfs.enableErasureCodingPolicy(addedErasureCodingPolicyName);
// Create a reasonable namespace
for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
Path dir = new Path("/dir" + i);
@ -272,10 +302,11 @@ public class TestOfflineImageViewer {
}
LOG.debug("original FS image file is " + originalFsimage);
} finally {
if (cluster != null)
if (cluster != null) {
cluster.shutdown();
}
}
}
@AfterClass
public static void deleteOriginalFSImage() throws IOException {
@ -585,6 +616,7 @@ public class TestOfflineImageViewer {
IOUtils.closeStream(out);
}
}
private void testPBDelimitedWriter(String db)
throws IOException, InterruptedException {
final String DELIMITER = "\t";
@ -808,4 +840,71 @@ public class TestOfflineImageViewer {
IOUtils.closeStream(out);
}
}
private static String getXmlString(Element element, String name) {
NodeList id = element.getElementsByTagName(name);
Element line = (Element) id.item(0);
if (line == null) {
return "";
}
Node first = line.getFirstChild();
// handle empty <key></key>
if (first == null) {
return "";
}
String val = first.getNodeValue();
if (val == null) {
return "";
}
return val;
}
@Test
public void testOfflineImageViewerForECPolicies() throws Exception {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), o);
v.visit(new RandomAccessFile(originalFsimage, "r"));
final String xml = output.toString();
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(xml));
Document dom = db.parse(is);
NodeList ecSection = dom.getElementsByTagName(ERASURE_CODING_SECTION_NAME);
assertEquals(1, ecSection.getLength());
NodeList policies =
dom.getElementsByTagName(ERASURE_CODING_SECTION_POLICY);
assertEquals(1 + SystemErasureCodingPolicies.getPolicies().size(),
policies.getLength());
for (int i = 0; i < policies.getLength(); i++) {
Element policy = (Element) policies.item(i);
String name = getXmlString(policy, ERASURE_CODING_SECTION_POLICY_NAME);
if (name.equals(addedErasureCodingPolicyName)) {
String cellSize =
getXmlString(policy, ERASURE_CODING_SECTION_POLICY_CELL_SIZE);
assertEquals("1024", cellSize);
String state =
getXmlString(policy, ERASURE_CODING_SECTION_POLICY_STATE);
assertEquals(ErasureCodingPolicyState.ENABLED.toString(), state);
Element schema = (Element) policy
.getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA).item(0);
String codecName =
getXmlString(schema, ERASURE_CODING_SECTION_SCHEMA_CODEC_NAME);
assertEquals(ErasureCodeConstants.RS_CODEC_NAME, codecName);
NodeList options =
schema.getElementsByTagName(ERASURE_CODING_SECTION_SCHEMA_OPTION);
assertEquals(2, options.getLength());
Element option1 = (Element) options.item(0);
assertEquals("k1", getXmlString(option1, "key"));
assertEquals("v1", getXmlString(option1, "value"));
Element option2 = (Element) options.item(1);
assertEquals("k2", getXmlString(option2, "key"));
assertEquals("v2", getXmlString(option2, "value"));
}
}
}
}