big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-14622. [Dynamometer] Update XML FsImage parsing logic to ignore non-INodeSection entries to fix an issue caused by the presence of Centralized Cache Management functionality. Contributed by Erik Krogen.

Browse Source
This commit is contained in:
Erik Krogen 2019-07-08 15:10:21 -07:00
parent efb916457f
commit 90b10a0d54
2 changed files with 44 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/main/java/org/apache/hadoop/tools/dynamometer/blockgenerator/XMLParser.java
View File

@ -20,6 +20,7 @@
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
@ -67,6 +68,13 @@ class XMLParser {
* @return {@code BlockInfo}s for any blocks found. * @return {@code BlockInfo}s for any blocks found.
*/ */
List<BlockInfo> parseLine(String line) throws IOException { List<BlockInfo> parseLine(String line) throws IOException {
if (currentState == State.DEFAULT) {
if (line.contains("<INodeSection>")) {
transitionTo(State.INODE_SECTION);
} else {
return Collections.emptyList();
}
}
if (line.contains("<inode>")) { if (line.contains("<inode>")) {
transitionTo(State.INODE); transitionTo(State.INODE);
} }
@ -95,6 +103,9 @@ List<BlockInfo> parseLine(String line) throws IOException {
blockInfos.add(new BlockInfo(id, gs, size, currentReplication)); blockInfos.add(new BlockInfo(id, gs, size, currentReplication));
} }
if (line.contains("</inode>")) { if (line.contains("</inode>")) {
transitionTo(State.INODE_SECTION);
}
if (line.contains("</INodeSection>")) {
transitionTo(State.DEFAULT); transitionTo(State.DEFAULT);
} }
return blockInfos; return blockInfos;
@ -132,14 +143,19 @@ private static List<String> valuesFromXMLString(String xml, String field) {
} }
private enum State { private enum State {
DEFAULT, INODE, FILE, FILE_WITH_REPLICATION; DEFAULT,
INODE_SECTION,
INODE,
FILE,
FILE_WITH_REPLICATION;
private final Set<State> allowedTransitions = new HashSet<>(); private final Set<State> allowedTransitions = new HashSet<>();
static { static {
DEFAULT.addTransitions(DEFAULT, INODE); DEFAULT.addTransitions(DEFAULT, INODE_SECTION);
INODE.addTransitions(DEFAULT, FILE); INODE_SECTION.addTransitions(DEFAULT, INODE);
FILE.addTransitions(DEFAULT, FILE_WITH_REPLICATION); INODE.addTransitions(INODE_SECTION, FILE);
FILE_WITH_REPLICATION.addTransitions(DEFAULT); FILE.addTransitions(INODE_SECTION, FILE_WITH_REPLICATION);
FILE_WITH_REPLICATION.addTransitions(INODE_SECTION);
} }
private void addTransitions(State... nextState) { private void addTransitions(State... nextState) {

24
hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-blockgen/src/test/java/org/apache/hadoop/tools/dynamometer/blockgenerator/TestXMLParser.java
View File

@ -47,7 +47,8 @@ public void testBlocksFromLine() throws Exception {
"<replication>12</replication>", "<replication>12</replication>",
"<blocks><block><id>13</id><genstamp>14</genstamp>" "<blocks><block><id>13</id><genstamp>14</genstamp>"
+ "<numBytes>15</numBytes></block>", + "<numBytes>15</numBytes></block>",
"</inode>" "</inode>",
"</INodeSection>"
}; };
Map<BlockInfo, Short> expectedBlockCount = new HashMap<>(); Map<BlockInfo, Short> expectedBlockCount = new HashMap<>();
@ -67,4 +68,25 @@ public void testBlocksFromLine() throws Exception {
assertEquals(expect.getValue(), actualBlockCount.get(expect.getKey())); assertEquals(expect.getValue(), actualBlockCount.get(expect.getKey()));
} }
} }
@Test
public void testNonInodeSectionIgnored() throws Exception {
String[] lines = {
"<INodeSection>",
"</INodeSection>",
"<OtherSection>",
"<inode><id>1</id><type>FILE</type><name>fake-file</name>"
+ "<replication>1</replication>",
"<blocks><block><id>2</id><genstamp>1</genstamp>"
+ "<numBytes>1</numBytes></block>",
"</inode>",
"<replication>3</replication>",
"</OtherSection>"
};
XMLParser parser = new XMLParser();
for (String line : lines) {
assertTrue((parser.parseLine(line).isEmpty()));
}
}
} }