HADOOP-16116. Fix Spelling Mistakes - DECOMISSIONED. Contributed by BELUGA BEHR.

This commit is contained in:
Akira Ajisaka 2019-02-18 15:21:46 +09:00
parent ba56bc24f5
commit 0f2b65c3da
No known key found for this signature in database
GPG Key ID: C1EDBB9CA400FD50
11 changed files with 33 additions and 32 deletions

View File

@ -210,7 +210,7 @@ For the complete command usage, see [balancer](./HDFSCommands.html#balancer).
### Decommissioning ### Decommissioning
Decommissioning is similar to prior releases. The nodes that need to be decomissioned are added to the exclude file at all of the Namenodes. Each Namenode decommissions its Block Pool. When all the Namenodes finish decommissioning a Datanode, the Datanode is considered decommissioned. Decommissioning is similar to prior releases. The nodes that need to be decommissioned are added to the exclude file at all of the Namenodes. Each Namenode decommissions its Block Pool. When all the Namenodes finish decommissioning a Datanode, the Datanode is considered decommissioned.
**Step 1**: To distribute an exclude file to all the Namenodes, use the following command: **Step 1**: To distribute an exclude file to all the Namenodes, use the following command:

View File

@ -204,15 +204,15 @@ public void testDecommission2() throws IOException {
writeFile(fileSys, file1, replicas); writeFile(fileSys, file1, replicas);
int deadDecomissioned = ns.getNumDecomDeadDataNodes(); int deadDecommissioned = ns.getNumDecomDeadDataNodes();
int liveDecomissioned = ns.getNumDecomLiveDataNodes(); int liveDecommissioned = ns.getNumDecomLiveDataNodes();
// Decommission one node. Verify that node is decommissioned. // Decommission one node. Verify that node is decommissioned.
DatanodeInfo decomNode = takeNodeOutofService(0, null, 0, DatanodeInfo decomNode = takeNodeOutofService(0, null, 0,
decommissionedNodes, AdminStates.DECOMMISSIONED); decommissionedNodes, AdminStates.DECOMMISSIONED);
decommissionedNodes.add(decomNode); decommissionedNodes.add(decomNode);
assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes()); assertEquals(deadDecommissioned, ns.getNumDecomDeadDataNodes());
assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes()); assertEquals(liveDecommissioned + 1, ns.getNumDecomLiveDataNodes());
// Ensure decommissioned datanode is not automatically shutdown // Ensure decommissioned datanode is not automatically shutdown
DFSClient client = getDfsClient(0); DFSClient client = getDfsClient(0);
@ -378,15 +378,15 @@ private void testDecommission(int numNamenodes, int numDatanodes)
writeFile(fileSys, file1, replicas); writeFile(fileSys, file1, replicas);
int deadDecomissioned = ns.getNumDecomDeadDataNodes(); int deadDecommissioned = ns.getNumDecomDeadDataNodes();
int liveDecomissioned = ns.getNumDecomLiveDataNodes(); int liveDecommissioned = ns.getNumDecomLiveDataNodes();
// Decommission one node. Verify that node is decommissioned. // Decommission one node. Verify that node is decommissioned.
DatanodeInfo decomNode = takeNodeOutofService(i, null, 0, DatanodeInfo decomNode = takeNodeOutofService(i, null, 0,
decommissionedNodes, AdminStates.DECOMMISSIONED); decommissionedNodes, AdminStates.DECOMMISSIONED);
decommissionedNodes.add(decomNode); decommissionedNodes.add(decomNode);
assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes()); assertEquals(deadDecommissioned, ns.getNumDecomDeadDataNodes());
assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes()); assertEquals(liveDecommissioned + 1, ns.getNumDecomLiveDataNodes());
// Ensure decommissioned datanode is not automatically shutdown // Ensure decommissioned datanode is not automatically shutdown
DFSClient client = getDfsClient(i); DFSClient client = getDfsClient(i);

View File

@ -243,8 +243,8 @@ public void run() {
} }
}; };
}; };
int deadDecomissioned = fsn.getNumDecomDeadDataNodes(); int deadDecommissioned = fsn.getNumDecomDeadDataNodes();
int liveDecomissioned = fsn.getNumDecomLiveDataNodes(); int liveDecommissioned = fsn.getNumDecomLiveDataNodes();
decomTh.start(); decomTh.start();
decomStarted.await(5, TimeUnit.SECONDS); decomStarted.await(5, TimeUnit.SECONDS);
Thread.sleep(3000); // grace period to trigger decommissioning call Thread.sleep(3000); // grace period to trigger decommissioning call
@ -260,8 +260,8 @@ public void run() {
decomTh.join(20000); // waiting 20secs to finish decommission decomTh.join(20000); // waiting 20secs to finish decommission
LOG.info("Finished decommissioning node:{}", decommisionNodes); LOG.info("Finished decommissioning node:{}", decommisionNodes);
assertEquals(deadDecomissioned, fsn.getNumDecomDeadDataNodes()); assertEquals(deadDecommissioned, fsn.getNumDecomDeadDataNodes());
assertEquals(liveDecomissioned + decommisionNodes.size(), assertEquals(liveDecommissioned + decommisionNodes.size(),
fsn.getNumDecomLiveDataNodes()); fsn.getNumDecomLiveDataNodes());
// Ensure decommissioned datanode is not automatically shutdown // Ensure decommissioned datanode is not automatically shutdown
@ -328,8 +328,8 @@ private void testDecommission(int writeBytes, int storageCount,
List<DatanodeInfo> decommisionNodes = getDecommissionDatanode(dfs, ecFile, List<DatanodeInfo> decommisionNodes = getDecommissionDatanode(dfs, ecFile,
writeBytes, decomNodeCount); writeBytes, decomNodeCount);
int deadDecomissioned = fsn.getNumDecomDeadDataNodes(); int deadDecommissioned = fsn.getNumDecomDeadDataNodes();
int liveDecomissioned = fsn.getNumDecomLiveDataNodes(); int liveDecommissioned = fsn.getNumDecomLiveDataNodes();
List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile)) List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile))
.getAllBlocks(); .getAllBlocks();
@ -342,8 +342,8 @@ private void testDecommission(int writeBytes, int storageCount,
// Decommission node. Verify that node is decommissioned. // Decommission node. Verify that node is decommissioned.
decommissionNode(0, decommisionNodes, AdminStates.DECOMMISSIONED); decommissionNode(0, decommisionNodes, AdminStates.DECOMMISSIONED);
assertEquals(deadDecomissioned, fsn.getNumDecomDeadDataNodes()); assertEquals(deadDecommissioned, fsn.getNumDecomDeadDataNodes());
assertEquals(liveDecomissioned + decommisionNodes.size(), assertEquals(liveDecommissioned + decommisionNodes.size(),
fsn.getNumDecomLiveDataNodes()); fsn.getNumDecomLiveDataNodes());
// Ensure decommissioned datanode is not automatically shutdown // Ensure decommissioned datanode is not automatically shutdown

View File

@ -408,7 +408,7 @@ public void HelperFunction(String scriptFileName, int providedStorages)
storageIDs[i] = "storageID-" + i; storageIDs[i] = "storageID-" + i;
} }
// set first 2 locations as decomissioned // set first 2 locations as decommissioned
locs[0].setDecommissioned(); locs[0].setDecommissioned();
locs[1].setDecommissioned(); locs[1].setDecommissioned();

View File

@ -218,12 +218,12 @@ private void doTestStripedBlockPriorities(int dataBlkNum, int parityBlkNum)
private void assertAdded(LowRedundancyBlocks queues, private void assertAdded(LowRedundancyBlocks queues,
BlockInfo block, BlockInfo block,
int curReplicas, int curReplicas,
int decomissionedReplicas, int decommissionedReplicas,
int expectedReplicas) { int expectedReplicas) {
assertTrue("Failed to add " + block, assertTrue("Failed to add " + block,
queues.add(block, queues.add(block,
curReplicas, 0, curReplicas, 0,
decomissionedReplicas, decommissionedReplicas,
expectedReplicas)); expectedReplicas));
} }

View File

@ -452,7 +452,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
locs[index].setDecommissioned(); locs[index].setDecommissioned();
decommNodeInfo.add(locs[index].toString()); decommNodeInfo.add(locs[index].toString());
// Removing it from the list to ensure that all the given nodes are // Removing it from the list to ensure that all the given nodes are
// successfully marked as decomissioned. // successfully marked as decommissioned.
decommnNodeIndices.remove(new Integer(index)); decommnNodeIndices.remove(new Integer(index));
} }
} }
@ -471,7 +471,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
locs[j].setDecommissioned(); locs[j].setDecommissioned();
decommNodeInfo.add(locs[j].toString()); decommNodeInfo.add(locs[j].toString());
// Removing it from the list to ensure that all the given nodes are // Removing it from the list to ensure that all the given nodes are
// successfully marked as decomissioned. // successfully marked as decommissioned.
decommnNodeIndices.remove(new Integer(index)); decommnNodeIndices.remove(new Integer(index));
} }
} }
@ -493,7 +493,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
locs[index].setDecommissioned(); locs[index].setDecommissioned();
decommNodeInfo.add(locs[index].toString()); decommNodeInfo.add(locs[index].toString());
// Removing it from the list to ensure that all the given nodes are // Removing it from the list to ensure that all the given nodes are
// successfully marked as decomissioned. // successfully marked as decommissioned.
decommnNodeIndices.remove(new Integer(blkIndexPos)); decommnNodeIndices.remove(new Integer(blkIndexPos));
} }
} }

View File

@ -1146,7 +1146,7 @@ public void testDatanodeLifeCycle() throws Exception {
DataNode dn1 = cluster.getDataNodes().get(0); DataNode dn1 = cluster.getDataNodes().get(0);
DataNode dn2 = cluster.getDataNodes().get(1); DataNode dn2 = cluster.getDataNodes().get(1);
// stop the 1st DN while being decomissioned. // stop the 1st DN while being decommissioned.
MiniDFSCluster.DataNodeProperties dn1Properties = cluster.stopDataNode(0); MiniDFSCluster.DataNodeProperties dn1Properties = cluster.stopDataNode(0);
BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(), BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(),
dn1.getDatanodeId().getXferAddr()); dn1.getDatanodeId().getXferAddr());

View File

@ -113,7 +113,7 @@ protected void serviceInit(Configuration conf) throws Exception {
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
this.hostsReader = this.hostsReader =
createHostsFileReader(this.includesFile, this.excludesFile); createHostsFileReader(this.includesFile, this.excludesFile);
setDecomissionedNMs(); setDecommissionedNMs();
printConfiguredHosts(); printConfiguredHosts();
} catch (YarnException ex) { } catch (YarnException ex) {
disableHostsFileReader(ex); disableHostsFileReader(ex);
@ -245,7 +245,7 @@ private void refreshHostsReader(
handleExcludeNodeList(graceful, timeout); handleExcludeNodeList(graceful, timeout);
} }
private void setDecomissionedNMs() { private void setDecommissionedNMs() {
Set<String> excludeList = hostsReader.getExcludedHosts(); Set<String> excludeList = hostsReader.getExcludedHosts();
for (final String host : excludeList) { for (final String host : excludeList) {
NodeId nodeId = createUnknownNodeId(host); NodeId nodeId = createUnknownNodeId(host);
@ -526,7 +526,7 @@ private void disableHostsFileReader(Exception ex) {
conf.get(YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); conf.get(YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
this.hostsReader = this.hostsReader =
createHostsFileReader(this.includesFile, this.excludesFile); createHostsFileReader(this.includesFile, this.excludesFile);
setDecomissionedNMs(); setDecommissionedNMs();
} catch (IOException ioe2) { } catch (IOException ioe2) {
// Should *never* happen // Should *never* happen
this.hostsReader = null; this.hostsReader = null;

View File

@ -2019,7 +2019,7 @@ private void assertQueueMetrics(QueueMetrics qm, int appsSubmitted,
} }
@Test (timeout = 60000) @Test (timeout = 60000)
public void testDecomissionedNMsMetricsOnRMRestart() throws Exception { public void testDecommissionedNMsMetricsOnRMRestart() throws Exception {
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
hostFile.getAbsolutePath()); hostFile.getAbsolutePath());
writeToHostsFile(""); writeToHostsFile("");

View File

@ -523,12 +523,13 @@ public void testAddNewExcludePathToConfiguration() throws Exception {
checkDecommissionedNMCount(rm, ++initialMetricCount); checkDecommissionedNMCount(rm, ++initialMetricCount);
nodeHeartbeat = nm1.nodeHeartbeat(true); nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertEquals( Assert.assertEquals(
"Node should not have been decomissioned.", "Node should not have been decommissioned.",
NodeAction.NORMAL, NodeAction.NORMAL,
nodeHeartbeat.getNodeAction()); nodeHeartbeat.getNodeAction());
nodeHeartbeat = nm2.nodeHeartbeat(true); nodeHeartbeat = nm2.nodeHeartbeat(true);
Assert.assertEquals("Node should have been decomissioned but is in state" + Assert.assertEquals(
nodeHeartbeat.getNodeAction(), "Node should have been decommissioned but is in state"
+ nodeHeartbeat.getNodeAction(),
NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction()); NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
} }

View File

@ -73,7 +73,7 @@ test('Testing fields', function(assert) {
assert.deepEqual(model.get('getRunningAppsDataForDonutChart'), assert.deepEqual(model.get('getRunningAppsDataForDonutChart'),
[{label: "Pending", value: 0}, {label: "Running", value: 0}]); [{label: "Pending", value: 0}, {label: "Running", value: 0}]);
assert.deepEqual(model.get('getNodesDataForDonutChart'), assert.deepEqual(model.get('getNodesDataForDonutChart'),
[{label: "Active", value: 4}, {label: "Unhealthy", value: 0}, {label: "Decomissioned", value: 0}]); [{label: "Active", value: 4}, {label: "Unhealthy", value: 0}, {label: "Decommissioned", value: 0}]);
assert.deepEqual(model.get('getMemoryDataForDonutChart'), assert.deepEqual(model.get('getMemoryDataForDonutChart'),
[{label: "Allocated", value: 0}, {label: "Reserved", value: 0}, {label: "Available", value: 32768}]); [{label: "Allocated", value: 0}, {label: "Reserved", value: 0}, {label: "Available", value: 32768}]);
assert.deepEqual(model.get('getVCoreDataForDonutChart'), assert.deepEqual(model.get('getVCoreDataForDonutChart'),