HADOOP-16116. Fix Spelling Mistakes - DECOMISSIONED. Contributed by BELUGA BEHR.
This commit is contained in:
parent
ba56bc24f5
commit
0f2b65c3da
@ -210,7 +210,7 @@ For the complete command usage, see [balancer](./HDFSCommands.html#balancer).
|
|||||||
|
|
||||||
### Decommissioning
|
### Decommissioning
|
||||||
|
|
||||||
Decommissioning is similar to prior releases. The nodes that need to be decomissioned are added to the exclude file at all of the Namenodes. Each Namenode decommissions its Block Pool. When all the Namenodes finish decommissioning a Datanode, the Datanode is considered decommissioned.
|
Decommissioning is similar to prior releases. The nodes that need to be decommissioned are added to the exclude file at all of the Namenodes. Each Namenode decommissions its Block Pool. When all the Namenodes finish decommissioning a Datanode, the Datanode is considered decommissioned.
|
||||||
|
|
||||||
**Step 1**: To distribute an exclude file to all the Namenodes, use the following command:
|
**Step 1**: To distribute an exclude file to all the Namenodes, use the following command:
|
||||||
|
|
||||||
|
@ -204,15 +204,15 @@ public void testDecommission2() throws IOException {
|
|||||||
|
|
||||||
writeFile(fileSys, file1, replicas);
|
writeFile(fileSys, file1, replicas);
|
||||||
|
|
||||||
int deadDecomissioned = ns.getNumDecomDeadDataNodes();
|
int deadDecommissioned = ns.getNumDecomDeadDataNodes();
|
||||||
int liveDecomissioned = ns.getNumDecomLiveDataNodes();
|
int liveDecommissioned = ns.getNumDecomLiveDataNodes();
|
||||||
|
|
||||||
// Decommission one node. Verify that node is decommissioned.
|
// Decommission one node. Verify that node is decommissioned.
|
||||||
DatanodeInfo decomNode = takeNodeOutofService(0, null, 0,
|
DatanodeInfo decomNode = takeNodeOutofService(0, null, 0,
|
||||||
decommissionedNodes, AdminStates.DECOMMISSIONED);
|
decommissionedNodes, AdminStates.DECOMMISSIONED);
|
||||||
decommissionedNodes.add(decomNode);
|
decommissionedNodes.add(decomNode);
|
||||||
assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes());
|
assertEquals(deadDecommissioned, ns.getNumDecomDeadDataNodes());
|
||||||
assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes());
|
assertEquals(liveDecommissioned + 1, ns.getNumDecomLiveDataNodes());
|
||||||
|
|
||||||
// Ensure decommissioned datanode is not automatically shutdown
|
// Ensure decommissioned datanode is not automatically shutdown
|
||||||
DFSClient client = getDfsClient(0);
|
DFSClient client = getDfsClient(0);
|
||||||
@ -378,15 +378,15 @@ private void testDecommission(int numNamenodes, int numDatanodes)
|
|||||||
|
|
||||||
writeFile(fileSys, file1, replicas);
|
writeFile(fileSys, file1, replicas);
|
||||||
|
|
||||||
int deadDecomissioned = ns.getNumDecomDeadDataNodes();
|
int deadDecommissioned = ns.getNumDecomDeadDataNodes();
|
||||||
int liveDecomissioned = ns.getNumDecomLiveDataNodes();
|
int liveDecommissioned = ns.getNumDecomLiveDataNodes();
|
||||||
|
|
||||||
// Decommission one node. Verify that node is decommissioned.
|
// Decommission one node. Verify that node is decommissioned.
|
||||||
DatanodeInfo decomNode = takeNodeOutofService(i, null, 0,
|
DatanodeInfo decomNode = takeNodeOutofService(i, null, 0,
|
||||||
decommissionedNodes, AdminStates.DECOMMISSIONED);
|
decommissionedNodes, AdminStates.DECOMMISSIONED);
|
||||||
decommissionedNodes.add(decomNode);
|
decommissionedNodes.add(decomNode);
|
||||||
assertEquals(deadDecomissioned, ns.getNumDecomDeadDataNodes());
|
assertEquals(deadDecommissioned, ns.getNumDecomDeadDataNodes());
|
||||||
assertEquals(liveDecomissioned + 1, ns.getNumDecomLiveDataNodes());
|
assertEquals(liveDecommissioned + 1, ns.getNumDecomLiveDataNodes());
|
||||||
|
|
||||||
// Ensure decommissioned datanode is not automatically shutdown
|
// Ensure decommissioned datanode is not automatically shutdown
|
||||||
DFSClient client = getDfsClient(i);
|
DFSClient client = getDfsClient(i);
|
||||||
|
@ -243,8 +243,8 @@ public void run() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
int deadDecomissioned = fsn.getNumDecomDeadDataNodes();
|
int deadDecommissioned = fsn.getNumDecomDeadDataNodes();
|
||||||
int liveDecomissioned = fsn.getNumDecomLiveDataNodes();
|
int liveDecommissioned = fsn.getNumDecomLiveDataNodes();
|
||||||
decomTh.start();
|
decomTh.start();
|
||||||
decomStarted.await(5, TimeUnit.SECONDS);
|
decomStarted.await(5, TimeUnit.SECONDS);
|
||||||
Thread.sleep(3000); // grace period to trigger decommissioning call
|
Thread.sleep(3000); // grace period to trigger decommissioning call
|
||||||
@ -260,8 +260,8 @@ public void run() {
|
|||||||
decomTh.join(20000); // waiting 20secs to finish decommission
|
decomTh.join(20000); // waiting 20secs to finish decommission
|
||||||
LOG.info("Finished decommissioning node:{}", decommisionNodes);
|
LOG.info("Finished decommissioning node:{}", decommisionNodes);
|
||||||
|
|
||||||
assertEquals(deadDecomissioned, fsn.getNumDecomDeadDataNodes());
|
assertEquals(deadDecommissioned, fsn.getNumDecomDeadDataNodes());
|
||||||
assertEquals(liveDecomissioned + decommisionNodes.size(),
|
assertEquals(liveDecommissioned + decommisionNodes.size(),
|
||||||
fsn.getNumDecomLiveDataNodes());
|
fsn.getNumDecomLiveDataNodes());
|
||||||
|
|
||||||
// Ensure decommissioned datanode is not automatically shutdown
|
// Ensure decommissioned datanode is not automatically shutdown
|
||||||
@ -328,8 +328,8 @@ private void testDecommission(int writeBytes, int storageCount,
|
|||||||
List<DatanodeInfo> decommisionNodes = getDecommissionDatanode(dfs, ecFile,
|
List<DatanodeInfo> decommisionNodes = getDecommissionDatanode(dfs, ecFile,
|
||||||
writeBytes, decomNodeCount);
|
writeBytes, decomNodeCount);
|
||||||
|
|
||||||
int deadDecomissioned = fsn.getNumDecomDeadDataNodes();
|
int deadDecommissioned = fsn.getNumDecomDeadDataNodes();
|
||||||
int liveDecomissioned = fsn.getNumDecomLiveDataNodes();
|
int liveDecommissioned = fsn.getNumDecomLiveDataNodes();
|
||||||
List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile))
|
List<LocatedBlock> lbs = ((HdfsDataInputStream) dfs.open(ecFile))
|
||||||
.getAllBlocks();
|
.getAllBlocks();
|
||||||
|
|
||||||
@ -342,8 +342,8 @@ private void testDecommission(int writeBytes, int storageCount,
|
|||||||
// Decommission node. Verify that node is decommissioned.
|
// Decommission node. Verify that node is decommissioned.
|
||||||
decommissionNode(0, decommisionNodes, AdminStates.DECOMMISSIONED);
|
decommissionNode(0, decommisionNodes, AdminStates.DECOMMISSIONED);
|
||||||
|
|
||||||
assertEquals(deadDecomissioned, fsn.getNumDecomDeadDataNodes());
|
assertEquals(deadDecommissioned, fsn.getNumDecomDeadDataNodes());
|
||||||
assertEquals(liveDecomissioned + decommisionNodes.size(),
|
assertEquals(liveDecommissioned + decommisionNodes.size(),
|
||||||
fsn.getNumDecomLiveDataNodes());
|
fsn.getNumDecomLiveDataNodes());
|
||||||
|
|
||||||
// Ensure decommissioned datanode is not automatically shutdown
|
// Ensure decommissioned datanode is not automatically shutdown
|
||||||
|
@ -408,7 +408,7 @@ public void HelperFunction(String scriptFileName, int providedStorages)
|
|||||||
storageIDs[i] = "storageID-" + i;
|
storageIDs[i] = "storageID-" + i;
|
||||||
}
|
}
|
||||||
|
|
||||||
// set first 2 locations as decomissioned
|
// set first 2 locations as decommissioned
|
||||||
locs[0].setDecommissioned();
|
locs[0].setDecommissioned();
|
||||||
locs[1].setDecommissioned();
|
locs[1].setDecommissioned();
|
||||||
|
|
||||||
|
@ -218,12 +218,12 @@ private void doTestStripedBlockPriorities(int dataBlkNum, int parityBlkNum)
|
|||||||
private void assertAdded(LowRedundancyBlocks queues,
|
private void assertAdded(LowRedundancyBlocks queues,
|
||||||
BlockInfo block,
|
BlockInfo block,
|
||||||
int curReplicas,
|
int curReplicas,
|
||||||
int decomissionedReplicas,
|
int decommissionedReplicas,
|
||||||
int expectedReplicas) {
|
int expectedReplicas) {
|
||||||
assertTrue("Failed to add " + block,
|
assertTrue("Failed to add " + block,
|
||||||
queues.add(block,
|
queues.add(block,
|
||||||
curReplicas, 0,
|
curReplicas, 0,
|
||||||
decomissionedReplicas,
|
decommissionedReplicas,
|
||||||
expectedReplicas));
|
expectedReplicas));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -452,7 +452,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
|
|||||||
locs[index].setDecommissioned();
|
locs[index].setDecommissioned();
|
||||||
decommNodeInfo.add(locs[index].toString());
|
decommNodeInfo.add(locs[index].toString());
|
||||||
// Removing it from the list to ensure that all the given nodes are
|
// Removing it from the list to ensure that all the given nodes are
|
||||||
// successfully marked as decomissioned.
|
// successfully marked as decommissioned.
|
||||||
decommnNodeIndices.remove(new Integer(index));
|
decommnNodeIndices.remove(new Integer(index));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -471,7 +471,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
|
|||||||
locs[j].setDecommissioned();
|
locs[j].setDecommissioned();
|
||||||
decommNodeInfo.add(locs[j].toString());
|
decommNodeInfo.add(locs[j].toString());
|
||||||
// Removing it from the list to ensure that all the given nodes are
|
// Removing it from the list to ensure that all the given nodes are
|
||||||
// successfully marked as decomissioned.
|
// successfully marked as decommissioned.
|
||||||
decommnNodeIndices.remove(new Integer(index));
|
decommnNodeIndices.remove(new Integer(index));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -493,7 +493,7 @@ private LocatedStripedBlock createEachLocatedBlock(int numDataBlk,
|
|||||||
locs[index].setDecommissioned();
|
locs[index].setDecommissioned();
|
||||||
decommNodeInfo.add(locs[index].toString());
|
decommNodeInfo.add(locs[index].toString());
|
||||||
// Removing it from the list to ensure that all the given nodes are
|
// Removing it from the list to ensure that all the given nodes are
|
||||||
// successfully marked as decomissioned.
|
// successfully marked as decommissioned.
|
||||||
decommnNodeIndices.remove(new Integer(blkIndexPos));
|
decommnNodeIndices.remove(new Integer(blkIndexPos));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1146,7 +1146,7 @@ public void testDatanodeLifeCycle() throws Exception {
|
|||||||
DataNode dn1 = cluster.getDataNodes().get(0);
|
DataNode dn1 = cluster.getDataNodes().get(0);
|
||||||
DataNode dn2 = cluster.getDataNodes().get(1);
|
DataNode dn2 = cluster.getDataNodes().get(1);
|
||||||
|
|
||||||
// stop the 1st DN while being decomissioned.
|
// stop the 1st DN while being decommissioned.
|
||||||
MiniDFSCluster.DataNodeProperties dn1Properties = cluster.stopDataNode(0);
|
MiniDFSCluster.DataNodeProperties dn1Properties = cluster.stopDataNode(0);
|
||||||
BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(),
|
BlockManagerTestUtil.noticeDeadDatanode(cluster.getNameNode(),
|
||||||
dn1.getDatanodeId().getXferAddr());
|
dn1.getDatanodeId().getXferAddr());
|
||||||
|
@ -113,7 +113,7 @@ protected void serviceInit(Configuration conf) throws Exception {
|
|||||||
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
|
YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
|
||||||
this.hostsReader =
|
this.hostsReader =
|
||||||
createHostsFileReader(this.includesFile, this.excludesFile);
|
createHostsFileReader(this.includesFile, this.excludesFile);
|
||||||
setDecomissionedNMs();
|
setDecommissionedNMs();
|
||||||
printConfiguredHosts();
|
printConfiguredHosts();
|
||||||
} catch (YarnException ex) {
|
} catch (YarnException ex) {
|
||||||
disableHostsFileReader(ex);
|
disableHostsFileReader(ex);
|
||||||
@ -245,7 +245,7 @@ private void refreshHostsReader(
|
|||||||
handleExcludeNodeList(graceful, timeout);
|
handleExcludeNodeList(graceful, timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void setDecomissionedNMs() {
|
private void setDecommissionedNMs() {
|
||||||
Set<String> excludeList = hostsReader.getExcludedHosts();
|
Set<String> excludeList = hostsReader.getExcludedHosts();
|
||||||
for (final String host : excludeList) {
|
for (final String host : excludeList) {
|
||||||
NodeId nodeId = createUnknownNodeId(host);
|
NodeId nodeId = createUnknownNodeId(host);
|
||||||
@ -526,7 +526,7 @@ private void disableHostsFileReader(Exception ex) {
|
|||||||
conf.get(YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
|
conf.get(YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH);
|
||||||
this.hostsReader =
|
this.hostsReader =
|
||||||
createHostsFileReader(this.includesFile, this.excludesFile);
|
createHostsFileReader(this.includesFile, this.excludesFile);
|
||||||
setDecomissionedNMs();
|
setDecommissionedNMs();
|
||||||
} catch (IOException ioe2) {
|
} catch (IOException ioe2) {
|
||||||
// Should *never* happen
|
// Should *never* happen
|
||||||
this.hostsReader = null;
|
this.hostsReader = null;
|
||||||
|
@ -2019,7 +2019,7 @@ private void assertQueueMetrics(QueueMetrics qm, int appsSubmitted,
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 60000)
|
@Test (timeout = 60000)
|
||||||
public void testDecomissionedNMsMetricsOnRMRestart() throws Exception {
|
public void testDecommissionedNMsMetricsOnRMRestart() throws Exception {
|
||||||
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
|
conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH,
|
||||||
hostFile.getAbsolutePath());
|
hostFile.getAbsolutePath());
|
||||||
writeToHostsFile("");
|
writeToHostsFile("");
|
||||||
|
@ -523,12 +523,13 @@ public void testAddNewExcludePathToConfiguration() throws Exception {
|
|||||||
checkDecommissionedNMCount(rm, ++initialMetricCount);
|
checkDecommissionedNMCount(rm, ++initialMetricCount);
|
||||||
nodeHeartbeat = nm1.nodeHeartbeat(true);
|
nodeHeartbeat = nm1.nodeHeartbeat(true);
|
||||||
Assert.assertEquals(
|
Assert.assertEquals(
|
||||||
"Node should not have been decomissioned.",
|
"Node should not have been decommissioned.",
|
||||||
NodeAction.NORMAL,
|
NodeAction.NORMAL,
|
||||||
nodeHeartbeat.getNodeAction());
|
nodeHeartbeat.getNodeAction());
|
||||||
nodeHeartbeat = nm2.nodeHeartbeat(true);
|
nodeHeartbeat = nm2.nodeHeartbeat(true);
|
||||||
Assert.assertEquals("Node should have been decomissioned but is in state" +
|
Assert.assertEquals(
|
||||||
nodeHeartbeat.getNodeAction(),
|
"Node should have been decommissioned but is in state"
|
||||||
|
+ nodeHeartbeat.getNodeAction(),
|
||||||
NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
|
NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,7 +73,7 @@ test('Testing fields', function(assert) {
|
|||||||
assert.deepEqual(model.get('getRunningAppsDataForDonutChart'),
|
assert.deepEqual(model.get('getRunningAppsDataForDonutChart'),
|
||||||
[{label: "Pending", value: 0}, {label: "Running", value: 0}]);
|
[{label: "Pending", value: 0}, {label: "Running", value: 0}]);
|
||||||
assert.deepEqual(model.get('getNodesDataForDonutChart'),
|
assert.deepEqual(model.get('getNodesDataForDonutChart'),
|
||||||
[{label: "Active", value: 4}, {label: "Unhealthy", value: 0}, {label: "Decomissioned", value: 0}]);
|
[{label: "Active", value: 4}, {label: "Unhealthy", value: 0}, {label: "Decommissioned", value: 0}]);
|
||||||
assert.deepEqual(model.get('getMemoryDataForDonutChart'),
|
assert.deepEqual(model.get('getMemoryDataForDonutChart'),
|
||||||
[{label: "Allocated", value: 0}, {label: "Reserved", value: 0}, {label: "Available", value: 32768}]);
|
[{label: "Allocated", value: 0}, {label: "Reserved", value: 0}, {label: "Available", value: 32768}]);
|
||||||
assert.deepEqual(model.get('getVCoreDataForDonutChart'),
|
assert.deepEqual(model.get('getVCoreDataForDonutChart'),
|
||||||
|
Loading…
Reference in New Issue
Block a user