big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-8740. Clear node attribute path after each test run. Contributed by Bibin A Chundatt.

Browse Source
This commit is contained in:
Sunil G 2018-09-03 12:11:19 +05:30
parent c44088ac19
commit 5e64e62dee
6 changed files with 93 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@ -151,7 +152,8 @@ private NodeAttributeStore getAttributeStoreClass(Configuration conf) {
}
}
private void internalUpdateAttributesOnNodes(
@VisibleForTesting
protected void internalUpdateAttributesOnNodes(
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
AttributeMappingOperationType op,
Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
@ -195,7 +197,7 @@ private void internalUpdateAttributesOnNodes(
logMsg.append(" NM = ");
logMsg.append(entry.getKey());
logMsg.append(", attributes=[ ");
logMsg.append(StringUtils.join(entry.getValue().entrySet(), ","));
logMsg.append(StringUtils.join(entry.getValue().keySet(), ","));
logMsg.append("] ,");
}
@ -700,9 +702,10 @@ private void processMapping(
new HashMap<>();
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
internalUpdateAttributesOnNodes(validMapping, mappingType,
newAttributesToBeAdded, attributePrefix);
if (validMapping.size() > 0) {
internalUpdateAttributesOnNodes(validMapping, mappingType,
newAttributesToBeAdded, attributePrefix);
}
}
protected void stopDispatcher() {

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java Normal file
View File

@ -0,0 +1,54 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Test Utils for NodeAttribute.
*/
public final class NodeAttributeTestUtils {
private NodeAttributeTestUtils() {
}
public static YarnConfiguration getRandomDirConf(Configuration conf)
throws IOException {
YarnConfiguration newConf;
if (conf == null) {
newConf = new YarnConfiguration();
} else {
newConf = new YarnConfiguration(conf);
}
File tempDir = GenericTestUtils.getRandomizedTestDir();
FileUtils.deleteDirectory(tempDir);
tempDir.mkdirs();
tempDir.deleteOnExit();
newConf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
tempDir.getAbsolutePath());
return newConf;
}
}

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -2012,7 +2012,8 @@ protected ClientRMService createClientRMService() {
@Test(timeout = 120000)
public void testGetClusterNodeAttributes() throws IOException, YarnException {
MockRM rm = new MockRM() {
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
@ -2061,7 +2062,8 @@ protected ClientRMService createClientRMService() {
@Test(timeout = 120000)
public void testGetAttributesToNodes() throws IOException, YarnException {
MockRM rm = new MockRM() {
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
@ -2152,7 +2154,8 @@ private boolean findHostnameAndValInMapping(String hostname, String attrVal,
@Test(timeout = 120000)
public void testGetNodesToAttributes() throws IOException, YarnException {
MockRM rm = new MockRM() {
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
View File

@ -1557,7 +1557,8 @@ public void testMapAttributesToNodes() throws Exception, YarnException {
// 1.5. Test with unknown node when failOnUnknownNodes is false
// also test : 3. Ensure Appropriate manager Method call is done
rm = new MockRM();
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
rm = new MockRM(newConf);
NodeAttributesManager spiedAttributesManager =
Mockito.spy(rm.getRMContext().getNodeAttributesManager());

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
View File

@ -17,6 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
@ -25,12 +30,12 @@
import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.nodelabels.AttributeValue;
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@ -68,12 +73,7 @@ public void before() throws IOException {
conf = new Configuration();
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
File tempDir = File.createTempFile("nattr", ".tmp");
tempDir.delete();
tempDir.mkdirs();
tempDir.deleteOnExit();
conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
tempDir.getAbsolutePath());
conf = NodeAttributeTestUtils.getRandomDirConf(conf);
mgr.init(conf);
mgr.start();
}
@ -86,6 +86,20 @@ public void after() throws IOException {
mgr.stop();
}
@Test(timeout = 10000)
public void testEmptyRecoverSkipInternalUdpate() throws Exception {
// Stop manager
mgr.stop();
// Start new attribute manager with same path
mgr = spy(new MockNodeAttrbuteManager());
mgr.init(conf);
mgr.start();
verify(mgr, times(0))
.internalUpdateAttributesOnNodes(any(), any(), any(), any());
}
@Test(timeout = 10000)
public void testRecoverWithMirror() throws Exception {

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
View File

@ -28,12 +28,12 @@
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
import org.junit.Test;
import org.junit.Before;
import org.junit.After;
import org.junit.Assert;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@ -57,12 +57,7 @@ public void init() throws IOException {
attributesManager = new NodeAttributesManagerImpl();
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
File tempDir = File.createTempFile("nattr", ".tmp");
tempDir.delete();
tempDir.mkdirs();
tempDir.deleteOnExit();
conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
tempDir.getAbsolutePath());
conf = NodeAttributeTestUtils.getRandomDirConf(conf);
attributesManager.init(conf);
attributesManager.start();
}