YARN-8740. Clear node attribute path after each test run. Contributed by Bibin A Chundatt.
This commit is contained in:
parent
c44088ac19
commit
5e64e62dee
@ -18,6 +18,7 @@
|
|||||||
|
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
||||||
|
|
||||||
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
@ -151,7 +152,8 @@ private NodeAttributeStore getAttributeStoreClass(Configuration conf) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void internalUpdateAttributesOnNodes(
|
@VisibleForTesting
|
||||||
|
protected void internalUpdateAttributesOnNodes(
|
||||||
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
|
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
|
||||||
AttributeMappingOperationType op,
|
AttributeMappingOperationType op,
|
||||||
Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
|
Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
|
||||||
@ -195,7 +197,7 @@ private void internalUpdateAttributesOnNodes(
|
|||||||
logMsg.append(" NM = ");
|
logMsg.append(" NM = ");
|
||||||
logMsg.append(entry.getKey());
|
logMsg.append(entry.getKey());
|
||||||
logMsg.append(", attributes=[ ");
|
logMsg.append(", attributes=[ ");
|
||||||
logMsg.append(StringUtils.join(entry.getValue().entrySet(), ","));
|
logMsg.append(StringUtils.join(entry.getValue().keySet(), ","));
|
||||||
logMsg.append("] ,");
|
logMsg.append("] ,");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -700,9 +702,10 @@ private void processMapping(
|
|||||||
new HashMap<>();
|
new HashMap<>();
|
||||||
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
|
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
|
||||||
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
|
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
|
||||||
|
if (validMapping.size() > 0) {
|
||||||
internalUpdateAttributesOnNodes(validMapping, mappingType,
|
internalUpdateAttributesOnNodes(validMapping, mappingType,
|
||||||
newAttributesToBeAdded, attributePrefix);
|
newAttributesToBeAdded, attributePrefix);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void stopDispatcher() {
|
protected void stopDispatcher() {
|
||||||
|
@ -0,0 +1,54 @@
|
|||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.yarn.server.resourcemanager;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import org.apache.commons.io.FileUtils;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test Utils for NodeAttribute.
|
||||||
|
*/
|
||||||
|
public final class NodeAttributeTestUtils {
|
||||||
|
|
||||||
|
private NodeAttributeTestUtils() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public static YarnConfiguration getRandomDirConf(Configuration conf)
|
||||||
|
throws IOException {
|
||||||
|
YarnConfiguration newConf;
|
||||||
|
if (conf == null) {
|
||||||
|
newConf = new YarnConfiguration();
|
||||||
|
} else {
|
||||||
|
newConf = new YarnConfiguration(conf);
|
||||||
|
}
|
||||||
|
File tempDir = GenericTestUtils.getRandomizedTestDir();
|
||||||
|
FileUtils.deleteDirectory(tempDir);
|
||||||
|
tempDir.mkdirs();
|
||||||
|
tempDir.deleteOnExit();
|
||||||
|
newConf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
|
||||||
|
tempDir.getAbsolutePath());
|
||||||
|
return newConf;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -2012,7 +2012,8 @@ protected ClientRMService createClientRMService() {
|
|||||||
|
|
||||||
@Test(timeout = 120000)
|
@Test(timeout = 120000)
|
||||||
public void testGetClusterNodeAttributes() throws IOException, YarnException {
|
public void testGetClusterNodeAttributes() throws IOException, YarnException {
|
||||||
MockRM rm = new MockRM() {
|
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
|
||||||
|
MockRM rm = new MockRM(newConf) {
|
||||||
protected ClientRMService createClientRMService() {
|
protected ClientRMService createClientRMService() {
|
||||||
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||||
this.applicationACLsManager, this.queueACLsManager,
|
this.applicationACLsManager, this.queueACLsManager,
|
||||||
@ -2061,7 +2062,8 @@ protected ClientRMService createClientRMService() {
|
|||||||
|
|
||||||
@Test(timeout = 120000)
|
@Test(timeout = 120000)
|
||||||
public void testGetAttributesToNodes() throws IOException, YarnException {
|
public void testGetAttributesToNodes() throws IOException, YarnException {
|
||||||
MockRM rm = new MockRM() {
|
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
|
||||||
|
MockRM rm = new MockRM(newConf) {
|
||||||
protected ClientRMService createClientRMService() {
|
protected ClientRMService createClientRMService() {
|
||||||
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||||
this.applicationACLsManager, this.queueACLsManager,
|
this.applicationACLsManager, this.queueACLsManager,
|
||||||
@ -2152,7 +2154,8 @@ private boolean findHostnameAndValInMapping(String hostname, String attrVal,
|
|||||||
|
|
||||||
@Test(timeout = 120000)
|
@Test(timeout = 120000)
|
||||||
public void testGetNodesToAttributes() throws IOException, YarnException {
|
public void testGetNodesToAttributes() throws IOException, YarnException {
|
||||||
MockRM rm = new MockRM() {
|
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
|
||||||
|
MockRM rm = new MockRM(newConf) {
|
||||||
protected ClientRMService createClientRMService() {
|
protected ClientRMService createClientRMService() {
|
||||||
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
|
||||||
this.applicationACLsManager, this.queueACLsManager,
|
this.applicationACLsManager, this.queueACLsManager,
|
||||||
|
@ -1557,7 +1557,8 @@ public void testMapAttributesToNodes() throws Exception, YarnException {
|
|||||||
// 1.5. Test with unknown node when failOnUnknownNodes is false
|
// 1.5. Test with unknown node when failOnUnknownNodes is false
|
||||||
|
|
||||||
// also test : 3. Ensure Appropriate manager Method call is done
|
// also test : 3. Ensure Appropriate manager Method call is done
|
||||||
rm = new MockRM();
|
Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
|
||||||
|
rm = new MockRM(newConf);
|
||||||
|
|
||||||
NodeAttributesManager spiedAttributesManager =
|
NodeAttributesManager spiedAttributesManager =
|
||||||
Mockito.spy(rm.getRMContext().getNodeAttributesManager());
|
Mockito.spy(rm.getRMContext().getNodeAttributesManager());
|
||||||
|
@ -17,6 +17,11 @@
|
|||||||
*/
|
*/
|
||||||
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
|
||||||
|
|
||||||
|
import static org.mockito.Matchers.any;
|
||||||
|
import static org.mockito.Mockito.spy;
|
||||||
|
import static org.mockito.Mockito.times;
|
||||||
|
import static org.mockito.Mockito.verify;
|
||||||
|
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
import org.apache.hadoop.yarn.api.records.NodeAttribute;
|
||||||
@ -25,12 +30,12 @@
|
|||||||
import org.apache.hadoop.yarn.event.InlineDispatcher;
|
import org.apache.hadoop.yarn.event.InlineDispatcher;
|
||||||
import org.apache.hadoop.yarn.nodelabels.AttributeValue;
|
import org.apache.hadoop.yarn.nodelabels.AttributeValue;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
|
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
@ -68,12 +73,7 @@ public void before() throws IOException {
|
|||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
|
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
|
||||||
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
|
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
|
||||||
File tempDir = File.createTempFile("nattr", ".tmp");
|
conf = NodeAttributeTestUtils.getRandomDirConf(conf);
|
||||||
tempDir.delete();
|
|
||||||
tempDir.mkdirs();
|
|
||||||
tempDir.deleteOnExit();
|
|
||||||
conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
|
|
||||||
tempDir.getAbsolutePath());
|
|
||||||
mgr.init(conf);
|
mgr.init(conf);
|
||||||
mgr.start();
|
mgr.start();
|
||||||
}
|
}
|
||||||
@ -86,6 +86,20 @@ public void after() throws IOException {
|
|||||||
mgr.stop();
|
mgr.stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testEmptyRecoverSkipInternalUdpate() throws Exception {
|
||||||
|
// Stop manager
|
||||||
|
mgr.stop();
|
||||||
|
|
||||||
|
// Start new attribute manager with same path
|
||||||
|
mgr = spy(new MockNodeAttrbuteManager());
|
||||||
|
mgr.init(conf);
|
||||||
|
mgr.start();
|
||||||
|
|
||||||
|
verify(mgr, times(0))
|
||||||
|
.internalUpdateAttributesOnNodes(any(), any(), any(), any());
|
||||||
|
}
|
||||||
|
|
||||||
@Test(timeout = 10000)
|
@Test(timeout = 10000)
|
||||||
public void testRecoverWithMirror() throws Exception {
|
public void testRecoverWithMirror() throws Exception {
|
||||||
|
|
||||||
|
@ -28,12 +28,12 @@
|
|||||||
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
|
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
|
||||||
import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
|
import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
|
||||||
|
import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
@ -57,12 +57,7 @@ public void init() throws IOException {
|
|||||||
attributesManager = new NodeAttributesManagerImpl();
|
attributesManager = new NodeAttributesManagerImpl();
|
||||||
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
|
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
|
||||||
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
|
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
|
||||||
File tempDir = File.createTempFile("nattr", ".tmp");
|
conf = NodeAttributeTestUtils.getRandomDirConf(conf);
|
||||||
tempDir.delete();
|
|
||||||
tempDir.mkdirs();
|
|
||||||
tempDir.deleteOnExit();
|
|
||||||
conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
|
|
||||||
tempDir.getAbsolutePath());
|
|
||||||
attributesManager.init(conf);
|
attributesManager.init(conf);
|
||||||
attributesManager.start();
|
attributesManager.start();
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user