From c6282df3e59eb1e5481158184c344034872d2a89 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Mon, 6 Feb 2012 22:06:58 +0000 Subject: [PATCH] MAPREDUCE-3813. Added a cache for resolved racks. Contributed by Vinod K V. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1241209 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../apache/hadoop/yarn/util/RackResolver.java | 10 ++- .../hadoop/yarn/util/TestRackResolver.java | 71 +++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b56c7cdc3d..140c82ac8d 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -713,6 +713,8 @@ Release 0.23.1 - Unreleased MAPREDUCE-3810. Performance tweaks - reduced logging in AM and defined hascode/equals for ResourceRequest & Priority. (vinodkv via acmurthy) + MAPREDUCE-3813. Added a cache for resolved racks. (vinodkv via acmurthy) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java index 4b70afe74e..efbc90e2bb 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/RackResolver.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.net.CachedDNSToSwitchMapping; import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.Node; import org.apache.hadoop.net.NodeBase; @@ -50,7 +51,14 @@ public synchronized static void init(Configuration conf) { try { Constructor dnsToSwitchMappingConstructor = dnsToSwitchMappingClass.getConstructor(); - dnsToSwitchMapping = dnsToSwitchMappingConstructor.newInstance(); + DNSToSwitchMapping newInstance = + dnsToSwitchMappingConstructor.newInstance(); + // Wrap around the configured class with the Cached implementation so as + // to save on repetitive lookups. + // Check if the impl is already caching, to avoid double caching. + dnsToSwitchMapping = + ((newInstance instanceof CachedDNSToSwitchMapping) ? newInstance + : new CachedDNSToSwitchMapping(newInstance)); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java new file mode 100644 index 0000000000..a82f129d51 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java @@ -0,0 +1,71 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.util; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.net.DNSToSwitchMapping; +import org.apache.hadoop.net.Node; +import org.junit.Assert; +import org.junit.Test; + +public class TestRackResolver { + + public static final class MyResolver implements DNSToSwitchMapping { + + int numHost1 = 0; + + @Override + public List resolve(List hostList) { + // Only one host at a time + Assert.assertTrue("hostList size is " + hostList.size(), + hostList.size() <= 1); + List returnList = new ArrayList(); + if (hostList.isEmpty()) { + return returnList; + } + if (hostList.get(0).equals("host1")) { + numHost1++; + returnList.add("/rack1"); + } + // I should not be reached again as RackResolver is supposed to do + // caching. + Assert.assertTrue(numHost1 <= 1); + return returnList; + } + + } + + @Test + public void testCaching() { + Configuration conf = new Configuration(); + conf.setClass( + CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + MyResolver.class, DNSToSwitchMapping.class); + RackResolver.init(conf); + Node node = RackResolver.resolve("host1"); + Assert.assertEquals("/rack1", node.getNetworkLocation()); + node = RackResolver.resolve("host1"); + Assert.assertEquals("/rack1", node.getNetworkLocation()); + } + +}