big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MAPREDUCE-3239. Use new createSocketAddr API in MRv2 to give better error messages on misconfig (Todd Lipcon via mahadev)

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1187556 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Mahadev Konar 2011-10-21 21:36:58 +00:00
parent d2e1a71918
commit 4186121c08
16 changed files with 74 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -420,6 +420,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3163. JobClient spews errors when killing MR2 job.
(mahadev)
MAPREDUCE-3239. Use new createSocketAddr API in MRv2 to give better
error messages on misconfig (Todd Lipcon via mahadev)
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and

7
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
View File

@ -33,7 +33,9 @@ public class JHAdminConfig {
/** host:port address for History Server API.*/
public static final String MR_HISTORY_ADDRESS = MR_HISTORY_PREFIX + "address";
public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:10020";
public static final int DEFAULT_MR_HISTORY_PORT = 10020;
public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:" +
DEFAULT_MR_HISTORY_PORT;
/** If history cleaning should be enabled or not.*/
public static final String MR_HISTORY_CLEANER_ENABLE =
@ -106,6 +108,7 @@ public class JHAdminConfig {
/**The address the history server webapp is on.*/
public static final String MR_HISTORY_WEBAPP_ADDRESS =
MR_HISTORY_PREFIX + "webapp.address";
public static final int DEFAULT_MR_HISTORY_WEBAPP_PORT = 19888;
public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS =
"0.0.0.0:19888";
"0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT;
}

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
View File

@ -480,7 +480,9 @@ public static String getHistoryUrl(Configuration conf, ApplicationId appId)
//construct the history url for job
String hsAddress = conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
InetSocketAddress address = NetUtils.createSocketAddr(hsAddress);
InetSocketAddress address = NetUtils.createSocketAddr(
hsAddress, JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
StringBuffer sb = new StringBuffer();
if (address.getAddress().isAnyLocalAddress() ||
address.getAddress().isLoopbackAddress()) {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
View File

@ -48,12 +48,14 @@ public static InetSocketAddress getMasterAddress(Configuration conf) {
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
}
else {
masterAddress = conf.get(YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS);
return NetUtils.createSocketAddr(masterAddress, YarnConfiguration.DEFAULT_RM_PORT,
YarnConfiguration.RM_ADDRESS);
}
return NetUtils.createSocketAddr(masterAddress);
}
public static String getMasterPrincipal(Configuration conf)

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
View File

@ -42,7 +42,7 @@ public void testGetMasterAddress() {
// Trying invalid master address for classic
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(MRConfig.MASTER_ADDRESS, "local");
conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
// should throw an exception for invalid value
try {

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
View File

@ -106,7 +106,9 @@ public void start() {
initializeWebApp(conf);
String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr);
InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr,
JHAdminConfig.DEFAULT_MR_HISTORY_PORT,
JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
InetAddress hostNameResolved = null;
try {
hostNameResolved = InetAddress.getLocalHost(); //address.getAddress().getLocalHost();

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
View File

@ -89,7 +89,9 @@ public ResourceMgrDelegate(YarnConfiguration conf) {
InetSocketAddress rmAddress =
NetUtils.createSocketAddr(this.conf.get(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS));
YarnConfiguration.DEFAULT_RM_ADDRESS),
YarnConfiguration.DEFAULT_RM_PORT,
YarnConfiguration.RM_ADDRESS);
LOG.info("Connecting to ResourceManager at " + rmAddress);
applicationsManager =
(ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,

29
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -83,9 +83,10 @@ public class YarnConfiguration extends Configuration {
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
public static final int DEFAULT_RM_PORT = 8040;
public static final String DEFAULT_RM_ADDRESS =
"0.0.0.0:8040";
"0.0.0.0:" + DEFAULT_RM_PORT;
/** The number of threads used to handle applications manager requests.*/
public static final String RM_CLIENT_THREAD_COUNT =
RM_PREFIX + "client.thread-count";
@ -103,7 +104,9 @@ public class YarnConfiguration extends Configuration {
/** The address of the scheduler interface.*/
public static final String RM_SCHEDULER_ADDRESS =
RM_PREFIX + "scheduler.address";
public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:8030";
public static final int DEFAULT_RM_SCHEDULER_PORT = 8030;
public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
DEFAULT_RM_SCHEDULER_PORT;
/** Number of threads to handle scheduler interface.*/
public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
@ -113,12 +116,16 @@ public class YarnConfiguration extends Configuration {
/** The address of the RM web application.*/
public static final String RM_WEBAPP_ADDRESS =
RM_PREFIX + "webapp.address";
public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:8088";
public static final int DEFAULT_RM_WEBAPP_PORT = 8088;
public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" +
DEFAULT_RM_WEBAPP_PORT;
public static final String RM_RESOURCE_TRACKER_ADDRESS =
RM_PREFIX + "resource-tracker.address";
public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8025;
public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
"0.0.0.0:8025";
"0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT;
/** Are acls enabled.*/
public static final String YARN_ACL_ENABLE =
@ -133,7 +140,9 @@ public class YarnConfiguration extends Configuration {
/** The address of the RM admin interface.*/
public static final String RM_ADMIN_ADDRESS =
RM_PREFIX + "admin.address";
public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:8141";
public static final int DEFAULT_RM_ADMIN_PORT = 8141;
public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
DEFAULT_RM_ADMIN_PORT;
/**Number of threads used to handle RM admin interface.*/
public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
@ -250,7 +259,9 @@ public class YarnConfiguration extends Configuration {
/** Address where the localizer IPC is.*/
public static final String NM_LOCALIZER_ADDRESS =
NM_PREFIX + "localizer.address";
public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:4344";
public static final int DEFAULT_NM_LOCALIZER_PORT = 4344;
public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
DEFAULT_NM_LOCALIZER_PORT;
/** Interval in between cache cleanups.*/
public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
@ -288,7 +299,9 @@ public class YarnConfiguration extends Configuration {
/** NM Webapp address.**/
public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:9999";
public static final int DEFAULT_NM_WEBAPP_PORT = 9999;
public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
DEFAULT_NM_WEBAPP_PORT;
/** How often to monitor containers.*/
public final static String NM_CONTAINER_MON_INTERVAL_MS =

8
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
View File

@ -117,7 +117,9 @@ public void start() {
getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
InetSocketAddress httpBindAddress =
NetUtils.createSocketAddr(httpBindAddressStr);
NetUtils.createSocketAddr(httpBindAddressStr,
YarnConfiguration.DEFAULT_NM_WEBAPP_PORT,
YarnConfiguration.NM_WEBAPP_ADDRESS);
try {
// this.hostName = InetAddress.getLocalHost().getCanonicalHostName();
this.httpPort = httpBindAddress.getPort();
@ -141,7 +143,9 @@ public synchronized void stop() {
protected ResourceTracker getRMClient() {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress);
InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress,
conf);
}

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
View File

@ -192,7 +192,9 @@ public void start() {
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress initialAddress = NetUtils.createSocketAddr(conf.get(
YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS));
YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS),
YarnConfiguration.DEFAULT_NM_PORT,
YarnConfiguration.NM_ADDRESS);
server =
rpc.getServer(ContainerManager.class, this, initialAddress, conf,

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -216,7 +216,9 @@ public void init(Configuration conf) {
cacheCleanupPeriod =
conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
localizationServerAddress = NetUtils.createSocketAddr(
conf.get(YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS));
conf.get(YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS),
YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT,
YarnConfiguration.NM_LOCALIZER_ADDRESS);
localizerTracker = createLocalizerTracker(conf);
addService(localizerTracker);
dispatcher.register(LocalizerEventType.class, localizerTracker);

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
View File

@ -83,7 +83,9 @@ public void init(Configuration conf) {
String bindAddress =
conf.get(YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS);
masterServiceAddress = NetUtils.createSocketAddr(bindAddress);
masterServiceAddress = NetUtils.createSocketAddr(bindAddress,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT,
YarnConfiguration.RM_ADMIN_ADDRESS);
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
View File

@ -92,7 +92,9 @@ public void init(Configuration conf) {
String bindAddress =
conf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
masterServiceAddress = NetUtils.createSocketAddr(bindAddress);
masterServiceAddress = NetUtils.createSocketAddr(bindAddress,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT,
YarnConfiguration.RM_SCHEDULER_ADDRESS);
super.init(conf);
}

4
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -111,7 +111,9 @@ public void init(Configuration conf) {
conf.get(YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS);
clientBindAddress =
NetUtils.createSocketAddr(clientServiceBindAddress);
NetUtils.createSocketAddr(clientServiceBindAddress,
YarnConfiguration.DEFAULT_RM_PORT,
YarnConfiguration.RM_ADDRESS);
super.init(conf);
}

5
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
View File

@ -98,7 +98,10 @@ public synchronized void init(Configuration conf) {
String resourceTrackerBindAddress =
conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
resourceTrackerAddress = NetUtils.createSocketAddr(resourceTrackerBindAddress);
resourceTrackerAddress = NetUtils.createSocketAddr(
resourceTrackerBindAddress,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
RackResolver.init(conf);
super.init(conf);

9
hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.tools;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import org.apache.hadoop.conf.Configuration;
@ -145,7 +146,11 @@ private RMAdminProtocol createAdminProtocol() throws IOException {
// Create the client
final String adminAddress =
conf.get(YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.RM_ADMIN_ADDRESS);
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS);
final InetSocketAddress addr =
NetUtils.createSocketAddr(adminAddress,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT,
YarnConfiguration.RM_ADMIN_ADDRESS);
final YarnRPC rpc = YarnRPC.create(conf);
RMAdminProtocol adminProtocol =
@ -153,7 +158,7 @@ private RMAdminProtocol createAdminProtocol() throws IOException {
@Override
public RMAdminProtocol run() {
return (RMAdminProtocol) rpc.getProxy(RMAdminProtocol.class,
NetUtils.createSocketAddr(adminAddress), conf);
addr, conf);
}
});