From a2c150a7369cc629bbfaa2dfa3a8495b6f9c42e2 Mon Sep 17 00:00:00 2001 From: Robert Kanter Date: Thu, 9 Nov 2017 12:07:46 -0800 Subject: [PATCH] YARN-7386. Duplicate Strings in various places in Yarn memory (misha@cloudera.com via rkanter) --- .../impl/pb/ContainerLaunchContextPBImpl.java | 16 ++++++++++++---- .../api/records/impl/pb/ContainerPBImpl.java | 2 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 7 ++++--- .../rmapp/attempt/RMAppAttemptImpl.java | 3 ++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java index d722cc58db..a9f2ee3d5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerRetryContext; @@ -392,7 +393,8 @@ private void initEnv() { this.environment = new HashMap(); for (StringStringMapProto c : list) { - this.environment.put(c.getKey(), c.getValue()); + this.environment.put(StringInterner.weakIntern(c.getKey()), + StringInterner.weakIntern(c.getValue())); } } @@ -402,7 +404,10 @@ public void setEnvironment(final Map env) { return; initEnv(); this.environment.clear(); - this.environment.putAll(env); + for (Map.Entry e : env.entrySet()) { + this.environment.put(StringInterner.weakIntern(e.getKey()), + StringInterner.weakIntern(e.getValue())); + } } private void addEnvToProto() { @@ -464,7 +469,7 @@ private void initApplicationACLs() { for (ApplicationACLMapProto aclProto : list) { this.applicationACLS.put(ProtoUtils.convertFromProtoFormat(aclProto - .getAccessType()), aclProto.getAcl()); + .getAccessType()), StringInterner.weakIntern(aclProto.getAcl())); } } @@ -513,7 +518,10 @@ public void setApplicationACLs( return; initApplicationACLs(); this.applicationACLS.clear(); - this.applicationACLS.putAll(appACLs); + for (Map.Entry e : appACLs.entrySet()) { + this.applicationACLS.put(e.getKey(), + StringInterner.weakIntern(e.getValue())); + } } public ContainerRetryContext getContainerRetryContext() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index b6e22d15c6..be8493886b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -181,7 +181,7 @@ public void setNodeHttpAddress(String nodeHttpAddress) { builder.clearNodeHttpAddress(); return; } - builder.setNodeHttpAddress(nodeHttpAddress); + builder.setNodeHttpAddress(nodeHttpAddress.intern()); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index cfb8a74f59..ae5f6b487c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.ipc.CallerContext; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -425,12 +426,12 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, this.systemClock = SystemClock.getInstance(); this.applicationId = applicationId; - this.name = name; + this.name = StringInterner.weakIntern(name); this.rmContext = rmContext; this.dispatcher = rmContext.getDispatcher(); this.handler = dispatcher.getEventHandler(); this.conf = config; - this.user = user; + this.user = StringInterner.weakIntern(user); this.queue = queue; this.submissionContext = submissionContext; this.scheduler = scheduler; @@ -441,7 +442,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext, } else { this.startTime = startTime; } - this.applicationType = applicationType; + this.applicationType = StringInterner.weakIntern(applicationType); this.applicationTags = applicationTags; this.amReqs = amReqs; if (submissionContext.getPriority() != null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index d3fbcb5d86..2d5c6a3ab5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -44,6 +44,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -1614,7 +1615,7 @@ public void transition(RMAppAttemptImpl appAttempt, ClusterMetrics.getMetrics().addAMRegisterDelay(delay); RMAppAttemptRegistrationEvent registrationEvent = (RMAppAttemptRegistrationEvent) event; - appAttempt.host = registrationEvent.getHost(); + appAttempt.host = StringInterner.weakIntern(registrationEvent.getHost()); appAttempt.rpcPort = registrationEvent.getRpcport(); appAttempt.originalTrackingUrl = sanitizeTrackingUrl(registrationEvent.getTrackingurl());