big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-7386. Duplicate Strings in various places in Yarn memory (misha@cloudera.com via rkanter)

Browse Source
This commit is contained in:
Robert Kanter 2017-11-09 12:07:46 -08:00
parent ba8136615a
commit a2c150a736
4 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
@ -392,7 +393,8 @@ private void initEnv() {
this.environment = new HashMap<String, String>();
for (StringStringMapProto c : list) {
this.environment.put(c.getKey(), c.getValue());
this.environment.put(StringInterner.weakIntern(c.getKey()),
StringInterner.weakIntern(c.getValue()));
}
}
@ -402,7 +404,10 @@ public void setEnvironment(final Map<String, String> env) {
return;
initEnv();
this.environment.clear();
this.environment.putAll(env);
for (Map.Entry<String, String> e : env.entrySet()) {
this.environment.put(StringInterner.weakIntern(e.getKey()),
StringInterner.weakIntern(e.getValue()));
}
}
private void addEnvToProto() {
@ -464,7 +469,7 @@ private void initApplicationACLs() {
for (ApplicationACLMapProto aclProto : list) {
this.applicationACLS.put(ProtoUtils.convertFromProtoFormat(aclProto
.getAccessType()), aclProto.getAcl());
.getAccessType()), StringInterner.weakIntern(aclProto.getAcl()));
}
}
@ -513,7 +518,10 @@ public void setApplicationACLs(
return;
initApplicationACLs();
this.applicationACLS.clear();
this.applicationACLS.putAll(appACLs);
for (Map.Entry<ApplicationAccessType, String> e : appACLs.entrySet()) {
this.applicationACLS.put(e.getKey(),
StringInterner.weakIntern(e.getValue()));
}
}
public ContainerRetryContext getContainerRetryContext() {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
View File

@ -181,7 +181,7 @@ public void setNodeHttpAddress(String nodeHttpAddress) {
builder.clearNodeHttpAddress();
return;
}
builder.setNodeHttpAddress(nodeHttpAddress);
builder.setNodeHttpAddress(nodeHttpAddress.intern());
}
@Override

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@ -425,12 +426,12 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
this.systemClock = SystemClock.getInstance();
this.applicationId = applicationId;
this.name = name;
this.name = StringInterner.weakIntern(name);
this.rmContext = rmContext;
this.dispatcher = rmContext.getDispatcher();
this.handler = dispatcher.getEventHandler();
this.conf = config;
this.user = user;
this.user = StringInterner.weakIntern(user);
this.queue = queue;
this.submissionContext = submissionContext;
this.scheduler = scheduler;
@ -441,7 +442,7 @@ public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
} else {
this.startTime = startTime;
}
this.applicationType = applicationType;
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
if (submissionContext.getPriority() != null) {

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -1614,7 +1615,7 @@ public void transition(RMAppAttemptImpl appAttempt,
ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
RMAppAttemptRegistrationEvent registrationEvent
= (RMAppAttemptRegistrationEvent) event;
appAttempt.host = registrationEvent.getHost();
appAttempt.host = StringInterner.weakIntern(registrationEvent.getHost());
appAttempt.rpcPort = registrationEvent.getRpcport();
appAttempt.originalTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());