big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-10779. Add option to disable lowercase conversion in GetApplicationsRequestPBImpl and ApplicationSubmissionContextPBImpl. Contributed by Peter Bacsko

Browse Source
This commit is contained in:
Szilard Nemeth 2021-05-21 18:04:24 +02:00
parent c8073811b2
commit 43e77caa71
6 changed files with 211 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1954,7 +1954,10 @@ public static boolean isAclEnabled(Configuration conf) {
false;
public static final String APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST =
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".username.whitelist";
public static final String APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".force-lowercase";
public static final boolean DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
true;
/** Enable switch for container log monitoring. */
public static final String NM_CONTAINER_LOG_MONITOR_ENABLED =
NM_PREFIX + "container-log-monitor.enable";

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
View File

@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@ -25,6 +28,7 @@
import org.apache.commons.lang3.Range;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@ -33,12 +37,14 @@
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
@Private
@Unstable
public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
private static volatile Boolean forceLowerCaseTags;
GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance();
GetApplicationsRequestProto.Builder builder = null;
boolean viaProto = false;
@ -56,11 +62,23 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder();
initLowerCaseConfig();
}
public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) {
this.proto = proto;
viaProto = true;
initLowerCaseConfig();
}
private static void initLowerCaseConfig() {
if (forceLowerCaseTags == null) {
Configuration conf = new Configuration();
forceLowerCaseTags =
conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
}
}
public synchronized GetApplicationsRequestProto getProto() {
@ -210,7 +228,8 @@ public synchronized void setApplicationTags(Set<String> tags) {
// Convert applicationTags to lower case and add
this.applicationTags = new HashSet<String>();
for (String tag : tags) {
this.applicationTags.add(StringUtils.toLowerCase(tag));
this.applicationTags.add(
forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
}
}
@ -406,4 +425,9 @@ public boolean equals(Object other) {
public String toString() {
return TextFormat.shortDebugString(getProto());
}
@VisibleForTesting
static void setForceLowerCaseTags(boolean convert) {
GetApplicationsRequestPBImpl.forceLowerCaseTags = convert;
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
View File

@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -29,6 +32,7 @@
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@ -50,13 +54,15 @@
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
@Private
@Unstable
public class ApplicationSubmissionContextPBImpl
extends ApplicationSubmissionContext {
private static volatile Boolean forceLowerCaseTags;
ApplicationSubmissionContextProto proto =
ApplicationSubmissionContextProto.getDefaultInstance();
ApplicationSubmissionContextProto.Builder builder = null;
@ -75,14 +81,26 @@ public class ApplicationSubmissionContextPBImpl
public ApplicationSubmissionContextPBImpl() {
builder = ApplicationSubmissionContextProto.newBuilder();
initLowerCaseConfig();
}
public ApplicationSubmissionContextPBImpl(
ApplicationSubmissionContextProto proto) {
this.proto = proto;
viaProto = true;
initLowerCaseConfig();
}
private static void initLowerCaseConfig() {
if (forceLowerCaseTags == null) {
Configuration conf = new Configuration();
forceLowerCaseTags =
conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
}
}
public synchronized ApplicationSubmissionContextProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
@ -287,7 +305,8 @@ public synchronized void setApplicationTags(Set<String> tags) {
// Convert applicationTags to lower case and add
this.applicationTags = new TreeSet<>();
for (String tag : tags) {
this.applicationTags.add(StringUtils.toLowerCase(tag));
this.applicationTags.add(
forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
}
}
@ -719,4 +738,9 @@ public synchronized void setApplicationSchedulingPropertiesMap(
this.schedulingProperties.clear();
this.schedulingProperties.putAll(schedulingPropertyMap);
}
@VisibleForTesting
static void setForceLowerCaseTags(boolean convert) {
ApplicationSubmissionContextPBImpl.forceLowerCaseTags = convert;
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -4773,4 +4773,10 @@
<name>yarn.resourcemanager.application.max-tag.length</name>
<value>100</value>
</property>
<property>
<description>Specifies whether application tags should be converted to lowercase or not.</description>
<name>yarn.resourcemanager.application-tag-based-placement.force-lowercase</name>
<value>true</value>
</property>
</configuration>

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/TestGetApplicationsRequestPBImpl.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestGetApplicationsRequestPBImpl {
@Parameter
@SuppressWarnings("checkstyle:visibilitymodifier")
public GetApplicationsRequestPBImpl impl;
@Test
public void testAppTagsLowerCaseConversionDefault() {
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertEquals(s, s.toLowerCase()));
}
@Test
public void testAppTagsLowerCaseConversionDisabled() {
GetApplicationsRequestPBImpl.setForceLowerCaseTags(false);
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertNotEquals(s, s.toLowerCase()));
}
@Test
public void testAppTagsLowerCaseConversionEnabled() {
GetApplicationsRequestPBImpl.setForceLowerCaseTags(true);
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertEquals(s, s.toLowerCase()));
}
@Parameters
public static Collection<Object[]> data() {
List<Object[]> list = new ArrayList<>();
list.add(new Object[] {new GetApplicationsRequestPBImpl()});
list.add(new Object[] {new GetApplicationsRequestPBImpl(
GetApplicationsRequestProto.newBuilder().build())});
return list;
}
}

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationSubmissionContextPBImpl.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.api.records.impl.pb;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class TestApplicationSubmissionContextPBImpl {
@Parameter
@SuppressWarnings("checkstyle:visibilitymodifier")
public ApplicationSubmissionContextPBImpl impl;
@Test
public void testAppTagsLowerCaseConversionDefault() {
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertEquals(s, s.toLowerCase()));
}
@Test
public void testAppTagsLowerCaseConversionDisabled() {
ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(false);
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertNotEquals(s, s.toLowerCase()));
}
@Test
public void testAppTagsLowerCaseConversionEnabled() {
ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(true);
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
impl.getApplicationTags().forEach(s ->
assertEquals(s, s.toLowerCase()));
}
@Parameters
public static Collection<Object[]> data() {
List<Object[]> list = new ArrayList<>();
list.add(new Object[] {new ApplicationSubmissionContextPBImpl()});
list.add(new Object[] {new ApplicationSubmissionContextPBImpl(
ApplicationSubmissionContextProto.newBuilder().build())});
return list;
}
}