YARN-10779. Add option to disable lowercase conversion in GetApplicationsRequestPBImpl and ApplicationSubmissionContextPBImpl. Contributed by Peter Bacsko
This commit is contained in:
parent
c8073811b2
commit
43e77caa71
@ -1954,7 +1954,10 @@ public static boolean isAclEnabled(Configuration conf) {
|
||||
false;
|
||||
public static final String APPLICATION_TAG_BASED_PLACEMENT_USER_WHITELIST =
|
||||
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".username.whitelist";
|
||||
|
||||
public static final String APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
|
||||
APPLICATION_TAG_BASED_PLACEMENT_PREFIX + ".force-lowercase";
|
||||
public static final boolean DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION =
|
||||
true;
|
||||
/** Enable switch for container log monitoring. */
|
||||
public static final String NM_CONTAINER_LOG_MONITOR_ENABLED =
|
||||
NM_PREFIX + "container-log-monitor.enable";
|
||||
|
@ -18,6 +18,9 @@
|
||||
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
|
||||
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
@ -25,6 +28,7 @@
|
||||
import org.apache.commons.lang3.Range;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
|
||||
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
|
||||
@ -33,12 +37,14 @@
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
|
||||
private static volatile Boolean forceLowerCaseTags;
|
||||
|
||||
GetApplicationsRequestProto proto = GetApplicationsRequestProto.getDefaultInstance();
|
||||
GetApplicationsRequestProto.Builder builder = null;
|
||||
boolean viaProto = false;
|
||||
@ -56,11 +62,23 @@ public class GetApplicationsRequestPBImpl extends GetApplicationsRequest {
|
||||
|
||||
public GetApplicationsRequestPBImpl() {
|
||||
builder = GetApplicationsRequestProto.newBuilder();
|
||||
initLowerCaseConfig();
|
||||
}
|
||||
|
||||
public GetApplicationsRequestPBImpl(GetApplicationsRequestProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
initLowerCaseConfig();
|
||||
}
|
||||
|
||||
private static void initLowerCaseConfig() {
|
||||
if (forceLowerCaseTags == null) {
|
||||
Configuration conf = new Configuration();
|
||||
|
||||
forceLowerCaseTags =
|
||||
conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
|
||||
DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized GetApplicationsRequestProto getProto() {
|
||||
@ -210,7 +228,8 @@ public synchronized void setApplicationTags(Set<String> tags) {
|
||||
// Convert applicationTags to lower case and add
|
||||
this.applicationTags = new HashSet<String>();
|
||||
for (String tag : tags) {
|
||||
this.applicationTags.add(StringUtils.toLowerCase(tag));
|
||||
this.applicationTags.add(
|
||||
forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -406,4 +425,9 @@ public boolean equals(Object other) {
|
||||
public String toString() {
|
||||
return TextFormat.shortDebugString(getProto());
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static void setForceLowerCaseTags(boolean convert) {
|
||||
GetApplicationsRequestPBImpl.forceLowerCaseTags = convert;
|
||||
}
|
||||
}
|
||||
|
@ -18,6 +18,9 @@
|
||||
|
||||
package org.apache.hadoop.yarn.api.records.impl.pb;
|
||||
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
|
||||
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
@ -29,6 +32,7 @@
|
||||
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
|
||||
@ -50,13 +54,15 @@
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
|
||||
import org.apache.hadoop.thirdparty.protobuf.TextFormat;
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
public class ApplicationSubmissionContextPBImpl
|
||||
extends ApplicationSubmissionContext {
|
||||
private static volatile Boolean forceLowerCaseTags;
|
||||
|
||||
ApplicationSubmissionContextProto proto =
|
||||
ApplicationSubmissionContextProto.getDefaultInstance();
|
||||
ApplicationSubmissionContextProto.Builder builder = null;
|
||||
@ -75,12 +81,24 @@ public class ApplicationSubmissionContextPBImpl
|
||||
|
||||
public ApplicationSubmissionContextPBImpl() {
|
||||
builder = ApplicationSubmissionContextProto.newBuilder();
|
||||
initLowerCaseConfig();
|
||||
}
|
||||
|
||||
public ApplicationSubmissionContextPBImpl(
|
||||
ApplicationSubmissionContextProto proto) {
|
||||
this.proto = proto;
|
||||
viaProto = true;
|
||||
initLowerCaseConfig();
|
||||
}
|
||||
|
||||
private static void initLowerCaseConfig() {
|
||||
if (forceLowerCaseTags == null) {
|
||||
Configuration conf = new Configuration();
|
||||
|
||||
forceLowerCaseTags =
|
||||
conf.getBoolean(APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION,
|
||||
DEFAULT_APPLICATION_TAG_FORCE_LOWERCASE_CONVERSION);
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized ApplicationSubmissionContextProto getProto() {
|
||||
@ -287,7 +305,8 @@ public synchronized void setApplicationTags(Set<String> tags) {
|
||||
// Convert applicationTags to lower case and add
|
||||
this.applicationTags = new TreeSet<>();
|
||||
for (String tag : tags) {
|
||||
this.applicationTags.add(StringUtils.toLowerCase(tag));
|
||||
this.applicationTags.add(
|
||||
forceLowerCaseTags ? StringUtils.toLowerCase(tag) : tag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -719,4 +738,9 @@ public synchronized void setApplicationSchedulingPropertiesMap(
|
||||
this.schedulingProperties.clear();
|
||||
this.schedulingProperties.putAll(schedulingPropertyMap);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static void setForceLowerCaseTags(boolean convert) {
|
||||
ApplicationSubmissionContextPBImpl.forceLowerCaseTags = convert;
|
||||
}
|
||||
}
|
||||
|
@ -4773,4 +4773,10 @@
|
||||
<name>yarn.resourcemanager.application.max-tag.length</name>
|
||||
<value>100</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Specifies whether application tags should be converted to lowercase or not.</description>
|
||||
<name>yarn.resourcemanager.application-tag-based-placement.force-lowercase</name>
|
||||
<value>true</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameter;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestGetApplicationsRequestPBImpl {
|
||||
@Parameter
|
||||
@SuppressWarnings("checkstyle:visibilitymodifier")
|
||||
public GetApplicationsRequestPBImpl impl;
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionDefault() {
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionDisabled() {
|
||||
GetApplicationsRequestPBImpl.setForceLowerCaseTags(false);
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertNotEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionEnabled() {
|
||||
GetApplicationsRequestPBImpl.setForceLowerCaseTags(true);
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Parameters
|
||||
public static Collection<Object[]> data() {
|
||||
List<Object[]> list = new ArrayList<>();
|
||||
list.add(new Object[] {new GetApplicationsRequestPBImpl()});
|
||||
list.add(new Object[] {new GetApplicationsRequestPBImpl(
|
||||
GetApplicationsRequestProto.newBuilder().build())});
|
||||
|
||||
return list;
|
||||
}
|
||||
}
|
@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.api.records.impl.pb;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
|
||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationSubmissionContextProto;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.junit.runners.Parameterized;
|
||||
import org.junit.runners.Parameterized.Parameter;
|
||||
import org.junit.runners.Parameterized.Parameters;
|
||||
|
||||
@RunWith(Parameterized.class)
|
||||
public class TestApplicationSubmissionContextPBImpl {
|
||||
@Parameter
|
||||
@SuppressWarnings("checkstyle:visibilitymodifier")
|
||||
public ApplicationSubmissionContextPBImpl impl;
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionDefault() {
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionDisabled() {
|
||||
ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(false);
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertNotEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAppTagsLowerCaseConversionEnabled() {
|
||||
ApplicationSubmissionContextPBImpl.setForceLowerCaseTags(true);
|
||||
impl.setApplicationTags(Sets.newHashSet("ABcd", "efgH"));
|
||||
impl.getApplicationTags().forEach(s ->
|
||||
assertEquals(s, s.toLowerCase()));
|
||||
}
|
||||
|
||||
@Parameters
|
||||
public static Collection<Object[]> data() {
|
||||
List<Object[]> list = new ArrayList<>();
|
||||
list.add(new Object[] {new ApplicationSubmissionContextPBImpl()});
|
||||
list.add(new Object[] {new ApplicationSubmissionContextPBImpl(
|
||||
ApplicationSubmissionContextProto.newBuilder().build())});
|
||||
|
||||
return list;
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user