YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke
This commit is contained in:
Szilard Nemeth
parent
c392d9022a
commit
6e3d7050a9
1
.gitignore
vendored
1
.gitignore
vendored
@ -51,6 +51,7 @@ patchprocess/
|
||||
.history/
|
||||
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
|
||||
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
|
||||
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
|
||||
|
||||
#robotframework outputs
|
||||
log.html
|
||||
|
||||
@ -207,6 +207,7 @@
|
||||
<powermock.version>1.5.6</powermock.version>
|
||||
<solr.version>7.7.0</solr.version>
|
||||
<openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
|
||||
<jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
@ -1165,6 +1166,11 @@
|
||||
<artifactId>jackson-dataformat-cbor</artifactId>
|
||||
<version>${jackson2.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-yaml</artifactId>
|
||||
<version>${jackson2.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
@ -1728,6 +1734,49 @@
|
||||
<artifactId>jna</artifactId>
|
||||
<version>${jna.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.jsonschema2pojo</groupId>
|
||||
<artifactId>jsonschema2pojo-core</artifactId>
|
||||
<version>${jsonschema2pojo.version}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
<artifactId>*</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.gson</groupId>
|
||||
<artifactId>gson</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>annotations</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.scala-lang</groupId>
|
||||
<artifactId>scala-library</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.jsonschema2pojo</groupId>
|
||||
<artifactId>jsonschema2pojo-scalagen</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>com.google.code.javaparser</groupId>
|
||||
<artifactId>javaparser</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>javax.validation</groupId>
|
||||
<artifactId>validation-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
||||
@ -711,4 +711,9 @@
|
||||
<Class name="~org\.apache\.hadoop\.yarn\.server\.timelineservice\.reader\.TestTimelineReaderWebServicesHBaseStorage.*" />
|
||||
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
|
||||
</Match>
|
||||
|
||||
<!-- Generated files from JSON schema -->
|
||||
<Match>
|
||||
<Package name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema" />
|
||||
</Match>
|
||||
</FindBugsFilter>
|
||||
|
||||
@ -250,6 +250,21 @@
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Necessary to include jackson separately to avoid issues with
|
||||
maven enforcer plugin in "hadoop-client-check-test-invariants"
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>org.jsonschema2pojo</groupId>
|
||||
<artifactId>jsonschema2pojo-core</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.dataformat</groupId>
|
||||
<artifactId>jackson-dataformat-yaml</artifactId>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
@ -380,6 +395,22 @@
|
||||
</excludes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.jsonschema2pojo</groupId>
|
||||
<artifactId>jsonschema2pojo-maven-plugin</artifactId>
|
||||
<configuration>
|
||||
<sourceDirectory>${basedir}/src/main/json_schema</sourceDirectory>
|
||||
<targetPackage>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema</targetPackage>
|
||||
<outputDirectory>${basedir}/src/main/java</outputDirectory>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>generate</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
|
||||
@ -0,0 +1,76 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import org.jsonschema2pojo.DefaultGenerationConfig;
|
||||
import org.jsonschema2pojo.GenerationConfig;
|
||||
import org.jsonschema2pojo.Jackson2Annotator;
|
||||
import org.jsonschema2pojo.SchemaGenerator;
|
||||
import org.jsonschema2pojo.SchemaMapper;
|
||||
import org.jsonschema2pojo.SchemaStore;
|
||||
import org.jsonschema2pojo.rules.RuleFactory;
|
||||
|
||||
import com.sun.codemodel.JCodeModel;
|
||||
|
||||
/**
|
||||
* Helper class to re-generate java POJOs based on the JSON schema.
|
||||
*/
|
||||
public final class GeneratePojos {
|
||||
@SuppressWarnings("checkstyle:linelength")
|
||||
private static final String TARGET_PACKAGE =
|
||||
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema";
|
||||
|
||||
private GeneratePojos() {
|
||||
// no instances
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
JCodeModel codeModel = new JCodeModel();
|
||||
URL schemaURL = Paths.get(
|
||||
"src/main/json_schema/MappingRulesDescription.json").toUri().toURL();
|
||||
|
||||
GenerationConfig config = new DefaultGenerationConfig() {
|
||||
@Override
|
||||
public boolean isGenerateBuilders() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isUsePrimitives() {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
SchemaMapper mapper =
|
||||
new SchemaMapper(
|
||||
new RuleFactory(config,
|
||||
new Jackson2Annotator(config),
|
||||
new SchemaStore()),
|
||||
new SchemaGenerator());
|
||||
|
||||
mapper.generate(codeModel, "ignore", TARGET_PACKAGE, schemaURL);
|
||||
|
||||
codeModel.build(new File("src/main/java"));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,23 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Contains classes that are related to the newer
|
||||
* version of CS placement engine.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
|
||||
@ -0,0 +1,36 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft/2019-09/schema#",
|
||||
|
||||
"type" : "object",
|
||||
"properties" : {
|
||||
"rules" : {
|
||||
"type" : "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": { "type": "string",
|
||||
"enum": ["user", "group", "application"] },
|
||||
"matches": { "type": "string" },
|
||||
"policy": { "type": "string",
|
||||
"enum": [
|
||||
"specified",
|
||||
"reject",
|
||||
"defaultQueue",
|
||||
"user",
|
||||
"primaryGroup",
|
||||
"secondaryGroup",
|
||||
"primaryGroupUser",
|
||||
"secondaryGroupUser",
|
||||
"setDefaultQueue",
|
||||
"custom" ] },
|
||||
"parentQueue": { "type": "string" },
|
||||
"fallbackResult": { "type": "string",
|
||||
"enum": ["skip", "reject", "placeDefault"] },
|
||||
"create": { "type": "boolean" },
|
||||
"value": { "type": "string" },
|
||||
"customPlacement": { "type" : "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user