YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke

.gitignore

@@ -51,6 +51,7 @@ patchprocess/
 .history/
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
 
 #robotframework outputs
 log.html

hadoop-project/pom.xml

@@ -207,6 +207,7 @@
     <powermock.version>1.5.6</powermock.version>
     <solr.version>7.7.0</solr.version>
     <openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
+    <jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
   </properties>
 
   <dependencyManagement>
@@ -1165,6 +1166,11 @@
         <artifactId>jackson-dataformat-cbor</artifactId>
         <version>${jackson2.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.dataformat</groupId>
+        <artifactId>jackson-dataformat-yaml</artifactId>
+        <version>${jackson2.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-core</artifactId>
@@ -1728,6 +1734,49 @@
         <artifactId>jna</artifactId>
         <version>${jna.version}</version>
       </dependency>
+      <dependency>
+         <groupId>org.jsonschema2pojo</groupId>
+         <artifactId>jsonschema2pojo-core</artifactId>
+         <version>${jsonschema2pojo.version}</version>
+         <exclusions>
+           <exclusion>
+             <groupId>com.fasterxml.jackson.core</groupId>
+             <artifactId>*</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>com.fasterxml.jackson.dataformat</groupId>
+             <artifactId>*</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>org.codehaus.jackson</groupId>
+             <artifactId>*</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>com.google.code.gson</groupId>
+             <artifactId>gson</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>com.google.code.findbugs</groupId>
+             <artifactId>annotations</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>org.scala-lang</groupId>
+             <artifactId>scala-library</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>org.jsonschema2pojo</groupId>
+             <artifactId>jsonschema2pojo-scalagen</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>com.google.code.javaparser</groupId>
+             <artifactId>javaparser</artifactId>
+           </exclusion>
+           <exclusion>
+             <groupId>javax.validation</groupId>
+             <artifactId>validation-api</artifactId>
+           </exclusion>
+         </exclusions>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml

@@ -711,4 +711,9 @@
     <Class name="~org\.apache\.hadoop\.yarn\.server\.timelineservice\.reader\.TestTimelineReaderWebServicesHBaseStorage.*" />
     <Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
   </Match>
+
+  <!-- Generated files from JSON schema -->
+  <Match>
+    <Package name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema" />
+  </Match>
 </FindBugsFilter>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -250,6 +250,21 @@
       <scope>test</scope>
     </dependency>
 
+    <!-- Necessary to include jackson separately to avoid issues with
+         maven enforcer plugin in "hadoop-client-check-test-invariants"
+    -->
+     <dependency>
+         <groupId>org.jsonschema2pojo</groupId>
+         <artifactId>jsonschema2pojo-core</artifactId>
+     </dependency>
+     <dependency>
+         <groupId>com.fasterxml.jackson.core</groupId>
+         <artifactId>jackson-databind</artifactId>
+     </dependency>
+     <dependency>
+         <groupId>com.fasterxml.jackson.dataformat</groupId>
+         <artifactId>jackson-dataformat-yaml</artifactId>
+     </dependency>
   </dependencies>
 
   <build>
@@ -380,6 +395,22 @@
           </excludes>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.jsonschema2pojo</groupId>
+        <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+        <configuration>
+          <sourceDirectory>${basedir}/src/main/json_schema</sourceDirectory>
+          <targetPackage>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema</targetPackage>
+          <outputDirectory>${basedir}/src/main/java</outputDirectory>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>generate</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/GeneratePojos.java

@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Paths;
+
+import org.jsonschema2pojo.DefaultGenerationConfig;
+import org.jsonschema2pojo.GenerationConfig;
+import org.jsonschema2pojo.Jackson2Annotator;
+import org.jsonschema2pojo.SchemaGenerator;
+import org.jsonschema2pojo.SchemaMapper;
+import org.jsonschema2pojo.SchemaStore;
+import org.jsonschema2pojo.rules.RuleFactory;
+
+import com.sun.codemodel.JCodeModel;
+
+/**
+ * Helper class to re-generate java POJOs based on the JSON schema.
+ */
+public final class GeneratePojos {
+  @SuppressWarnings("checkstyle:linelength")
+  private static final String TARGET_PACKAGE =
+      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema";
+
+  private GeneratePojos() {
+    // no instances
+  }
+
+  public static void main(String[] args) throws IOException {
+    JCodeModel codeModel = new JCodeModel();
+    URL schemaURL = Paths.get(
+        "src/main/json_schema/MappingRulesDescription.json").toUri().toURL();
+
+    GenerationConfig config = new DefaultGenerationConfig() {
+      @Override
+      public boolean isGenerateBuilders() {
+        return false;
+      }
+
+      @Override
+      public boolean isUsePrimitives() {
+          return true;
+      }
+    };
+
+    SchemaMapper mapper =
+        new SchemaMapper(
+            new RuleFactory(config,
+                new Jackson2Annotator(config),
+                new SchemaStore()),
+            new SchemaGenerator());
+
+    mapper.generate(codeModel, "ignore", TARGET_PACKAGE, schemaURL);
+
+    codeModel.build(new File("src/main/java"));
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/package-info.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains classes that are related to the newer
+ * version of CS placement engine.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/json_schema/MappingRulesDescription.json

@@ -0,0 +1,36 @@
+{
+    "$schema": "http://json-schema.org/draft/2019-09/schema#",
+
+    "type" : "object",
+    "properties" : {
+        "rules" : {
+             "type" : "array",
+             "items": {
+                 "type": "object",
+                 "properties": {
+                     "type": { "type": "string",
+                          "enum": ["user", "group", "application"] },
+                     "matches": { "type": "string" },
+                     "policy": { "type": "string",
+                         "enum": [
+                              "specified",
+                              "reject",
+                              "defaultQueue",
+                              "user",
+                              "primaryGroup",
+                              "secondaryGroup",
+                              "primaryGroupUser",
+                              "secondaryGroupUser",
+                              "setDefaultQueue",
+                              "custom" ] },
+                     "parentQueue": { "type": "string" },
+                     "fallbackResult": { "type": "string",
+                         "enum": ["skip", "reject", "placeDefault"] },
+                     "create": { "type": "boolean" },
+                     "value": { "type": "string" },
+                     "customPlacement": { "type" : "string" }
+                }
+            }
+        }
+    }
+}