YARN-6853. Add MySql Scripts for FederationStateStore. (Contributed by Giovanni Matteo Fumarola via curino)

(cherry picked from commit 874ddbf0b5b1d34aca70ee7fc303cbffdde67236)

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to create a new Database in MySQL for the Federation StateStore
+
+CREATE database FederationStateStore;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql

@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to generate all the stored procedures for the Federation StateStore in MySQL
+
+USE FederationStateStore
+
+DELIMITER //
+
+CREATE PROCEDURE sp_registerSubCluster(
+   IN subClusterId_IN varchar(256),
+   IN amRMServiceAddress_IN varchar(256),
+   IN clientRMServiceAddress_IN varchar(256),
+   IN rmAdminServiceAddress_IN varchar(256),
+   IN rmWebServiceAddress_IN varchar(256),
+   IN state_IN varchar(256),
+   IN lastStartTime_IN bigint, IN capability_IN varchar(6000),
+   OUT rowCount_OUT int)
+BEGIN
+   DELETE FROM membership WHERE (subClusterId = subClusterId_IN);
+   INSERT INTO membership (subClusterId, amRMServiceAddress, clientRMServiceAddress,
+          rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability)
+      VALUES (subClusterId_IN, amRMServiceAddress_IN, clientRMServiceAddress_IN,
+      rmAdminServiceAddress_IN, rmWebServiceAddress_IN, NOW(), state_IN, lastStartTime_IN, capability_IN);
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_deregisterSubCluster(
+    IN subClusterId_IN varchar(256),
+    IN state_IN varchar(64),
+    OUT rowCount_OUT int)
+BEGIN
+    UPDATE membership SET state = state_IN
+    WHERE (subClusterId = subClusterId_IN AND state != state_IN);
+    SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_subClusterHeartbeat(
+    IN subClusterId_IN varchar(256), IN state_IN varchar(64),
+    IN capability_IN varchar(6000), OUT rowCount_OUT int)
+BEGIN
+   UPDATE membership
+   SET capability = capability_IN,
+       state = state_IN,
+       lastHeartBeat = NOW()
+   WHERE subClusterId = subClusterId_IN;
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_getSubCluster(
+   IN subClusterId_IN varchar(256),
+   OUT amRMServiceAddress_OUT varchar(256),
+   OUT clientRMServiceAddress_OUT varchar(256),
+   OUT rmAdminServiceAddress_OUT varchar(256),
+   OUT rmWebServiceAddress_OUT varchar(256),
+   OUT lastHeartBeat_OUT datetime, OUT state_OUT varchar(64),
+   OUT lastStartTime_OUT bigint,
+   OUT capability_OUT varchar(6000))
+BEGIN
+   SELECT amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
+          lastHeartBeat, state, lastStartTime, capability
+   INTO amRMServiceAddress_OUT, clientRMServiceAddress_OUT, rmAdminServiceAddress_OUT,
+        rmWebServiceAddress_OUT, lastHeartBeat_OUT, state_OUT, lastStartTime_OUT, capability_OUT
+   FROM membership WHERE subClusterId = subClusterId_IN;
+END //
+
+CREATE PROCEDURE sp_getSubClusters()
+BEGIN
+   SELECT subClusterId, amRMServiceAddress, clientRMServiceAddress,
+          rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat,
+          state, lastStartTime, capability
+   FROM membership;
+END //
+
+CREATE PROCEDURE sp_addApplicationHomeSubCluster(
+   IN applicationId_IN varchar(64), IN homeSubCluster_IN varchar(256),
+   OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)
+BEGIN
+   INSERT INTO applicationsHomeSubCluster
+      (applicationId,homeSubCluster)
+      (SELECT applicationId_IN, homeSubCluster_IN
+       FROM applicationsHomeSubCluster
+       WHERE applicationId = applicationId_IN
+       HAVING COUNT(*) = 0 );
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+   SELECT homeSubCluster INTO storedHomeSubCluster_OUT
+   FROM applicationsHomeSubCluster
+   WHERE applicationId = applicationID_IN;
+END //
+
+CREATE PROCEDURE sp_updateApplicationHomeSubCluster(
+   IN applicationId_IN varchar(64),
+   IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)
+BEGIN
+   UPDATE applicationsHomeSubCluster
+     SET homeSubCluster = homeSubCluster_IN
+   WHERE applicationId = applicationId_IN;
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_getApplicationHomeSubCluster(
+   IN applicationId_IN varchar(64),
+   OUT homeSubCluster_OUT varchar(256))
+BEGIN
+   SELECT homeSubCluster INTO homeSubCluster_OUT
+   FROM applicationsHomeSubCluster
+   WHERE applicationId = applicationID_IN;
+END //
+
+CREATE PROCEDURE sp_getApplicationsHomeSubCluster()
+BEGIN
+   SELECT applicationId, homeSubCluster
+   FROM applicationsHomeSubCluster;
+END //
+
+CREATE PROCEDURE sp_deleteApplicationHomeSubCluster(
+   IN applicationId_IN varchar(64), OUT rowCount_OUT int)
+BEGIN
+   DELETE FROM applicationsHomeSubCluster
+   WHERE applicationId = applicationId_IN;
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_setPolicyConfiguration(
+   IN queue_IN varchar(256), IN policyType_IN varchar(256),
+   IN params_IN varbinary(32768), OUT rowCount_OUT int)
+BEGIN
+   DELETE FROM policies WHERE queue = queue_IN;
+   INSERT INTO policies (queue, policyType, params)
+   VALUES (queue_IN, policyType_IN, params_IN);
+   SELECT ROW_COUNT() INTO rowCount_OUT;
+END //
+
+CREATE PROCEDURE sp_getPoliciesConfigurations()
+BEGIN
+   SELECT queue, policyType, params FROM policies;
+END //
+
+CREATE PROCEDURE sp_getPolicyConfiguration(
+   IN queue_IN varchar(256), OUT policyType_OUT varchar(256),
+   OUT params_OUT varbinary(32768))
+BEGIN
+   SELECT policyType, params INTO policyType_OUT, params_OUT
+   FROM policies WHERE queue = queue_IN;
+END //
+
+DELIMITER ;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to generate all the tables for the Federation StateStore in MySQL
+
+USE FederationStateStore
+
+CREATE TABLE applicationsHomeSubCluster(
+   applicationId varchar(64) NOT NULL,
+   subClusterId varchar(256) NULL,
+   CONSTRAINT pk_applicationId PRIMARY KEY (applicationId)
+);
+
+CREATE TABLE membership(
+   subClusterId varchar(256) NOT NULL,
+   amRMServiceAddress varchar(256) NOT NULL,
+   clientRMServiceAddress varchar(256) NOT NULL,
+   rmAdminServiceAddress varchar(256) NOT NULL,
+   rmWebServiceAddress varchar(256) NOT NULL,
+   lastHeartBeat datetime NOT NULL,
+   state varchar(32) NOT NULL,
+   lastStartTime bigint NULL,
+   capability varchar(6000),
+   CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId)
+);
+
+CREATE TABLE policies(
+   queue varchar(256) NOT NULL,
+   policyType varchar(256) NOT NULL,
+   params varbinary(32768),
+   CONSTRAINT pk_queue PRIMARY KEY (queue)
+);

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql

@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to create a new User in MySQL for the Federation StateStore
+
+CREATE USER 'FederationUser'@'%' IDENTIFIED BY 'FederationPassword';
+
+GRANT ALL PRIVILEGES ON FederationStateStore.* TO 'FederationUser'@'%';
+
+FLUSH PRIVILEGES;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop the Federation StateStore in MySQL
+
+DROP DATABASE FederationStateStore;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop all the stored procedures for the Federation StateStore in MySQL
+
+USE FederationStateStore
+
+DROP PROCEDURE sp_registerSubCluster;
+
+DROP PROCEDURE sp_deregisterSubCluster;
+
+DROP PROCEDURE sp_subClusterHeartbeat;
+
+DROP PROCEDURE sp_getSubCluster;
+
+DROP PROCEDURE sp_getSubClusters;
+
+DROP PROCEDURE sp_addApplicationHomeSubCluster;
+
+DROP PROCEDURE sp_updateApplicationHomeSubCluster;
+
+DROP PROCEDURE sp_getApplicationHomeSubCluster;
+
+DROP PROCEDURE sp_getApplicationsHomeSubCluster;
+
+DROP PROCEDURE sp_deleteApplicationHomeSubCluster;
+
+DROP PROCEDURE sp_setPolicyConfiguration;
+
+DROP PROCEDURE sp_getPolicyConfiguration;
+
+DROP PROCEDURE sp_getPoliciesConfigurations;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop all the tables from the Federation StateStore in MySQL
+
+USE FederationStateStore
+
+DROP TABLE applicationsHomeSubCluster;
+
+DROP TABLE membership;
+
+DROP TABLE policies;

hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql

@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- Script to drop the user from Federation StateStore in MySQL
+
+DROP USER 'FederationUser'@'%';

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md

@@ -161,8 +161,8 @@ These are common configurations that should appear in the **conf/yarn-site.xml**
 |:---- |:---- |
 |`yarn.federation.enabled` | `true` | Whether federation is enabled or not |
 |`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. |
-|`yarn.federation.state-store.sql.url` | `jdbc:sqlserver://<host>:<port>;databaseName=FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
+|`yarn.federation.state-store.sql.url` | `jdbc:mysql://<host>:<port>/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
-|`yarn.federation.state-store.sql.jdbc-class` | `com.microsoft.sqlserver.jdbc.SQLServerDataSource` | For SQLFederationStateStore the jdbc class to use. |
+|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. |
 |`yarn.federation.state-store.sql.username` | `<dbuser>` | For SQLFederationStateStore the username for the DB connection. |
 |`yarn.federation.state-store.sql.password` | `<dbpass>` | For SQLFederationStateStore the password for the DB connection. |
 |`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
@@ -238,7 +238,19 @@ Optional:
 
 ###State-Store:
 
-Currently, the only supported implementation of the state-store is Microsoft SQL Server. After [setting up](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) such an instance of SQL Server, set up the database for use by the federation system. This can be done by running the following SQL files in the database: **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql** and **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreTables.sql**
+Currently, we support only SQL based implementation of state-store (ZooKeeper is in the works), i.e. either MySQL or Microsoft SQL Server.
+
+For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
+Then the DB schema is created by executing the following SQL scripts in the database:
+1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**.
+2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**.
+3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**.
+4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**.
+In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database.
+**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
+
+For SQL-Server, the process is similar, but the jdbc driver is already included in the pom (license allows it).
+SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
 
 Running a Sample Job
 --------------------