From 1a9faf123d920b7b0170a18b51cb274dca6aaed6 Mon Sep 17 00:00:00 2001 From: slfan1989 <55643692+slfan1989@users.noreply.github.com> Date: Thu, 6 Oct 2022 04:25:20 +0800 Subject: [PATCH] YARN-11313. [Federation] Add SQLServer Script and Supported DB Version in Federation.md. (#4927) --- .../FederationStateStoreDatabase.sql | 26 +++++++ .../SQLServer/FederationStateStoreUser.sql | 31 ++++++++ .../SQLServer/dropDatabase.sql | 23 ++++++ .../SQLServer/dropStoreProcedures.sql | 76 +++++++++++++++++++ .../SQLServer/dropTables.sql | 34 +++++++++ .../SQLServer/dropUser.sql | 22 ++++++ .../src/site/markdown/Federation.md | 18 ++++- 7 files changed, 229 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql new file mode 100644 index 0000000000..ae0a1c5b77 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreDatabase.sql @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to create a new Database in SQLServer for the Federation StateStore + +IF DB_ID ( '[FederationStateStore]') IS NOT NULL + DROP DATABASE [FederationStateStore]; +GO + +CREATE database FederationStateStore; +GO diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql new file mode 100644 index 0000000000..3f9553fbe3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/FederationStateStoreUser.sql @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to create a new User in SQLServer for the Federation StateStore + +USE [FederationStateStore] +GO + +CREATE LOGIN 'FederationUser' with password = 'FederationPassword', default_database=[FederationStateStore] ; +GO + +CREATE USER 'FederationUser' FOR LOGIN 'FederationUser' WITH default_schema=dbo; +GO + +EXEC sp_addrolemember 'db_owner', 'FederationUser'; +GO diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql new file mode 100644 index 0000000000..8d434ab4d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropDatabase.sql @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop the Federation StateStore in SQLServer + +IF DB_ID ( '[FederationStateStore]') IS NOT NULL + DROP DATABASE [FederationStateStore]; +GO diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql new file mode 100644 index 0000000000..6204df2f41 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropStoreProcedures.sql @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop all the stored procedures for the Federation StateStore in SQLServer + +USE [FederationStateStore] +GO + +DROP PROCEDURE IF EXISTS [sp_addApplicationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_updateApplicationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_getApplicationsHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_getApplicationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_deleteApplicationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_registerSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_getSubClusters]; +GO + +DROP PROCEDURE IF EXISTS [sp_getSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_subClusterHeartbeat]; +GO + +DROP PROCEDURE IF EXISTS [sp_deregisterSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_setPolicyConfiguration]; +GO + +DROP PROCEDURE IF EXISTS [sp_getPolicyConfiguration]; +GO + +DROP PROCEDURE IF EXISTS [sp_getPoliciesConfigurations]; +GO + +DROP PROCEDURE IF EXISTS [sp_addApplicationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_updateReservationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_getReservationsHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_getReservationHomeSubCluster]; +GO + +DROP PROCEDURE IF EXISTS [sp_deleteReservationHomeSubCluster]; +GO diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql new file mode 100644 index 0000000000..9bcacb7f88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropTables.sql @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop all the tables from the Federation StateStore in SQLServer + +USE [FederationStateStore] +GO + +DROP TABLE [applicationsHomeSubCluster]; +GO + +DROP TABLE [membership]; +GO + +DROP TABLE [policies]; +GO + +DROP TABLE [reservationsHomeSubCluster]; +GO diff --git a/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql new file mode 100644 index 0000000000..6d5203a52e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/SQLServer/dropUser.sql @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Script to drop the user from Federation StateStore in MySQL + +DROP USER 'FederationUser'; +GO \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md index 3f2e05aee6..eae1d8d6fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md @@ -191,7 +191,9 @@ SQL: one must setup the following parameters: |`yarn.federation.state-store.sql.username` | `` | For SQLFederationStateStore the username for the DB connection. | |`yarn.federation.state-store.sql.password` | `` | For SQLFederationStateStore the password for the DB connection. | -We provide scripts for MySQL and Microsoft SQL Server. +We provide scripts for **MySQL** and **Microsoft SQL Server**. + +> MySQL For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH. Then the DB schema is created by executing the following SQL scripts in the database: @@ -205,9 +207,23 @@ In the same directory we provide scripts to drop the Stored Procedures, the Tabl **Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password. +**The versions supported by MySQL are MySQL 5.7 and above:** + +1. MySQL 5.7 +2. MySQL 8.0 + +> Microsoft SQL Server + For SQL-Server, the process is similar, but the jdbc driver is already included. SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**. +**The versions supported by SQL-Server are SQL Server 2008 R2 and above:** + +1. SQL Server 2008 R2 Enterprise +2. SQL Server 2012 Enterprise +3. SQL Server 2016 Enterprise +4. SQL Server 2017 Enterprise +5. SQL Server 2019 Enterprise ####Optional: