big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-6853. Add MySql Scripts for FederationStateStore. (Contributed by Giovanni Matteo Fumarola via curino)

Browse Source 
(cherry picked from commit 874ddbf0b5b1d34aca70ee7fc303cbffdde67236)
This commit is contained in:
Carlo Curino 2017-08-01 17:18:20 -07:00
parent 894ff83dd6
commit 9625a030de
9 changed files with 386 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to create a new Database in MySQL for the Federation StateStore
CREATE database FederationStateStore;

162
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql Normal file
View File

@ -0,0 +1,162 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to generate all the stored procedures for the Federation StateStore in MySQL
USE FederationStateStore
DELIMITER //
CREATE PROCEDURE sp_registerSubCluster(
IN subClusterId_IN varchar(256),
IN amRMServiceAddress_IN varchar(256),
IN clientRMServiceAddress_IN varchar(256),
IN rmAdminServiceAddress_IN varchar(256),
IN rmWebServiceAddress_IN varchar(256),
IN state_IN varchar(256),
IN lastStartTime_IN bigint, IN capability_IN varchar(6000),
OUT rowCount_OUT int)
BEGIN
DELETE FROM membership WHERE (subClusterId = subClusterId_IN);
INSERT INTO membership (subClusterId, amRMServiceAddress, clientRMServiceAddress,
rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability)
VALUES (subClusterId_IN, amRMServiceAddress_IN, clientRMServiceAddress_IN,
rmAdminServiceAddress_IN, rmWebServiceAddress_IN, NOW(), state_IN, lastStartTime_IN, capability_IN);
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_deregisterSubCluster(
IN subClusterId_IN varchar(256),
IN state_IN varchar(64),
OUT rowCount_OUT int)
BEGIN
UPDATE membership SET state = state_IN
WHERE (subClusterId = subClusterId_IN AND state != state_IN);
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_subClusterHeartbeat(
IN subClusterId_IN varchar(256), IN state_IN varchar(64),
IN capability_IN varchar(6000), OUT rowCount_OUT int)
BEGIN
UPDATE membership
SET capability = capability_IN,
state = state_IN,
lastHeartBeat = NOW()
WHERE subClusterId = subClusterId_IN;
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_getSubCluster(
IN subClusterId_IN varchar(256),
OUT amRMServiceAddress_OUT varchar(256),
OUT clientRMServiceAddress_OUT varchar(256),
OUT rmAdminServiceAddress_OUT varchar(256),
OUT rmWebServiceAddress_OUT varchar(256),
OUT lastHeartBeat_OUT datetime, OUT state_OUT varchar(64),
OUT lastStartTime_OUT bigint,
OUT capability_OUT varchar(6000))
BEGIN
SELECT amRMServiceAddress, clientRMServiceAddress, rmAdminServiceAddress, rmWebServiceAddress,
lastHeartBeat, state, lastStartTime, capability
INTO amRMServiceAddress_OUT, clientRMServiceAddress_OUT, rmAdminServiceAddress_OUT,
rmWebServiceAddress_OUT, lastHeartBeat_OUT, state_OUT, lastStartTime_OUT, capability_OUT
FROM membership WHERE subClusterId = subClusterId_IN;
END //
CREATE PROCEDURE sp_getSubClusters()
BEGIN
SELECT subClusterId, amRMServiceAddress, clientRMServiceAddress,
rmAdminServiceAddress, rmWebServiceAddress, lastHeartBeat,
state, lastStartTime, capability
FROM membership;
END //
CREATE PROCEDURE sp_addApplicationHomeSubCluster(
IN applicationId_IN varchar(64), IN homeSubCluster_IN varchar(256),
OUT storedHomeSubCluster_OUT varchar(256), OUT rowCount_OUT int)
BEGIN
INSERT INTO applicationsHomeSubCluster
(applicationId,homeSubCluster)
(SELECT applicationId_IN, homeSubCluster_IN
FROM applicationsHomeSubCluster
WHERE applicationId = applicationId_IN
HAVING COUNT(*) = 0 );
SELECT ROW_COUNT() INTO rowCount_OUT;
SELECT homeSubCluster INTO storedHomeSubCluster_OUT
FROM applicationsHomeSubCluster
WHERE applicationId = applicationID_IN;
END //
CREATE PROCEDURE sp_updateApplicationHomeSubCluster(
IN applicationId_IN varchar(64),
IN homeSubCluster_IN varchar(256), OUT rowCount_OUT int)
BEGIN
UPDATE applicationsHomeSubCluster
SET homeSubCluster = homeSubCluster_IN
WHERE applicationId = applicationId_IN;
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_getApplicationHomeSubCluster(
IN applicationId_IN varchar(64),
OUT homeSubCluster_OUT varchar(256))
BEGIN
SELECT homeSubCluster INTO homeSubCluster_OUT
FROM applicationsHomeSubCluster
WHERE applicationId = applicationID_IN;
END //
CREATE PROCEDURE sp_getApplicationsHomeSubCluster()
BEGIN
SELECT applicationId, homeSubCluster
FROM applicationsHomeSubCluster;
END //
CREATE PROCEDURE sp_deleteApplicationHomeSubCluster(
IN applicationId_IN varchar(64), OUT rowCount_OUT int)
BEGIN
DELETE FROM applicationsHomeSubCluster
WHERE applicationId = applicationId_IN;
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_setPolicyConfiguration(
IN queue_IN varchar(256), IN policyType_IN varchar(256),
IN params_IN varbinary(32768), OUT rowCount_OUT int)
BEGIN
DELETE FROM policies WHERE queue = queue_IN;
INSERT INTO policies (queue, policyType, params)
VALUES (queue_IN, policyType_IN, params_IN);
SELECT ROW_COUNT() INTO rowCount_OUT;
END //
CREATE PROCEDURE sp_getPoliciesConfigurations()
BEGIN
SELECT queue, policyType, params FROM policies;
END //
CREATE PROCEDURE sp_getPolicyConfiguration(
IN queue_IN varchar(256), OUT policyType_OUT varchar(256),
OUT params_OUT varbinary(32768))
BEGIN
SELECT policyType, params INTO policyType_OUT, params_OUT
FROM policies WHERE queue = queue_IN;
END //
DELIMITER ;

47
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreTables.sql Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to generate all the tables for the Federation StateStore in MySQL
USE FederationStateStore
CREATE TABLE applicationsHomeSubCluster(
applicationId varchar(64) NOT NULL,
subClusterId varchar(256) NULL,
CONSTRAINT pk_applicationId PRIMARY KEY (applicationId)
);
CREATE TABLE membership(
subClusterId varchar(256) NOT NULL,
amRMServiceAddress varchar(256) NOT NULL,
clientRMServiceAddress varchar(256) NOT NULL,
rmAdminServiceAddress varchar(256) NOT NULL,
rmWebServiceAddress varchar(256) NOT NULL,
lastHeartBeat datetime NOT NULL,
state varchar(32) NOT NULL,
lastStartTime bigint NULL,
capability varchar(6000),
CONSTRAINT pk_subClusterId PRIMARY KEY (subClusterId)
);
CREATE TABLE policies(
queue varchar(256) NOT NULL,
policyType varchar(256) NOT NULL,
params varbinary(32768),
CONSTRAINT pk_queue PRIMARY KEY (queue)
);

25
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/FederationStateStoreUser.sql Normal file
View File

@ -0,0 +1,25 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to create a new User in MySQL for the Federation StateStore
CREATE USER 'FederationUser'@'%' IDENTIFIED BY 'FederationPassword';
GRANT ALL PRIVILEGES ON FederationStateStore.* TO 'FederationUser'@'%';
FLUSH PRIVILEGES;

21
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropDatabase.sql Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to drop the Federation StateStore in MySQL
DROP DATABASE FederationStateStore;

47
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropStoreProcedures.sql Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to drop all the stored procedures for the Federation StateStore in MySQL
USE FederationStateStore
DROP PROCEDURE sp_registerSubCluster;
DROP PROCEDURE sp_deregisterSubCluster;
DROP PROCEDURE sp_subClusterHeartbeat;
DROP PROCEDURE sp_getSubCluster;
DROP PROCEDURE sp_getSubClusters;
DROP PROCEDURE sp_addApplicationHomeSubCluster;
DROP PROCEDURE sp_updateApplicationHomeSubCluster;
DROP PROCEDURE sp_getApplicationHomeSubCluster;
DROP PROCEDURE sp_getApplicationsHomeSubCluster;
DROP PROCEDURE sp_deleteApplicationHomeSubCluster;
DROP PROCEDURE sp_setPolicyConfiguration;
DROP PROCEDURE sp_getPolicyConfiguration;
DROP PROCEDURE sp_getPoliciesConfigurations;

27
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropTables.sql Normal file
View File

@ -0,0 +1,27 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to drop all the tables from the Federation StateStore in MySQL
USE FederationStateStore
DROP TABLE applicationsHomeSubCluster;
DROP TABLE membership;
DROP TABLE policies;

21
hadoop-yarn-project/hadoop-yarn/bin/FederationStateStore/MySQL/dropUser.sql Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-- Script to drop the user from Federation StateStore in MySQL
DROP USER 'FederationUser'@'%';

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
View File

@ -161,8 +161,8 @@ These are common configurations that should appear in the **conf/yarn-site.xml**
|:---- |:---- |
|`yarn.federation.enabled` | `true` | Whether federation is enabled or not |
|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. |
|`yarn.federation.state-store.sql.url` | `jdbc:sqlserver://<host>:<port>;databaseName=FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
|`yarn.federation.state-store.sql.jdbc-class` | `com.microsoft.sqlserver.jdbc.SQLServerDataSource` | For SQLFederationStateStore the jdbc class to use. |
|`yarn.federation.state-store.sql.url` | `jdbc:mysql://<host>:<port>/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. |
|`yarn.federation.state-store.sql.username` | `<dbuser>` | For SQLFederationStateStore the username for the DB connection. |
|`yarn.federation.state-store.sql.password` | `<dbpass>` | For SQLFederationStateStore the password for the DB connection. |
|`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
@ -238,7 +238,19 @@ Optional:
###State-Store:
Currently, the only supported implementation of the state-store is Microsoft SQL Server. After [setting up](https://www.microsoft.com/en-us/sql-server/sql-server-downloads) such an instance of SQL Server, set up the database for use by the federation system. This can be done by running the following SQL files in the database: **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreProcs.sql** and **sbin/FederationStateStore/SQLServer/FederationStateStoreStoreTables.sql**
Currently, we support only SQL based implementation of state-store (ZooKeeper is in the works), i.e. either MySQL or Microsoft SQL Server.
For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
Then the DB schema is created by executing the following SQL scripts in the database:
1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**.
2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**.
3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**.
4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**.
In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database.
**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
For SQL-Server, the process is similar, but the jdbc driver is already included in the pom (license allows it).
SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
Running a Sample Job
--------------------