big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph

Browse Source
This commit is contained in:
Sunil G 2019-10-09 15:53:14 +05:30
parent 1d27930407
commit be901f4962
13 changed files with 258 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
View File

@ -65,6 +65,12 @@ void logAndApplyMutation(UserGroupInformation user, SchedConfUpdateInfo
*/
Configuration getConfiguration();
/**
* Get the last updated scheduler config version.
* @return Last updated scheduler config version.
*/
long getConfigVersion() throws Exception;
void formatConfigurationInStore(Configuration conf) throws Exception;
/**

43
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
View File

@ -29,6 +29,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@ -62,6 +63,7 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
private volatile Configuration schedConf;
private volatile Configuration oldConf;
private Path tempConfigPath;
private Path configVersionFile;
@Override
public void initialize(Configuration conf, Configuration vSchedConf,
@ -99,9 +101,17 @@ public boolean accept(Path path) {
}
}
this.configVersionFile = new Path(schedulerConfPathStr, "ConfigVersion");
if (!fileSystem.exists(configVersionFile)) {
fileSystem.createNewFile(configVersionFile);
writeConfigVersion(0L);
}
// create capacity-schedule.xml.ts file if not existing
if (this.getConfigFileInputStream() == null) {
writeConfigurationToFileSystem(vSchedConf);
long configVersion = getConfigVersion() + 1L;
writeConfigVersion(configVersion);
}
this.schedConf = this.getConfigurationFromFileSystem();
@ -141,6 +151,8 @@ public void confirmMutation(boolean isValid) throws Exception {
}
if (isValid) {
finalizeFileSystemFile();
long configVersion = getConfigVersion() + 1L;
writeConfigVersion(configVersion);
} else {
schedConf = oldConf;
removeTmpConfigFile();
@ -158,7 +170,15 @@ private void finalizeFileSystemFile() throws IOException {
@Override
public void format() throws Exception {
fileSystem.delete(schedulerConfDir, true);
FileStatus[] fileStatuses = fileSystem.listStatus(this.schedulerConfDir,
this.configFilePathFilter);
if (fileStatuses == null) {
return;
}
for (int i = 0; i < fileStatuses.length; i++) {
fileSystem.delete(fileStatuses[i].getPath(), false);
LOG.info("delete config file " + fileStatuses[i].getPath());
}
}
private Path getFinalConfigPath(Path tempPath) {
@ -222,6 +242,27 @@ private Path getLatestConfigPath() throws IOException {
return fileStatuses[fileStatuses.length - 1].getPath();
}
private void writeConfigVersion(long configVersion) throws IOException {
try (FSDataOutputStream out = fileSystem.create(configVersionFile, true)) {
out.writeLong(configVersion);
} catch (IOException e) {
LOG.info("Failed to write config version at {}", configVersionFile, e);
throw e;
}
}
@Override
public long getConfigVersion() throws Exception {
try (FSDataInputStream in = fileSystem.open(configVersionFile)) {
return in.readLong();
} catch (IOException e) {
LOG.info("Failed to read config version at {}", configVersionFile, e);
throw e;
}
}
@VisibleForTesting
private Path writeTmpConfig(Configuration vSchedConf) throws IOException {
long start = Time.monotonicNow();

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
View File

@ -33,11 +33,13 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
private Configuration schedConf;
private LogMutation pendingMutation;
private long configVersion;
@Override
public void initialize(Configuration conf, Configuration schedConf,
RMContext rmContext) {
this.schedConf = schedConf;
this.configVersion = 1L;
}
@Override
@ -56,6 +58,7 @@ public void confirmMutation(boolean isValid) {
schedConf.set(kv.getKey(), kv.getValue());
}
}
this.configVersion = this.configVersion + 1L;
}
pendingMutation = null;
}
@ -70,6 +73,11 @@ public synchronized Configuration retrieve() {
return schedConf;
}
@Override
public long getConfigVersion() {
return configVersion;
}
@Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
// Unimplemented.

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
View File

@ -68,8 +68,11 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
private static final String DB_NAME = "yarn-conf-store";
private static final String LOG_KEY = "log";
private static final String VERSION_KEY = "version";
private static final String CONF_VERSION_NAME = "conf-version-store";
private static final String CONF_VERSION_KEY = "conf-version";
private DB db;
private DB versiondb;
private long maxLogs;
private Configuration conf;
private LogMutation pendingMutation;
@ -102,11 +105,11 @@ public void initialize(Configuration config, Configuration schedConf,
public void format() throws Exception {
close();
FileSystem fs = FileSystem.getLocal(conf);
fs.delete(getStorageDir(), true);
fs.delete(getStorageDir(DB_NAME), true);
}
private void initDatabase(Configuration config) throws Exception {
Path storeRoot = createStorageDir();
Path storeRoot = createStorageDir(DB_NAME);
Options options = new Options();
options.createIfMissing(false);
options.comparator(new DBComparator() {
@ -142,6 +145,29 @@ public byte[] findShortSuccessor(byte[] key) {
}
});
Path confVersion = createStorageDir(CONF_VERSION_NAME);
Options confOptions = new Options();
confOptions.createIfMissing(false);
LOG.info("Using conf version at " + confVersion);
File confVersionFile = new File(confVersion.toString());
try {
versiondb = JniDBFactory.factory.open(confVersionFile, confOptions);
} catch (NativeDB.DBException e) {
if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
LOG.info("Creating conf version at " + confVersionFile);
confOptions.createIfMissing(true);
try {
versiondb = JniDBFactory.factory.open(confVersionFile, confOptions);
versiondb.put(bytes(CONF_VERSION_KEY), bytes(String.valueOf(0)));
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
}
} else {
throw e;
}
}
LOG.info("Using conf database at " + storeRoot);
File dbfile = new File(storeRoot.toString());
try {
@ -158,6 +184,9 @@ public byte[] findShortSuccessor(byte[] key) {
initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
}
db.write(initBatch);
long configVersion = getConfigVersion() + 1L;
versiondb.put(bytes(CONF_VERSION_KEY),
bytes(String.valueOf(configVersion)));
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
}
@ -167,20 +196,20 @@ public byte[] findShortSuccessor(byte[] key) {
}
}
private Path createStorageDir() throws IOException {
Path root = getStorageDir();
private Path createStorageDir(String storageName) throws IOException {
Path root = getStorageDir(storageName);
FileSystem fs = FileSystem.getLocal(conf);
fs.mkdirs(root, new FsPermission((short) 0700));
return root;
}
private Path getStorageDir() throws IOException {
private Path getStorageDir(String storageName) throws IOException {
String storePath = conf.get(YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
if (storePath == null) {
throw new IOException("No store location directory configured in " +
YarnConfiguration.RM_SCHEDCONF_STORE_PATH);
}
return new Path(storePath, DB_NAME);
return new Path(storePath, storageName);
}
@Override
@ -188,6 +217,9 @@ public void close() throws IOException {
if (db != null) {
db.close();
}
if (versiondb != null) {
versiondb.close();
}
}
@Override
@ -213,6 +245,9 @@ public void confirmMutation(boolean isValid) throws IOException {
updateBatch.put(bytes(changes.getKey()), bytes(changes.getValue()));
}
}
long configVersion = getConfigVersion() + 1L;
versiondb.put(bytes(CONF_VERSION_KEY),
bytes(String.valueOf(configVersion)));
}
db.write(updateBatch);
pendingMutation = null;
@ -258,6 +293,13 @@ public synchronized Configuration retrieve() {
return config;
}
@Override
public long getConfigVersion() {
String version = new String(versiondb.get(bytes(CONF_VERSION_KEY)),
StandardCharsets.UTF_8);
return Long.parseLong(version);
}
@Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
View File

@ -134,6 +134,11 @@ public Configuration getConfiguration() {
return new Configuration(schedConf);
}
@Override
public long getConfigVersion() throws Exception {
return confStore.getConfigVersion();
}
@Override
public ConfigurationMutationACLPolicy getAclMutationPolicy() {
return aclMutationPolicy;

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
View File

@ -132,6 +132,12 @@ public void close() throws IOException {}
*/
public abstract void format() throws Exception;
/**
* Get the last updated config version.
* @return Last updated config version.
*/
public abstract long getConfigVersion() throws Exception;
/**
* Get a list of confirmed configuration mutations starting from a given id.
* @param fromId id from which to start getting mutations, inclusive

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
View File

@ -62,11 +62,13 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
private static final String LOGS_PATH = "LOGS";
private static final String CONF_STORE_PATH = "CONF_STORE";
private static final String FENCING_PATH = "FENCING";
private static final String CONF_VERSION_PATH = "CONF_VERSION";
private String zkVersionPath;
private String logsPath;
private String confStorePath;
private String fencingNodePath;
private String confVersionPath;
@VisibleForTesting
protected ZKCuratorManager zkManager;
@ -89,6 +91,7 @@ public void initialize(Configuration config, Configuration schedConf,
this.logsPath = getNodePath(znodeParentPath, LOGS_PATH);
this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH);
this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH);
this.confVersionPath = getNodePath(znodeParentPath, CONF_VERSION_PATH);
zkManager.createRootDirRecursively(znodeParentPath, zkAcl);
zkManager.delete(fencingNodePath);
@ -99,6 +102,11 @@ public void initialize(Configuration config, Configuration schedConf,
serializeObject(new LinkedList<LogMutation>()), -1);
}
if (!zkManager.exists(confVersionPath)) {
zkManager.create(confVersionPath);
zkManager.setData(confVersionPath, String.valueOf(0), -1);
}
if (!zkManager.exists(confStorePath)) {
zkManager.create(confStorePath);
HashMap<String, String> mapSchedConf = new HashMap<>();
@ -106,6 +114,8 @@ public void initialize(Configuration config, Configuration schedConf,
mapSchedConf.put(entry.getKey(), entry.getValue());
}
zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1);
long configVersion = getConfigVersion() + 1L;
zkManager.setData(confVersionPath, String.valueOf(configVersion), -1);
}
}
@ -185,6 +195,9 @@ public void confirmMutation(boolean isValid)
}
zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1,
zkAcl, fencingNodePath);
long configVersion = getConfigVersion() + 1L;
zkManager.setData(confVersionPath, String.valueOf(configVersion), -1);
}
pendingMutation = null;
}
@ -213,6 +226,11 @@ public synchronized Configuration retrieve() {
return null;
}
@Override
public long getConfigVersion() throws Exception {
return Long.parseLong(zkManager.getStringData(confVersionPath));
}
@Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
View File

@ -51,6 +51,9 @@ public final class RMWSConsts {
/** Path for {@code RMWebServices#formatSchedulerConfiguration}. */
public static final String FORMAT_SCHEDULER_CONF = "/scheduler-conf/format";
/** Path for {@code RMWebServices#getSchedulerConfigurationVersion}. */
public static final String SCHEDULER_CONF_VERSION = "/scheduler-conf/version";
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
public static final String SCHEDULER_LOGS = "/scheduler/logs";

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -196,6 +196,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@ -2590,7 +2591,7 @@ public Response formatSchedulerConfiguration(@Context HttpServletRequest hsr)
}
} else {
return Response.status(Status.BAD_REQUEST)
.entity("Configuration change only supported by " +
.entity("Scheduler Configuration format only supported by " +
"MutableConfScheduler.").build();
}
}
@ -2680,6 +2681,39 @@ public Response getSchedulerConfiguration(@Context HttpServletRequest hsr)
}
}
@GET
@Path(RMWSConsts.SCHEDULER_CONF_VERSION)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public Response getSchedulerConfigurationVersion(@Context
HttpServletRequest hsr) throws AuthorizationException {
// Only admin user is allowed to get scheduler conf version
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
initForWritableEndpoints(callerUGI, true);
ResourceScheduler scheduler = rm.getResourceScheduler();
if (scheduler instanceof MutableConfScheduler
&& ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
MutableConfigurationProvider mutableConfigurationProvider =
((MutableConfScheduler) scheduler).getMutableConfProvider();
try {
long configVersion = mutableConfigurationProvider
.getConfigVersion();
return Response.status(Status.OK)
.entity(new ConfigVersionInfo(configVersion)).build();
} catch (Exception e) {
LOG.error("Exception thrown when fetching configuration version.", e);
return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
.build();
}
} else {
return Response.status(Status.BAD_REQUEST)
.entity("Configuration Version only supported by "
+ "MutableConfScheduler.").build();
}
}
@GET
@Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfigVersionInfo.java Normal file
View File

@ -0,0 +1,44 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Version of Scheduler Config.
*/
@XmlRootElement(name = "configversion")
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfigVersionInfo {
private long versionID;
public ConfigVersionInfo() {
} // JAXB needs this
public ConfigVersionInfo(long version) {
this.versionID = version;
}
public long getVersionID() {
return this.versionID;
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
View File

@ -37,7 +37,6 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
/**
@ -140,7 +139,6 @@ public void confirmMutationWithInValid() throws Exception {
@Test
public void testFormatConfiguration() throws Exception {
assertTrue(testSchedulerConfigurationDir.exists());
Configuration schedulerConf = new Configuration();
schedulerConf.set("a", "a");
writeConf(schedulerConf);
@ -148,7 +146,15 @@ public void testFormatConfiguration() throws Exception {
Configuration storedConfig = configurationStore.retrieve();
assertEquals("a", storedConfig.get("a"));
configurationStore.format();
assertFalse(testSchedulerConfigurationDir.exists());
boolean exceptionCaught = false;
try {
storedConfig = configurationStore.retrieve();
} catch (IOException e) {
if (e.getMessage().contains("no capacity scheduler file in")) {
exceptionCaught = true;
}
}
assertTrue(exceptionCaught);
}
@Test

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
View File

@ -137,6 +137,21 @@ public void testFormatConfiguration() throws Exception {
assertNull(confStore.retrieve());
}
@Test
public void testGetConfigurationVersion() throws Exception {
confStore.initialize(conf, schedConf, rmContext);
long v1 = confStore.getConfigVersion();
assertEquals(1, v1);
Map<String, String> update = new HashMap<>();
update.put("keyver", "valver");
YarnConfigurationStore.LogMutation mutation =
new YarnConfigurationStore.LogMutation(update, TEST_USER);
confStore.logMutation(mutation);
confStore.confirmMutation(true);
long v2 = confStore.getConfigVersion();
assertEquals(2, v2);
}
@Test
public void testPersistUpdatedConfiguration() throws Exception {
confStore.initialize(conf, schedConf, rmContext);

19
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
View File

@ -202,6 +202,25 @@ public void testFormatSchedulerConf() throws Exception {
assertEquals(3, orgConf.getQueues("root").length);
}
private long getConfigVersion() throws Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
.queryParam("user.name", userName)
.path(RMWSConsts.SCHEDULER_CONF_VERSION)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
JSONObject json = response.getEntity(JSONObject.class);
return Long.parseLong(json.get("versionID").toString());
}
@Test
public void testSchedulerConfigVersion() throws Exception {
assertEquals(1, getConfigVersion());
testAddNestedQueue();
assertEquals(2, getConfigVersion());
}
@Test
public void testAddNestedQueue() throws Exception {
CapacitySchedulerConfiguration orgConf = getSchedulerConf();