From 8733012ae35f2762d704f94975a762885d116795 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 25 May 2018 13:06:14 -0700 Subject: [PATCH] HDDS-96. Add an option in ozone script to generate a site file with minimally required ozone configs. Contributed by Dinesh Chitlangia. --- .../hadoop/hdds/conf/OzoneConfiguration.java | 6 +- hadoop-ozone/common/src/main/bin/ozone | 4 + ...stGenerateOzoneRequiredConfigurations.java | 100 ++++++++++ .../GenerateOzoneRequiredConfigurations.java | 174 ++++++++++++++++++ .../hadoop/ozone/genconf/package-info.java | 24 +++ 5 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index f07718c0c8..36d953c960 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -137,7 +137,7 @@ public int compareTo(Property o) { @Override public String toString() { - return this.getName() + " " + this.getValue() + this.getTag(); + return this.getName() + " " + this.getValue() + " " + this.getTag(); } @Override @@ -152,11 +152,11 @@ public boolean equals(Object obj) { } } - public static void activate(){ + public static void activate() { // adds the default resources Configuration.addDefaultResource("hdfs-default.xml"); Configuration.addDefaultResource("hdfs-site.xml"); Configuration.addDefaultResource("ozone-default.xml"); Configuration.addDefaultResource("ozone-site.xml"); } -} \ No newline at end of file +} diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 00261c73cc..6843bddca5 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -47,6 +47,7 @@ function hadoop_usage hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager " hadoop_add_subcommand "version" client "print the version" + hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false } @@ -118,6 +119,9 @@ function ozonecmd_case version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; + genconf) + HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations + ;; *) HADOOP_CLASSNAME="${subcmd}" if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java new file mode 100644 index 0000000000..82582a676b --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + + +/** + * Tests GenerateOzoneRequiredConfigurations. + */ +public class TestGenerateOzoneRequiredConfigurations { + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf; + + /** + * Create a MiniDFSCluster for testing. + *

+ * Ozone is made active by setting OZONE_ENABLED = true and + * OZONE_HANDLER_TYPE_KEY = "distributed" + * + * @throws IOException + */ + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); + cluster.waitForClusterToBeReady(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + /** + * Tests a valid path and generates ozone-site.xml. + * @throws Exception + */ + @Test + public void generateConfigurationsSuccess() throws Exception { + String[] args = new String[]{"-output", "."}; + GenerateOzoneRequiredConfigurations.main(args); + + Assert.assertEquals("Path is valid", + true, GenerateOzoneRequiredConfigurations.isValidPath(args[1])); + + Assert.assertEquals("Permission is valid", + true, GenerateOzoneRequiredConfigurations.canWrite(args[1])); + + Assert.assertEquals("Config file generated", + 0, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1])); + } + + /** + * Test to avoid generating ozone-site.xml when invalid permission. + * @throws Exception + */ + @Test + public void generateConfigurationsFailure() throws Exception { + String[] args = new String[]{"-output", "/"}; + GenerateOzoneRequiredConfigurations.main(args); + + Assert.assertEquals("Path is valid", + true, GenerateOzoneRequiredConfigurations.isValidPath(args[1])); + + Assert.assertEquals("Invalid permission", + false, GenerateOzoneRequiredConfigurations.canWrite(args[1])); + + Assert.assertEquals("Config file not generated", + 1, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1])); + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java new file mode 100644 index 0000000000..6296c9dc57 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.File; +import java.net.URL; +import java.nio.file.InvalidPathException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * GenerateOzoneRequiredConfigurations - A tool to generate ozone-site.xml
+ * This tool generates an ozone-site.xml with minimally required configs. + * This tool can be invoked as follows:
+ *

+ */ +public final class GenerateOzoneRequiredConfigurations { + + private static final String OUTPUT = "-output"; + private static final String HELP = "-help"; + private static final String USAGE = "Usage: \nozone genconf " + + OUTPUT + " \n" + + "ozone genconf " + + HELP; + private static final int SUCCESS = 0; + private static final int FAILURE = 1; + + private GenerateOzoneRequiredConfigurations() { + + } + /** + * Entry point for using genconf tool. + * + * @param args + * @throws JAXBException + */ + public static void main(String[] args) { + + try { + if (args.length == 0) { + System.out.println(USAGE); + System.exit(1); + } + + switch (args[0]) { + case OUTPUT: + if (args.length > 1) { + int result = generateConfigurations(args[1]); + } else { + System.out.println("Path to output file is mandatory"); + System.out.println(USAGE); + System.exit(1); + } + break; + + case HELP: + System.out.println(USAGE); + System.exit(0); + break; + + default: + System.out.println(USAGE); + System.exit(1); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Check if the path is valid. + * + * @param path + * @return true, if path is valid, else return false + */ + public static boolean isValidPath(String path) { + try { + Paths.get(path); + } catch (InvalidPathException | NullPointerException ex) { + return false; + } + return true; + } + + /** + * Check if user has permission to write in the specified path. + * + * @param path + * @return true, if the user has permission to write, else returns false + */ + public static boolean canWrite(String path) { + File file = new File(path); + return file.canWrite(); + } + + /** + * Generate ozone-site.xml at specified path. + * + * @param path + * @return SUCCESS(0) if file can be generated, else returns FAILURE(1) + * @throws JAXBException + */ + public static int generateConfigurations(String path) throws JAXBException { + + if (!isValidPath(path)) { + System.out.println("Invalid path or insufficient permission"); + return FAILURE; + } + + if (!canWrite(path)) { + System.out.println("Invalid path or insufficient permission"); + return FAILURE; + } + + OzoneConfiguration oc = new OzoneConfiguration(); + + ClassLoader cL = Thread.currentThread().getContextClassLoader(); + if (cL == null) { + cL = OzoneConfiguration.class.getClassLoader(); + } + URL url = cL.getResource("ozone-default.xml"); + + List allProperties = + oc.readPropertyFromXml(url); + + List requiredProperties = new ArrayList<>(); + + for (OzoneConfiguration.Property p : allProperties) { + if (p.getTag() != null && p.getTag().contains("REQUIRED")) { + requiredProperties.add(p); + } + } + + OzoneConfiguration.XMLConfiguration requiredConfig = + new OzoneConfiguration.XMLConfiguration(); + requiredConfig.setProperties(requiredProperties); + + JAXBContext context = + JAXBContext.newInstance(OzoneConfiguration.XMLConfiguration.class); + Marshaller m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.marshal(requiredConfig, new File(path, "ozone-site.xml")); + + System.out.println("ozone-site.xml has been generated at " + path); + + return SUCCESS; + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java new file mode 100644 index 0000000000..4817d392db --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.genconf; + +/** + * Command line tool to generate required Ozone configs to an ozone-site.xml. + */ +