From 8ef3bc8113d2cdce67fd0110ee0a7d13182f2ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Tue, 9 Apr 2019 17:35:04 +0200 Subject: [PATCH] HDDS-1383. Create project skeleton with CLI interface for In-place upgrade. Closes #694 --- hadoop-ozone/common/src/main/bin/ozone | 5 ++ hadoop-ozone/dist/pom.xml | 11 ++++ hadoop-ozone/pom.xml | 6 +++ hadoop-ozone/upgrade/pom.xml | 54 +++++++++++++++++++ .../apache/hadoop/ozone/upgrade/Balance.java | 38 +++++++++++++ .../apache/hadoop/ozone/upgrade/Execute.java | 37 +++++++++++++ .../hadoop/ozone/upgrade/InPlaceUpgrade.java | 45 ++++++++++++++++ .../org/apache/hadoop/ozone/upgrade/Plan.java | 38 +++++++++++++ .../hadoop/ozone/upgrade/package-info.java | 23 ++++++++ 9 files changed, 257 insertions(+) create mode 100644 hadoop-ozone/upgrade/pom.xml create mode 100644 hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Balance.java create mode 100644 hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Execute.java create mode 100644 hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/InPlaceUpgrade.java create mode 100644 hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Plan.java create mode 100644 hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index a6ad1eb263..26906c475c 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -52,6 +52,7 @@ function hadoop_usage hadoop_add_subcommand "s3" client "command line interface for s3 related operations" hadoop_add_subcommand "version" client "print the version" hadoop_add_subcommand "dtutil" client "operations related to delegation tokens" + hadoop_add_subcommand "upgrade" client "HDFS to Ozone in-place upgrade tool" hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false } @@ -174,6 +175,10 @@ function ozonecmd_case HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-tools" ;; + upgrade) + HADOOP_CLASSNAME=org.apache.hadoop.ozone.upgrade.InPlaceUpgrade + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-upgrade" + ;; *) HADOOP_CLASSNAME="${subcmd}" if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index ace1eb08d0..b31c5b9c8d 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -110,6 +110,13 @@ classpath hadoop-ozone-recon.classpath + + org.apache.hadoop + hadoop-ozone-upgrade + ${ozone.version} + classpath + hadoop-ozone-upgrade.classpath + @@ -255,5 +262,9 @@ org.apache.hadoop hadoop-hdds-docs + + org.apache.hadoop + hadoop-ozone-upgrade + diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 7010878f62..c24e51146d 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -50,6 +50,7 @@ dist ozone-recon ozone-recon-codegen + upgrade @@ -165,6 +166,11 @@ hadoop-ozone-recon ${ozone.version} + + org.apache.hadoop + hadoop-ozone-upgrade + ${ozone.version} + org.apache.hadoop hadoop-hdds-container-service diff --git a/hadoop-ozone/upgrade/pom.xml b/hadoop-ozone/upgrade/pom.xml new file mode 100644 index 0000000000..072f2b18c3 --- /dev/null +++ b/hadoop-ozone/upgrade/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone + 0.5.0-SNAPSHOT + + hadoop-ozone-upgrade + 0.5.0-SNAPSHOT + Apache Hadoop Ozone In-Place Upgrade + Apache Hadoop Ozone In-Place Upgrade + jar + + + + org.apache.hadoop + hadoop-hdds-common + + + com.google.code.findbugs + findbugs + 3.0.1 + provided + + + junit + junit + test + + + org.mockito + mockito-core + 2.15.0 + test + + + diff --git a/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Balance.java b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Balance.java new file mode 100644 index 0000000000..149273862a --- /dev/null +++ b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Balance.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.upgrade; + +import java.util.concurrent.Callable; + +import picocli.CommandLine.Command; + +/** + * Command to move blocks between HDFS datanodes. + */ +@Command(name = "balance", + description = "Move the HDFS blocks for a better distribution " + + "usage.") +public class Balance implements Callable { + + @Override + public Void call() throws Exception { + System.err.println("[In-Place upgrade : balance] is not yet supported."); + return null; + } + +} \ No newline at end of file diff --git a/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Execute.java b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Execute.java new file mode 100644 index 0000000000..0837200c1f --- /dev/null +++ b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Execute.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.upgrade; + +import java.util.concurrent.Callable; + +import picocli.CommandLine.Command; + +/** + * Execute Ozone specific HDFS ballanced.. + */ +@Command(name = "execute", + description = "Start/restart upgrade from HDFS to Ozone cluster.") +public class Execute implements Callable { + + @Override + public Void call() throws Exception { + System.err.println("In-Place upgrade : execute] is not yet supported."); + return null; + } + +} \ No newline at end of file diff --git a/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/InPlaceUpgrade.java b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/InPlaceUpgrade.java new file mode 100644 index 0000000000..b307f44d02 --- /dev/null +++ b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/InPlaceUpgrade.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.upgrade; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; + +import picocli.CommandLine.Command; + +/** + * Command line interface for the In-Place upgrade utility. + *

+ * In-Place upgrade can convert HDFS cluster data to Ozone data without + * (or minimal) data moving. + */ +@Command(name = "ozone upgrade", + description = "Convert raw HDFS data to Ozone data without data movement.", + subcommands = { + Plan.class, + Balance.class, + Execute.class, + }, + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +public class InPlaceUpgrade extends GenericCli { + + public static void main(String[] args) { + new InPlaceUpgrade().run(args); + } +} diff --git a/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Plan.java b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Plan.java new file mode 100644 index 0000000000..efd6092a0b --- /dev/null +++ b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/Plan.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.upgrade; + +import java.util.concurrent.Callable; + +import picocli.CommandLine.Command; + +/** + * Command to calculate statistics and estimate the upgrade. + */ +@Command(name = "plan", + description = "Plan existing HDFS block distribution and give." + + "estimation.") +public class Plan implements Callable { + + @Override + public Void call() throws Exception { + System.err.println("[In-Place upgrade : plan] is not yet supported."); + return null; + } + +} diff --git a/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java new file mode 100644 index 0000000000..b14768329c --- /dev/null +++ b/hadoop-ozone/upgrade/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +/** + * In-Place upgrade utility to upgrade HDDS to Ozone cluster.. + */ \ No newline at end of file