diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java index bdc0418441..f3112c9a82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java @@ -357,6 +357,11 @@ public Pipeline allocateContainer(String containerName) throws IOException { ScmClient.ReplicationFactor.ONE); } + @VisibleForTesting + Pipeline getContainer(String containerName) throws IOException { + return scmContainerManager.getContainer(containerName); + } + /** * Asks SCM where a container should be allocated. SCM responds with the set * of datanodes that should be used creating this container. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java new file mode 100644 index 0000000000..fdd0a4985c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.ozone.web.exceptions.OzoneException; +import org.apache.hadoop.util.Tool; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * This class is the base CLI for scm, ksm and scmadm. + */ +public abstract class OzoneBaseCLI extends Configured implements Tool { + + protected abstract int dispatch(CommandLine cmd, Options opts) + throws IOException, OzoneException, URISyntaxException; + + protected abstract CommandLine parseArgs(String[] argv, Options opts) + throws ParseException; + + protected abstract Options getOptions(); + + protected abstract void displayHelp(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java new file mode 100644 index 0000000000..5abc82428a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.hadoop.scm.client.ScmClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** + * The abstract class of all SCM CLI commands. + */ +public abstract class OzoneCommandHandler { + + private ScmClient scmClient; + protected static final Logger LOG = + LoggerFactory.getLogger(OzoneCommandHandler.class); + + /** + * Constructs a handler object. + */ + public OzoneCommandHandler(ScmClient scmClient) { + this.scmClient = scmClient; + } + + protected ScmClient getScmClient() { + return scmClient; + } + + /** + * Executes the Client command. + * + * @param cmd - CommandLine. + * @throws IOException throws exception. + */ + public abstract void execute(CommandLine cmd) throws IOException; + + /** + * Display a help message describing the options the command takes. + * TODO : currently only prints to standard out, may want to change this. + */ + public abstract void displayHelp(); +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java new file mode 100644 index 0000000000..7b8704aa7f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli; + +/** + * The possible result code of SCM CLI. + */ +public final class ResultCode { + public static final int SUCCESS = 1; + + public static final int UNRECOGNIZED_CMD = 2; + + public static final int EXECUTION_ERROR = 3; + + private ResultCode() {} +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java new file mode 100644 index 0000000000..ee26d7cbcf --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java @@ -0,0 +1,237 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli; + +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler; +import org.apache.hadoop.ozone.web.exceptions.OzoneException; +import org.apache.hadoop.scm.XceiverClientManager; +import org.apache.hadoop.scm.client.ContainerOperationClient; +import org.apache.hadoop.scm.client.ScmClient; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ToolRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.PrintStream; +import java.net.InetSocketAddress; +import java.net.URISyntaxException; +import java.util.Arrays; + +import static org.apache.hadoop.ozone.scm.cli.ResultCode.EXECUTION_ERROR; +import static org.apache.hadoop.ozone.scm.cli.ResultCode.SUCCESS; +import static org.apache.hadoop.ozone.scm.cli.ResultCode.UNRECOGNIZED_CMD; +import static org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler.CONTAINER_CMD; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT; +import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB; + +/** + * This class is the CLI of SCM. + */ +public class SCMCLI extends OzoneBaseCLI { + + private static final Logger LOG = LoggerFactory.getLogger(SCMCLI.class); + + public static final String HELP_OP = "help"; + public static final int CMD_WIDTH = 80; + + private final ScmClient scmClient; + private final PrintStream out; + private final PrintStream err; + + private final Options options; + + public SCMCLI(ScmClient scmClient) { + this(scmClient, System.out, System.err); + } + + public SCMCLI(ScmClient scmClient, PrintStream out, PrintStream err) { + this.scmClient = scmClient; + this.out = out; + this.err = err; + this.options = getOptions(); + } + + /** + * Main for the scm shell Command handling. + * + * @param argv - System Args Strings[] + * @throws Exception + */ + public static void main(String[] argv) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + ScmClient scmClient = getScmClient(conf); + SCMCLI shell = new SCMCLI(scmClient); + conf.setQuietMode(false); + shell.setConf(conf); + int res = 0; + try { + res = ToolRunner.run(shell, argv); + } catch (Exception ex) { + System.exit(1); + } + System.exit(res); + } + + private static ScmClient getScmClient(OzoneConfiguration ozoneConf) + throws IOException { + long version = RPC.getProtocolVersion( + StorageContainerLocationProtocolPB.class); + String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY, + OZONE_SCM_CLIENT_BIND_HOST_DEFAULT); + int scmPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY, + OZONE_SCM_CLIENT_PORT_DEFAULT); + int containerSizeGB = ozoneConf.getInt(OZONE_SCM_CONTAINER_SIZE_GB, + OZONE_SCM_CONTAINER_SIZE_DEFAULT); + ContainerOperationClient.setContainerSizeB(containerSizeGB*OzoneConsts.GB); + InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort); + + RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class, + ProtobufRpcEngine.class); + StorageContainerLocationProtocolClientSideTranslatorPB client = + new StorageContainerLocationProtocolClientSideTranslatorPB( + RPC.getProxy(StorageContainerLocationProtocolPB.class, version, + address, UserGroupInformation.getCurrentUser(), ozoneConf, + NetUtils.getDefaultSocketFactory(ozoneConf), + Client.getRpcTimeout(ozoneConf))); + ScmClient storageClient = new ContainerOperationClient( + client, new XceiverClientManager(ozoneConf)); + return storageClient; + } + + /** + * Adds ALL the options that hdfs scm command supports. Given the hierarchy + * of commands, the options are added in a cascading manner, e.g.: + * {@link SCMCLI} asks {@link ContainerCommandHandler} to add it's options, + * which then asks it's sub command, such as + * {@link org.apache.hadoop.ozone.scm.cli.container.CreateContainerHandler} + * to add it's own options. + * + * We need to do this because {@link BasicParser} need to take all the options + * when paring args. + * @return ALL the options supported by this CLI. + */ + @Override + protected Options getOptions() { + Options newOptions = new Options(); + // add the options + addTopLevelOptions(newOptions); + ContainerCommandHandler.addOptions(newOptions); + // TODO : add pool, node and pipeline commands. + addHelpOption(newOptions); + return newOptions; + } + + private static void addTopLevelOptions(Options options) { + Option containerOps = new Option( + CONTAINER_CMD, false, "Container related options"); + options.addOption(containerOps); + // TODO : add pool, node and pipeline commands. + } + + private static void addHelpOption(Options options) { + Option helpOp = new Option(HELP_OP, false, "display help message"); + options.addOption(helpOp); + } + + @Override + protected void displayHelp() { + HelpFormatter helpFormatter = new HelpFormatter(); + Options topLevelOptions = new Options(); + addTopLevelOptions(topLevelOptions); + helpFormatter.printHelp(CMD_WIDTH, "hdfs scm []", + "where can be one of the following", + topLevelOptions, ""); + } + + @Override + public int run(String[] args) throws Exception { + CommandLine cmd = parseArgs(args, options); + if (cmd == null) { + err.println("Unrecognized options:" + Arrays.asList(args)); + displayHelp(); + return UNRECOGNIZED_CMD; + } + return dispatch(cmd, options); + } + + /** + * This function parses all command line arguments + * and returns the appropriate values. + * + * @param argv - Argv from main + * + * @return CommandLine + */ + @Override + protected CommandLine parseArgs(String[] argv, Options opts) + throws ParseException { + try { + BasicParser parser = new BasicParser(); + return parser.parse(opts, argv); + } catch (ParseException ex) { + LOG.error(ex.getMessage()); + } + return null; + } + + @Override + protected int dispatch(CommandLine cmd, Options opts) + throws IOException, OzoneException, URISyntaxException { + OzoneCommandHandler handler = null; + try { + if (cmd.hasOption(CONTAINER_CMD)) { + handler = new ContainerCommandHandler(scmClient); + } + if (handler == null) { + if (cmd.hasOption(HELP_OP)) { + displayHelp(); + return SUCCESS; + } else { + displayHelp(); + err.println("Unrecognized command: " + Arrays.asList(cmd.getArgs())); + return UNRECOGNIZED_CMD; + } + } else { + handler.execute(cmd); + return SUCCESS; + } + } catch (IOException ioe) { + err.println("Error executing command:" + ioe); + return EXECUTION_ERROR; + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java new file mode 100644 index 0000000000..2da252a795 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli.container; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler; +import org.apache.hadoop.scm.client.ScmClient; + +import java.io.IOException; +import java.util.Arrays; + +import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH; +import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP; +import static org.apache.hadoop.ozone.scm.cli.container.CreateContainerHandler.CONTAINER_CREATE; + +/** + * The handler class of container-specific commands, e.g. createContainer. + */ +public class ContainerCommandHandler extends OzoneCommandHandler { + + public static final String CONTAINER_CMD = "container"; + + public ContainerCommandHandler(ScmClient scmClient) { + super(scmClient); + } + + @Override + public void execute(CommandLine cmd) throws IOException { + // all container commands should contain -container option + if (!cmd.hasOption(CONTAINER_CMD)) { + throw new IOException("Expecting container cmd"); + } + // check which each the sub command it is + OzoneCommandHandler handler = null; + if (cmd.hasOption(CONTAINER_CREATE)) { + handler = new CreateContainerHandler(getScmClient()); + } + // execute the sub command, throw exception if no sub command found + // unless -help option is given. + if (handler != null) { + handler.execute(cmd); + } else { + displayHelp(); + if (!cmd.hasOption(HELP_OP)) { + throw new IOException("Unrecognized command " + + Arrays.asList(cmd.getArgs())); + } + } + } + + @Override + public void displayHelp() { + Options options = new Options(); + addCommandsOption(options); + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.printHelp(CMD_WIDTH, + "hdfs scm -container ", + "where can be one of the following", options, ""); + } + + private static void addCommandsOption(Options options) { + Option createContainer = + new Option(CONTAINER_CREATE, false, "Create container"); + options.addOption(createContainer); + // TODO : add other options such as delete, close etc. + } + + public static void addOptions(Options options) { + addCommandsOption(options); + // for create container options. + CreateContainerHandler.addOptions(options); + // TODO : add other options such as delete, close etc. + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java new file mode 100644 index 0000000000..9b30f0d822 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.cli.container; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler; +import org.apache.hadoop.scm.client.ScmClient; + +import java.io.IOException; + +import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH; +import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP; + +/** + * This is the handler that process container creation command. + */ +public class CreateContainerHandler extends OzoneCommandHandler { + + public static final String CONTAINER_CREATE = "create"; + public static final String PIPELINE_ID = "p"; + + public CreateContainerHandler(ScmClient scmClient) { + super(scmClient); + } + + @Override + public void execute(CommandLine cmd) throws IOException { + if (!cmd.hasOption(CONTAINER_CREATE)) { + throw new IOException("Expecting container create"); + } + // TODO requires pipeline id (instead of optional as in the design) for now + if (!cmd.hasOption(PIPELINE_ID)) { + displayHelp(); + if (!cmd.hasOption(HELP_OP)) { + throw new IOException("Expecting container ID"); + } else { + return; + } + } + String pipelineID = cmd.getOptionValue(PIPELINE_ID); + LOG.info("Create container :" + pipelineID + " " + getScmClient()); + getScmClient().createContainer(pipelineID); + LOG.debug("Container creation returned"); + } + + @Override + public void displayHelp() { + // TODO : may need to change this if we decide to make -p optional later + Options options = new Options(); + addOptions(options); + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -create