HDDS-1192. Support -conf command line argument in GenericCli

Closes #713

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java

@@ -22,6 +22,7 @@
 import java.util.concurrent.Callable;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.fs.Path;
 
 import com.google.common.annotations.VisibleForTesting;
 import picocli.CommandLine;
@@ -41,6 +42,9 @@ public class GenericCli implements Callable<Void>, GenericParentCommand {
   @Option(names = {"-D", "--set"})
   private Map<String, String> configurationOverrides = new HashMap<>();
 
+  @Option(names = {"-conf"})
+  private String configurationPath;
+
   private final CommandLine cmd;
 
   public GenericCli() {
@@ -70,19 +74,19 @@ protected void printError(Throwable error) {
     } else {
       System.err.println(error.getMessage().split("\n")[0]);
     }
-    if(error instanceof MissingSubcommandException){
-      System.err.println(((MissingSubcommandException) error).getUsage());
-    }
   }
 
   @Override
   public Void call() throws Exception {
-    throw new MissingSubcommandException(cmd.getUsageMessage());
+    throw new MissingSubcommandException(cmd);
   }
 
   @Override
   public OzoneConfiguration createOzoneConfiguration() {
     OzoneConfiguration ozoneConf = new OzoneConfiguration();
+    if (configurationPath != null) {
+      ozoneConf.addResource(new Path(configurationPath));
+    }
     if (configurationOverrides != null) {
       for (Entry<String, String> entry : configurationOverrides.entrySet()) {
         ozoneConf.set(entry.getKey(), entry.getValue());

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java

@@ -17,19 +17,15 @@
  */
 package org.apache.hadoop.hdds.cli;
 
+import picocli.CommandLine;
+
 /**
  * Exception to throw if subcommand is not selected but required.
  */
-public class MissingSubcommandException extends RuntimeException {
+public class MissingSubcommandException extends CommandLine.ParameterException {
 
-  private String usage;
+  public MissingSubcommandException(CommandLine cmd) {
-
+    super(cmd, "Incomplete command");
-  public MissingSubcommandException(String usage) {
-    super("Incomplete command");
-    this.usage = usage;
   }
 
-  public String getUsage() {
-    return usage;
-  }
 }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -53,6 +53,7 @@
 import java.net.InetAddress;
 import java.security.KeyPair;
 import java.security.cert.CertificateException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.UUID;
 
@@ -81,32 +82,12 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
   private CertificateClient dnCertClient;
   private String component;
   private HddsDatanodeHttpServer httpServer;
+  private boolean printBanner;
+  private String[] args;
 
-  /**
+  public HddsDatanodeService(boolean printBanner, String[] args) {
-   * Default constructor.
+    this.printBanner = printBanner;
-   */
+    this.args = args != null ? Arrays.copyOf(args, args.length) : null;
-  public HddsDatanodeService() {
-    this(null);
-  }
-
-  /**
-   * Constructs {@link HddsDatanodeService} using the provided {@code conf}
-   * value.
-   *
-   * @param conf OzoneConfiguration
-   */
-  public HddsDatanodeService(Configuration conf) {
-    if (conf == null) {
-      this.conf = new OzoneConfiguration();
-    } else {
-      this.conf = new OzoneConfiguration(conf);
-    }
-  }
-
-  @VisibleForTesting
-  public static HddsDatanodeService createHddsDatanodeService(
-      String[] args, Configuration conf) {
-    return createHddsDatanodeService(args, conf, false);
   }
 
   /**
@@ -116,29 +97,31 @@ public static HddsDatanodeService createHddsDatanodeService(
    * startup/shutdown message and skips registering Unix signal handlers.
    *
    * @param args      command line arguments.
-   * @param conf        HDDS configuration
+   * @return Datanode instance
+   */
+  @VisibleForTesting
+  public static HddsDatanodeService createHddsDatanodeService(
+      String[] args) {
+    return createHddsDatanodeService(args, false);
+  }
+
+  /**
+   * Create an Datanode instance based on the supplied command-line arguments.
+   *
+   * @param args        command line arguments.
    * @param printBanner if true, then log a verbose startup message.
    * @return Datanode instance
    */
   private static HddsDatanodeService createHddsDatanodeService(
-      String[] args, Configuration conf, boolean printBanner) {
+      String[] args, boolean printBanner) {
-    if (args.length == 0 && printBanner) {
+    return new HddsDatanodeService(printBanner, args);
-      StringUtils
-          .startupShutdownMessage(HddsDatanodeService.class, args, LOG);
-
-    }
-    return new HddsDatanodeService(conf);
   }
 
   public static void main(String[] args) {
     try {
-      Configuration conf = new OzoneConfiguration();
       HddsDatanodeService hddsDatanodeService =
-          createHddsDatanodeService(args, conf, true);
+          createHddsDatanodeService(args, true);
-      if (hddsDatanodeService != null) {
+      hddsDatanodeService.run(args);
-        hddsDatanodeService.start(null);
-        hddsDatanodeService.join();
-      }
     } catch (Throwable e) {
       LOG.error("Exception in HddsDatanodeService.", e);
       terminate(1, e);
@@ -149,6 +132,21 @@ public static Logger getLogger() {
     return LOG;
   }
 
+  @Override
+  public Void call() throws Exception {
+    if (printBanner) {
+      StringUtils
+          .startupShutdownMessage(HddsDatanodeService.class, args, LOG);
+    }
+    start(createOzoneConfiguration());
+    join();
+    return null;
+  }
+
+  public void setConfiguration(OzoneConfiguration configuration) {
+    this.conf = configuration;
+  }
+
   /**
    * Starts HddsDatanode services.
    *
@@ -156,12 +154,21 @@ public static Logger getLogger() {
    */
   @Override
   public void start(Object service) {
+    if (service instanceof Configurable) {
+      start(new OzoneConfiguration(((Configurable) service).getConf()));
+    } else {
+      start(new OzoneConfiguration());
+    }
+  }
 
+  public void start(OzoneConfiguration configuration) {
+    setConfiguration(configuration);
+    start();
+  }
+
+  public void start() {
     DefaultMetricsSystem.initialize("HddsDatanode");
     OzoneConfiguration.activate();
-    if (service instanceof Configurable) {
-      conf = new OzoneConfiguration(((Configurable) service).getConf());
-    }
     if (HddsUtils.isHddsEnabled(conf)) {
       try {
         String hostname = HddsUtils.getHostName(conf);
@@ -404,6 +411,7 @@ public DatanodeStateMachine getDatanodeStateMachine() {
   }
 
   public void join() {
+    if (datanodeStateMachine != null) {
       try {
         datanodeStateMachine.join();
       } catch (InterruptedException e) {
@@ -411,6 +419,7 @@ public void join() {
         LOG.info("Interrupted during StorageContainerManager join.");
       }
     }
+  }
 
   @Override
   public void stop() {

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java

@@ -24,7 +24,6 @@
 import java.io.File;
 import java.io.IOException;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -40,7 +39,7 @@
  */
 public class TestHddsDatanodeService {
   private File testDir;
-  private Configuration conf;
+  private OzoneConfiguration conf;
   private HddsDatanodeService service;
   private String[] args = new String[] {};
 
@@ -64,8 +63,8 @@ public void tearDown() {
 
   @Test
   public void testStartup() throws IOException {
-    service = HddsDatanodeService.createHddsDatanodeService(args, conf);
+    service = HddsDatanodeService.createHddsDatanodeService(args);
-    service.start(null);
+    service.start(conf);
     service.join();
 
     assertNotNull(service.getDatanodeDetails());

hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsSecureDatanodeInit.java

@@ -83,10 +83,10 @@ public static void setUp() throws Exception {
         ServicePlugin.class);
     securityConfig = new SecurityConfig(conf);
 
-    service = HddsDatanodeService.createHddsDatanodeService(args, conf);
+    service = HddsDatanodeService.createHddsDatanodeService(args);
     dnLogs = GenericTestUtils.LogCapturer.captureLogs(getLogger());
     callQuietly(() -> {
-      service.start(null);
+      service.start(conf);
       return null;
     });
     callQuietly(() -> {

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SafeModeCommands.java

@@ -54,7 +54,6 @@ public SCMCLI getParent() {
   @Override
   public Void call() throws Exception {
     throw new MissingSubcommandException(
-        this.parent.getCmd().getSubcommands().get("safemode").
+        this.parent.getCmd().getSubcommands().get("safemode"));
-        getUsageMessage());
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -23,7 +23,6 @@
 import java.util.Optional;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.StorageUnit;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -275,7 +274,7 @@ public void restartHddsDatanode(int i, boolean waitForDatanode)
     datanodeService.stop();
     datanodeService.join();
     // ensure same ports are used across restarts.
-    Configuration config = datanodeService.getConf();
+    OzoneConfiguration config = datanodeService.getConf();
     int currentPort = datanodeService.getDatanodeDetails()
         .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
     config.setInt(DFS_CONTAINER_IPC_PORT, currentPort);
@@ -291,9 +290,9 @@ public void restartHddsDatanode(int i, boolean waitForDatanode)
     }
     String[] args = new String[]{};
     HddsDatanodeService service =
-        HddsDatanodeService.createHddsDatanodeService(args, config);
+        HddsDatanodeService.createHddsDatanodeService(args);
     hddsDatanodes.add(i, service);
-    service.start(null);
+    service.start(config);
     if (waitForDatanode) {
       // wait for the node to be identified as a healthy node again.
       waitForClusterToBeReady();
@@ -371,7 +370,7 @@ public void startScm() throws IOException {
   public void startHddsDatanodes() {
     hddsDatanodes.forEach((datanode) -> {
       datanode.setCertificateClient(getCAClient());
-      datanode.start(null);
+      datanode.start();
     });
   }
 
@@ -537,7 +536,7 @@ List<HddsDatanodeService> createHddsDatanodes(
       conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, scmAddress);
       List<HddsDatanodeService> hddsDatanodes = new ArrayList<>();
       for (int i = 0; i < numOfDatanodes; i++) {
-        Configuration dnConf = new OzoneConfiguration(conf);
+        OzoneConfiguration dnConf = new OzoneConfiguration(conf);
         String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
         Path metaDir = Paths.get(datanodeBaseDir, "meta");
         Path dataDir = Paths.get(datanodeBaseDir, "data", "containers");
@@ -555,8 +554,10 @@ List<HddsDatanodeService> createHddsDatanodes(
         dnConf.set(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR,
             wrokDir.toString());
 
-        hddsDatanodes.add(
+        HddsDatanodeService datanode
-            HddsDatanodeService.createHddsDatanodeService(args, dnConf));
+            = HddsDatanodeService.createHddsDatanodeService(args);
+        datanode.setConfiguration(dnConf);
+        hddsDatanodes.add(datanode);
       }
       return hddsDatanodes;
     }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestScmSafeMode.java

@@ -270,7 +270,7 @@ public void testSCMSafeMode() throws Exception {
     assertFalse(logCapturer.getOutput().contains("SCM exiting safe mode."));
     assertTrue(scm.getCurrentContainerThreshold() == 0);
     for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
-      dn.start(null);
+      dn.start();
     }
     GenericTestUtils
         .waitFor(() -> scm.getCurrentContainerThreshold() == 1.0, 100, 20000);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java

@@ -27,7 +27,6 @@
 import java.util.List;
 
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdds.cli.MissingSubcommandException;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.HddsDatanodeService;
@@ -108,7 +107,7 @@ public static void init() throws Exception {
     baseDir = new File(path);
     baseDir.mkdirs();
 
-    datanode = new HddsDatanodeService();
+    datanode = HddsDatanodeService.createHddsDatanodeService(null);
 
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
@@ -177,7 +176,7 @@ public List<Object> handleExecutionException(ExecutionException ex,
    * was thrown and contains the specified usage string.
    */
   private void executeDatanodeWithError(HddsDatanodeService hdds, String[] args,
-      String expectedError, String usage) {
+      String expectedError) {
     if (Strings.isNullOrEmpty(expectedError)) {
       executeDatanode(hdds, args);
     } else {
@@ -197,24 +196,27 @@ private void executeDatanodeWithError(HddsDatanodeService hdds, String[] args,
                       "exception [%s] in [%s]",
                   expectedError, exceptionToCheck.getMessage()),
               exceptionToCheck.getMessage().contains(expectedError));
-          Assert.assertTrue(
-              exceptionToCheck instanceof MissingSubcommandException);
-          Assert.assertTrue(
-              ((MissingSubcommandException) exceptionToCheck)
-                  .getUsage().contains(usage));
         }
       }
     }
   }
 
   @Test
-  public void testDatanodeIncompleteCommand() {
+  public void testDatanodeCommand() {
     LOG.info("Running testDatanodeIncompleteCommand");
-    String expectedError = "Incomplete command";
     String[] args = new String[]{}; //executing 'ozone datanode'
 
-    executeDatanodeWithError(datanode, args, expectedError,
+    //'ozone datanode' command should not result in error
-        "Usage: ozone datanode [-hV] [--verbose] [-D=<String=String>]...");
+    executeDatanodeWithError(datanode, args, null);
+  }
 
+  @Test
+  public void testDatanodeInvalidParamCommand() {
+    LOG.info("Running testDatanodeIncompleteCommand");
+    String expectedError = "Unknown option: -invalidParam";
+    //executing 'ozone datanode -invalidParam'
+    String[] args = new String[]{"-invalidParam"};
+
+    executeDatanodeWithError(datanode, args, expectedError);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java

@@ -361,21 +361,19 @@ public void testShellIncompleteCommand() throws Exception {
     String expectedError = "Incomplete command";
     String[] args = new String[] {}; //executing 'ozone sh'
 
-    executeWithError(shell, args, expectedError,
+    executeWithError(shell, args, expectedError);
-        "Usage: ozone sh [-hV] [--verbose] [-D=<String=String>]..." +
-            " [COMMAND]");
 
     args = new String[] {"volume"}; //executing 'ozone sh volume'
-    executeWithError(shell, args, expectedError,
+    executeWithError(shell, args, MissingSubcommandException.class,
-        "Usage: ozone sh volume [-hV] [COMMAND]");
+        expectedError);
 
     args = new String[] {"bucket"}; //executing 'ozone sh bucket'
-    executeWithError(shell, args, expectedError,
+    executeWithError(shell, args, MissingSubcommandException.class,
-        "Usage: ozone sh bucket [-hV] [COMMAND]");
+        expectedError);
 
     args = new String[] {"key"}; //executing 'ozone sh key'
-    executeWithError(shell, args, expectedError,
+    executeWithError(shell, args, MissingSubcommandException.class,
-        "Usage: ozone sh key [-hV] [COMMAND]");
+        expectedError);
   }
 
   @Test
@@ -475,32 +473,11 @@ private void executeWithError(Shell ozoneShell, String[] args,
   }
 
   /**
-   * Execute command, assert exception message and returns true if error
+   * Execute command, assert exception message and exception class
-   * was thrown.
+   * and returns true if error was thrown.
    */
   private void executeWithError(Shell ozoneShell, String[] args,
-      Class exception) {
+                                Class expectedException, String expectedError) {
-    if (Objects.isNull(exception)) {
-      execute(ozoneShell, args);
-    } else {
-      try {
-        execute(ozoneShell, args);
-        fail("Exception is expected from command execution " + Arrays
-            .asList(args));
-      } catch (Exception ex) {
-        LOG.error("Exception: ", ex);
-        assertTrue(ex.getCause().getClass().getCanonicalName()
-            .equals(exception.getCanonicalName()));
-      }
-    }
-  }
-
-  /**
-   * Execute command, assert exception message and returns true if error
-   * was thrown and contains the specified usage string.
-   */
-  private void executeWithError(Shell ozoneShell, String[] args,
-      String expectedError, String usage) {
     if (Strings.isNullOrEmpty(expectedError)) {
       execute(ozoneShell, args);
     } else {
@@ -517,19 +494,37 @@ private void executeWithError(Shell ozoneShell, String[] args,
           Assert.assertTrue(
               String.format(
                   "Error of shell code doesn't contain the " +
-                      "exception [%s] in [%s]",
+                      "expectedException [%s] in [%s]",
                   expectedError, exceptionToCheck.getMessage()),
               exceptionToCheck.getMessage().contains(expectedError));
-          Assert.assertTrue(
+          assertTrue(ex.getClass().getCanonicalName()
-              exceptionToCheck instanceof MissingSubcommandException);
+              .equals(expectedException.getCanonicalName()));
-          Assert.assertTrue(
-              ((MissingSubcommandException)exceptionToCheck)
-                  .getUsage().contains(usage));
         }
       }
     }
   }
 
+  /**
+   * Execute command, assert exception cause message and returns true if error
+   * was thrown.
+   */
+  private void executeWithError(Shell ozoneShell, String[] args,
+      Class expectedCause) {
+    if (Objects.isNull(expectedCause)) {
+      execute(ozoneShell, args);
+    } else {
+      try {
+        execute(ozoneShell, args);
+        fail("Exception is expected from command execution " + Arrays
+            .asList(args));
+      } catch (Exception ex) {
+        LOG.error("Exception: ", ex);
+        assertTrue(ex.getCause().getClass().getCanonicalName()
+            .equals(expectedCause.getCanonicalName()));
+      }
+    }
+  }
+
   @Test
   public void testListVolume() throws Exception {
     LOG.info("Running testListVolume");

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/BucketCommands.java

@@ -51,7 +51,7 @@ public class BucketCommands implements GenericParentCommand, Callable<Void> {
   @Override
   public Void call() throws Exception {
     throw new MissingSubcommandException(
-        this.shell.getCmd().getSubcommands().get("bucket").getUsageMessage());
+        this.shell.getCmd().getSubcommands().get("bucket"));
   }
 
   @Override

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/KeyCommands.java

@@ -52,7 +52,7 @@ public class KeyCommands implements GenericParentCommand, Callable<Void> {
   @Override
   public Void call() throws Exception {
     throw new MissingSubcommandException(
-        this.shell.getCmd().getSubcommands().get("key").getUsageMessage());
+        this.shell.getCmd().getSubcommands().get("key"));
   }
 
   @Override

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/token/TokenCommands.java

@@ -49,7 +49,7 @@ public class TokenCommands implements GenericParentCommand, Callable<Void> {
   @Override
   public Void call() throws Exception {
     throw new MissingSubcommandException(
-        this.shell.getCmd().getSubcommands().get("token").getUsageMessage());
+        this.shell.getCmd().getSubcommands().get("token"));
   }
 
   @Override

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/VolumeCommands.java

@@ -52,7 +52,7 @@ public class VolumeCommands implements GenericParentCommand, Callable<Void> {
   @Override
   public Void call() throws Exception {
     throw new MissingSubcommandException(
-        this.shell.getCmd().getSubcommands().get("volume").getUsageMessage());
+        this.shell.getCmd().getSubcommands().get("volume"));
   }
 
   @Override

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/audit/parser/TestAuditParser.java

@@ -177,8 +177,9 @@ public void testQueryCommand() {
   public void testHelp() throws Exception {
     String[] args = new String[]{"--help"};
     execute(args,
-        "Usage: ozone auditparser [-hV] [--verbose] [-D=<String=String>]... " +
+        "Usage: ozone auditparser [-hV] [--verbose] " +
-            "<database>\n" +
+            "[-conf=<configurationPath>]\n" +
+            "                         [-D=<String=String>]... <database> " +
             "[COMMAND]");
   }