From 5c1f9460715d5231f5693bfd334ed1612a401439 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Sun, 24 Feb 2019 15:04:25 -0800 Subject: [PATCH] HDDS-1145. Add optional web server to the Ozone freon test tool. Contributed by Elek, Marton. --- .../apache/hadoop/ozone/OzoneConfigKeys.java | 20 +++++ .../src/main/resources/ozone-default.xml | 67 ++++++++++++++++- hadoop-ozone/tools/pom.xml | 4 + .../org/apache/hadoop/ozone/freon/Freon.java | 36 +++++++++ .../hadoop/ozone/freon/FreonHttpServer.java | 74 +++++++++++++++++++ .../ozone/freon/RandomKeyGenerator.java | 6 +- .../src/main/resources/webapps/freon/.gitkeep | 17 +++++ 7 files changed, 222 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java create mode 100644 hadoop-ozone/tools/src/main/resources/webapps/freon/.gitkeep diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 4ac6fda990..cd40f7c091 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -380,6 +380,26 @@ public final class OzoneConfigKeys { "ozone.fs.isolated-classloader"; + public static final String OZONE_FREON_HTTP_ENABLED_KEY = + "ozone.freon.http.enabled"; + public static final String OZONE_FREON_HTTP_BIND_HOST_KEY = + "ozone.freon.http-bind-host"; + public static final String OZONE_FREON_HTTPS_BIND_HOST_KEY = + "ozone.freon.https-bind-host"; + public static final String OZONE_FREON_HTTP_ADDRESS_KEY = + "ozone.freon.http-address"; + public static final String OZONE_FREON_HTTPS_ADDRESS_KEY = + "ozone.freon.https-address"; + + public static final String OZONE_FREON_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; + public static final int OZONE_FREON_HTTP_BIND_PORT_DEFAULT = 9884; + public static final int OZONE_FREON_HTTPS_BIND_PORT_DEFAULT = 9885; + public static final String + OZONE_FREON_HTTP_KERBEROS_PRINCIPAL_KEY = + "ozone.freon.http.kerberos.principal"; + public static final String + OZONE_FREON_HTTP_KERBEROS_KEYTAB_FILE_KEY = + "ozone.freon.http.kerberos.keytab"; /** * There is no need to instantiate this class. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 0eba789048..99f4e35c57 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1899,4 +1899,69 @@ the servlet. - \ No newline at end of file + + + ozone.freon.http-address + 0.0.0.0:9884 + OZONE, MANAGEMENT + + The address and the base port where the FREON web ui will listen on. + If the port is 0 then the server will start on a free port. + + + + ozone.freon.http-bind-host + 0.0.0.0 + OZONE, MANAGEMENT + + The actual address the Freon web server will bind to. If this + optional address is set, it overrides only the hostname portion of + ozone.freon.http-address. + + + + ozone.freon.http.enabled + true + OZONE, MANAGEMENT + + Property to enable or disable FREON web ui. + + + + ozone.freon.https-address + 0.0.0.0:9885 + OZONE, MANAGEMENT + + The address and the base port where the Freon web server will listen + on using HTTPS. + If the port is 0 then the server will start on a free port. + + + + ozone.freon.https-bind-host + 0.0.0.0 + OZONE, MANAGEMENT + + The actual address the Freon web server will bind to using HTTPS. + If this optional address is set, it overrides only the hostname portion of + ozone.freon.http-address. + + + + ozone.freon.http.kerberos.principal + HTTP/_HOST@EXAMPLE.COM + SECURITY + + Security principal used by freon. + + + + ozone.freon.http.kerberos.keytab + /etc/security/keytabs/HTTP.keytab + SECURITY + + Keytab used by Freon. + + + + diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 6838add203..3cef596c4e 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -45,6 +45,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop hadoop-ozone-filesystem + + org.apache.hadoop + hadoop-hdds-server-framework + org.apache.hadoop hadoop-common diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index 694373d86d..88ca098c8f 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -16,11 +16,16 @@ */ package org.apache.hadoop.ozone.freon; +import java.io.IOException; + import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; +import picocli.CommandLine.Option; /** * Ozone data generator and performance test tool. @@ -33,12 +38,43 @@ mixinStandardHelpOptions = true) public class Freon extends GenericCli { + public static final Logger LOG = LoggerFactory.getLogger(Freon.class); + + @Option(names = "--server", + description = "Enable internal http server to provide metric " + + "and profile endpoint") + private boolean httpServer = false; + + private FreonHttpServer freonHttpServer; + @Override public void execute(String[] argv) { TracingUtil.initTracing("freon"); super.execute(argv); } + public void stopHttpServer() { + if (freonHttpServer != null) { + try { + freonHttpServer.stop(); + } catch (Exception e) { + LOG.error("Freon http server can't be stopped", e); + } + } + } + + public void startHttpServer() { + if (httpServer) { + try { + freonHttpServer = new FreonHttpServer(createOzoneConfiguration()); + freonHttpServer.start(); + } catch (IOException e) { + LOG.error("Freon http server can't be started", e); + } + } + + } + public static void main(String[] args) { new Freon().run(args); } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java new file mode 100644 index 0000000000..dab4889550 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/FreonHttpServer.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.freon; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.server.BaseHttpServer; +import org.apache.hadoop.ozone.OzoneConfigKeys; + +/** + * Http server to provide metrics + profile endpoint. + */ +public class FreonHttpServer extends BaseHttpServer { + public FreonHttpServer(Configuration conf) throws IOException { + super(conf, "freon"); + } + + + @Override protected String getHttpAddressKey() { + return OzoneConfigKeys.OZONE_FREON_HTTP_ADDRESS_KEY; + } + + @Override protected String getHttpBindHostKey() { + return OzoneConfigKeys.OZONE_FREON_HTTP_BIND_HOST_KEY; + } + + @Override protected String getHttpsAddressKey() { + return OzoneConfigKeys.OZONE_FREON_HTTPS_ADDRESS_KEY; + } + + @Override protected String getHttpsBindHostKey() { + return OzoneConfigKeys.OZONE_FREON_HTTPS_BIND_HOST_KEY; + } + + @Override protected String getBindHostDefault() { + return OzoneConfigKeys.OZONE_FREON_HTTP_BIND_HOST_DEFAULT; + } + + @Override protected int getHttpBindPortDefault() { + return OzoneConfigKeys.OZONE_FREON_HTTP_BIND_PORT_DEFAULT; + } + + @Override protected int getHttpsBindPortDefault() { + return OzoneConfigKeys.OZONE_FREON_HTTPS_BIND_PORT_DEFAULT; + } + + @Override protected String getKeytabFile() { + return OzoneConfigKeys.OZONE_FREON_HTTP_KERBEROS_KEYTAB_FILE_KEY; + } + + @Override protected String getSpnegoPrincipal() { + return OzoneConfigKeys.OZONE_FREON_HTTP_KERBEROS_PRINCIPAL_KEY; + } + + @Override protected String getEnabledKey() { + return OzoneConfigKeys.OZONE_FREON_HTTP_ENABLED_KEY; + } +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java index 87029fa0aa..eb25b337b2 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java @@ -214,6 +214,7 @@ public void init(OzoneConfiguration configuration) throws IOException { for (FreonOps ops : FreonOps.values()) { histograms.add(ops.ordinal(), new Histogram(new UniformReservoir())); } + freon.startHttpServer(); } @Override @@ -292,7 +293,10 @@ public Void call() throws Exception { */ private void addShutdownHook() { Runtime.getRuntime().addShutdownHook( - new Thread(() -> printStats(System.out))); + new Thread(() -> { + printStats(System.out); + freon.stopHttpServer(); + })); } /** * Prints stats of {@link Freon} run to the PrintStream. diff --git a/hadoop-ozone/tools/src/main/resources/webapps/freon/.gitkeep b/hadoop-ozone/tools/src/main/resources/webapps/freon/.gitkeep new file mode 100644 index 0000000000..64853142e5 --- /dev/null +++ b/hadoop-ozone/tools/src/main/resources/webapps/freon/.gitkeep @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file