From 67173b4847908cdf27df143ca61a785aef9b835a Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 24 Sep 2018 13:42:55 -0700 Subject: [PATCH] HDDS-441. Create new s3gateway daemon. Contributed by Elek Marton. --- hadoop-ozone/common/src/main/bin/ozone | 7 +- hadoop-ozone/dist/pom.xml | 11 ++ .../main/compose/ozones3/docker-compose.yaml | 56 ++++++ .../src/main/compose/ozones3/docker-config | 73 ++++++++ hadoop-ozone/pom.xml | 1 + hadoop-ozone/s3gateway/pom.xml | 173 ++++++++++++++++++ .../org/apache/hadoop/ozone/s3/Gateway.java | 55 ++++++ hadoop-project/pom.xml | 5 + 8 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3/docker-config create mode 100644 hadoop-ozone/s3gateway/pom.xml create mode 100644 hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index d173ec6d58..4b50771176 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -44,6 +44,7 @@ function hadoop_usage hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" hadoop_add_subcommand "om" daemon "Ozone Manager" hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" + hadoop_add_subcommand "s3g" daemon "run the S3 compatible REST gateway" hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager" hadoop_add_subcommand "sh" client "command line interface for object store operations" hadoop_add_subcommand "version" client "print the version" @@ -114,7 +115,11 @@ function ozonecmd_case hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}" OZONE_RUN_ARTIFACT_NAME="hadoop-hdds-server-scm" - + ;; + s3g) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway' + OZONE_RUN_ARTIFACT_NAME="hadoop-ozone-s3gateway" ;; fs) HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index ba9f14fe91..7d1137274e 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -60,6 +60,13 @@ classpath hadoop-hdds-tools.classpath + + org.apache.hadoop + hadoop-ozone-s3gateway + ${ozone.version} + classpath + hadoop-ozone-s3gateway.classpath + org.apache.hadoop hadoop-ozone-ozone-manager @@ -164,6 +171,10 @@ org.apache.hadoop hadoop-hdds-container-service + + org.apache.hadoop + hadoop-ozone-s3gateway + org.apache.hadoop hadoop-ozone-ozone-manager diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml new file mode 100644 index 0000000000..10dc61e40b --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + datanode: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9864 + command: ["ozone","datanode"] + env_file: + - ./docker-config + ozoneManager: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION + env_file: + - ./docker-config + command: ["ozone","om"] + scm: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone","scm"] + s3g: + image: apache/hadoop-runner + volumes: + - ../..:/opt/hadoop + env_file: + - ./docker-config + command: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config new file mode 100644 index 0000000000..2b22874b02 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-config @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=ozoneManager +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=true +OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 8d3e1613dd..075c674432 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -43,6 +43,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> integration-test objectstore-service datanode + s3gateway dist docs diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml new file mode 100644 index 0000000000..7b02c85215 --- /dev/null +++ b/hadoop-ozone/s3gateway/pom.xml @@ -0,0 +1,173 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-ozone + 0.3.0-SNAPSHOT + + hadoop-ozone-s3gateway + Apache Hadoop Ozone S3 Gatway + jar + 0.3.0-SNAPSHOT + + UTF-8 + true + + + + + + + + + + + org.apache.hadoop + hadoop-ozone-common + compile + + + org.jboss.weld.servlet + weld-servlet + 2.4.7.Final + + + org.glassfish.jersey.containers + jersey-container-servlet-core + 2.27 + + + org.glassfish.jersey.ext.cdi + jersey-cdi1x + 2.27 + + + org.glassfish.jersey.inject + jersey-hk2 + 2.27 + + + javax.enterprise + cdi-api + 1.2 + + + org.apache.hadoop + hadoop-common + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + + org.apache.hadoop + hadoop-hdfs + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + + org.apache.hadoop + hadoop-common + test-jar + test + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + + org.apache.hadoop + hadoop-hdfs + test-jar + test + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-servlet + + + com.sun.jersey + jersey-json + + + com.sun.jersey + jersey-server + + + + + org.apache.hadoop + hadoop-ozone-client + + + org.mockito + mockito-core + 2.15.0 + test + + + diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java new file mode 100644 index 0000000000..4e3e48e1f7 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import org.apache.hadoop.hdds.cli.GenericCli; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; + +/** + * This class is used to start/stop S3 compatible rest server. + */ +@Command(name = "ozone s3g", + hidden = true, description = "S3 compatible rest server.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true) +public class Gateway extends GenericCli { + + private static final Logger LOG = LoggerFactory.getLogger(Gateway.class); + + public static void main(String[] args) throws Exception { + new Gateway().run(args); + } + + @Override + public Void call() throws Exception { + start(); + return null; + } + + public void start() { + LOG.info("Starting Ozone S3 gateway"); + } + + public void stop() { + LOG.info("Stoping Ozone S3 gateway"); + } +} diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 8d08703af9..e7baee003c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -621,6 +621,11 @@ hadoop-ozone-datanode ${hdds.version} + + org.apache.hadoop + hadoop-ozone-s3gateway + ${hdds.version} + org.apache.hadoop hadoop-ozone-integration-test