ozone.client.checksum.type
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index 4f6558c522..5065fcad8b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -449,11 +449,12 @@ private static void printUsage(PrintStream out) {
* @param args command line arguments.
* @param conf HDDS configuration
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
@VisibleForTesting
- public static StorageContainerManager createSCM(String[] args,
- OzoneConfiguration conf) throws IOException, AuthenticationException {
+ public static StorageContainerManager createSCM(
+ String[] args, OzoneConfiguration conf)
+ throws IOException, AuthenticationException {
return createSCM(args, conf, false);
}
@@ -464,7 +465,7 @@ public static StorageContainerManager createSCM(String[] args,
* @param conf HDDS configuration
* @param printBanner if true, then log a verbose startup message.
* @return SCM instance
- * @throws IOException
+ * @throws IOException, AuthenticationException
*/
private static StorageContainerManager createSCM(
String[] args,
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
index da936ad5bf..41dd89a909 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManagerHttpServer.java
@@ -62,11 +62,11 @@ public StorageContainerManagerHttpServer(Configuration conf)
}
@Override protected String getKeytabFile() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY;
}
@Override protected String getSpnegoPrincipal() {
- return ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
+ return ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
index 32547356cb..199d52ce30 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/.env
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-OZONEDIR=../../../hadoop-dist/target/ozone
\ No newline at end of file
+OZONEDIR=../../../../../../hadoop-dist/target/ozone-0.3.0-SNAPSHOT
diff --git a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
index 360b69a886..cdf72ced54 100644
--- a/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
+++ b/hadoop-ozone/acceptance-test/src/test/compose/compose-secure/docker-config
@@ -22,7 +22,7 @@ OZONE-SITE.XML_hdds.scm.block.client.address=scm
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
OZONE-SITE.XML_ozone.handler.type=distributed
OZONE-SITE.XML_hdds.scm.client.address=scm
-OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService
+OZONE-SITE.XML_ozone.replication=1
OZONE-SITE.XML_hdds.scm.kerberos.principal=scm/scm@EXAMPLE.COM
OZONE-SITE.XML_hdds.scm.kerberos.keytab.file=/etc/security/keytabs/scm.keytab
OZONE-SITE.XML_ozone.om.kerberos.principal=om/om@EXAMPLE.COM
@@ -32,21 +32,12 @@ OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.principal=HTTP/scm@EXAMPLE.C
OZONE-SITE.XML_hdds.scm.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.principal=HTTP/om@EXAMPLE.COM
OZONE-SITE.XML_ozone.om.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
-OZONE-SITE.XML_ozone.scm.block.client.address=scm
-OZONE-SITE.XML_ozone.scm.client.address=scm
-HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
-HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService
-HDFS-SITE.XML_dfs.block.access.token.enable=true
-HDFS-SITE.XML_dfs.namenode.kerberos.principal=nn/namenode@EXAMPLE.COM
-HDFS-SITE.XML_dfs.namenode.keytab.file=/etc/security/keytabs/nn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/datanode@EXAMPLE.COM
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
-HDFS-SITE.XML_dfs.namenode.kerberos.internal.spnego.principal=HTTP/namenode@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/_HOST@EXAMPLE.COM
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
-HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*)s/.*/root/
@@ -55,9 +46,51 @@ LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
+#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
+
+#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
+LOG4J2.PROPERTIES_monitorInterval=30
+LOG4J2.PROPERTIES_filter=read,write
+LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.read.marker=READ
+LOG4J2.PROPERTIES_filter.read.onMatch=DENY
+LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
+LOG4J2.PROPERTIES_filter.write.marker=WRITE
+LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
+LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
+LOG4J2.PROPERTIES_appenders=console, rolling
+LOG4J2.PROPERTIES_appender.console.type=Console
+LOG4J2.PROPERTIES_appender.console.name=STDOUT
+LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
+LOG4J2.PROPERTIES_appender.rolling.fileName =${sys:hadoop.log.dir}/om-audit-${hostName}.log
+LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
+LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
+LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
+LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
+LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
+LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
+LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
+LOG4J2.PROPERTIES_loggers=audit
+LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
+LOG4J2.PROPERTIES_logger.audit.name=OMAudit
+LOG4J2.PROPERTIES_logger.audit.level=INFO
+LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
+LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
+LOG4J2.PROPERTIES_rootLogger.level=INFO
+LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
+LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT
+
+
OZONE_DATANODE_SECURE_USER=root
CONF_DIR=/etc/security/keytabs
-KERBEROS_KEYTABS=dn nn om scm HTTP testuser
+KERBEROS_KEYTABS=dn om scm HTTP testuser
KERBEROS_KEYSTORES=hadoop
KERBEROS_SERVER=ozone.kdc
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index cc25dbe017..0000000000
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
- /**
- * Never constructed.
- */
- private KSMConfigKeys() {
- }
-
-
- public static final String OZONE_KSM_HANDLER_COUNT_KEY =
- "ozone.ksm.handler.count.key";
- public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
- public static final String OZONE_KSM_ADDRESS_KEY =
- "ozone.ksm.address";
- public static final String OZONE_KSM_BIND_HOST_DEFAULT =
- "0.0.0.0";
- public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
- public static final String OZONE_KSM_HTTP_ENABLED_KEY =
- "ozone.ksm.http.enabled";
- public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
- "ozone.ksm.http-bind-host";
- public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
- "ozone.ksm.https-bind-host";
- public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
- "ozone.ksm.http-address";
- public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
- "ozone.ksm.https-address";
- public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
- public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
- public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
- "ozone.ksm.db.cache.size.mb";
- public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_KSM_USER_MAX_VOLUME =
- "ozone.ksm.user.max.volume";
- public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
- // KSM Default user/group permissions
- public static final String OZONE_KSM_USER_RIGHTS =
- "ozone.ksm.user.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KSM_GROUP_RIGHTS =
- "ozone.ksm.group.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
- "ozone.key.deleting.limit.per.task";
- public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.om.web.authentication.kerberos.principal";
- public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
- "ozone.om.web.authentication.kerberos.keytab";
-}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 64021d362e..a38787009f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -150,7 +150,8 @@ static Builder newBuilder(OzoneConfiguration conf) {
* @throws TimeoutException
* @throws InterruptedException
*/
- void restartStorageContainerManager() throws InterruptedException, TimeoutException, IOException, AuthenticationException;
+ void restartStorageContainerManager() throws InterruptedException,
+ TimeoutException, IOException, AuthenticationException;
/**
* Restarts OzoneManager instance.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index a41b9a27c7..cc9459b13d 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -233,8 +233,9 @@ public OzoneClient getRestClient() throws IOException {
}
@Override
- public void restartStorageContainerManager() throws TimeoutException,
- InterruptedException, IOException, AuthenticationException {
+ public void restartStorageContainerManager()
+ throws TimeoutException, InterruptedException, IOException,
+ AuthenticationException {
scm.stop();
scm.join();
scm = StorageContainerManager.createSCM(null, conf);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
index 2fe00e3bf2..809a354917 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
@@ -91,10 +91,6 @@ public void init() {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
-
- clusterId = UUID.randomUUID().toString();
- scmId = UUID.randomUUID().toString();
- omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@@ -122,11 +118,12 @@ private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
- conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
- conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
- createPrincipal(omKeyTab,
+ conf.get(ScmConfigKeys
+ .HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
conf.get(OMConfigKeys
- .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
+ .OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY);
+ createPrincipal(omKeyTab,
+ conf.get(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@@ -158,7 +155,7 @@ private void setSecureConfig(Configuration conf) throws IOException {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY,
"scm/" + host + "@" + realm);
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
@@ -172,7 +169,7 @@ private void setSecureConfig(Configuration conf) throws IOException {
conf.set(ScmConfigKeys.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
- conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
+ conf.set(ScmConfigKeys.HDDS_SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
@@ -191,6 +188,11 @@ public void testSecureScmStartupSuccess() throws Exception {
private void initSCM()
throws IOException, AuthenticationException {
+
+ clusterId = UUID.randomUUID().toString();
+ scmId = UUID.randomUUID().toString();
+ omId = UUID.randomUUID().toString();
+
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");