diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 736272d1a4..f09f03d3fd 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -52,6 +52,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> test + + io.dropwizard.metrics + metrics-core + test + diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java index ac245d511c..f04d392532 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.statemachine .DatanodeStateMachine; @@ -107,6 +108,12 @@ public DatanodeStateMachine.DatanodeStates call() throws Exception { */ private void persistContainerDatanodeDetails() throws IOException { String dataNodeIDPath = HddsUtils.getDatanodeIdFilePath(conf); + if (Strings.isNullOrEmpty(dataNodeIDPath)) { + LOG.error("A valid file path is needed for config setting {}", + ScmConfigKeys.OZONE_SCM_DATANODE_ID); + this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN); + return; + } File idPath = new File(dataNodeIDPath); DatanodeDetails datanodeDetails = this.context.getParent() .getDatanodeDetails(); diff --git a/hadoop-hdds/container-service/src/test/resources/log4j.properties b/hadoop-hdds/container-service/src/test/resources/log4j.properties new file mode 100644 index 0000000000..bb5cbe5ec3 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# log4j configuration used during build and unit tests + +log4j.rootLogger=INFO,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 35975f4e49..f742a00b1c 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -100,6 +100,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> hamcrest-all 1.3 + + org.bouncycastle + bcprov-jdk16 + test + diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java index 5d9139d72c..5ca9dd7a7c 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java @@ -76,7 +76,7 @@ public TestStorageContainerManagerHttpServer(Policy policy) { conf = new Configuration(); keystoresDir = new File(BASEDIR).getAbsolutePath(); sslConfDir = KeyStoreTestUtil.getClasspathDir( - org.apache.hadoop.hdfs.server.namenode.TestNameNodeHttpServer.class); + TestStorageContainerManagerHttpServer.class); KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); connectionFactory = URLConnectionFactory.newDefaultURLConnectionFactory(conf); diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java index 7237ab4a55..77030cdd54 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java @@ -262,7 +262,7 @@ public void testDeletedBlockTransactions() throws IOException { int count = 0; String containerName = null; DatanodeDetails dnDd1 = DatanodeDetails.newBuilder() - .setUuid("node1") + .setUuid(UUID.randomUUID().toString()) .setIpAddress("127.0.0.1") .setHostName("localhost") .setContainerPort(0) @@ -270,7 +270,7 @@ public void testDeletedBlockTransactions() throws IOException { .setOzoneRestPort(0) .build(); DatanodeDetails dnId2 = DatanodeDetails.newBuilder() - .setUuid("node2") + .setUuid(UUID.randomUUID().toString()) .setIpAddress("127.0.0.1") .setHostName("localhost") .setContainerPort(0) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java index e3473b3634..f38b6f94a8 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java @@ -315,7 +315,7 @@ public void addDatanodeCommand(UUID dnId, SCMCommand command) { // Returns the number of commands that is queued to this node manager. public int getCommandCount(DatanodeDetails dd) { - List list = commandMap.get(dd); + List list = commandMap.get(dd.getUuid()); return (list == null) ? 0 : list.size(); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java index a8d3362e6c..4c465d36b8 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java @@ -294,6 +294,9 @@ private void allocateNewBlock(int index) throws IOException { @Override public synchronized void flush() throws IOException { checkNotClosed(); + if (streamEntries.size() == 0) { + return; + } for (int i = 0; i <= currentStreamIndex; i++) { streamEntries.get(i).flush(); } diff --git a/hadoop-tools/hadoop-ozone/pom.xml b/hadoop-tools/hadoop-ozone/pom.xml index 2464c32c1e..c688af15b3 100644 --- a/hadoop-tools/hadoop-ozone/pom.xml +++ b/hadoop-tools/hadoop-ozone/pom.xml @@ -85,6 +85,31 @@ hadoop-hdds-common provided + + org.apache.hadoop + hadoop-hdds-server-scm + test + + + org.apache.hadoop + hadoop-hdds-server-framework + test + + + org.apache.hadoop + hadoop-ozone-ozone-manager + test + + + org.apache.hadoop + hadoop-hdds-container-service + test + + + org.apache.hadoop + hadoop-hdds-client + test + org.apache.hadoop hadoop-ozone-common diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index 6be9f919cc..f09dd2a937 100644 --- a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -93,8 +93,8 @@ public static void init() throws Exception { String host = dataNode.getDatanodeHostname(); // Set the fs.defaultFS and start the filesystem - String uri = String.format("%s://%s:%d/%s/%s", - Constants.OZONE_URI_SCHEME, host, port, volumeName, bucketName); + String uri = String.format("%s://%s.%s/", + Constants.OZONE_URI_SCHEME, bucketName, volumeName); conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri); fs = FileSystem.get(conf); int fileLen = 100 * 1024 * 1024; diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java similarity index 95% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java index 6b65dd76f8..a1c9404bc4 100644 --- a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java +++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -32,14 +32,13 @@ import org.apache.commons.lang.RandomStringUtils; import org.junit.After; -import org.apache.hadoop.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; import org.apache.hadoop.ozone.MiniOzoneClassicCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -99,7 +98,7 @@ public TestOzoneFileInterfaces(boolean setDefaultFs, public void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); cluster = new MiniOzoneClassicCluster.Builder(conf) - .numDataNodes(10) + .numDataNodes(3) .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED) .build(); storageHandler = @@ -119,11 +118,6 @@ public void init() throws Exception { BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs); storageHandler.createBucket(bucketArgs); - // Fetch the host and port for File System init - DataNode dataNode = cluster.getDataNodes().get(0); - int port = dataNode.getInfoPort(); - String host = dataNode.getDatanodeHostname(); - rootPath = String .format("%s://%s.%s/", Constants.OZONE_URI_SCHEME, bucketName, volumeName); @@ -147,7 +141,7 @@ public void teardown() throws IOException { public void testFileSystemInit() throws IOException { if (setDefaultFs) { assertTrue( - "The initialized file system is not OzoneFileSysetem but " + + "The initialized file system is not OzoneFileSystem but " + fs.getClass(), fs instanceof OzoneFileSystem); assertEquals(Constants.OZONE_URI_SCHEME, fs.getUri().getScheme()); diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractCreate.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDelete.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCp.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractGetFileStatus.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractMkdir.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractOpen.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRename.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractRootDir.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractSeek.java diff --git a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java similarity index 95% rename from hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java rename to hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java index 97ec3f4614..2cbe0f44d6 100644 --- a/hadoop-tools/hadoop-ozone/src/todo/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java +++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java @@ -20,12 +20,11 @@ import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.ozone.Constants; -import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; import org.apache.hadoop.ozone.MiniOzoneClassicCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -105,8 +104,6 @@ public FileSystem getTestFileSystem() throws IOException { } catch (OzoneException e) { throw new IOException(e.getMessage()); } - DataNode dataNode = cluster.getDataNodes().get(0); - final int port = dataNode.getInfoPort(); String uri = String.format("%s://%s.%s/", Constants.OZONE_URI_SCHEME, bucketName, volumeName); diff --git a/hadoop-tools/hadoop-ozone/src/todo/resources/contract/ozone.xml b/hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/resources/contract/ozone.xml rename to hadoop-tools/hadoop-ozone/src/test/resources/contract/ozone.xml diff --git a/hadoop-tools/hadoop-ozone/src/todo/resources/log4j.properties b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties similarity index 100% rename from hadoop-tools/hadoop-ozone/src/todo/resources/log4j.properties rename to hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties