From e83719c830dd4927c8eef26062c56c0d62b2f04f Mon Sep 17 00:00:00 2001 From: Nanda kumar Date: Thu, 2 Aug 2018 19:02:25 +0530 Subject: [PATCH] HDDS-290. putKey is failing with KEY_ALLOCATION_ERROR. Contributed by Xiaoyu Yao. --- .../src/main/compose/ozone/docker-config | 1 + .../acceptance/ozonefs/ozonesinglenode.robot | 49 +++++++++++++++++++ .../hadoop/ozone/web/ozShell/Shell.java | 4 ++ .../ozone/web/ozShell/keys/PutKeyHandler.java | 16 ++++-- 4 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config index 50abb18e1a..1b75c01cbe 100644 --- a/hadoop-dist/src/main/compose/ozone/docker-config +++ b/hadoop-dist/src/main/compose/ozone/docker-config @@ -22,6 +22,7 @@ OZONE-SITE.XML_ozone.scm.block.client.address=scm OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata OZONE-SITE.XML_ozone.handler.type=distributed OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout diff --git a/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot new file mode 100644 index 0000000000..b844cee798 --- /dev/null +++ b/hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonesinglenode.robot @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Ozonefs Single Node Test +Library OperatingSystem +Suite Setup Startup Ozone cluster with size 1 +Suite Teardown Teardown Ozone cluster +Resource ../commonlib.robot + +*** Variables *** +${COMPOSEFILE} ${CURDIR}/docker-compose.yaml +${PROJECTDIR} ${CURDIR}/../../../../../.. + + +*** Test Cases *** +Create volume and bucket + Execute on datanode ozone oz -createVolume http://ozoneManager/fstest -user bilbo -quota 100TB -root + Execute on datanode ozone oz -createBucket http://ozoneManager/fstest/bucket1 + +Check volume from ozonefs + ${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/ + +Create directory from ozonefs + Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep + ${result} = Execute on ozoneManager ozone oz -listKey o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName' + Should contain ${result} testdir/deep +Test key handling + Execute on datanode ozone oz -putKey o3://ozoneManager/fstest/bucket1/key1 -file NOTICE.txt -replicationFactor 1 + Execute on datanode rm -f NOTICE.txt.1 + Execute on datanode ozone oz -getKey o3://ozoneManager/fstest/bucket1/key1 -file NOTICE.txt.1 + Execute on datanode ls -l NOTICE.txt.1 + ${result} = Execute on datanode ozone oz -infoKey o3://ozoneManager/fstest/bucket1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")' + Should contain ${result} createdOn + ${result} = Execute on datanode ozone oz -listKey o3://ozoneManager/fstest/bucket1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName' + Should Be Equal ${result} key1 + Execute on datanode ozone oz -deleteKey o3://ozoneManager/fstest/bucket1/key1 -v diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java index 726f4cab90..41eef1abf9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java @@ -92,6 +92,7 @@ public class Shell extends Configured implements Tool { public static final String DELETE_KEY = "deleteKey"; public static final String LIST_KEY = "listKey"; public static final String FILE = "file"; + public static final String REPLICATION_FACTOR = "replicationFactor"; // Listing related command line arguments public static final String LIST_LENGTH = "length"; @@ -292,6 +293,9 @@ private void addKeyCommands(Options opts) { new Option(FILE, true, "Data file path"); opts.addOption(fileArgument); + Option repFactor = + new Option(REPLICATION_FACTOR, true, "Replication factor (1 or 3)"); + opts.addOption(repFactor); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java index ed8cc8822d..c73307dd82 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java @@ -44,7 +44,9 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_DEFAULT; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT; /** * Puts a file into an ozone bucket. @@ -103,11 +105,17 @@ protected void execute(CommandLine cmd) } Configuration conf = new OzoneConfiguration(); - ReplicationFactor replicationFactor = ReplicationFactor.valueOf( - conf.getInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue())); - ReplicationType replicationType = ReplicationType.valueOf( - conf.get(OZONE_REPLICATION_TYPE, ReplicationType.RATIS.toString())); + ReplicationFactor replicationFactor; + if (cmd.hasOption(Shell.REPLICATION_FACTOR)) { + replicationFactor = ReplicationFactor.valueOf(Integer.parseInt(cmd + .getOptionValue(Shell.REPLICATION_FACTOR))); + } else { + replicationFactor = ReplicationFactor.valueOf( + conf.getInt(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT)); + } + ReplicationType replicationType = ReplicationType.valueOf( + conf.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)); OzoneVolume vol = client.getObjectStore().getVolume(volumeName); OzoneBucket bucket = vol.getBucket(bucketName); OzoneOutputStream outputStream = bucket