From f1e2592bf806285b34de002005a2702e0fe84005 Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Fri, 21 Jul 2017 20:43:57 +0800 Subject: [PATCH] HDFS-12127. Ozone: Ozone shell: Add more testing for key shell commands. Contributed by Yiqun Lin. --- ...ManagerProtocolClientSideTranslatorPB.java | 2 +- .../hadoop/ozone/ksm/KeyManagerImpl.java | 2 + .../web/ozShell/keys/DeleteKeyHandler.java | 2 +- .../web/ozShell/keys/ListKeyHandler.java | 2 + .../hadoop/ozone/ozShell/TestOzoneShell.java | 265 ++++++++++++++---- .../hadoop/ozone/web/client/TestKeys.java | 4 +- 6 files changed, 226 insertions(+), 51 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java index 0f5ca95d06..2ff6be9427 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java @@ -521,7 +521,7 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException { throw ProtobufHelper.getRemoteException(e); } if (resp.getStatus() != Status.OK) { - throw new IOException("Get key failed, error:" + + throw new IOException("Create key failed, error:" + resp.getStatus()); } return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java index d12bef1525..aee3f7c5a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java @@ -108,6 +108,8 @@ public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException { LOG.debug("Key {} allocated in volume {} bucket {}", keyName, volumeName, bucketName); return keyBlock; + } catch (KSMException e) { + throw e; } catch (IOException ex) { LOG.error("Key allocation failed for volume:{} bucket:{} key:{}", volumeName, bucketName, keyName, ex); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java index 1ec857839c..b1912c6a95 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java @@ -89,7 +89,7 @@ protected void execute(CommandLine cmd) OzoneVolume vol = client.getVolume(volumeName); - OzoneBucket bucket = vol.createBucket(bucketName); + OzoneBucket bucket = vol.getBucket(bucketName); bucket.deleteKey(keyName); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java index 0e8a7739d0..c51fbd3ed6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.web.ozShell.Handler; import org.apache.hadoop.ozone.web.ozShell.Shell; import org.apache.hadoop.ozone.web.utils.JsonUtils; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; import java.io.IOException; import java.net.URI; @@ -63,6 +64,7 @@ protected void execute(CommandLine cmd) String length = null; if (cmd.hasOption(Shell.LIST_LENGTH)) { length = cmd.getOptionValue(Shell.LIST_LENGTH); + OzoneUtils.verifyMaxKeyLength(length); } String startKey = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java index baec765598..c1597ebb68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java @@ -24,6 +24,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -36,6 +37,7 @@ import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneAcl; @@ -44,8 +46,8 @@ import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status; import org.apache.hadoop.ozone.web.client.OzoneBucket; +import org.apache.hadoop.ozone.web.client.OzoneKey; import org.apache.hadoop.ozone.web.client.OzoneRestClient; import org.apache.hadoop.ozone.web.client.OzoneVolume; import org.apache.hadoop.ozone.web.exceptions.OzoneException; @@ -238,7 +240,7 @@ public void testUpdateVolume() throws Exception { } @Test - public void testListVolumes() throws Exception { + public void testListVolume() throws Exception { final int volCount = 20; final String user1 = "test-user-a"; final String user2 = "test-user-b"; @@ -447,7 +449,7 @@ public void testUpdateBucket() throws Exception { } @Test - public void testListBuckets() throws Exception { + public void testListBucket() throws Exception { int bucketCount = 11; OzoneVolume vol = creatVolume(); @@ -485,7 +487,8 @@ public void testListBuckets() throws Exception { buckets = getValueLines("bucketName", out.toString()); assertEquals(3, buckets.size()); - // return volume names should be [test-vol0, test-vol1, test-vol10] + // return bucket names should be [test-bucket0, + // test-bucket1, test-bucket10] assertTrue(buckets.get(0).contains("test-bucket0") && buckets.get(1).contains("test-bucket1") && buckets.get(2).contains("test-bucket10")); @@ -498,7 +501,7 @@ public void testListBuckets() throws Exception { buckets = getValueLines("bucketName", out.toString()); assertEquals(2, buckets.size()); - // return volume names should be [test-vol1, test-vol10] + // return bucket names should be [test-bucket1, test-bucket10] assertTrue(buckets.get(0).contains("test-bucket1") && buckets.get(1).contains("test-bucket10")); @@ -523,21 +526,220 @@ public void testListBuckets() throws Exception { } @Test - public void testGetKeyInfo() throws Exception { - // create a volume - String volume = "volume" + RandomStringUtils.randomNumeric(5); - String[] args = new String[] {"-createVolume", url + "/" + volume, "-user", - "bilbo", "-root"}; + public void testPutKey() throws Exception { + OzoneBucket bucket = creatBucket(); + String volumeName = bucket.getBucketInfo().getVolumeName(); + String bucketName = bucket.getBucketName(); + String keyName = "key" + RandomStringUtils.randomNumeric(5); + + String[] args = new String[] {"-putKey", + url + "/" + volumeName + "/" + bucketName + "/" + keyName, "-file", + createTmpFile()}; assertEquals(0, ToolRunner.run(shell, args)); - // create a bucket - String bucket = "bucket" + RandomStringUtils.randomNumeric(5); - args = new String[] {"-createBucket", url + "/" + volume + "/" + bucket}; + OzoneKey keyInfo = bucket.getKeyInfo(keyName); + assertEquals(keyName, keyInfo.getObjectInfo().getKeyName()); + + // test put key in a non-exist bucket + args = new String[] {"-putKey", + url + "/" + volumeName + "/invalid-bucket/" + keyName, "-file", + createTmpFile()}; + assertEquals(1, ToolRunner.run(shell, args)); + assertTrue(err.toString().contains( + "Create key failed, error:BUCKET_NOT_FOUND")); + } + + @Test + public void testGetKey() throws Exception { + String keyName = "key" + RandomStringUtils.randomNumeric(5); + OzoneBucket bucket = creatBucket(); + String volumeName = bucket.getBucketInfo().getVolumeName(); + String bucketName = bucket.getBucketName(); + + String dataStr = "test-data"; + bucket.putKey(keyName, dataStr); + + String tmpPath = baseDir.getAbsolutePath() + "/testfile-" + + UUID.randomUUID().toString(); + String[] args = new String[] {"-getKey", + url + "/" + volumeName + "/" + bucketName + "/" + keyName, "-file", + tmpPath}; assertEquals(0, ToolRunner.run(shell, args)); + byte[] dataBytes = new byte[dataStr.length()]; + try (FileInputStream randFile = new FileInputStream(new File(tmpPath))) { + randFile.read(dataBytes); + } + assertEquals(dataStr, DFSUtil.bytes2String(dataBytes)); + } + + @Test + public void testDeleteKey() throws Exception { + String keyName = "key" + RandomStringUtils.randomNumeric(5); + OzoneBucket bucket = creatBucket(); + String volumeName = bucket.getBucketInfo().getVolumeName(); + String bucketName = bucket.getBucketName(); + bucket.putKey(keyName, "test-data"); + + OzoneKey keyInfo = bucket.getKeyInfo(keyName); + assertEquals(keyName, keyInfo.getObjectInfo().getKeyName()); + + String[] args = new String[] {"-deleteKey", + url + "/" + volumeName + "/" + bucketName + "/" + keyName}; + assertEquals(0, ToolRunner.run(shell, args)); + + // verify if key has been deleted in the bucket + try { + bucket.getKeyInfo(keyName); + fail("Get key should have thrown."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + "Lookup key failed, error:KEY_NOT_FOUND", e); + } + + // test delete key in a non-exist bucket + args = new String[] {"-deleteKey", + url + "/" + volumeName + "/invalid-bucket/" + keyName}; + assertEquals(1, ToolRunner.run(shell, args)); + assertTrue(err.toString().contains( + "Info Bucket failed, error: BUCKET_NOT_FOUND")); + + err.reset(); + // test delete a non-exist key in bucket + args = new String[] {"-deleteKey", + url + "/" + volumeName + "/" + bucketName + "/invalid-key"}; + assertEquals(1, ToolRunner.run(shell, args)); + assertTrue(err.toString().contains( + "Delete key failed, error:KEY_NOT_FOUND")); + } + + @Test + public void testInfoKey() throws Exception { + String keyName = "key" + RandomStringUtils.randomNumeric(5); + OzoneBucket bucket = creatBucket(); + String volumeName = bucket.getBucketInfo().getVolumeName(); + String bucketName = bucket.getBucketName(); + bucket.putKey(keyName, "test-data"); + + String[] args = new String[] {"-infoKey", + url + "/" + volumeName + "/" + bucketName + "/" + keyName}; + + // verify the response output + assertEquals(0, ToolRunner.run(shell, args)); + assertTrue(out.toString().contains(keyName)); + + // reset stream + out.reset(); + err.reset(); + + // get the info of a non-exist key + args = new String[] {"-infoKey", + url + "/" + volumeName + "/" + bucketName + "/invalid-key"}; + + // verify the response output + // get the non-exist key info should be failed + assertEquals(1, ToolRunner.run(shell, args)); + assertTrue(err.toString().contains( + "Lookup key failed, error:KEY_NOT_FOUND")); + } + + @Test + public void testListKey() throws Exception { + int keyCount = 11; + OzoneBucket bucket = creatBucket(); + String volumeName = bucket.getBucketInfo().getVolumeName(); + String bucketName = bucket.getBucketName(); + + String keyName; + List keyNames = new ArrayList<>(); + for (int i = 0; i < keyCount; i++) { + keyName = "test-key" + i; + keyNames.add(keyName); + bucket.putKey(keyName, "test-data" + i); + } + + // test -length option + String[] args = new String[] {"-listKey", + url + "/" + volumeName + "/" + bucketName, "-length", "100"}; + assertEquals(0, ToolRunner.run(shell, args)); + + List keys = getValueLines("keyName", out.toString()); + assertEquals(11, keys.size()); + // sort key names since the return keys isn't in created order + Collections.sort(keyNames); + // return key names should be [test-key0, test-key1, + // test-key10, test-key2, ,..., test-key9] + for (int i = 0; i < keys.size(); i++) { + assertTrue(keys.get(i).contains(keyNames.get(i))); + } + + out.reset(); + args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName, + "-length", "3"}; + assertEquals(0, ToolRunner.run(shell, args)); + + keys = getValueLines("keyName", out.toString()); + assertEquals(3, keys.size()); + // return key names should be [test-key0, test-key1, test-key10] + assertTrue(keys.get(0).contains("test-key0") + && keys.get(1).contains("test-key1") + && keys.get(2).contains("test-key10")); + + // test -prefix option + out.reset(); + args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName, + "-length", "100", "-prefix", "test-key1"}; + assertEquals(0, ToolRunner.run(shell, args)); + + keys = getValueLines("keyName", out.toString()); + assertEquals(2, keys.size()); + // return key names should be [test-key1, test-key10] + assertTrue(keys.get(0).contains("test-key1") + && keys.get(1).contains("test-key10")); + + // test -start option + out.reset(); + args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName, + "-length", "100", "-start", "test-key7"}; + assertEquals(0, ToolRunner.run(shell, args)); + + keys = getValueLines("keyName", out.toString()); + assertTrue(keys.get(0).contains("test-key8") + && keys.get(1).contains("test-key9")); + + // test error conditions + err.reset(); + args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName, + "-length", "-1"}; + assertEquals(1, ToolRunner.run(shell, args)); + assertTrue(err.toString().contains( + "the vaule should be a positive number")); + } + + private OzoneVolume creatVolume() throws OzoneException { + String volumeName = UUID.randomUUID().toString() + "volume"; + OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); + + return vol; + } + + private OzoneBucket creatBucket() throws OzoneException { + OzoneVolume vol = creatVolume(); + String bucketName = UUID.randomUUID().toString() + "bucket"; + OzoneBucket bucketInfo = vol.createBucket(bucketName); + + return bucketInfo; + } + + /** + * Create a temporary file used for putting key. + * @return the created file's path string + * @throws Exception + */ + private String createTmpFile() throws Exception { // write a new file that used for putting key - String key = "key" + RandomStringUtils.randomNumeric(5); - File tmpFile = new File(baseDir, "/testfile"); + File tmpFile = new File(baseDir, + "/testfile-" + UUID.randomUUID().toString()); FileOutputStream randFile = new FileOutputStream(tmpFile); Random r = new Random(); for (int x = 0; x < 10; x++) { @@ -546,38 +748,7 @@ public void testGetKeyInfo() throws Exception { } randFile.close(); - // create the key in above bucket - args = new String[] {"-putKey", - url + "/" + volume + "/" + bucket + "/" + key, "-file", - tmpFile.getAbsolutePath()}; - assertEquals(0, ToolRunner.run(shell, args)); - - args = new String[] {"-infoKey", - url + "/" + volume + "/" + bucket + "/" + key}; - - // verify the response output - assertEquals(0, ToolRunner.run(shell, args)); - assertTrue(out.toString().contains(key)); - - // reset stream - out.reset(); - err.reset(); - - // get the info of a non-exist key - args = new String[] {"-infoKey", - url + "/" + volume + "/" + bucket + "/invalid-key"}; - - // verify the response output - // get the non-exist key info should be failed - assertEquals(1, ToolRunner.run(shell, args)); - assertTrue(err.toString().contains(Status.KEY_NOT_FOUND.toString())); - } - - private OzoneVolume creatVolume() throws OzoneException { - String volumeName = UUID.randomUUID().toString() + "volume"; - OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); - - return vol; + return tmpFile.getAbsolutePath(); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index 8efef0b69b..516c5713ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -200,7 +200,7 @@ static void runTestPutKey(PutHelper helper) throws OzoneException { + " when using invalid volume name."); } catch(OzoneException e) { GenericTestUtils.assertExceptionContains( - Status.INTERNAL_ERROR.toString(), e); + Status.VOLUME_NOT_FOUND.toString(), e); } try { @@ -210,7 +210,7 @@ static void runTestPutKey(PutHelper helper) throws OzoneException { + "when using invalid bucket name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - Status.INTERNAL_ERROR.toString(), e); + Status.BUCKET_NOT_FOUND.toString(), e); } }