From a4b3160efbc87626d51a4b30312ca15b58df15aa Mon Sep 17 00:00:00 2001 From: Weiwei Yang Date: Thu, 27 Jul 2017 12:40:40 +0800 Subject: [PATCH] HDFS-12145. Ozone: OzoneFileSystem: Ozone & KSM should support / delimited key names. Contributed by Mukul Kumar Singh. (cherry picked from commit fe0a957e92d506330ba005abba18f665eb5ac1cc) --- .../hadoop/scm/storage/ChunkOutputStream.java | 4 +- .../hadoop/ozone/web/interfaces/Keys.java | 2 +- .../hadoop/ozone/web/client/TestKeys.java | 56 +++++++++++++++++-- .../ozone/web/client/TestKeysRatis.java | 20 +++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java index d93d2cf620..8daded5e83 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java @@ -28,6 +28,7 @@ import com.google.protobuf.ByteString; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo; import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData; import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; @@ -209,7 +210,8 @@ private synchronized void writeChunkToContainer() throws IOException { ChunkInfo chunk = ChunkInfo .newBuilder() .setChunkName( - key + "_stream_" + streamId + "_chunk_" + ++chunkIndex) + DigestUtils.md5Hex(key) + "_stream_" + + streamId + "_chunk_" + ++chunkIndex) .setOffset(0) .setLen(data.size()) .build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java index b6885a7933..e64c8bab38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java @@ -40,7 +40,7 @@ * This interface defines operations permitted on a key. */ -@Path("/{volume}/{bucket}/{keys}") +@Path("/{volume}/{bucket}/{keys:.*}") public interface Keys { /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index 516c5713ed..d9168ab20d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.web.client; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.RandomStringUtils; +import org.apache.commons.lang.math.RandomUtils; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.ozone.MiniOzoneCluster; @@ -28,6 +30,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.AfterClass; @@ -122,17 +125,41 @@ static File createRandomDataFile(String dir, String fileName, long size) { return tmpFile; } + /** + * This function generates multi part key which are delimited by a certain + * delimiter. Different parts of key are random string of random length + * between 0 - 4. Number of parts of the keys are between 0 and 5. + * + * @param delimiter delimiter used to delimit parts of string + * @return Key composed of multiple parts delimited by "/" + */ + static String getMultiPartKey(String delimiter) { + int numParts = RandomUtils.nextInt(5) + 1; + String[] nameParts = new String[numParts]; + for (int i = 0; i < numParts; i++) { + int stringLength = numParts == 1 ? 5 : RandomUtils.nextInt(5); + nameParts[i] = RandomStringUtils.randomAlphanumeric(stringLength); + } + return StringUtils.join(delimiter, nameParts); + } + static class PutHelper { private final OzoneRestClient client; private final String dir; + private final String keyName; OzoneVolume vol; OzoneBucket bucket; File file; PutHelper(OzoneRestClient client, String dir) { + this(client, dir, OzoneUtils.getRequestID().toLowerCase()); + } + + PutHelper(OzoneRestClient client, String dir, String key) { this.client = client; this.dir = dir; + this.keyName = key; } public OzoneVolume getVol() { @@ -163,8 +190,8 @@ private String putKey() throws String bucketName = OzoneUtils.getRequestID().toLowerCase(); bucket = vol.createBucket(bucketName, acls, StorageType.DEFAULT); - String keyName = OzoneUtils.getRequestID().toLowerCase(); - file = createRandomDataFile(dir, keyName, 1024); + String fileName = OzoneUtils.getRequestID().toLowerCase(); + file = createRandomDataFile(dir, fileName, 1024); bucket.putKey(keyName, file); return keyName; @@ -174,7 +201,12 @@ private String putKey() throws @Test public void testPutKey() throws OzoneException { + // Test non-delimited keys runTestPutKey(new PutHelper(ozoneRestClient, path)); + // Test key delimited by a random delimiter + String delimiter = RandomStringUtils.randomAscii(1); + runTestPutKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } static void runTestPutKey(PutHelper helper) throws OzoneException { @@ -229,6 +261,10 @@ public void testPutAndGetKeyWithDnRestart() throws OzoneException, IOException, URISyntaxException { runTestPutAndGetKeyWithDnRestart( new PutHelper(ozoneRestClient, path), ozoneCluster); + String delimiter = RandomStringUtils.randomAscii(1); + runTestPutAndGetKeyWithDnRestart( + new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter)), ozoneCluster); } static void runTestPutAndGetKeyWithDnRestart( @@ -263,6 +299,9 @@ static void runTestPutAndGetKeyWithDnRestart( @Test public void testPutAndGetKey() throws OzoneException, IOException { runTestPutAndGetKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAscii(1); + runTestPutAndGetKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } static void runTestPutAndGetKey(PutHelper helper) @@ -326,6 +365,9 @@ static void runTestPutAndGetKey(PutHelper helper) @Test public void testPutAndDeleteKey() throws OzoneException, IOException { runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAscii(1); + runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } static void runTestPutAndDeleteKey(PutHelper helper) @@ -347,6 +389,9 @@ static void runTestPutAndDeleteKey(PutHelper helper) @Test public void testPutAndListKey() throws OzoneException, IOException { runTestPutAndListKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAscii(1); + runTestPutAndListKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } static void runTestPutAndListKey(PutHelper helper) @@ -378,9 +423,9 @@ static void runTestPutAndListKey(PutHelper helper) Assert.assertEquals(keyList2.size(), 1); // test startKey parameter of list keys - keyList1 = helper.getBucket().listKeys("100", "list-key4", null); + keyList1 = helper.getBucket().listKeys("100", "list-key4", "list-key"); keyList2 = client.listKeys(helper.getVol().getVolumeName(), - helper.getBucket().getBucketName(), "100", "list-key4", null); + helper.getBucket().getBucketName(), "100", "list-key4", "list-key"); Assert.assertEquals(keyList1.size(), 5); Assert.assertEquals(keyList2.size(), 5); @@ -416,6 +461,9 @@ static void runTestPutAndListKey(PutHelper helper) @Test public void testGetKeyInfo() throws OzoneException, IOException { runTestGetKeyInfo(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAscii(1); + runTestGetKeyInfo(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } static void runTestGetKeyInfo(PutHelper helper) throws OzoneException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java index b0fea9090a..1d33644e76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.ozone.web.client; +import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.RatisTestHelper; import org.apache.hadoop.ozone.web.exceptions.OzoneException; @@ -57,6 +58,9 @@ public static void shutdown() { @Test public void testPutKey() throws OzoneException { runTestPutKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestPutKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } @Test @@ -64,25 +68,41 @@ public void testPutAndGetKeyWithDnRestart() throws OzoneException, IOException, URISyntaxException { runTestPutAndGetKeyWithDnRestart( new PutHelper(ozoneRestClient, path), suite.getCluster()); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestPutAndGetKeyWithDnRestart( + new PutHelper(ozoneRestClient, path, getMultiPartKey(delimiter)), + suite.getCluster()); } @Test public void testPutAndGetKey() throws OzoneException, IOException { runTestPutAndGetKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestPutAndGetKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } @Test public void testPutAndDeleteKey() throws OzoneException, IOException { runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestPutAndDeleteKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } @Test public void testPutAndListKey() throws OzoneException, IOException { runTestPutAndListKey(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestPutAndListKey(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } @Test public void testGetKeyInfo() throws OzoneException, IOException { runTestGetKeyInfo(new PutHelper(ozoneRestClient, path)); + String delimiter = RandomStringUtils.randomAlphanumeric(1); + runTestGetKeyInfo(new PutHelper(ozoneRestClient, path, + getMultiPartKey(delimiter))); } }