big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-12389. Ozone: oz commandline list calls should return valid JSON format output. Contributed by Weiwei Yang.

Browse Source
This commit is contained in:
Weiwei Yang 2017-09-12 11:05:51 +08:00 committed by Owen O'Malley
parent fec04f1158
commit 0c17776646
6 changed files with 123 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
View File

@ -95,6 +95,13 @@ public void setVolumeInfo(String volInfoString) throws IOException {
this.volumeInfo = VolumeInfo.parse(volInfoString);
}
/**
* @return the volume info.
*/
public VolumeInfo getVolumeInfo() {
return this.volumeInfo;
}
/**
* Returns volume Name.
*

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
View File

@ -25,6 +25,7 @@
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -34,6 +35,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
/**
* Executes List Bucket.
@ -102,10 +104,10 @@ protected void execute(CommandLine cmd)
OzoneVolume vol = client.getVolume(volumeName);
List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
for (OzoneBucket bucket : bucketList) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
bucket.getBucketInfo().toJsonString()));
}
List<BucketInfo> jsonData = bucketList.stream()
.map(OzoneBucket::getBucketInfo).collect(Collectors.toList());
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
JsonUtils.toJsonString(jsonData)));
}
}

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -33,6 +34,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.stream.Collectors;
/**
* Executes List Keys.
@ -104,10 +106,11 @@ protected void execute(CommandLine cmd)
List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
startKey, prefix);
for (OzoneKey key : keys) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
key.getObjectInfo().toJsonString()));
}
List<KeyInfo> jsonData = keys.stream()
.map(OzoneKey::getObjectInfo).collect(Collectors.toList());
System.out.printf(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
JsonUtils.toJsonString(jsonData)));
}
}

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.ozShell.Handler;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -31,6 +32,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.stream.Collectors;
/**
* Executes List Volume call.
@ -102,10 +104,10 @@ protected void execute(CommandLine cmd)
userName);
}
for (OzoneVolume vol : volumes) {
System.out.printf("%s%n", JsonUtils
.toJsonStringWithDefaultPrettyPrinter(vol.getJsonString()));
}
List<VolumeInfo> jsonData = volumes.stream()
.map(OzoneVolume::getVolumeInfo).collect(Collectors.toList());
System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
JsonUtils.toJsonString(jsonData)));
}
}
}

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
View File

@ -19,10 +19,12 @@
package org.apache.hadoop.ozone.web.utils;
import java.io.IOException;
import java.util.List;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.fasterxml.jackson.databind.type.CollectionType;
/**
* JSON Utility functions used in ozone.
@ -50,4 +52,20 @@ public static String toJsonStringWithDefaultPrettyPrinter(String jsonString)
public static String toJsonString(Object obj) throws IOException {
return MAPPER.writeValueAsString(obj);
}
/**
* Deserialize a list of elements from a given string,
* each element in the list is in the given type.
*
* @param str json string.
* @param elementType element type.
* @return
* @throws IOException
*/
public static List<?> toJsonList(String str, Class<?> elementType)
throws IOException {
CollectionType type = MAPPER.getTypeFactory()
.constructCollectionType(List.class, elementType);
return MAPPER.readValue(str, type);
}
}

163
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -53,6 +53,10 @@
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
@ -245,6 +249,8 @@ public void testUpdateVolume() throws Exception {
@Test
public void testListVolume() throws Exception {
String commandOutput;
List<VolumeInfo> volumes;
final int volCount = 20;
final String user1 = "test-user-a";
final String user2 = "test-user-b";
@ -271,28 +277,24 @@ public void testListVolume() throws Exception {
String[] args = new String[] {"-listVolume", url + "/", "-user",
user1, "-length", "100"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
.toJsonList(commandOutput, VolumeInfo.class);
List<String> names = getValueLines("name", out.toString());
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(10, volumes.size());
assertEquals(10, names.size());
assertEquals(10, creationTimes.size());
for (String user : names) {
assertTrue(user.contains(user1));
}
for (String time : creationTimes) {
assertTrue(time.contains(OzoneConsts.OZONE_TIME_ZONE));
for (VolumeInfo volume : volumes) {
assertEquals(volume.getOwner().getName(), user1);
assertTrue(volume.getCreatedOn().contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();
args = new String[] {"-listVolume", url + "/", "-user",
user1, "-length", "2"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
.toJsonList(commandOutput, VolumeInfo.class);
volumes = getValueLines("volumeName", out.toString());
assertEquals(2, volumes.size());
// test -prefix option
@ -300,15 +302,15 @@ public void testListVolume() throws Exception {
args = new String[] {"-listVolume", url + "/", "-user",
user1, "-length", "100", "-prefix", "test-vol1"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
.toJsonList(commandOutput, VolumeInfo.class);
names = getValueLines("name", out.toString());
volumes = getValueLines("volumeName", out.toString());
assertEquals(5, volumes.size());
assertEquals(5, names.size());
// return volume names should be [test-vol10, test-vol12, ..., test-vol18]
for (int i = 0; i < volumes.size(); i++) {
assertTrue(volumes.get(i).contains("test-vol" + ((i + 5) * 2)));
assertTrue(names.get(i).contains(user1));
assertEquals(volumes.get(i).getVolumeName(), "test-vol" + ((i + 5) * 2));
assertEquals(volumes.get(i).getOwner().getName(), user1);
}
// test -start option
@ -316,14 +318,16 @@ public void testListVolume() throws Exception {
args = new String[] {"-listVolume", url + "/", "-user",
user2, "-length", "100", "-start", "test-vol15"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
.toJsonList(commandOutput, VolumeInfo.class);
names = getValueLines("name", out.toString());
volumes = getValueLines("volumeName", out.toString());
assertEquals(2, volumes.size());
assertTrue(volumes.get(0).contains("test-vol17")
&& volumes.get(1).contains("test-vol19"));
assertTrue(names.get(0).contains(user2)
&& names.get(1).contains(user2));
assertEquals(volumes.get(0).getVolumeName(), "test-vol17");
assertEquals(volumes.get(1).getVolumeName(), "test-vol19");
assertEquals(volumes.get(0).getOwner().getName(), user2);
assertEquals(volumes.get(1).getOwner().getName(), user2);
// test error conditions
err.reset();
@ -468,6 +472,8 @@ public void testUpdateBucket() throws Exception {
@Test
public void testListBucket() throws Exception {
List<BucketInfo> buckets;
String commandOutput;
int bucketCount = 11;
OzoneVolume vol = creatVolume();
@ -484,59 +490,63 @@ public void testListBucket() throws Exception {
String[] args = new String[] {"-listBucket",
url + "/" + vol.getVolumeName(), "-length", "100"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
BucketInfo.class);
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> buckets = getValueLines("bucketName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(11, volumes.size());
assertEquals(11, buckets.size());
assertEquals(11, creationTimes.size());
// sort bucket names since the return buckets isn't in created order
Collections.sort(bucketNames);
// return bucket names should be [test-bucket0, test-bucket1,
// test-bucket10, test-bucket2, ,..., test-bucket9]
for (int i = 0; i < buckets.size(); i++) {
assertTrue(buckets.get(i).contains(bucketNames.get(i)));
assertTrue(volumes.get(i).contains(vol.getVolumeName()));
assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
assertEquals(buckets.get(i).getBucketName(), bucketNames.get(i));
assertEquals(buckets.get(i).getVolumeName(), vol.getVolumeName());
assertTrue(buckets.get(i).getCreatedOn()
.contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();
args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
"-length", "3"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
BucketInfo.class);
buckets = getValueLines("bucketName", out.toString());
assertEquals(3, buckets.size());
// return bucket names should be [test-bucket0,
// test-bucket1, test-bucket10]
assertTrue(buckets.get(0).contains("test-bucket0")
&& buckets.get(1).contains("test-bucket1")
&& buckets.get(2).contains("test-bucket10"));
assertEquals(buckets.get(0).getBucketName(), "test-bucket0");
assertEquals(buckets.get(1).getBucketName(), "test-bucket1");
assertEquals(buckets.get(2).getBucketName(), "test-bucket10");
// test -prefix option
out.reset();
args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
"-length", "100", "-prefix", "test-bucket1"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
BucketInfo.class);
buckets = getValueLines("bucketName", out.toString());
assertEquals(2, buckets.size());
// return bucket names should be [test-bucket1, test-bucket10]
assertTrue(buckets.get(0).contains("test-bucket1")
&& buckets.get(1).contains("test-bucket10"));
assertEquals(buckets.get(0).getBucketName(), "test-bucket1");
assertEquals(buckets.get(1).getBucketName(), "test-bucket10");
// test -start option
out.reset();
args = new String[] {"-listBucket", url + "/" + vol.getVolumeName(),
"-length", "100", "-start", "test-bucket7"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
buckets = (List<BucketInfo>) JsonUtils.toJsonList(commandOutput,
BucketInfo.class);
buckets = getValueLines("bucketName", out.toString());
assertEquals(2, buckets.size());
assertTrue(buckets.get(0).contains("test-bucket8")
&& buckets.get(1).contains("test-bucket9"));
assertEquals(buckets.get(0).getBucketName(), "test-bucket8");
assertEquals(buckets.get(1).getBucketName(), "test-bucket9");
// test error conditions
err.reset();
@ -671,6 +681,8 @@ public void testInfoKey() throws Exception {
@Test
public void testListKey() throws Exception {
String commandOutput;
List<KeyInfo> keys;
int keyCount = 11;
OzoneBucket bucket = creatBucket();
String volumeName = bucket.getBucketInfo().getVolumeName();
@ -688,61 +700,63 @@ public void testListKey() throws Exception {
String[] args = new String[] {"-listKey",
url + "/" + volumeName + "/" + bucketName, "-length", "100"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
keys = (List<KeyInfo>) JsonUtils.toJsonList(commandOutput,
KeyInfo.class);
List<String> keys = getValueLines("keyName", out.toString());
assertEquals(11, keys.size());
List<String> creationTime = getValueLines("createdOn", out.toString());
List<String> modificationTime = getValueLines("modifiedOn", out.toString());
assertEquals(11, creationTime.size());
assertEquals(11, modificationTime.size());
// sort key names since the return keys isn't in created order
Collections.sort(keyNames);
// return key names should be [test-key0, test-key1,
// test-key10, test-key2, ,..., test-key9]
for (int i = 0; i < keys.size(); i++) {
assertTrue(keys.get(i).contains(keyNames.get(i)));
assertEquals(keys.get(i).getKeyName(), keyNames.get(i));
// verify the creation/modification time of key
assertTrue(creationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
assertTrue(
modificationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
assertTrue(keys.get(i).getCreatedOn()
.contains(OzoneConsts.OZONE_TIME_ZONE));
assertTrue(keys.get(i).getModifiedOn()
.contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();
args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName,
"-length", "3"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
keys = (List<KeyInfo>) JsonUtils.toJsonList(commandOutput,
KeyInfo.class);
keys = getValueLines("keyName", out.toString());
assertEquals(3, keys.size());
// return key names should be [test-key0, test-key1, test-key10]
assertTrue(keys.get(0).contains("test-key0")
&& keys.get(1).contains("test-key1")
&& keys.get(2).contains("test-key10"));
assertEquals(keys.get(0).getKeyName(), "test-key0");
assertEquals(keys.get(1).getKeyName(), "test-key1");
assertEquals(keys.get(2).getKeyName(), "test-key10");
// test -prefix option
out.reset();
args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName,
"-length", "100", "-prefix", "test-key1"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
keys = (List<KeyInfo>) JsonUtils.toJsonList(commandOutput,
KeyInfo.class);
keys = getValueLines("keyName", out.toString());
assertEquals(2, keys.size());
// return key names should be [test-key1, test-key10]
assertTrue(keys.get(0).contains("test-key1")
&& keys.get(1).contains("test-key10"));
assertEquals(keys.get(0).getKeyName(), "test-key1");
assertEquals(keys.get(1).getKeyName(), "test-key10");
// test -start option
out.reset();
args = new String[] {"-listKey", url + "/" + volumeName + "/" + bucketName,
"-length", "100", "-start", "test-key7"};
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
keys = (List<KeyInfo>) JsonUtils.toJsonList(commandOutput,
KeyInfo.class);
keys = getValueLines("keyName", out.toString());
assertTrue(keys.get(0).contains("test-key8")
&& keys.get(1).contains("test-key9"));
assertEquals(keys.get(0).getKeyName(), "test-key8");
assertEquals(keys.get(1).getKeyName(), "test-key9");
// test error conditions
err.reset();
@ -787,23 +801,4 @@ private String createTmpFile() throws Exception {
return tmpFile.getAbsolutePath();
}
/**
* Extract lines from output string that contains specified key name.
* @param keyName Key name that line should contained.
* @param outputStr Response output content.
* @return List of string.
*/
private List<String> getValueLines(String keyName, String outputStr) {
List<String> nameLines = new ArrayList<>();
String[] lines = outputStr.split("\n");
for (String line : lines) {
if (line.contains(keyName)) {
nameLines.add(line);
}
}
return nameLines;
}
}