big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11984. Ozone: Ensures listKey lists all required key fields. Contributed by Yiqun Lin.

Browse Source
This commit is contained in:
Weiwei Yang 2017-08-01 13:47:44 +08:00 committed by Owen O'Malley
parent acfae2716a
commit 1ad95cf2a9
6 changed files with 110 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -51,11 +51,11 @@
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.ozone.web.response.VolumeInfo;
import org.apache.hadoop.ozone.web.response.VolumeOwner;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.util.Time;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -433,8 +433,10 @@ public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
keyInfo.setVersion(0);
keyInfo.setKeyName(ksmKeyInfo.getKeyName());
keyInfo.setSize(ksmKeyInfo.getDataSize());
keyInfo.setCreatedOn(Time.formatTime(ksmKeyInfo.getCreationTime()));
keyInfo.setModifiedOn(Time.formatTime(ksmKeyInfo.getModificationTime()));
keyInfo.setCreatedOn(
OzoneUtils.formatTime(ksmKeyInfo.getCreationTime()));
keyInfo.setModifiedOn(
OzoneUtils.formatTime(ksmKeyInfo.getModificationTime()));
return keyInfo;
}
@ -474,6 +476,10 @@ public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
tempInfo.setVersion(0);
tempInfo.setKeyName(info.getKeyName());
tempInfo.setSize(info.getDataSize());
tempInfo.setCreatedOn(
OzoneUtils.formatTime(info.getCreationTime()));
tempInfo.setModifiedOn(
OzoneUtils.formatTime(info.getModificationTime()));
result.addKey(tempInfo);
}

49
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.util.Time;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Request;
@ -58,6 +59,21 @@ private OzoneUtils() {
// Never constructed
}
/**
* Date format that used in ozone. Here the format is thread safe to use.
*/
private static final ThreadLocal<SimpleDateFormat> DATE_FORMAT =
new ThreadLocal<SimpleDateFormat>() {
@Override
protected SimpleDateFormat initialValue() {
SimpleDateFormat format = new SimpleDateFormat(
OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
return format;
}
};
/**
* verifies that bucket name / volume name is a valid DNS name.
*
@ -242,12 +258,8 @@ public static void validate(Request request, HttpHeaders headers,
public static synchronized Date parseDate(String dateString, String reqID,
String resource, String hostname)
throws OzoneException {
SimpleDateFormat format =
new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
try {
return format.parse(dateString);
return DATE_FORMAT.get().parse(dateString);
} catch (ParseException ex) {
OzoneException exp =
ErrorTable.newError(ErrorTable.BAD_DATE, reqID, resource, hostname);
@ -267,10 +279,7 @@ public static synchronized Date parseDate(String dateString, String reqID,
*/
public static Response getResponse(UserArgs args, int statusCode,
String payload) {
SimpleDateFormat format =
new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
String date = format.format(new Date(System.currentTimeMillis()));
String date = DATE_FORMAT.get().format(new Date(Time.now()));
return Response.ok(payload)
.header(Header.OZONE_SERVER_NAME, args.getHostName())
.header(Header.OZONE_REQUEST_ID, args.getRequestID())
@ -288,10 +297,7 @@ public static Response getResponse(UserArgs args, int statusCode,
*/
public static Response getResponse(UserArgs args, int statusCode,
LengthInputStream stream) {
SimpleDateFormat format =
new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
String date = format.format(new Date(System.currentTimeMillis()));
String date = DATE_FORMAT.get().format(new Date(Time.now()));
return Response.ok(stream, MediaType.APPLICATION_OCTET_STREAM)
.header(Header.OZONE_SERVER_NAME, args.getHostName())
.header(Header.OZONE_REQUEST_ID, args.getRequestID())
@ -319,4 +325,21 @@ public static File getScmMetadirPath(Configuration conf) {
}
return dirPath;
}
/**
* Convert time in millisecond to a human readable format required in ozone.
* @return a human readable string for the input time
*/
public static String formatTime(long millis) {
return DATE_FORMAT.get().format(millis);
}
/**
* Convert time in ozone date format to millisecond.
* @return time in milliseconds
*/
public static long formatDate(String date) throws ParseException {
Preconditions.checkNotNull(date, "Date string should not be null.");
return DATE_FORMAT.get().parse(date).getTime();
}
}

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
View File

@ -961,7 +961,7 @@ public void testGetKeyInfo() throws IOException,
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
long currentTime = Time.monotonicNow();
long currentTime = Time.now();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -982,8 +982,12 @@ public void testGetKeyInfo() throws IOException,
stream.close();
KeyInfo keyInfo = storageHandler.getKeyInfo(keyArgs);
Assert.assertTrue(Time.formatDate(keyInfo.getCreatedOn()) >= currentTime);
Assert.assertTrue(Time.formatDate(keyInfo.getModifiedOn()) >= currentTime);
// Compare the time in second unit since the date string reparsed to
// millisecond will lose precision.
Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getCreatedOn())
/ 1000) >= (currentTime / 1000));
Assert.assertTrue((OzoneUtils.formatDate(keyInfo.getModifiedOn())
/ 1000) >= (currentTime / 1000));
Assert.assertEquals(keyName, keyInfo.getKeyName());
Assert.assertEquals(4096, keyInfo.getSize());
}

17
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -626,7 +626,11 @@ public void testInfoKey() throws Exception {
// verify the response output
assertEquals(0, ToolRunner.run(shell, args));
assertTrue(out.toString().contains(keyName));
String output = out.toString();
assertTrue(output.contains(keyName));
assertTrue(output.contains("createdOn") && output.contains("modifiedOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
// reset stream
out.reset();
@ -665,12 +669,23 @@ public void testListKey() throws Exception {
List<String> keys = getValueLines("keyName", out.toString());
assertEquals(11, keys.size());
List<String> creationTime = getValueLines("createdOn", out.toString());
List<String> modificationTime = getValueLines("modifiedOn", out.toString());
assertEquals(11, creationTime.size());
assertEquals(11, modificationTime.size());
// sort key names since the return keys isn't in created order
Collections.sort(keyNames);
// return key names should be [test-key0, test-key1,
// test-key10, test-key2, ,..., test-key9]
for (int i = 0; i < keys.size(); i++) {
assertTrue(keys.get(i).contains(keyNames.get(i)));
// verify the creation/modification time of key
assertTrue(creationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
assertTrue(
modificationTime.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();

41
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
View File

@ -31,6 +31,7 @@
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
@ -47,11 +48,13 @@
import java.net.URISyntaxException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestKeys {
@ -387,7 +390,8 @@ static void runTestPutAndDeleteKey(PutHelper helper)
}
@Test
public void testPutAndListKey() throws OzoneException, IOException {
public void testPutAndListKey()
throws OzoneException, IOException, ParseException {
runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
String delimiter = RandomStringUtils.randomAscii(1);
runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
@ -395,12 +399,13 @@ public void testPutAndListKey() throws OzoneException, IOException {
}
static void runTestPutAndListKey(PutHelper helper)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
final OzoneRestClient client = helper.client;
helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
long currentTime = Time.now();
// add keys [list-key0, list-key1, ..., list-key9]
for (int x = 0; x < 10; x++) {
String newkeyName = "list-key" + x;
@ -414,6 +419,22 @@ static void runTestPutAndListKey(PutHelper helper)
Assert.assertEquals(keyList1.size(), 11);
Assert.assertEquals(keyList2.size(), 11);
// Verify the key creation/modification time. Here we compare the time in
// second unit since the date string reparsed to millisecond will
// lose precision.
for (OzoneKey key : keyList1) {
assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn())
/ 1000) >= (currentTime / 1000));
assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn())
/ 1000) >= (currentTime / 1000));
}
for (OzoneKey key : keyList2) {
assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getCreatedOn())
/ 1000) >= (currentTime / 1000));
assertTrue((OzoneUtils.formatDate(key.getObjectInfo().getModifiedOn())
/ 1000) >= (currentTime / 1000));
}
// test maxLength parameter of list keys
keyList1 = helper.getBucket().listKeys("1", null, null);
@ -459,14 +480,17 @@ static void runTestPutAndListKey(PutHelper helper)
}
@Test
public void testGetKeyInfo() throws OzoneException, IOException {
public void testGetKeyInfo()
throws OzoneException, IOException, ParseException {
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
String delimiter = RandomStringUtils.randomAscii(1);
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,
getMultiPartKey(delimiter)));
}
static void runTestGetKeyInfo(PutHelper helper) throws OzoneException {
static void runTestGetKeyInfo(PutHelper helper)
throws OzoneException, ParseException {
long currentTime = Time.now();
String keyName = helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
@ -474,5 +498,14 @@ static void runTestGetKeyInfo(PutHelper helper) throws OzoneException {
OzoneKey keyInfo = helper.getBucket().getKeyInfo(keyName);
assertNotNull(keyInfo.getObjectInfo());
assertEquals(keyName, keyInfo.getObjectInfo().getKeyName());
// Compare the time in second unit since the date string reparsed to
// millisecond will lose precision.
Assert.assertTrue(
(OzoneUtils.formatDate(keyInfo.getObjectInfo().getCreatedOn())
/ 1000) >= (currentTime / 1000));
Assert.assertTrue(
(OzoneUtils.formatDate(keyInfo.getObjectInfo().getModifiedOn())
/ 1000) >= (currentTime / 1000));
}
}

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeysRatis.java
View File

@ -29,6 +29,7 @@
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import static org.apache.hadoop.ozone.web.client.TestKeys.*;
@ -91,7 +92,8 @@ public void testPutAndDeleteKey() throws OzoneException, IOException {
}
@Test
public void testPutAndListKey() throws OzoneException, IOException {
public void testPutAndListKey()
throws OzoneException, IOException, ParseException {
runTestPutAndListKey(new PutHelper(ozoneRestClient, path));
String delimiter = RandomStringUtils.randomAlphanumeric(1);
runTestPutAndListKey(new PutHelper(ozoneRestClient, path,
@ -99,7 +101,8 @@ public void testPutAndListKey() throws OzoneException, IOException {
}
@Test
public void testGetKeyInfo() throws OzoneException, IOException {
public void testGetKeyInfo()
throws OzoneException, IOException, ParseException {
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path));
String delimiter = RandomStringUtils.randomAlphanumeric(1);
runTestGetKeyInfo(new PutHelper(ozoneRestClient, path,