big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham.

Browse Source
This commit is contained in:
Márton Elek 2018-11-06 18:19:40 +01:00
parent 349168c4b3
commit 3d5cc1138a
9 changed files with 85 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.client;
import org.apache.hadoop.hdds.client.ReplicationType;
/**
* A class that encapsulates OzoneKey.
*/
@ -48,19 +50,22 @@ public class OzoneKey {
*/
private long modificationTime;
private ReplicationType replicationType;
/**
* Constructs OzoneKey from OmKeyInfo.
*
*/
public OzoneKey(String volumeName, String bucketName,
String keyName, long size, long creationTime,
long modificationTime) {
long modificationTime, ReplicationType type) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.name = keyName;
this.dataSize = size;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
this.replicationType = type;
}
/**
@ -116,4 +121,13 @@ public long getCreationTime() {
public long getModificationTime() {
return modificationTime;
}
/**
* Returns the replication type of the key.
*
* @return replicationType
*/
public ReplicationType getReplicationType() {
return replicationType;
}
}

9
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.client;
import org.apache.hadoop.hdds.client.ReplicationType;
import java.util.List;
/**
@ -34,10 +36,11 @@ public class OzoneKeyDetails extends OzoneKey {
* Constructs OzoneKeyDetails from OmKeyInfo.
*/
public OzoneKeyDetails(String volumeName, String bucketName, String keyName,
long size, long creationTime, long modificationTime,
List<OzoneKeyLocation> ozoneKeyLocations) {
long size, long creationTime, long modificationTime,
List<OzoneKeyLocation> ozoneKeyLocations,
ReplicationType type) {
super(volumeName, bucketName, keyName, size, creationTime,
modificationTime);
modificationTime, type);
this.ozoneKeyLocations = ozoneKeyLocations;
}

6
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
View File

@ -777,7 +777,8 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
LOG.warn("Parse exception in getting creation time for volume", e);
}
return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
keyInfo.getSize(), creationTime, modificationTime);
keyInfo.getSize(), creationTime, modificationTime,
ReplicationType.valueOf(keyInfo.getType().toString()));
}).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IOException(e);
@ -812,7 +813,8 @@ public OzoneKeyDetails getKeyDetails(
keyInfo.getSize(),
HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()),
HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()),
ozoneKeyLocations);
ozoneKeyLocations, ReplicationType.valueOf(
keyInfo.getType().toString()));
EntityUtils.consume(response);
return key;
} catch (URISyntaxException | ParseException e) {

14
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -537,7 +537,8 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
key.getKeyName(),
key.getDataSize(),
key.getCreationTime(),
key.getModificationTime()))
key.getModificationTime(),
ReplicationType.valueOf(key.getType().toString())))
.collect(Collectors.toList());
}
@ -559,13 +560,10 @@ public OzoneKeyDetails getKeyDetails(
keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach(
(a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(),
a.getLocalID(), a.getLength(), a.getOffset())));
return new OzoneKeyDetails(keyInfo.getVolumeName(),
keyInfo.getBucketName(),
keyInfo.getKeyName(),
keyInfo.getDataSize(),
keyInfo.getCreationTime(),
keyInfo.getModificationTime(),
ozoneKeyLocations);
return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(),
keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(),
keyInfo.getModificationTime(), ozoneKeyLocations, ReplicationType
.valueOf(keyInfo.getType().toString()));
}
@Override

20
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
View File

@ -24,6 +24,7 @@
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.hdds.client.ReplicationType;
/**
* KeyInfo class is used used for parsing json response
@ -40,6 +41,25 @@ public class KeyInfo implements Comparable<KeyInfo> {
private String modifiedOn;
private long size;
private String keyName;
private ReplicationType type;
/**
* Return replication type of the key.
*
* @return replication type
*/
public ReplicationType getType() {
return type;
}
/**
* Set replication type of the key.
*
* @param replicationType
*/
public void setType(ReplicationType replicationType) {
this.type = replicationType;
}
/**
* When this key was created.

21
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
View File

@ -21,6 +21,7 @@
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
@ -75,6 +76,26 @@ class MixIn {
private String dataFileName;
private ReplicationType type;
/**
* Return replication type of the key.
*
* @return replication type
*/
public ReplicationType getType() {
return type;
}
/**
* Set replication type of the key.
*
* @param replicationType
*/
public void setType(ReplicationType replicationType) {
this.type = replicationType;
}
/**
* When this key was created.
*

4
hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot vendored
View File

@ -36,9 +36,13 @@ Delete file with multi delete
Should contain ${result} multidelete/f1
Should contain ${result} multidelete/f2
Should contain ${result} multidelete/f3
Should contain ${result} REDUCED_REDUNDANCY
Should not contain ${result} STANDARD
${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]'
Should not contain ${result} Error
${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/
Should not contain ${result} multidelete/f1
Should not contain ${result} multidelete/f2
Should contain ${result} multidelete/f3
Should contain ${result} REDUCED_REDUNDANCY
Should not contain ${result} STANDARD

9
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java
View File

@ -37,6 +37,7 @@
import java.time.Instant;
import java.util.Iterator;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata;
@ -48,6 +49,7 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.ozone.s3.util.S3StorageType;
import org.apache.hadoop.ozone.s3.util.S3utils;
import org.apache.http.HttpStatus;
import org.slf4j.Logger;
@ -296,7 +298,12 @@ private void addKey(ListObjectResponse response, OzoneKey next) {
keyMetadata.setKey(next.getName());
keyMetadata.setSize(next.getDataSize());
keyMetadata.setETag("" + next.getModificationTime());
keyMetadata.setStorageClass("STANDARD");
if (next.getReplicationType().toString().equals(ReplicationType
.STAND_ALONE.toString())) {
keyMetadata.setStorageClass(S3StorageType.REDUCED_REDUNDANCY.toString());
} else {
keyMetadata.setStorageClass(S3StorageType.STANDARD.toString());
}
keyMetadata.setLastModified(Instant.ofEpochMilli(
next.getModificationTime()));
response.addKey(keyMetadata);

2
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
View File

@ -93,7 +93,7 @@ public void close() throws IOException {
size,
System.currentTimeMillis(),
System.currentTimeMillis(),
new ArrayList<>()
new ArrayList<>(), type
));
super.close();
}