diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt index 8c4db07b97..78522d897e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-7240.txt @@ -24,3 +24,6 @@ HDFS-8680. OzoneHandler : Add Local StorageHandler support for volumes. (Anu Engineer via Arpit Agarwal) + + HDFS-8717. OzoneHandler : Add common bucket objects. (Anu Engineer via + Arpit Agarwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java new file mode 100644 index 0000000000..315ae3f8c8 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.web.handlers; + +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.web.request.OzoneAcl; +import org.apache.hadoop.ozone.web.utils.OzoneConsts; + +import java.util.LinkedList; +import java.util.List; + +/** + * BucketArgs packages all bucket related arguments to + * file system calls. + */ +public class BucketArgs extends VolumeArgs { + private final String bucketName; + private List addAcls; + private List removeAcls; + private OzoneConsts.Versioning versioning; + private StorageType storageType; + + /** + * Constructor for BucketArgs. + * + * @param volumeName - volumeName + * @param bucketName - bucket Name + * @param userArgs - userArgs + */ + public BucketArgs(String volumeName, String bucketName, UserArgs userArgs) { + super(volumeName, userArgs); + this.bucketName = bucketName; + this.versioning = OzoneConsts.Versioning.NOT_DEFINED; + this.storageType = null; + } + + + /** + * Constructor for BucketArgs. + * + * @param bucketName - bucket Name + * @param volumeArgs - volume Args + */ + public BucketArgs(String bucketName, VolumeArgs volumeArgs) { + super(volumeArgs); + this.bucketName = bucketName; + this.versioning = OzoneConsts.Versioning.NOT_DEFINED; + this.storageType = null; + } + + /** + * Constructor for BucketArgs. + * + * @param args - Bucket Args + */ + public BucketArgs(BucketArgs args) { + this(args.getBucketName(), args); + this.setAddAcls(args.getAddAcls()); + this.setRemoveAcls(args.getRemoveAcls()); + } + + /** + * Returns the Bucket Name. + * + * @return Bucket Name + */ + public String getBucketName() { + return bucketName; + } + + /** + * Returns Additive ACLs for the Bucket if specified. + * + * @return acls + */ + public List getAddAcls() { + return addAcls; + } + + /** + * Set Additive ACLs. + * + * @param acl - ACL + */ + public void setAddAcls(List acl) { + this.addAcls = acl; + } + + /** + * Returns remove ACLs for the Bucket if specified. + * + * @return acls + */ + public List getRemoveAcls() { + return removeAcls; + } + + /** + * Takes an ACL and sets the ACL object to ACL represented by the String. + * + * @param aclString - aclString + */ + public void addAcls(List aclString) throws IllegalArgumentException { + if (aclString == null) { + throw new IllegalArgumentException("ACLs cannot be null"); + } + if (this.addAcls == null) { + this.addAcls = new LinkedList<>(); + } + for (String s : aclString) { + this.addAcls.add(OzoneAcl.parseAcl(s)); + } + } + + /** + * Takes an ACL and sets the ACL object to ACL represented by the String. + * + * @param aclString - aclString + */ + public void removeAcls(List aclString) + throws IllegalArgumentException { + if (aclString == null) { + throw new IllegalArgumentException("ACLs cannot be null"); + } + if (this.removeAcls == null) { + this.removeAcls = new LinkedList<>(); + } + for (String s : aclString) { + this.removeAcls.add(OzoneAcl.parseAcl(s)); + } + } + + /** + * Set remove ACLs. + * + * @param acl - ACL + */ + public void setRemoveAcls(List acl) { + this.removeAcls = acl; + } + + + /** + * Returns Versioning Info. + * + * @return versioning + */ + public OzoneConsts.Versioning getVersioning() { + return versioning; + } + + /** + * Converts a valid String to Enum for ease of use. + * + * @param version version string. + */ + public void setVersioning(String version) { + if (version != null) { + this.versioning = OzoneConsts.Versioning.valueOf(version.toUpperCase()); + } + } + + /** + * SetVersioning Info. + * + * @param versioning - Enum value + */ + public void setVersioning(OzoneConsts.Versioning versioning) { + this.versioning = versioning; + } + + /** + * returns the current Storage Class. + * + * @return Storage Class + */ + public StorageType getStorageType() { + return storageType; + } + + /** + * Sets the Storage Class. + * + * @param storageType Set Storage Class + */ + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + /** + * returns - Volume/bucketName. + * + * @return String + */ + @Override + public String getResourceName() { + return getVolumeName() + "/" + getBucketName(); + } + + /** + * Returns User/Volume name which is the parent of this + * bucket. + * + * @return String + */ + public String getParentName() { + return getUserName() + "/" + getVolumeName(); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/headers/Header.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/headers/Header.java index f217038b7e..6400b44cd5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/headers/Header.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/headers/Header.java @@ -32,14 +32,14 @@ public final class Header { public static final String OZONE_QUOTA_REMOVE = "remove"; public static final String OZONE_QUOTA_UNDEFINED = "undefined"; - public static final String OZONE_LIST_QUERY_BUCKET = "bucket"; - public static final String OZONE_USER = "x-ozone-user"; public static final String OZONE_SIMPLE_AUTHENTICATION_SCHEME = "OZONE"; public static final String OZONE_VERSION_HEADER = "x-ozone-version"; public static final String OZONE_LIST_QUERY_SERVICE = "service"; public static final String OZONE_LIST_QUERY_VOLUME = "volume"; + public static final String OZONE_LIST_QUERY_BUCKET ="bucket"; + public static final String OZONE_LIST_QUERY_KEY ="key"; public static final String OZONE_REQUEST_ID = "x-ozone-request-id"; public static final String OZONE_SERVER_NAME = "x-ozone-server-name"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java new file mode 100644 index 0000000000..857cc61eb6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java @@ -0,0 +1,299 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.web.response; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.web.request.OzoneAcl; +import org.apache.hadoop.ozone.web.utils.OzoneConsts; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonMethod; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectWriter; +import org.codehaus.jackson.map.annotate.JsonFilter; +import org.codehaus.jackson.map.ser.FilterProvider; +import org.codehaus.jackson.map.ser.impl.SimpleBeanPropertyFilter; +import org.codehaus.jackson.map.ser.impl.SimpleFilterProvider; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +/** + * BucketInfo class, this is used as response class to send + * Json info about a bucket back to a client. + */ +public class BucketInfo implements Comparable { + static final String BUCKET_INFO = "BUCKET_INFO_FILTER"; + private String volumeName; + private String bucketName; + private List acls; + private OzoneConsts.Versioning versioning; + private StorageType storageType; + private long bytesUsed; + private long keyCount; + + /** + * Constructor for BucketInfo. + * + * @param volumeName + * @param bucketName + */ + public BucketInfo(String volumeName, String bucketName) { + this.volumeName = volumeName; + this.bucketName = bucketName; + } + + + /** + * Default constructor for BucketInfo. + */ + public BucketInfo() { + acls = new LinkedList(); + } + + /** + * Parse a JSON string into BucketInfo Object. + * + * @param jsonString - Json String + * + * @return - BucketInfo + * + * @throws IOException + */ + public static BucketInfo parse(String jsonString) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(jsonString, BucketInfo.class); + } + + /** + * Returns a List of ACL on the Bucket. + * + * @return List of Acls + */ + public List getAcls() { + return acls; + } + + /** + * Sets ACls. + * + * @param acls - Acls list + */ + public void setAcls(List acls) { + this.acls = acls; + } + + /** + * Returns Storage Type info. + * + * @return Storage Type of the bucket + */ + public StorageType getStorageType() { + return storageType; + } + + /** + * Sets the Storage Type. + * + * @param storageType - Storage Type + */ + public void setStorageType(StorageType storageType) { + this.storageType = storageType; + } + + /** + * Returns versioning. + * + * @return versioning Enum + */ + public OzoneConsts.Versioning getVersioning() { + return versioning; + } + + /** + * Sets Versioning. + * + * @param versioning + */ + public void setVersioning(OzoneConsts.Versioning versioning) { + this.versioning = versioning; + } + + + /** + * Gets bucket Name. + * + * @return String + */ + public String getBucketName() { + return bucketName; + } + + /** + * Sets bucket Name. + * + * @param bucketName - Name of the bucket + */ + public void setBucketName(String bucketName) { + this.bucketName = bucketName; + } + + /** + * Returns a JSON string of this object. + * After stripping out bytesUsed and keyCount + * + * @return String + */ + public String toJsonString() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + String[] ignorableFieldNames = {"bytesUsed", "keyCount"}; + + FilterProvider filters = new SimpleFilterProvider().addFilter( + BUCKET_INFO, + SimpleBeanPropertyFilter.serializeAllExcept(ignorableFieldNames)); + + mapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.getSerializationConfig() + .addMixInAnnotations(Object.class, MixIn.class); + ObjectWriter writer = mapper.writer(filters); + return writer.writeValueAsString(this); + } + + /** + * Returns the Object as a Json String. + * + * The reason why both toJSONString exists and toDBString exists + * is because toJSONString supports an external facing contract with + * REST clients. However server internally would want to add more + * fields to this class. The distinction helps in serializing all + * fields vs. only fields that are part of REST protocol. + */ + public String toDBString() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(this); + } + + /** + * Returns Volume Name. + * + * @return String volume name + */ + public String getVolumeName() { + return volumeName; + } + + /** + * Sets the Volume Name of the bucket. + * + * @param volumeName - volumeName + */ + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + + /** + * Compares this object with the specified object for order. Returns a + * negative integer, zero, or a positive integer as this object is less + * than, equal to, or greater than the specified object. + * + * Please note : BucketInfo compare functions are used only within the + * context of a volume, hence volume name is purposefully ignored in + * compareTo, equal and hashcode functions of this class. + */ + @Override + public int compareTo(BucketInfo o) { + Preconditions.checkState(o.getVolumeName().equals(this.getVolumeName())); + return this.bucketName.compareTo(o.getBucketName()); + } + + /** + * Checks if two bucketInfo's are equal. + * @param o Object BucketInfo + * @return True or False + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BucketInfo)) { + return false; + } + + BucketInfo that = (BucketInfo) o; + Preconditions.checkState(that.getVolumeName().equals(this.getVolumeName())); + return bucketName.equals(that.bucketName); + + } + + /** + * Hash Code for this object. + * @return int + */ + @Override + public int hashCode() { + return bucketName.hashCode(); + } + + /** + * Get the number of bytes used by this bucket. + * + * @return long + */ + public long getBytesUsed() { + return bytesUsed; + } + + /** + * Set bytes Used. + * + * @param bytesUsed - bytesUsed + */ + public void setBytesUsed(long bytesUsed) { + this.bytesUsed = bytesUsed; + } + + /** + * Get Key Count inside this bucket. + * + * @return - KeyCount + */ + public long getKeyCount() { + return keyCount; + } + + /** + * Set Key Count inside this bucket. + * + * @param keyCount - Sets the Key Count + */ + public void setKeyCount(long keyCount) { + this.keyCount = keyCount; + } + + /** + * This class allows us to create custom filters + * for the Json serialization. + */ + @JsonFilter(BUCKET_INFO) + class MixIn { + + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java new file mode 100644 index 0000000000..18b166c9da --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.web.response; + + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonMethod; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectWriter; +import org.codehaus.jackson.map.annotate.JsonFilter; +import org.codehaus.jackson.map.ser.FilterProvider; +import org.codehaus.jackson.map.ser.impl.SimpleBeanPropertyFilter; +import org.codehaus.jackson.map.ser.impl.SimpleFilterProvider; + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +/** + * List Bucket is the response for the ListBucket Query. + */ +public class ListBuckets { + static final String BUCKET_LIST = "BUCKET_LIST_FILTER"; + private List buckets; + + /** + * Constructor for ListBuckets. + * @param buckets - List of buckets owned by this user + */ + public ListBuckets(List buckets) { + this.buckets = buckets; + + } + + /** + * Constructor for ListBuckets. + */ + public ListBuckets() { + this.buckets = new LinkedList(); + } + + /** + * Parses a String to return ListBuckets object. + * + * @param data - Json String + * + * @return - ListBuckets + * + * @throws IOException + */ + public static ListBuckets parse(String data) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(data, ListBuckets.class); + } + + /** + * Returns a list of Buckets. + * + * @return Bucket list + */ + public List getBuckets() { + return Collections.unmodifiableList(buckets); + } + + /** + * Sets the list of buckets owned by this user. + * + * @param buckets - List of Buckets + */ + public void setBuckets(List buckets) { + this.buckets = buckets; + } + + + /** + * Returns a JSON string of this object. + * After stripping out bytesUsed and keyCount + * + * @return String + */ + public String toJsonString() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + String[] ignorableFieldNames = {"bytesUsed", "keyCount"}; + + FilterProvider filters = new SimpleFilterProvider() + .addFilter(BUCKET_LIST, SimpleBeanPropertyFilter + .serializeAllExcept(ignorableFieldNames)); + + mapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.getSerializationConfig() + .addMixInAnnotations(Object.class, MixIn.class); + ObjectWriter writer = mapper.writer(filters); + + return writer.writeValueAsString(this); + } + + /** + * Returns the Object as a Json String. + */ + public String toDBString() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(this); + } + + /** + * Sorts the buckets based on bucketName. + * This is useful when we return the list of buckets + */ + public void sort() { + Collections.sort(buckets); + } + + /** + * Add a new bucket to the list of buckets. + * @param bucketInfo - bucket Info + */ + public void addBucket(BucketInfo bucketInfo){ + this.buckets.add(bucketInfo); + } + + /** + * This class allows us to create custom filters + * for the Json serialization. + */ + @JsonFilter(BUCKET_LIST) + class MixIn { + + } + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneConsts.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneConsts.java index 35f2995c7a..fb0a7a6460 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneConsts.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneConsts.java @@ -47,6 +47,11 @@ public final class OzoneConsts { "EEE, dd MMM yyyy HH:mm:ss zzz"; public static final String OZONE_TIME_ZONE = "GMT"; + /** + * Supports Bucket Versioning. + */ + public enum Versioning {NOT_DEFINED, ENABLED, DISABLED} + private OzoneConsts() { // Never Constructed } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java new file mode 100644 index 0000000000..acf1d9397c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.web; + + +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.web.request.OzoneAcl; +import org.apache.hadoop.ozone.web.response.BucketInfo; +import org.apache.hadoop.ozone.web.utils.OzoneConsts; +import org.junit.Test; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; + +public class TestBucketInfo { + @Test + public void TestBucketInfoJson() throws IOException { + BucketInfo bucketInfo = new BucketInfo("volumeName", "bucketName"); + String bucketInfoString = bucketInfo.toJsonString(); + BucketInfo newBucketInfo = BucketInfo.parse(bucketInfoString); + assert(bucketInfo.equals(newBucketInfo)); + } + + @Test + public void TestBucketInfoDBString() throws IOException { + BucketInfo bucketInfo = new BucketInfo("volumeName", "bucketName"); + String bucketInfoString = bucketInfo.toDBString(); + BucketInfo newBucketInfo = BucketInfo.parse(bucketInfoString); + assert(bucketInfo.equals(newBucketInfo)); + } + + @Test + public void TestBucketInfoAddAcls() throws IOException { + BucketInfo bucketInfo = new BucketInfo("volumeName", "bucketName"); + String bucketInfoString = bucketInfo.toDBString(); + BucketInfo newBucketInfo = BucketInfo.parse(bucketInfoString); + assert(bucketInfo.equals(newBucketInfo)); + List aclList = new LinkedList<>(); + + aclList.add(OzoneAcl.parseAcl("user:bilbo:r")); + aclList.add(OzoneAcl.parseAcl("user:samwise:rw")); + newBucketInfo.setAcls(aclList); + + assert(newBucketInfo.getAcls() != null); + assert(newBucketInfo.getAcls().size() == 2); + } + + + @Test + public void TestBucketInfoVersionAndType() throws IOException { + BucketInfo bucketInfo = new BucketInfo("volumeName", "bucketName"); + bucketInfo.setVersioning(OzoneConsts.Versioning.ENABLED); + bucketInfo.setStorageType(StorageType.DISK); + + String bucketInfoString = bucketInfo.toDBString(); + + BucketInfo newBucketInfo = BucketInfo.parse(bucketInfoString); + assert(bucketInfo.equals(newBucketInfo)); + } + +}