HDFS-11880. Ozone: KSM: Remove protobuf formats from KSM wrappers. Contributed by Nandakumar Vadivelu.

This commit is contained in:
Xiaoyu Yao 2017-06-07 14:49:44 -07:00 committed by Owen O'Malley
parent 2007e85d5b
commit bacd1188f1
20 changed files with 163 additions and 135 deletions

View File

@ -18,14 +18,15 @@
package org.apache.hadoop.ksm.helpers; package org.apache.hadoop.ksm.helpers;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto import org.apache.hadoop.fs.StorageType;
.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketArgs; .KeySpaceManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
/** /**
* A class that encapsulates Bucket Arguments. * A class that encapsulates Bucket Arguments.
@ -42,11 +43,11 @@ public final class KsmBucketArgs {
/** /**
* ACL's that are to be added for the bucket. * ACL's that are to be added for the bucket.
*/ */
private List<OzoneAclInfo> addAcls; private List<OzoneAcl> addAcls;
/** /**
* ACL's that are to be removed from the bucket. * ACL's that are to be removed from the bucket.
*/ */
private List<OzoneAclInfo> removeAcls; private List<OzoneAcl> removeAcls;
/** /**
* Bucket Version flag. * Bucket Version flag.
*/ */
@ -55,7 +56,7 @@ public final class KsmBucketArgs {
* Type of storage to be used for this bucket. * Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE] * [RAM_DISK, SSD, DISK, ARCHIVE]
*/ */
private StorageTypeProto storageType; private StorageType storageType;
/** /**
* Private constructor, constructed via builder. * Private constructor, constructed via builder.
@ -67,8 +68,8 @@ public final class KsmBucketArgs {
* @param storageType - Storage type to be used. * @param storageType - Storage type to be used.
*/ */
private KsmBucketArgs(String volumeName, String bucketName, private KsmBucketArgs(String volumeName, String bucketName,
List<OzoneAclInfo> addAcls, List<OzoneAclInfo> removeAcls, List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
Boolean isVersionEnabled, StorageTypeProto storageType) { Boolean isVersionEnabled, StorageType storageType) {
this.volumeName = volumeName; this.volumeName = volumeName;
this.bucketName = bucketName; this.bucketName = bucketName;
this.addAcls = addAcls; this.addAcls = addAcls;
@ -97,7 +98,7 @@ public String getBucketName() {
* Returns the ACL's that are to be added. * Returns the ACL's that are to be added.
* @return List<OzoneAclInfo> * @return List<OzoneAclInfo>
*/ */
public List<OzoneAclInfo> getAddAcls() { public List<OzoneAcl> getAddAcls() {
return addAcls; return addAcls;
} }
@ -105,7 +106,7 @@ public List<OzoneAclInfo> getAddAcls() {
* Returns the ACL's that are to be removed. * Returns the ACL's that are to be removed.
* @return List<OzoneAclInfo> * @return List<OzoneAclInfo>
*/ */
public List<OzoneAclInfo> getRemoveAcls() { public List<OzoneAcl> getRemoveAcls() {
return removeAcls; return removeAcls;
} }
@ -121,7 +122,7 @@ public Boolean getIsVersionEnabled() {
* Returns the type of storage to be used. * Returns the type of storage to be used.
* @return StorageType * @return StorageType
*/ */
public StorageTypeProto getStorageType() { public StorageType getStorageType() {
return storageType; return storageType;
} }
@ -140,10 +141,10 @@ public static Builder newBuilder() {
public static class Builder { public static class Builder {
private String volumeName; private String volumeName;
private String bucketName; private String bucketName;
private List<OzoneAclInfo> addAcls; private List<OzoneAcl> addAcls;
private List<OzoneAclInfo> removeAcls; private List<OzoneAcl> removeAcls;
private Boolean isVersionEnabled; private Boolean isVersionEnabled;
private StorageTypeProto storageType; private StorageType storageType;
public Builder setVolumeName(String volume) { public Builder setVolumeName(String volume) {
this.volumeName = volume; this.volumeName = volume;
@ -155,12 +156,12 @@ public Builder setBucketName(String bucket) {
return this; return this;
} }
public Builder setAddAcls(List<OzoneAclInfo> acls) { public Builder setAddAcls(List<OzoneAcl> acls) {
this.addAcls = acls; this.addAcls = acls;
return this; return this;
} }
public Builder setRemoveAcls(List<OzoneAclInfo> acls) { public Builder setRemoveAcls(List<OzoneAcl> acls) {
this.removeAcls = acls; this.removeAcls = acls;
return this; return this;
} }
@ -170,7 +171,7 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
return this; return this;
} }
public Builder setStorageType(StorageTypeProto storage) { public Builder setStorageType(StorageType storage) {
this.storageType = storage; this.storageType = storage;
return this; return this;
} }
@ -195,16 +196,19 @@ public BucketArgs getProtobuf() {
builder.setVolumeName(volumeName) builder.setVolumeName(volumeName)
.setBucketName(bucketName); .setBucketName(bucketName);
if(addAcls != null && !addAcls.isEmpty()) { if(addAcls != null && !addAcls.isEmpty()) {
builder.addAllAddAcls(addAcls); builder.addAllAddAcls(addAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
} }
if(removeAcls != null && !removeAcls.isEmpty()) { if(removeAcls != null && !removeAcls.isEmpty()) {
builder.addAllRemoveAcls(removeAcls); builder.addAllRemoveAcls(removeAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
} }
if(isVersionEnabled != null) { if(isVersionEnabled != null) {
builder.setIsVersionEnabled(isVersionEnabled); builder.setIsVersionEnabled(isVersionEnabled);
} }
if(storageType != null) { if(storageType != null) {
builder.setStorageType(storageType); builder.setStorageType(
PBHelperClient.convertStorageType(storageType));
} }
return builder.build(); return builder.build();
} }
@ -217,10 +221,13 @@ public BucketArgs getProtobuf() {
public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) { public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
return new KsmBucketArgs(bucketArgs.getVolumeName(), return new KsmBucketArgs(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(), bucketArgs.getBucketName(),
bucketArgs.getAddAclsList(), bucketArgs.getAddAclsList().stream().map(
bucketArgs.getRemoveAclsList(), KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.hasIsVersionEnabled() ? bucketArgs.getIsVersionEnabled() : bucketArgs.getRemoveAclsList().stream().map(
null, KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.hasStorageType() ? bucketArgs.getStorageType() : null); bucketArgs.hasIsVersionEnabled() ?
bucketArgs.getIsVersionEnabled() : null,
bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
bucketArgs.getStorageType()) : null);
} }
} }

View File

@ -18,15 +18,16 @@
package org.apache.hadoop.ksm.helpers; package org.apache.hadoop.ksm.helpers;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto import org.apache.hadoop.fs.StorageType;
.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo; .KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* A class that encapsulates Bucket Info. * A class that encapsulates Bucket Info.
@ -43,7 +44,7 @@ public final class KsmBucketInfo {
/** /**
* ACL Information. * ACL Information.
*/ */
private List<OzoneAclInfo> acls; private List<OzoneAcl> acls;
/** /**
* Bucket Version flag. * Bucket Version flag.
*/ */
@ -52,7 +53,7 @@ public final class KsmBucketInfo {
* Type of storage to be used for this bucket. * Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE] * [RAM_DISK, SSD, DISK, ARCHIVE]
*/ */
private StorageTypeProto storageType; private StorageType storageType;
/** /**
* Private constructor, constructed via builder. * Private constructor, constructed via builder.
@ -63,8 +64,8 @@ public final class KsmBucketInfo {
* @param storageType - Storage type to be used. * @param storageType - Storage type to be used.
*/ */
private KsmBucketInfo(String volumeName, String bucketName, private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAclInfo> acls, boolean isVersionEnabled, List<OzoneAcl> acls, boolean isVersionEnabled,
StorageTypeProto storageType) { StorageType storageType) {
this.volumeName = volumeName; this.volumeName = volumeName;
this.bucketName = bucketName; this.bucketName = bucketName;
this.acls = acls; this.acls = acls;
@ -90,9 +91,9 @@ public String getBucketName() {
/** /**
* Returns the ACL's associated with this bucket. * Returns the ACL's associated with this bucket.
* @return List<OzoneAclInfo> * @return List<OzoneAcl>
*/ */
public List<OzoneAclInfo> getAcls() { public List<OzoneAcl> getAcls() {
return acls; return acls;
} }
@ -106,9 +107,9 @@ public boolean getIsVersionEnabled() {
/** /**
* Returns the type of storage to be used. * Returns the type of storage to be used.
* @return StorageTypeProto * @return StorageType
*/ */
public StorageTypeProto getStorageType() { public StorageType getStorageType() {
return storageType; return storageType;
} }
@ -127,15 +128,15 @@ public static Builder newBuilder() {
public static class Builder { public static class Builder {
private String volumeName; private String volumeName;
private String bucketName; private String bucketName;
private List<OzoneAclInfo> acls; private List<OzoneAcl> acls;
private Boolean isVersionEnabled; private Boolean isVersionEnabled;
private StorageTypeProto storageType; private StorageType storageType;
Builder() { Builder() {
//Default values //Default values
this.acls = new LinkedList<>(); this.acls = new LinkedList<>();
this.isVersionEnabled = false; this.isVersionEnabled = false;
this.storageType = StorageTypeProto.DISK; this.storageType = StorageType.DISK;
} }
public Builder setVolumeName(String volume) { public Builder setVolumeName(String volume) {
@ -148,7 +149,7 @@ public Builder setBucketName(String bucket) {
return this; return this;
} }
public Builder setAcls(List<OzoneAclInfo> listOfAcls) { public Builder setAcls(List<OzoneAcl> listOfAcls) {
this.acls = listOfAcls; this.acls = listOfAcls;
return this; return this;
} }
@ -158,7 +159,7 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
return this; return this;
} }
public Builder setStorageType(StorageTypeProto storage) { public Builder setStorageType(StorageType storage) {
this.storageType = storage; this.storageType = storage;
return this; return this;
} }
@ -185,9 +186,11 @@ public BucketInfo getProtobuf() {
return BucketInfo.newBuilder() return BucketInfo.newBuilder()
.setVolumeName(volumeName) .setVolumeName(volumeName)
.setBucketName(bucketName) .setBucketName(bucketName)
.addAllAcls(acls) .addAllAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
.setIsVersionEnabled(isVersionEnabled) .setIsVersionEnabled(isVersionEnabled)
.setStorageType(storageType) .setStorageType(PBHelperClient.convertStorageType(
storageType))
.build(); .build();
} }
@ -200,8 +203,10 @@ public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
return new KsmBucketInfo( return new KsmBucketInfo(
bucketInfo.getVolumeName(), bucketInfo.getVolumeName(),
bucketInfo.getBucketName(), bucketInfo.getBucketName(),
bucketInfo.getAclsList(), bucketInfo.getAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(), bucketInfo.getIsVersionEnabled(),
bucketInfo.getStorageType()); PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
} }
} }

View File

@ -17,9 +17,7 @@
*/ */
package org.apache.hadoop.ozone.web.request; package org.apache.hadoop.ozone;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.Objects; import java.util.Objects;

View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone;
/**
This package contains ozone client side libraries.
*/

View File

@ -17,7 +17,7 @@
*/ */
package org.apache.hadoop.ozone.protocolPB; package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo; .KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto

View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.protocolPB;
/**
* This package contains classes for the Protocol Buffers binding of Ozone
* protocols.
*/

View File

@ -16,9 +16,8 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.ozone.web; package org.apache.hadoop.ozone;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.junit.Test; import org.junit.Test;
import java.util.HashMap; import java.util.HashMap;

View File

@ -17,24 +17,19 @@
package org.apache.hadoop.ozone.ksm; package org.apache.hadoop.ozone.ksm;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto import org.apache.hadoop.fs.StorageType;
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs; import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo; import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException; import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo; .KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.iq80.leveldb.DBException; import org.iq80.leveldb.DBException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* KSM bucket manager. * KSM bucket manager.
@ -179,10 +174,8 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
//Check ACLs to update //Check ACLs to update
if(args.getAddAcls() != null || args.getRemoveAcls() != null) { if(args.getAddAcls() != null || args.getRemoveAcls() != null) {
List<OzoneAcl> acls = getUpdatedAclList(oldBucketInfo.getAcls(), bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(),
args.getRemoveAcls(), args.getAddAcls()); args.getRemoveAcls(), args.getAddAcls()));
bucketInfoBuilder.setAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
LOG.debug("Updating ACLs for bucket: {} in volume: {}", LOG.debug("Updating ACLs for bucket: {} in volume: {}",
bucketName, volumeName); bucketName, volumeName);
} else { } else {
@ -190,9 +183,9 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
} }
//Check StorageType to update //Check StorageType to update
StorageTypeProto storageTypeProto = args.getStorageType(); StorageType storageType = args.getStorageType();
if(storageTypeProto != null) { if(storageType != null) {
bucketInfoBuilder.setStorageType(storageTypeProto); bucketInfoBuilder.setStorageType(storageType);
LOG.debug("Updating bucket storage type for bucket: {} in volume: {}", LOG.debug("Updating bucket storage type for bucket: {} in volume: {}",
bucketName, volumeName); bucketName, volumeName);
} else { } else {
@ -226,26 +219,20 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
* Remove is done before Add. * Remove is done before Add.
* *
* @param existingAcls - old ACL list. * @param existingAcls - old ACL list.
* @param removeAclInfos - ACLs to be removed. * @param removeAcls - ACLs to be removed.
* @param addAclInfos - ACLs to be added. * @param addAcls - ACLs to be added.
* @return updated ACL list. * @return updated ACL list.
*/ */
private List<OzoneAcl> getUpdatedAclList(List<OzoneAclInfo> existingAcls, private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls,
List<OzoneAclInfo> removeAclInfos, List<OzoneAclInfo> addAclInfos) { List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) {
List<OzoneAcl> acls = existingAcls.stream().map( if(removeAcls != null && !removeAcls.isEmpty()) {
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()); existingAcls.removeAll(removeAcls);
if(removeAclInfos != null && !removeAclInfos.isEmpty()) {
List<OzoneAcl> removeAcls = removeAclInfos.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
acls.removeAll(removeAcls);
} }
if(addAclInfos != null && !addAclInfos.isEmpty()) { if(addAcls != null && !addAcls.isEmpty()) {
List<OzoneAcl> addAcls = addAclInfos.stream().map( addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()); existingAcls::add);
addAcls.stream().filter(acl -> !acls.contains(acl)).forEach(
acls::add);
} }
return acls; return existingAcls;
} }
/** /**

View File

@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.ksm; package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
/** /**
* KSM Constants. * KSM Constants.
*/ */

View File

@ -25,7 +25,7 @@
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.headers.Header; import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo; import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListKeys; import org.apache.hadoop.ozone.web.response.ListKeys;

View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.web.handlers; package org.apache.hadoop.ozone.web.handlers;
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import java.util.LinkedList; import java.util.LinkedList;

View File

@ -25,7 +25,7 @@
import org.apache.hadoop.ozone.web.handlers.KeyArgs; import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs; import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs; import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys; import org.apache.hadoop.ozone.web.response.ListKeys;

View File

@ -27,7 +27,7 @@
import org.apache.hadoop.ozone.web.handlers.ListArgs; import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs; import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListBuckets;

View File

@ -31,7 +31,7 @@
import org.apache.hadoop.ozone.web.handlers.ListArgs; import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs; import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs; import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo; import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets; import org.apache.hadoop.ozone.web.response.ListBuckets;

View File

@ -23,7 +23,7 @@
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.utils.JsonUtils; import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect;

View File

@ -25,7 +25,6 @@
import org.apache.hadoop.hdfs.ozone.protocol.proto import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.KeyData; .ContainerProtos.KeyData;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.datanode.fsdataset import org.apache.hadoop.hdfs.server.datanode.fsdataset
.LengthInputStream; .LengthInputStream;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs; import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
@ -40,7 +39,7 @@
import org.apache.hadoop.ozone.OzoneConsts.Versioning; import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys; import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.scm.container.common.helpers.Pipeline; import org.apache.hadoop.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.scm.ScmConfigKeys; import org.apache.hadoop.scm.ScmConfigKeys;
@ -78,7 +77,6 @@
import java.util.HashSet; import java.util.HashSet;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* A {@link StorageHandler} implementation that distributes object storage * A {@link StorageHandler} implementation that distributes object storage
@ -205,12 +203,10 @@ public void createBucket(final BucketArgs args)
builder.setVolumeName(args.getVolumeName()) builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName()); .setBucketName(args.getBucketName());
if(args.getAddAcls() != null) { if(args.getAddAcls() != null) {
builder.setAcls(args.getAddAcls().stream().map( builder.setAcls(args.getAddAcls());
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
} }
if(args.getStorageType() != null) { if(args.getStorageType() != null) {
builder.setStorageType(PBHelperClient.convertStorageType( builder.setStorageType(args.getStorageType());
args.getStorageType()));
} }
if(args.getVersioning() != null) { if(args.getVersioning() != null) {
builder.setIsVersionEnabled(getBucketVersioningProtobuf( builder.setIsVersionEnabled(getBucketVersioningProtobuf(
@ -250,12 +246,10 @@ public void setBucketAcls(BucketArgs args)
builder.setVolumeName(args.getVolumeName()) builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName()); .setBucketName(args.getBucketName());
if(removeAcls != null && !removeAcls.isEmpty()) { if(removeAcls != null && !removeAcls.isEmpty()) {
builder.setRemoveAcls(args.getRemoveAcls().stream().map( builder.setRemoveAcls(args.getRemoveAcls());
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
} }
if(addAcls != null && !addAcls.isEmpty()) { if(addAcls != null && !addAcls.isEmpty()) {
builder.setAddAcls(args.getAddAcls().stream().map( builder.setAddAcls(args.getAddAcls());
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
} }
keySpaceManagerClient.setBucketProperty(builder.build()); keySpaceManagerClient.setBucketProperty(builder.build());
} }
@ -278,8 +272,7 @@ public void setBucketStorageClass(BucketArgs args)
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName()) builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName()) .setBucketName(args.getBucketName())
.setStorageType(PBHelperClient.convertStorageType( .setStorageType(args.getStorageType());
args.getStorageType()));
keySpaceManagerClient.setBucketProperty(builder.build()); keySpaceManagerClient.setBucketProperty(builder.build());
} }
@ -317,10 +310,8 @@ public BucketInfo getBucketInfo(BucketArgs args)
} else { } else {
bucketInfo.setVersioning(Versioning.DISABLED); bucketInfo.setVersioning(Versioning.DISABLED);
} }
bucketInfo.setStorageType(PBHelperClient.convertStorageType( bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
ksmBucketInfo.getStorageType())); bucketInfo.setAcls(ksmBucketInfo.getAcls());
bucketInfo.setAcls(ksmBucketInfo.getAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
return bucketInfo; return bucketInfo;
} }

View File

@ -16,18 +16,15 @@
*/ */
package org.apache.hadoop.ozone.ksm; package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs; import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo; import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException; import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.ksm.exceptions import org.apache.hadoop.ozone.ksm.exceptions
.KSMException.ResultCodes; .KSMException.ResultCodes;
import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.OzoneAcl;
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@ -203,7 +200,7 @@ public void testGetBucketInfo() throws IOException {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder() KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK) .setStorageType(StorageType.DISK)
.setIsVersionEnabled(false) .setIsVersionEnabled(false)
.build(); .build();
bucketManager.createBucket(bucketInfo); bucketManager.createBucket(bucketInfo);
@ -211,7 +208,7 @@ public void testGetBucketInfo() throws IOException {
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals("sampleVol", result.getVolumeName()); Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName()); Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK, Assert.assertEquals(StorageType.DISK,
result.getStorageType()); result.getStorageType());
Assert.assertEquals(false, result.getIsVersionEnabled()); Assert.assertEquals(false, result.getIsVersionEnabled());
} }
@ -219,16 +216,16 @@ public void testGetBucketInfo() throws IOException {
@Test @Test
public void testSetBucketPropertyAddACL() throws IOException { public void testSetBucketPropertyAddACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol"); MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>(); List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl ozoneAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, OzoneAcl ozoneAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ); "root", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(ozoneAcl)); acls.add(ozoneAcl);
BucketManager bucketManager = new BucketManagerImpl(metaMgr); BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder() KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
.setAcls(acls) .setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK) .setStorageType(StorageType.DISK)
.setIsVersionEnabled(false) .setIsVersionEnabled(false)
.build(); .build();
bucketManager.createBucket(bucketInfo); bucketManager.createBucket(bucketInfo);
@ -237,10 +234,10 @@ public void testSetBucketPropertyAddACL() throws IOException {
Assert.assertEquals("sampleVol", result.getVolumeName()); Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName()); Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(1, result.getAcls().size()); Assert.assertEquals(1, result.getAcls().size());
List<OzoneAclInfo> addAcls = new LinkedList<>(); List<OzoneAcl> addAcls = new LinkedList<>();
OzoneAcl newAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, OzoneAcl newAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ); "ozone", OzoneAcl.OzoneACLRights.READ);
addAcls.add(KSMPBHelper.convertOzoneAcl(newAcl)); addAcls.add(newAcl);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder() KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
@ -250,34 +247,33 @@ public void testSetBucketPropertyAddACL() throws IOException {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo( KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals(2, updatedResult.getAcls().size()); Assert.assertEquals(2, updatedResult.getAcls().size());
Assert.assertTrue(updatedResult.getAcls().contains( Assert.assertTrue(updatedResult.getAcls().contains(newAcl));
KSMPBHelper.convertOzoneAcl(newAcl)));
} }
@Test @Test
public void testSetBucketPropertyRemoveACL() throws IOException { public void testSetBucketPropertyRemoveACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol"); MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>(); List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl aclOne = new OzoneAcl(OzoneAcl.OzoneACLType.USER, OzoneAcl aclOne = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ); "root", OzoneAcl.OzoneACLRights.READ);
OzoneAcl aclTwo = new OzoneAcl(OzoneAcl.OzoneACLType.USER, OzoneAcl aclTwo = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ); "ozone", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(aclOne)); acls.add(aclOne);
acls.add(KSMPBHelper.convertOzoneAcl(aclTwo)); acls.add(aclTwo);
BucketManager bucketManager = new BucketManagerImpl(metaMgr); BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder() KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
.setAcls(acls) .setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK) .setStorageType(StorageType.DISK)
.setIsVersionEnabled(false) .setIsVersionEnabled(false)
.build(); .build();
bucketManager.createBucket(bucketInfo); bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo( KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals(2, result.getAcls().size()); Assert.assertEquals(2, result.getAcls().size());
List<OzoneAclInfo> removeAcls = new LinkedList<>(); List<OzoneAcl> removeAcls = new LinkedList<>();
removeAcls.add(KSMPBHelper.convertOzoneAcl(aclTwo)); removeAcls.add(aclTwo);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder() KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
@ -287,8 +283,7 @@ public void testSetBucketPropertyRemoveACL() throws IOException {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo( KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals(1, updatedResult.getAcls().size()); Assert.assertEquals(1, updatedResult.getAcls().size());
Assert.assertFalse(updatedResult.getAcls().contains( Assert.assertFalse(updatedResult.getAcls().contains(aclTwo));
KSMPBHelper.convertOzoneAcl(aclTwo)));
} }
@Test @Test
@ -298,22 +293,22 @@ public void testSetBucketPropertyChangeStorageType() throws IOException {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder() KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK) .setStorageType(StorageType.DISK)
.build(); .build();
bucketManager.createBucket(bucketInfo); bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo( KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK, Assert.assertEquals(StorageType.DISK,
result.getStorageType()); result.getStorageType());
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder() KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol") .setVolumeName("sampleVol")
.setBucketName("bucketOne") .setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.SSD) .setStorageType(StorageType.SSD)
.build(); .build();
bucketManager.setBucketProperty(bucketArgs); bucketManager.setBucketProperty(bucketArgs);
KsmBucketInfo updatedResult = bucketManager.getBucketInfo( KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne"); "sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.SSD, Assert.assertEquals(StorageType.SSD,
updatedResult.getStorageType()); updatedResult.getStorageType());
} }

View File

@ -31,7 +31,7 @@
import org.apache.hadoop.ozone.web.handlers.UserArgs; import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs; import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota; import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo; import org.apache.hadoop.ozone.web.response.VolumeInfo;

View File

@ -20,7 +20,7 @@
import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl; import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo; import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Test; import org.junit.Test;