big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11880. Ozone: KSM: Remove protobuf formats from KSM wrappers. Contributed by Nandakumar Vadivelu.

Browse Source
This commit is contained in:
Xiaoyu Yao 2017-06-07 14:49:44 -07:00 committed by Owen O'Malley
parent 2007e85d5b
commit bacd1188f1
20 changed files with 163 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java
View File

@ -18,14 +18,15 @@
package org.apache.hadoop.ksm.helpers;
import java.util.List;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
/**
* A class that encapsulates Bucket Arguments.
@ -42,11 +43,11 @@ public final class KsmBucketArgs {
/**
* ACL's that are to be added for the bucket.
*/
private List<OzoneAclInfo> addAcls;
private List<OzoneAcl> addAcls;
/**
* ACL's that are to be removed from the bucket.
*/
private List<OzoneAclInfo> removeAcls;
private List<OzoneAcl> removeAcls;
/**
* Bucket Version flag.
*/
@ -55,7 +56,7 @@ public final class KsmBucketArgs {
* Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageTypeProto storageType;
private StorageType storageType;
/**
* Private constructor, constructed via builder.
@ -67,8 +68,8 @@ public final class KsmBucketArgs {
* @param storageType - Storage type to be used.
*/
private KsmBucketArgs(String volumeName, String bucketName,
List<OzoneAclInfo> addAcls, List<OzoneAclInfo> removeAcls,
Boolean isVersionEnabled, StorageTypeProto storageType) {
List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
Boolean isVersionEnabled, StorageType storageType) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.addAcls = addAcls;
@ -97,7 +98,7 @@ public String getBucketName() {
* Returns the ACL's that are to be added.
* @return List<OzoneAclInfo>
*/
public List<OzoneAclInfo> getAddAcls() {
public List<OzoneAcl> getAddAcls() {
return addAcls;
}
@ -105,7 +106,7 @@ public List<OzoneAclInfo> getAddAcls() {
* Returns the ACL's that are to be removed.
* @return List<OzoneAclInfo>
*/
public List<OzoneAclInfo> getRemoveAcls() {
public List<OzoneAcl> getRemoveAcls() {
return removeAcls;
}
@ -121,7 +122,7 @@ public Boolean getIsVersionEnabled() {
* Returns the type of storage to be used.
* @return StorageType
*/
public StorageTypeProto getStorageType() {
public StorageType getStorageType() {
return storageType;
}
@ -140,10 +141,10 @@ public static Builder newBuilder() {
public static class Builder {
private String volumeName;
private String bucketName;
private List<OzoneAclInfo> addAcls;
private List<OzoneAclInfo> removeAcls;
private List<OzoneAcl> addAcls;
private List<OzoneAcl> removeAcls;
private Boolean isVersionEnabled;
private StorageTypeProto storageType;
private StorageType storageType;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
@ -155,12 +156,12 @@ public Builder setBucketName(String bucket) {
return this;
}
public Builder setAddAcls(List<OzoneAclInfo> acls) {
public Builder setAddAcls(List<OzoneAcl> acls) {
this.addAcls = acls;
return this;
}
public Builder setRemoveAcls(List<OzoneAclInfo> acls) {
public Builder setRemoveAcls(List<OzoneAcl> acls) {
this.removeAcls = acls;
return this;
}
@ -170,7 +171,7 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
return this;
}
public Builder setStorageType(StorageTypeProto storage) {
public Builder setStorageType(StorageType storage) {
this.storageType = storage;
return this;
}
@ -195,16 +196,19 @@ public BucketArgs getProtobuf() {
builder.setVolumeName(volumeName)
.setBucketName(bucketName);
if(addAcls != null && !addAcls.isEmpty()) {
builder.addAllAddAcls(addAcls);
builder.addAllAddAcls(addAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(removeAcls != null && !removeAcls.isEmpty()) {
builder.addAllRemoveAcls(removeAcls);
builder.addAllRemoveAcls(removeAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(isVersionEnabled != null) {
builder.setIsVersionEnabled(isVersionEnabled);
}
if(storageType != null) {
builder.setStorageType(storageType);
builder.setStorageType(
PBHelperClient.convertStorageType(storageType));
}
return builder.build();
}
@ -217,10 +221,13 @@ public BucketArgs getProtobuf() {
public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
return new KsmBucketArgs(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(),
bucketArgs.getAddAclsList(),
bucketArgs.getRemoveAclsList(),
bucketArgs.hasIsVersionEnabled() ? bucketArgs.getIsVersionEnabled() :
null,
bucketArgs.hasStorageType() ? bucketArgs.getStorageType() : null);
bucketArgs.getAddAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.getRemoveAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.hasIsVersionEnabled() ?
bucketArgs.getIsVersionEnabled() : null,
bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
bucketArgs.getStorageType()) : null);
}
}

47
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java
View File

@ -18,15 +18,16 @@
package org.apache.hadoop.ksm.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
/**
* A class that encapsulates Bucket Info.
@ -43,7 +44,7 @@ public final class KsmBucketInfo {
/**
* ACL Information.
*/
private List<OzoneAclInfo> acls;
private List<OzoneAcl> acls;
/**
* Bucket Version flag.
*/
@ -52,7 +53,7 @@ public final class KsmBucketInfo {
* Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageTypeProto storageType;
private StorageType storageType;
/**
* Private constructor, constructed via builder.
@ -63,8 +64,8 @@ public final class KsmBucketInfo {
* @param storageType - Storage type to be used.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAclInfo> acls, boolean isVersionEnabled,
StorageTypeProto storageType) {
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
@ -90,9 +91,9 @@ public String getBucketName() {
/**
* Returns the ACL's associated with this bucket.
* @return List<OzoneAclInfo>
* @return List<OzoneAcl>
*/
public List<OzoneAclInfo> getAcls() {
public List<OzoneAcl> getAcls() {
return acls;
}
@ -106,9 +107,9 @@ public boolean getIsVersionEnabled() {
/**
* Returns the type of storage to be used.
* @return StorageTypeProto
* @return StorageType
*/
public StorageTypeProto getStorageType() {
public StorageType getStorageType() {
return storageType;
}
@ -127,15 +128,15 @@ public static Builder newBuilder() {
public static class Builder {
private String volumeName;
private String bucketName;
private List<OzoneAclInfo> acls;
private List<OzoneAcl> acls;
private Boolean isVersionEnabled;
private StorageTypeProto storageType;
private StorageType storageType;
Builder() {
//Default values
this.acls = new LinkedList<>();
this.isVersionEnabled = false;
this.storageType = StorageTypeProto.DISK;
this.storageType = StorageType.DISK;
}
public Builder setVolumeName(String volume) {
@ -148,7 +149,7 @@ public Builder setBucketName(String bucket) {
return this;
}
public Builder setAcls(List<OzoneAclInfo> listOfAcls) {
public Builder setAcls(List<OzoneAcl> listOfAcls) {
this.acls = listOfAcls;
return this;
}
@ -158,7 +159,7 @@ public Builder setIsVersionEnabled(Boolean versionFlag) {
return this;
}
public Builder setStorageType(StorageTypeProto storage) {
public Builder setStorageType(StorageType storage) {
this.storageType = storage;
return this;
}
@ -185,9 +186,11 @@ public BucketInfo getProtobuf() {
return BucketInfo.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.addAllAcls(acls)
.addAllAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(storageType)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
.build();
}
@ -200,8 +203,10 @@ public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
return new KsmBucketInfo(
bucketInfo.getVolumeName(),
bucketInfo.getBucketName(),
bucketInfo.getAclsList(),
bucketInfo.getAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
bucketInfo.getStorageType());
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneAcl.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
View File

@ -17,9 +17,7 @@
*/
package org.apache.hadoop.ozone.web.request;
import org.apache.hadoop.ozone.OzoneConsts;
package org.apache.hadoop.ozone;
import java.util.Objects;

0
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

22
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java Normal file
View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone;
/**
This package contains ozone client side libraries.
*/

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
View File

@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto

24
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.protocolPB;
/**
* This package contains classes for the Protocol Buffers binding of Ozone
* protocols.
*/

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneAcls.java → hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
View File

@ -16,9 +16,8 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.web;
package org.apache.hadoop.ozone;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.junit.Test;
import java.util.HashMap;

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -17,24 +17,19 @@
package org.apache.hadoop.ozone.ksm;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
/**
* KSM bucket manager.
@ -179,10 +174,8 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
//Check ACLs to update
if(args.getAddAcls() != null || args.getRemoveAcls() != null) {
List<OzoneAcl> acls = getUpdatedAclList(oldBucketInfo.getAcls(),
args.getRemoveAcls(), args.getAddAcls());
bucketInfoBuilder.setAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(),
args.getRemoveAcls(), args.getAddAcls()));
LOG.debug("Updating ACLs for bucket: {} in volume: {}",
bucketName, volumeName);
} else {
@ -190,9 +183,9 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
}
//Check StorageType to update
StorageTypeProto storageTypeProto = args.getStorageType();
if(storageTypeProto != null) {
bucketInfoBuilder.setStorageType(storageTypeProto);
StorageType storageType = args.getStorageType();
if(storageType != null) {
bucketInfoBuilder.setStorageType(storageType);
LOG.debug("Updating bucket storage type for bucket: {} in volume: {}",
bucketName, volumeName);
} else {
@ -226,26 +219,20 @@ public void setBucketProperty(KsmBucketArgs args) throws IOException {
* Remove is done before Add.
*
* @param existingAcls - old ACL list.
* @param removeAclInfos - ACLs to be removed.
* @param addAclInfos - ACLs to be added.
* @param removeAcls - ACLs to be removed.
* @param addAcls - ACLs to be added.
* @return updated ACL list.
*/
private List<OzoneAcl> getUpdatedAclList(List<OzoneAclInfo> existingAcls,
List<OzoneAclInfo> removeAclInfos, List<OzoneAclInfo> addAclInfos) {
List<OzoneAcl> acls = existingAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
if(removeAclInfos != null && !removeAclInfos.isEmpty()) {
List<OzoneAcl> removeAcls = removeAclInfos.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
acls.removeAll(removeAcls);
private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls,
List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) {
if(removeAcls != null && !removeAcls.isEmpty()) {
existingAcls.removeAll(removeAcls);
}
if(addAclInfos != null && !addAclInfos.isEmpty()) {
List<OzoneAcl> addAcls = addAclInfos.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
addAcls.stream().filter(acl -> !acls.contains(acl)).forEach(
acls::add);
if(addAcls != null && !addAcls.isEmpty()) {
addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach(
existingAcls::add);
}
return acls;
return existingAcls;
}
/**

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
View File

@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
/**
* KSM Constants.
*/

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -25,7 +25,7 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListKeys;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.web.handlers;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.LinkedList;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
View File

@ -25,7 +25,7 @@
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
View File

@ -27,7 +27,7 @@
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
View File

@ -31,7 +31,7 @@
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -23,7 +23,7 @@
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -25,7 +25,6 @@
import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.KeyData;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.datanode.fsdataset
.LengthInputStream;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
@ -40,7 +39,7 @@
import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.scm.ScmConfigKeys;
@ -78,7 +77,6 @@
import java.util.HashSet;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* A {@link StorageHandler} implementation that distributes object storage
@ -205,12 +203,10 @@ public void createBucket(final BucketArgs args)
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(args.getAddAcls() != null) {
builder.setAcls(args.getAddAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setAcls(args.getAddAcls());
}
if(args.getStorageType() != null) {
builder.setStorageType(PBHelperClient.convertStorageType(
args.getStorageType()));
builder.setStorageType(args.getStorageType());
}
if(args.getVersioning() != null) {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
@ -250,12 +246,10 @@ public void setBucketAcls(BucketArgs args)
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(removeAcls != null && !removeAcls.isEmpty()) {
builder.setRemoveAcls(args.getRemoveAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setRemoveAcls(args.getRemoveAcls());
}
if(addAcls != null && !addAcls.isEmpty()) {
builder.setAddAcls(args.getAddAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setAddAcls(args.getAddAcls());
}
keySpaceManagerClient.setBucketProperty(builder.build());
}
@ -278,8 +272,7 @@ public void setBucketStorageClass(BucketArgs args)
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setStorageType(PBHelperClient.convertStorageType(
args.getStorageType()));
.setStorageType(args.getStorageType());
keySpaceManagerClient.setBucketProperty(builder.build());
}
@ -317,10 +310,8 @@ public BucketInfo getBucketInfo(BucketArgs args)
} else {
bucketInfo.setVersioning(Versioning.DISABLED);
}
bucketInfo.setStorageType(PBHelperClient.convertStorageType(
ksmBucketInfo.getStorageType()));
bucketInfo.setAcls(ksmBucketInfo.getAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
return bucketInfo;
}

47
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestBucketManagerImpl.java
View File

@ -16,18 +16,15 @@
*/
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.ksm.exceptions
.KSMException.ResultCodes;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@ -203,7 +200,7 @@ public void testGetBucketInfo() throws IOException {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
@ -211,7 +208,7 @@ public void testGetBucketInfo() throws IOException {
"sampleVol", "bucketOne");
Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK,
Assert.assertEquals(StorageType.DISK,
result.getStorageType());
Assert.assertEquals(false, result.getIsVersionEnabled());
}
@ -219,16 +216,16 @@ public void testGetBucketInfo() throws IOException {
@Test
public void testSetBucketPropertyAddACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>();
List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl ozoneAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(ozoneAcl));
acls.add(ozoneAcl);
BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
@ -237,10 +234,10 @@ public void testSetBucketPropertyAddACL() throws IOException {
Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(1, result.getAcls().size());
List<OzoneAclInfo> addAcls = new LinkedList<>();
List<OzoneAcl> addAcls = new LinkedList<>();
OzoneAcl newAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ);
addAcls.add(KSMPBHelper.convertOzoneAcl(newAcl));
addAcls.add(newAcl);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
@ -250,34 +247,33 @@ public void testSetBucketPropertyAddACL() throws IOException {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(2, updatedResult.getAcls().size());
Assert.assertTrue(updatedResult.getAcls().contains(
KSMPBHelper.convertOzoneAcl(newAcl)));
Assert.assertTrue(updatedResult.getAcls().contains(newAcl));
}
@Test
public void testSetBucketPropertyRemoveACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>();
List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl aclOne = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ);
OzoneAcl aclTwo = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(aclOne));
acls.add(KSMPBHelper.convertOzoneAcl(aclTwo));
acls.add(aclOne);
acls.add(aclTwo);
BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(2, result.getAcls().size());
List<OzoneAclInfo> removeAcls = new LinkedList<>();
removeAcls.add(KSMPBHelper.convertOzoneAcl(aclTwo));
List<OzoneAcl> removeAcls = new LinkedList<>();
removeAcls.add(aclTwo);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
@ -287,8 +283,7 @@ public void testSetBucketPropertyRemoveACL() throws IOException {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(1, updatedResult.getAcls().size());
Assert.assertFalse(updatedResult.getAcls().contains(
KSMPBHelper.convertOzoneAcl(aclTwo)));
Assert.assertFalse(updatedResult.getAcls().contains(aclTwo));
}
@Test
@ -298,22 +293,22 @@ public void testSetBucketPropertyChangeStorageType() throws IOException {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.build();
bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK,
Assert.assertEquals(StorageType.DISK,
result.getStorageType());
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.SSD)
.setStorageType(StorageType.SSD)
.build();
bucketManager.setBucketProperty(bucketArgs);
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.SSD,
Assert.assertEquals(StorageType.SSD,
updatedResult.getStorageType());
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
View File

@ -31,7 +31,7 @@
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java
View File

@ -20,7 +20,7 @@
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Test;