big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-11768. Ozone: KSM: Create Key Space manager service. Contributed by Anu Engineer.

Browse Source
This commit is contained in:
Anu Engineer 2017-05-11 21:11:32 -07:00 committed by Owen O'Malley
parent 169713f3f3
commit edff6c6a24
30 changed files with 1099 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java Normal file
View File

@ -0,0 +1,140 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.helpers;
import com.google.common.base.Preconditions;
import java.util.HashMap;
import java.util.Map;
/**
* A class that encapsulates the createVolume Args.
*/
public final class VolumeArgs {
private final String adminName;
private final String ownerName;
private final String volume;
private final long quotaInBytes;
private final Map<String, String> extendedAttributes;
/**
* Private constructor, constructed via builder.
*
* @param adminName - Administrator name.
* @param ownerName - Volume owner's name
* @param volume - volume name
* @param quotaInBytes - Volume Quota in bytes.
* @param keyValueMap - keyValue map.
*/
private VolumeArgs(String adminName, String ownerName, String volume,
long quotaInBytes, Map<String, String> keyValueMap) {
this.adminName = adminName;
this.ownerName = ownerName;
this.volume = volume;
this.quotaInBytes = quotaInBytes;
this.extendedAttributes = keyValueMap;
}
/**
* Returns the Admin Name.
*
* @return String.
*/
public String getAdminName() {
return adminName;
}
/**
* Returns the owner Name.
*
* @return String
*/
public String getOwnerName() {
return ownerName;
}
/**
* Returns the volume Name.
*
* @return String
*/
public String getVolume() {
return volume;
}
/**
* Returns Quota in Bytes.
*
* @return long, Quota in bytes.
*/
public long getQuotaInBytes() {
return quotaInBytes;
}
public Map<String, String> getExtendedAttributes() {
return extendedAttributes;
}
static class Builder {
private String adminName;
private String ownerName;
private String volume;
private long quotaInBytes;
private Map<String, String> extendedAttributes;
/**
* Constructs a builder.
*/
Builder() {
extendedAttributes = new HashMap<>();
}
public void setAdminName(String adminName) {
this.adminName = adminName;
}
public void setOwnerName(String ownerName) {
this.ownerName = ownerName;
}
public void setVolume(String volume) {
this.volume = volume;
}
public void setQuotaInBytes(long quotaInBytes) {
this.quotaInBytes = quotaInBytes;
}
public void addMetadata(String key, String value) {
extendedAttributes.put(key, value); // overwrite if present.
}
/**
* Constructs a CreateVolumeArgument.
*
* @return CreateVolumeArgs.
*/
public VolumeArgs build() {
Preconditions.checkNotNull(adminName);
Preconditions.checkNotNull(ownerName);
Preconditions.checkNotNull(volume);
return new VolumeArgs(adminName, ownerName, volume, quotaInBytes,
extendedAttributes);
}
}
}

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java Normal file
View File

@ -0,0 +1,18 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.helpers;

21
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm;
/**
This package contains client side protocol library to communicate with KSM.
*/

98
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeyspaceManagerProtocol.java Normal file
View File

@ -0,0 +1,98 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.protocol;
import org.apache.hadoop.ksm.helpers.VolumeArgs;
import java.io.IOException;
import java.util.List;
/**
* Protocol to talk to KSM.
*/
public interface KeyspaceManagerProtocol {
/**
* Creates a volume.
* @param args - Arguments to create Volume.
* @throws IOException
*/
void createVolume(VolumeArgs args) throws IOException;
/**
* Changes the owner of a volume.
* @param volume - Name of the volume.
* @param owner - Name of the owner.
* @throws IOException
*/
void setOwner(String volume, String owner) throws IOException;
/**
* Changes the Quota on a volume.
* @param volume - Name of the volume.
* @param quota - Quota in bytes.
* @throws IOException
*/
void setQuota(String volume, long quota) throws IOException;
/**
* Checks if the specified user can access this volume.
* @param volume - volume
* @param userName - user name
* @throws IOException
*/
void checkVolumeAccess(String volume, String userName) throws IOException;
/**
* Gets the volume information.
* @param volume - Volume name.s
* @return VolumeArgs or exception is thrown.
* @throws IOException
*/
VolumeArgs getVolumeinfo(String volume) throws IOException;
/**
* Deletes the an exisiting empty volume.
* @param volume - Name of the volume.
* @throws IOException
*/
void deleteVolume(String volume) throws IOException;
/**
* Lists volume owned by a specific user.
* @param userName - user name
* @param prefix - Filter prefix -- Return only entries that match this.
* @param prevKey - Previous key -- List starts from the next from the prevkey
* @param maxKeys - Max number of keys to return.
* @return List of Volumes.
* @throws IOException
*/
List<VolumeArgs> listVolumeByUser(String userName, String prefix, String
prevKey, long maxKeys) throws IOException;
/**
* Lists volume all volumes in the cluster.
* @param prefix - Filter prefix -- Return only entries that match this.
* @param prevKey - Previous key -- List starts from the next from the prevkey
* @param maxKeys - Max number of keys to return.
* @return List of Volumes.
* @throws IOException
*/
List<VolumeArgs> listAllVolumes(String prefix, String
prevKey, long maxKeys) throws IOException;
}

19
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java Normal file
View File

@ -0,0 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.protocol;

33
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/KeySpaceManagerProtocolPB.java Normal file
View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyspaceManagerService;
/**
* Protocol used to communicate with KSM.
*/
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.KeyspaceManagerProtocol",
protocolVersion = 1)
@InterfaceAudience.Private
public interface KeySpaceManagerProtocolPB
extends KeyspaceManagerService.BlockingInterface {
}

19
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocolPB/package-info.java Normal file
View File

@ -0,0 +1,19 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ksm.protocolPB;

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
View File

@ -152,6 +152,7 @@ public final class ScmConfigKeys {
"ozone.scm.container.provision_batch_size";
public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 10;
/**
* Never constructed.
*/

10
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
View File

@ -19,7 +19,7 @@
package org.apache.hadoop.scm.container.common.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
@ -60,7 +60,7 @@ public Pipeline(String leaderID) {
* @param pipeline - ProtoBuf definition for the pipeline.
* @return Pipeline Object
*/
public static Pipeline getFromProtoBuf(ContainerProtos.Pipeline pipeline) {
public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) {
Preconditions.checkNotNull(pipeline);
Pipeline newPipeline = new Pipeline(pipeline.getLeaderID());
for (HdfsProtos.DatanodeIDProto dataID : pipeline.getMembersList()) {
@ -104,9 +104,9 @@ public List<DatanodeID> getMachines() {
*
* @return Protobuf message
*/
public ContainerProtos.Pipeline getProtobufMessage() {
ContainerProtos.Pipeline.Builder builder =
ContainerProtos.Pipeline.newBuilder();
public OzoneProtos.Pipeline getProtobufMessage() {
OzoneProtos.Pipeline.Builder builder =
OzoneProtos.Pipeline.newBuilder();
for (DatanodeID datanode : datanodes.values()) {
builder.addMembers(datanode.getProtoBufMessage());
}

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
View File

@ -30,7 +30,7 @@
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.XceiverClientSpi;

11
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
View File

@ -29,6 +29,7 @@ option java_outer_classname = "ContainerProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs.ozone;
import "hdfs.proto";
import "Ozone.proto";
/**
* Commands that are used to manipulate the state of containers on a datanode.
@ -190,17 +191,7 @@ message ContainerCommandResponseProto {
}
// A pipeline is composed of one or more datanodes that back a container.
message Pipeline {
required string leaderID = 1;
repeated DatanodeIDProto members = 2;
required string containerName = 3;
}
message KeyValue {
required string key = 1;
optional string value = 2;
}
message ContainerData {
required string name = 1;

189
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto Normal file
View File

@ -0,0 +1,189 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* These .proto interfaces are private and unstable.
* Please see http://wiki.apache.org/hadoop/Compatibility
* for what changes are allowed for a *unstable* .proto interface.
*/
option java_package = "org.apache.hadoop.ozone.protocol.proto";
option java_outer_classname = "KeySpaceManagerProtocolProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.hdfs.ozone;
/**
This is file contains the protocol to communicate with
Ozone key space manager. Ozone KSM manages the namespace for ozone.
This is similar to Namenode for Ozone.
*/
import "Ozone.proto";
enum Status {
OK = 1;
VOLUME_NOT_UNIQUE = 2;
VOLUME_NOT_FOUND = 3;
VOLUME_NOT_EMPTY = 4;
USER_NOT_FOUND = 5;
ACCESS_DENIED = 6;
}
message VolumeInfo {
required string adminName = 1;
required string ownerName = 2;
required string volume = 3;
optional uint64 quotaInBytes = 4;
repeated KeyValue metadata = 5;
}
/**
Creates a volume
*/
message CreateVolumeRequest {
required VolumeInfo volumeInfo = 1;
}
message CreateVolumeResponse {
required Status status = 1;
}
/**
Changes the Volume Properties -- like ownership and quota for a volume.
*/
message SetVolumePropertyRequest {
required string volumeName = 1;
optional string ownerName = 2;
optional uint64 quotaInBytes = 3;
}
message SetVolumePropertyResponse {
required Status status = 1;
}
/**
Checks if a specified user has access to the volume.
*/
message CheckVolumeAccessRequest {
required string volumeName = 1;
required string userName = 2;
}
message CheckVolumeAccessResponse {
required Status status = 1;
}
/**
Returns information about a volume.
*/
message InfoVolumeRequest {
required string volumeName = 1;
}
message InfoVolumeResponse {
required Status status = 1;
optional VolumeInfo volumeInfo = 2;
}
/**
Deletes an existing volume.
*/
message DeleteVolumeRequest {
required string volumeName = 1;
}
message DeleteVolumeResponse {
required Status status = 1;
}
/**
List Volumes -- List all volumes in the cluster or by user.
*/
message ListVolumeRequest {
enum Scope {
USER_VOLUMES = 1; // User volumes -- called by user
VOLUMES_BY_USER = 2; // User volumes - called by Admin
VOLUMES_BY_CLUSTER = 3; // All volumes in the cluster
}
required Scope scope = 1;
required string volumeName = 2;
optional string userName = 3;
optional string prefix = 4;
optional string prevKey = 5;
optional uint64 maxKeys = 6;
}
message ListVolumeResponse {
enum Status {
OK = 1;
ACCESS_DENIED = 2;
REQUIRED_ARG_MISSING = 3;
}
required Status status = 1;
repeated VolumeInfo volumeInfo = 2;
}
/**
The KSM service that takes care of Ozone namespace.
*/
service KeyspaceManagerService {
/**
Creates a Volume.
*/
rpc createVolume(CreateVolumeRequest)
returns(CreateVolumeResponse);
/**
Allows modificiation of volume properties.
*/
rpc setVolumeProperty(SetVolumePropertyRequest)
returns (SetVolumePropertyResponse);
/**
Checks if the specified volume is accesible by the specified user.
*/
rpc checkVolumeAccess(CheckVolumeAccessRequest)
returns (CheckVolumeAccessResponse);
/**
Gets Volume information.
*/
rpc infoVolume(InfoVolumeRequest)
returns(InfoVolumeResponse);
/**
Deletes a volume if it is empty.
*/
rpc deleteVolume(DeleteVolumeRequest)
returns (DeleteVolumeResponse);
/**
Lists Volumes
*/
rpc listVolumes(ListVolumeRequest)
returns (ListVolumeResponse);
}

43
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto Normal file
View File

@ -0,0 +1,43 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* These .proto interfaces are private and unstable.
* Please see http://wiki.apache.org/hadoop/Compatibility
* for what changes are allowed for a *unstable* .proto interface.
*/
option java_package = "org.apache.hadoop.ozone.protocol.proto";
option java_outer_classname = "OzoneProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.hdfs.ozone;
import "hdfs.proto";
// A pipeline is composed of one or more datanodes that back a container.
message Pipeline {
required string leaderID = 1;
repeated DatanodeIDProto members = 2;
required string containerName = 3;
}
message KeyValue {
required string key = 1;
optional string value = 2;
}

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
View File

@ -29,7 +29,7 @@ option java_generate_equals_and_hash = true;
package hadoop.hdfs;
import "hdfs.proto";
import "DatanodeContainerProtocol.proto";
import "Ozone.proto";
/**
* keys - batch of object keys to find

24
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientUtils.java
View File

@ -35,6 +35,10 @@
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_BIND_HOST_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
import static org.apache.hadoop.scm.ScmConfigKeys
@ -224,6 +228,26 @@ public static InetSocketAddress getScmDataNodeBindAddress(
port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
}
/**
* Retrieve the socket address that is used by KSM.
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
public static InetSocketAddress getKsmAddress(
Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_KSM_ADDRESS_KEY);
// If no port number is specified then we'll just try the defaultBindPort.
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_KSM_ADDRESS_KEY);
return NetUtils.createSocketAddr(
host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" +
port.or(OZONE_KSM_PORT_DEFAULT));
}
/**
* Retrieve the hostname, trying the supplied config keys in order.
* Each config value may be absent, or if present in the format

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
View File

@ -20,6 +20,7 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import java.io.IOException;
import java.util.Map;
@ -110,8 +111,8 @@ public ContainerProtos.ChunkInfo getProtoBufMessage() {
}
for (Map.Entry<String, String> entry : metadata.entrySet()) {
ContainerProtos.KeyValue.Builder keyValBuilder =
ContainerProtos.KeyValue.newBuilder();
OzoneProtos.KeyValue.Builder keyValBuilder =
OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
View File

@ -20,6 +20,7 @@
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.util.Time;
import java.io.IOException;
@ -113,8 +114,8 @@ public ContainerProtos.ContainerData getProtoBufMessage() {
builder.setOpen(this.isOpen());
for (Map.Entry<String, String> entry : metadata.entrySet()) {
ContainerProtos.KeyValue.Builder keyValBuilder =
ContainerProtos.KeyValue.newBuilder();
OzoneProtos.KeyValue.Builder keyValBuilder =
OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.container.common.helpers;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import java.io.IOException;
import java.util.Collections;
@ -81,8 +82,8 @@ public ContainerProtos.KeyData getProtoBufMessage() {
builder.setName(this.getKeyName());
builder.addAllChunks(this.chunks);
for (Map.Entry<String, String> entry : metadata.entrySet()) {
ContainerProtos.KeyValue.Builder keyValBuilder =
ContainerProtos.KeyValue.newBuilder();
OzoneProtos.KeyValue.Builder keyValBuilder =
OzoneProtos.KeyValue.newBuilder();
builder.addMetadata(keyValBuilder.setKey(entry.getKey())
.setValue(entry.getValue()).build());
}

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
/**
* KSM Constants.
*/
public final class KSMConfigKeys {
/**
* Never constructed.
*/
private KSMConfigKeys() {
}
public static final String OZONE_KSM_HANDLER_COUNT_KEY =
"ozone.scm.handler.count.key";
public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 200;
public static final String OZONE_KSM_ADDRESS_KEY =
"ozone.ksm.address";
public static final String OZONE_KSM_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_KSM_PORT_DEFAULT = 9862;
}

289
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java Normal file
View File

@ -0,0 +1,289 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ksm.helpers.VolumeArgs;
import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol;
import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.protocolPB
.KeyspaceManagerProtocolServerSideTranslatorPB;
import org.apache.hadoop.ozone.scm.StorageContainerManager;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.List;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_HANDLER_COUNT_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_HANDLER_COUNT_KEY;
import static org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
.KeyspaceManagerService.newReflectiveBlockingService;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
* Ozone Keyspace manager is the metadata manager of ozone.
*/
@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
public class KeySpaceManager implements KeyspaceManagerProtocol {
// TODO: Support JMX
private static final Logger LOG =
LoggerFactory.getLogger(KeySpaceManager.class);
private final RPC.Server ksmRpcServer;
private final InetSocketAddress ksmRpcAddress;
public KeySpaceManager(OzoneConfiguration conf) throws IOException {
final int handlerCount = conf.getInt(OZONE_KSM_HANDLER_COUNT_KEY,
OZONE_KSM_HANDLER_COUNT_DEFAULT);
RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
ProtobufRpcEngine.class);
BlockingService ksmService = newReflectiveBlockingService(
new KeyspaceManagerProtocolServerSideTranslatorPB(this));
final InetSocketAddress ksmNodeRpcAddr = OzoneClientUtils.
getKsmAddress(conf);
ksmRpcServer = startRpcServer(conf, ksmNodeRpcAddr,
KeySpaceManagerProtocolPB.class, ksmService,
handlerCount);
ksmRpcAddress = updateListenAddress(conf,
OZONE_KSM_ADDRESS_KEY, ksmNodeRpcAddr, ksmRpcServer);
//TODO : Add call to register MXBean for JMX.
}
/**
* Starts an RPC server, if configured.
*
* @param conf configuration
* @param addr configured address of RPC server
* @param protocol RPC protocol provided by RPC server
* @param instance RPC protocol implementation instance
* @param handlerCount RPC server handler count
*
* @return RPC server
* @throws IOException if there is an I/O error while creating RPC server
*/
private static RPC.Server startRpcServer(OzoneConfiguration conf,
InetSocketAddress addr, Class<?> protocol, BlockingService instance,
int handlerCount) throws IOException {
RPC.Server rpcServer = new RPC.Builder(conf)
.setProtocol(protocol)
.setInstance(instance)
.setBindAddress(addr.getHostString())
.setPort(addr.getPort())
.setNumHandlers(handlerCount)
.setVerbose(false)
.setSecretManager(null)
.build();
DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
return rpcServer;
}
/**
* Main entry point for starting KeySpaceManager.
*
* @param argv arguments
* @throws IOException if startup fails due to I/O error
*/
public static void main(String[] argv) throws IOException {
StringUtils.startupShutdownMessage(StorageContainerManager.class,
argv, LOG);
try {
KeySpaceManager ksm = new KeySpaceManager(new OzoneConfiguration());
ksm.start();
ksm.join();
} catch (Throwable t) {
LOG.error("Failed to start the KeyspaceManager.", t);
terminate(1, t);
}
}
/**
* Builds a message for logging startup information about an RPC server.
*
* @param description RPC server description
* @param addr RPC server listening address
* @return server startup message
*/
private static String buildRpcServerStartMessage(String description,
InetSocketAddress addr) {
return addr != null ? String.format("%s is listening at %s",
description, addr.toString()) :
String.format("%s not started", description);
}
/**
* After starting an RPC server, updates configuration with the actual
* listening address of that server. The listening address may be different
* from the configured address if, for example, the configured address uses
* port 0 to request use of an ephemeral port.
*
* @param conf configuration to update
* @param rpcAddressKey configuration key for RPC server address
* @param addr configured address
* @param rpcServer started RPC server.
*/
private static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
String rpcAddressKey, InetSocketAddress addr, RPC.Server rpcServer) {
InetSocketAddress listenAddr = rpcServer.getListenerAddress();
InetSocketAddress updatedAddr = new InetSocketAddress(
addr.getHostString(), listenAddr.getPort());
conf.set(rpcAddressKey,
listenAddr.getHostString() + ":" + listenAddr.getPort());
return updatedAddr;
}
/**
* Start service.
*/
public void start() {
LOG.info(buildRpcServerStartMessage("KeyspaceManager RPC server",
ksmRpcAddress));
ksmRpcServer.start();
}
/**
* Wait until service has completed shutdown.
*/
public void join() {
try {
ksmRpcServer.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.info("Interrupted during KeyspaceManager join.");
}
}
/**
* Creates a volume.
*
* @param args - Arguments to create Volume.
* @throws IOException
*/
@Override
public void createVolume(VolumeArgs args) throws IOException {
}
/**
* Changes the owner of a volume.
*
* @param volume - Name of the volume.
* @param owner - Name of the owner.
* @throws IOException
*/
@Override
public void setOwner(String volume, String owner) throws IOException {
}
/**
* Changes the Quota on a volume.
*
* @param volume - Name of the volume.
* @param quota - Quota in bytes.
* @throws IOException
*/
@Override
public void setQuota(String volume, long quota) throws IOException {
}
/**
* Checks if the specified user can access this volume.
*
* @param volume - volume
* @param userName - user name
* @throws IOException
*/
@Override
public void checkVolumeAccess(String volume, String userName) throws
IOException {
}
/**
* Gets the volume information.
*
* @param volume - Volume name.s
* @return VolumeArgs or exception is thrown.
* @throws IOException
*/
@Override
public VolumeArgs getVolumeinfo(String volume) throws IOException {
return null;
}
/**
* Deletes the an exisiting empty volume.
*
* @param volume - Name of the volume.
* @throws IOException
*/
@Override
public void deleteVolume(String volume) throws IOException {
}
/**
* Lists volume owned by a specific user.
*
* @param userName - user name
* @param prefix - Filter prefix -- Return only entries that match this.
* @param prevKey - Previous key -- List starts from the next from the
* prevkey
* @param maxKeys - Max number of keys to return.
* @return List of Volumes.
* @throws IOException
*/
@Override
public List<VolumeArgs> listVolumeByUser(String userName, String prefix,
String prevKey, long maxKeys) throws IOException {
return null;
}
/**
* Lists volume all volumes in the cluster.
*
* @param prefix - Filter prefix -- Return only entries that match this.
* @param prevKey - Previous key -- List starts from the next from the
* prevkey
* @param maxKeys - Max number of keys to return.
* @return List of Volumes.
* @throws IOException
*/
@Override
public List<VolumeArgs> listAllVolumes(String prefix, String prevKey, long
maxKeys) throws IOException {
return null;
}
}

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java Normal file
View File

@ -0,0 +1,21 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
/*
This package contains the keyspace manager classes.
*/

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
View File

@ -1,3 +1,4 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@ -18,7 +19,7 @@
package org.apache.hadoop.ozone.protocol;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.ozone.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;

89
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeyspaceManagerProtocolServerSideTranslatorPB.java Normal file
View File

@ -0,0 +1,89 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.protocolPB;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.ksm.protocol.KeyspaceManagerProtocol;
import org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
/**
* This class is the server-side translator that forwards requests received on
* {@link org.apache.hadoop.ksm.protocolPB.KeySpaceManagerProtocolPB} to the
* KeyspaceManagerService server implementation.
*/
public class KeyspaceManagerProtocolServerSideTranslatorPB implements
KeySpaceManagerProtocolPB {
private final KeyspaceManagerProtocol impl;
/**
* Constructs an instance of the server handler.
*
* @param impl KeySpaceManagerProtocolPB
*/
public KeyspaceManagerProtocolServerSideTranslatorPB(
KeyspaceManagerProtocol impl) {
this.impl = impl;
}
@Override
public KeySpaceManagerProtocolProtos.CreateVolumeResponse createVolume(
RpcController controller, KeySpaceManagerProtocolProtos
.CreateVolumeRequest
request) throws ServiceException {
return null;
}
@Override
public KeySpaceManagerProtocolProtos.SetVolumePropertyResponse
setVolumeProperty(RpcController controller, KeySpaceManagerProtocolProtos
.SetVolumePropertyRequest request) throws ServiceException {
return null;
}
@Override
public KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse
checkVolumeAccess(RpcController controller, KeySpaceManagerProtocolProtos
.CheckVolumeAccessRequest request) throws ServiceException {
return null;
}
@Override
public KeySpaceManagerProtocolProtos.InfoVolumeResponse infoVolume(
RpcController controller,
KeySpaceManagerProtocolProtos.InfoVolumeRequest request)
throws ServiceException {
return null;
}
@Override
public KeySpaceManagerProtocolProtos.DeleteVolumeResponse deleteVolume(
RpcController controller, KeySpaceManagerProtocolProtos
.DeleteVolumeRequest
request) throws ServiceException {
return null;
}
@Override
public KeySpaceManagerProtocolProtos.ListVolumeResponse listVolumes(
RpcController controller,
KeySpaceManagerProtocolProtos.ListVolumeRequest request)
throws ServiceException {
return null;
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
View File

@ -26,8 +26,8 @@
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Pipeline;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.Pipeline;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
View File

@ -1,3 +1,4 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
@ -19,7 +20,7 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@ -172,7 +173,7 @@ public Pipeline getContainer(final String containerName) throws IOException {
containerName);
}
pipeline = Pipeline.getFromProtoBuf(
ContainerProtos.Pipeline.PARSER.parseFrom(pipelineBytes));
OzoneProtos.Pipeline.PARSER.parseFrom(pipelineBytes));
return pipeline;
} finally {
lock.unlock();

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java
View File

@ -22,7 +22,7 @@
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
View File

@ -27,7 +27,7 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.cblock;
import "DatanodeContainerProtocol.proto";
import "Ozone.proto";
/**
* This message is sent from CBlock client side to CBlock server to

2
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
View File

@ -38,7 +38,7 @@ import "HdfsServer.proto";
import "DatanodeProtocol.proto";
import "DatanodeContainerProtocol.proto";
import "Ozone.proto";
/**

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.scm.ScmConfigKeys;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@ -321,4 +322,29 @@ public void testGetSCMAddresses() {
assertTrue(e instanceof IllegalArgumentException);
}
}
@Test
public void testGetKSMAddress() {
final Configuration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
// back to the default port.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = OzoneClientUtils.getKsmAddress(conf);
assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
// Next try a client address with just a host name and port. Verify the port
// is ignored and the default KSM port is used.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100");
addr = OzoneClientUtils.getKsmAddress(conf);
assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(100));
// Assert the we are able to use default configs if no value is specified.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "");
addr = OzoneClientUtils.getKsmAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java
View File

@ -27,6 +27,7 @@
.ContainerCommandRequestProto;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
.ContainerCommandResponseProto;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@ -361,8 +362,8 @@ public static ContainerCommandRequestProto getUpdateContainerRequest(
containerData.setName(containerName);
String[] keys = metaData.keySet().toArray(new String[]{});
for(int i=0; i<keys.length; i++) {
ContainerProtos.KeyValue.Builder kvBuilder =
ContainerProtos.KeyValue.newBuilder();
OzoneProtos.KeyValue.Builder kvBuilder =
OzoneProtos.KeyValue.newBuilder();
kvBuilder.setKey(keys[i]);
kvBuilder.setValue(metaData.get(keys[i]));
containerData.addMetadata(i, kvBuilder.build());