diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java index 36cdfc995d..abad9e38f4 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java @@ -28,6 +28,8 @@ .ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos + .CloseContainerRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .DatanodeBlockID; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -86,15 +88,18 @@ public static GetKeyResponseProto getKey(XceiverClientSpi xceiverClient, .newBuilder() .setBlockID(datanodeBlockID); String id = xceiverClient.getPipeline().getLeader().getUuidString(); + ContainerCommandRequestProto request = ContainerCommandRequestProto .newBuilder() .setCmdType(Type.GetKey) + .setContainerID(datanodeBlockID.getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setGetKey(readKeyRequest) .build(); ContainerCommandResponseProto response = xceiverClient.sendCommand(request); validateContainerResponse(response); + return response.getGetKey(); } @@ -118,7 +123,9 @@ public static GetKeyResponseProto getKey(XceiverClientSpi xceiverClient, String id = xceiverClient.getPipeline().getLeader().getUuidString(); ContainerCommandRequestProto request = ContainerCommandRequestProto.newBuilder() - .setCmdType(Type.GetCommittedBlockLength).setTraceID(traceID) + .setCmdType(Type.GetCommittedBlockLength) + .setContainerID(blockID.getContainerID()) + .setTraceID(traceID) .setDatanodeUuid(id) .setGetCommittedBlockLength(getBlockLengthRequestBuilder).build(); ContainerCommandResponseProto response = xceiverClient.sendCommand(request); @@ -143,6 +150,7 @@ public static void putKey(XceiverClientSpi xceiverClient, ContainerCommandRequestProto request = ContainerCommandRequestProto .newBuilder() .setCmdType(Type.PutKey) + .setContainerID(containerKeyData.getBlockID().getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setPutKey(createKeyRequest) @@ -171,6 +179,7 @@ public static ReadChunkResponseProto readChunk(XceiverClientSpi xceiverClient, ContainerCommandRequestProto request = ContainerCommandRequestProto .newBuilder() .setCmdType(Type.ReadChunk) + .setContainerID(blockID.getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setReadChunk(readChunkRequest) @@ -202,6 +211,7 @@ public static void writeChunk(XceiverClientSpi xceiverClient, ChunkInfo chunk, ContainerCommandRequestProto request = ContainerCommandRequestProto .newBuilder() .setCmdType(Type.WriteChunk) + .setContainerID(blockID.getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setWriteChunk(writeChunkRequest) @@ -250,6 +260,7 @@ public static void writeSmallFile(XceiverClientSpi client, ContainerCommandRequestProto request = ContainerCommandRequestProto.newBuilder() .setCmdType(Type.PutSmallFile) + .setContainerID(blockID.getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setPutSmallFile(putSmallFileRequest) @@ -270,7 +281,6 @@ public static void createContainer(XceiverClientSpi client, long containerID, ContainerProtos.CreateContainerRequestProto.Builder createRequest = ContainerProtos.CreateContainerRequestProto .newBuilder(); - createRequest.setContainerID(containerID); createRequest.setContainerType(ContainerProtos.ContainerType .KeyValueContainer); @@ -278,6 +288,7 @@ public static void createContainer(XceiverClientSpi client, long containerID, ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.CreateContainer); + request.setContainerID(containerID); request.setCreateContainer(createRequest.build()); request.setDatanodeUuid(id); request.setTraceID(traceID); @@ -298,12 +309,13 @@ public static void deleteContainer(XceiverClientSpi client, long containerID, boolean force, String traceID) throws IOException { ContainerProtos.DeleteContainerRequestProto.Builder deleteRequest = ContainerProtos.DeleteContainerRequestProto.newBuilder(); - deleteRequest.setContainerID(containerID); deleteRequest.setForceDelete(force); String id = client.getPipeline().getLeader().getUuidString(); + ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.DeleteContainer); + request.setContainerID(containerID); request.setDeleteContainer(deleteRequest); request.setTraceID(traceID); request.setDatanodeUuid(id); @@ -322,15 +334,13 @@ public static void deleteContainer(XceiverClientSpi client, long containerID, */ public static void closeContainer(XceiverClientSpi client, long containerID, String traceID) throws IOException { - ContainerProtos.CloseContainerRequestProto.Builder closeRequest = - ContainerProtos.CloseContainerRequestProto.newBuilder(); - closeRequest.setContainerID(containerID); - String id = client.getPipeline().getLeader().getUuidString(); + ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(Type.CloseContainer); - request.setCloseContainer(closeRequest); + request.setContainerID(containerID); + request.setCloseContainer(CloseContainerRequestProto.getDefaultInstance()); request.setTraceID(traceID); request.setDatanodeUuid(id); ContainerCommandResponseProto response = @@ -348,19 +358,19 @@ public static void closeContainer(XceiverClientSpi client, public static ReadContainerResponseProto readContainer( XceiverClientSpi client, long containerID, String traceID) throws IOException { - ReadContainerRequestProto.Builder readRequest = - ReadContainerRequestProto.newBuilder(); - readRequest.setContainerID(containerID); String id = client.getPipeline().getLeader().getUuidString(); + ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(Type.ReadContainer); - request.setReadContainer(readRequest); + request.setContainerID(containerID); + request.setReadContainer(ReadContainerRequestProto.getDefaultInstance()); request.setDatanodeUuid(id); request.setTraceID(traceID); ContainerCommandResponseProto response = client.sendCommand(request.build()); validateContainerResponse(response); + return response.getReadContainer(); } @@ -383,15 +393,18 @@ public static GetSmallFileResponseProto readSmallFile(XceiverClientSpi client, .newBuilder().setKey(getKey) .build(); String id = client.getPipeline().getLeader().getUuidString(); + ContainerCommandRequestProto request = ContainerCommandRequestProto .newBuilder() .setCmdType(Type.GetSmallFile) + .setContainerID(blockID.getContainerID()) .setTraceID(traceID) .setDatanodeUuid(id) .setGetSmallFile(getSmallFileRequest) .build(); ContainerCommandResponseProto response = client.sendCommand(request); validateContainerResponse(response); + return response.getGetSmallFile(); } diff --git a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto index 6969fa6696..af0634672e 100644 --- a/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto +++ b/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto @@ -165,80 +165,81 @@ enum ContainerLifeCycleState { } message ContainerCommandRequestProto { - required Type cmdType = 1; // Type of the command + required Type cmdType = 1; // Type of the command // A string that identifies this command, we generate Trace ID in Ozone // frontend and this allows us to trace that command all over ozone. - optional string traceID = 2; + optional string traceID = 2; + + required int64 containerID = 3; + required string datanodeUuid = 4; // One of the following command is available when the corresponding // cmdType is set. At the protocol level we allow only // one command in each packet. // TODO : Upgrade to Protobuf 2.6 or later. - optional CreateContainerRequestProto createContainer = 3; - optional ReadContainerRequestProto readContainer = 4; - optional UpdateContainerRequestProto updateContainer = 5; - optional DeleteContainerRequestProto deleteContainer = 6; - optional ListContainerRequestProto listContainer = 7; + optional CreateContainerRequestProto createContainer = 5; + optional ReadContainerRequestProto readContainer = 6; + optional UpdateContainerRequestProto updateContainer = 7; + optional DeleteContainerRequestProto deleteContainer = 8; + optional ListContainerRequestProto listContainer = 9; + optional CloseContainerRequestProto closeContainer = 10; - optional PutKeyRequestProto putKey = 8; - optional GetKeyRequestProto getKey = 9; - optional DeleteKeyRequestProto deleteKey = 10; - optional ListKeyRequestProto listKey = 11; + optional PutKeyRequestProto putKey = 11; + optional GetKeyRequestProto getKey = 12; + optional DeleteKeyRequestProto deleteKey = 13; + optional ListKeyRequestProto listKey = 14; - optional ReadChunkRequestProto readChunk = 12; - optional WriteChunkRequestProto writeChunk = 13; - optional DeleteChunkRequestProto deleteChunk = 14; - optional ListChunkRequestProto listChunk = 15; + optional ReadChunkRequestProto readChunk = 15; + optional WriteChunkRequestProto writeChunk = 16; + optional DeleteChunkRequestProto deleteChunk = 17; + optional ListChunkRequestProto listChunk = 18; - optional PutSmallFileRequestProto putSmallFile = 16; - optional GetSmallFileRequestProto getSmallFile = 17; - optional CloseContainerRequestProto closeContainer = 18; - optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 19; - required string datanodeUuid = 20; + optional PutSmallFileRequestProto putSmallFile = 19; + optional GetSmallFileRequestProto getSmallFile = 20; + + optional GetCommittedBlockLengthRequestProto getCommittedBlockLength = 21; } message ContainerCommandResponseProto { - required Type cmdType = 1; - optional string traceID = 2; + required Type cmdType = 1; + optional string traceID = 2; - optional CreateContainerResponseProto createContainer = 3; - optional ReadContainerResponseProto readContainer = 4; - optional UpdateContainerResponseProto updateContainer = 5; - optional DeleteContainerResponseProto deleteContainer = 6; - optional ListContainerResponseProto listContainer = 7; + required Result result = 3; + optional string message = 4; - optional PutKeyResponseProto putKey = 8; - optional GetKeyResponseProto getKey = 9; - optional DeleteKeyResponseProto deleteKey = 10; - optional ListKeyResponseProto listKey = 11; + optional CreateContainerResponseProto createContainer = 5; + optional ReadContainerResponseProto readContainer = 6; + optional UpdateContainerResponseProto updateContainer = 7; + optional DeleteContainerResponseProto deleteContainer = 8; + optional ListContainerResponseProto listContainer = 9; + optional CloseContainerResponseProto closeContainer = 10; - optional WriteChunkResponseProto writeChunk = 12; - optional ReadChunkResponseProto readChunk = 13; - optional DeleteChunkResponseProto deleteChunk = 14; - optional ListChunkResponseProto listChunk = 15; + optional PutKeyResponseProto putKey = 11; + optional GetKeyResponseProto getKey = 12; + optional DeleteKeyResponseProto deleteKey = 13; + optional ListKeyResponseProto listKey = 14; - required Result result = 17; - optional string message = 18; + optional WriteChunkResponseProto writeChunk = 15; + optional ReadChunkResponseProto readChunk = 16; + optional DeleteChunkResponseProto deleteChunk = 17; + optional ListChunkResponseProto listChunk = 18; - optional PutSmallFileResponseProto putSmallFile = 19; - optional GetSmallFileResponseProto getSmallFile = 20; - optional CloseContainerResponseProto closeContainer = 21; - optional GetCommittedBlockLengthResponseProto getCommittedBlockLength = 22; + optional PutSmallFileResponseProto putSmallFile = 19; + optional GetSmallFileResponseProto getSmallFile = 20; + optional GetCommittedBlockLengthResponseProto getCommittedBlockLength = 21; } message ContainerData { required int64 containerID = 1; repeated KeyValue metadata = 2; - optional string dbPath = 3; optional string containerPath = 4; optional int64 bytesUsed = 6; optional int64 size = 7; - optional int64 keyCount = 8; + optional int64 blockCount = 8; optional ContainerLifeCycleState state = 9 [default = OPEN]; optional ContainerType containerType = 10 [default = KeyValueContainer]; - optional string containerDBType = 11; } enum ContainerType { @@ -248,7 +249,6 @@ enum ContainerType { // Container Messages. message CreateContainerRequestProto { - required int64 containerID = 1; repeated KeyValue metadata = 2; optional ContainerType containerType = 3 [default = KeyValueContainer]; } @@ -257,7 +257,6 @@ message CreateContainerResponseProto { } message ReadContainerRequestProto { - required int64 containerID = 1; } message ReadContainerResponseProto { @@ -265,7 +264,6 @@ message ReadContainerResponseProto { } message UpdateContainerRequestProto { - required int64 containerID = 1; repeated KeyValue metadata = 2; optional bool forceUpdate = 3 [default = false]; } @@ -274,7 +272,6 @@ message UpdateContainerResponseProto { } message DeleteContainerRequestProto { - required int64 containerID = 1; optional bool forceDelete = 2 [default = false]; } @@ -282,7 +279,6 @@ message DeleteContainerResponseProto { } message ListContainerRequestProto { - required int64 startContainerID = 1; optional uint32 count = 2; // Max Results to return } @@ -291,7 +287,6 @@ message ListContainerResponseProto { } message CloseContainerRequestProto { - required int64 containerID = 1; } message CloseContainerResponseProto { @@ -341,7 +336,6 @@ message DeleteKeyResponseProto { } message ListKeyRequestProto { - required int64 containerID = 1; optional int64 startLocalID = 2; required uint32 count = 3; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java index 6d11abb29d..3d418e5ec4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java @@ -98,13 +98,16 @@ public ContainerCommandResponseProto dispatch( long startTime = System.nanoTime(); ContainerProtos.Type cmdType = msg.getCmdType(); try { - long containerID = getContainerID(msg); + long containerID = msg.getContainerID(); metrics.incContainerOpsMetrics(cmdType); if (cmdType != ContainerProtos.Type.CreateContainer) { container = getContainer(containerID); containerType = getContainerType(container); } else { + if (!msg.hasCreateContainer()) { + return ContainerUtils.malformedRequest(msg); + } containerType = msg.getCreateContainer().getContainerType(); } } catch (StorageContainerException ex) { @@ -143,52 +146,6 @@ public void setScmId(String scmId) { } } - private long getContainerID(ContainerCommandRequestProto request) - throws StorageContainerException { - ContainerProtos.Type cmdType = request.getCmdType(); - - switch(cmdType) { - case CreateContainer: - return request.getCreateContainer().getContainerID(); - case ReadContainer: - return request.getReadContainer().getContainerID(); - case UpdateContainer: - return request.getUpdateContainer().getContainerID(); - case DeleteContainer: - return request.getDeleteContainer().getContainerID(); - case ListContainer: - return request.getListContainer().getStartContainerID(); - case CloseContainer: - return request.getCloseContainer().getContainerID(); - case PutKey: - return request.getPutKey().getKeyData().getBlockID().getContainerID(); - case GetKey: - return request.getGetKey().getBlockID().getContainerID(); - case DeleteKey: - return request.getDeleteKey().getBlockID().getContainerID(); - case ListKey: - return request.getListKey().getContainerID(); - case ReadChunk: - return request.getReadChunk().getBlockID().getContainerID(); - case DeleteChunk: - return request.getDeleteChunk().getBlockID().getContainerID(); - case WriteChunk: - return request.getWriteChunk().getBlockID().getContainerID(); - case ListChunk: - return request.getListChunk().getBlockID().getContainerID(); - case PutSmallFile: - return request.getPutSmallFile().getKey().getKeyData().getBlockID() - .getContainerID(); - case GetSmallFile: - return request.getGetSmallFile().getKey().getBlockID().getContainerID(); - case GetCommittedBlockLength: - return request.getGetCommittedBlockLength().getBlockID().getContainerID(); - } - - throw new StorageContainerException( - ContainerProtos.Result.UNSUPPORTED_REQUEST); - } - @VisibleForTesting public Container getContainer(long containerID) throws StorageContainerException { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java index f58cbae343..a3bddfc22c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java @@ -76,14 +76,12 @@ public void handle(SCMCommand command, OzoneContainer container, HddsProtos.ReplicationType replicationType = closeContainerProto.getReplicationType(); - ContainerProtos.CloseContainerRequestProto.Builder closeRequest = - ContainerProtos.CloseContainerRequestProto.newBuilder(); - closeRequest.setContainerID(containerID); - ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.CloseContainer); - request.setCloseContainer(closeRequest); + request.setContainerID(containerID); + request.setCloseContainer( + ContainerProtos.CloseContainerRequestProto.getDefaultInstance()); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid( context.getParent().getDatanodeDetails().getUuidString()); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index fc7635ee95..ac7aa5761c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -207,8 +207,7 @@ private CompletableFuture handleWriteChunk( private CompletableFuture handleCreateContainer( ContainerCommandRequestProto requestProto) { - long containerID = - requestProto.getCreateContainer().getContainerID(); + long containerID = requestProto.getContainerID(); createContainerFutureMap. computeIfAbsent(containerID, k -> new CompletableFuture<>()); return CompletableFuture.completedFuture(() -> ByteString.EMPTY); @@ -264,8 +263,7 @@ public CompletableFuture applyTransaction(TransactionContext trx) { } else { Message message = runCommand(requestProto); if (cmdType == ContainerProtos.Type.CreateContainer) { - long containerID = - requestProto.getCreateContainer().getContainerID(); + long containerID = requestProto.getContainerID(); createContainerFutureMap.remove(containerID).complete(message); } return CompletableFuture.completedFuture(message); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java index 34035c8088..0705cf480a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java @@ -177,7 +177,6 @@ public ContainerProtos.ContainerData getProtoBufMessage() { ContainerProtos.ContainerData.Builder builder = ContainerProtos .ContainerData.newBuilder(); builder.setContainerID(this.getContainerID()); - builder.setDbPath(this.getDbFile().getPath()); builder.setContainerPath(this.getMetadataPath()); builder.setState(this.getState()); @@ -196,10 +195,6 @@ public ContainerProtos.ContainerData getProtoBufMessage() { builder.setContainerType(ContainerProtos.ContainerType.KeyValueContainer); } - if(this.getContainerDBType() != null) { - builder.setContainerDBType(containerDBType); - } - return builder.build(); } @@ -239,10 +234,6 @@ public static KeyValueContainerData getFromProtoBuf( data.setBytesUsed(protoData.getBytesUsed()); } - if(protoData.hasContainerDBType()) { - data.setContainerDBType(protoData.getContainerDBType()); - } - return data; } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 0b26a14a9c..a4e124b705 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -231,13 +231,7 @@ ContainerCommandResponseProto handleCreateContainer( // container would be created here. Preconditions.checkArgument(kvContainer == null); - CreateContainerRequestProto createContainerReq = - request.getCreateContainer(); - long containerID = createContainerReq.getContainerID(); - if (createContainerReq.hasContainerType()) { - Preconditions.checkArgument(createContainerReq.getContainerType() - .equals(ContainerType.KeyValueContainer)); - } + long containerID = request.getContainerID(); KeyValueContainerData newContainerData = new KeyValueContainerData( containerID, maxContainerSizeGB); @@ -381,15 +375,15 @@ ContainerCommandResponseProto handleCloseContainer( try { checkContainerOpen(kvContainer); + KeyValueContainerData kvData = kvContainer.getContainerData(); + // remove the container from open block map once, all the blocks // have been committed and the container is closed - kvContainer.getContainerData() - .setState(ContainerProtos.ContainerLifeCycleState.CLOSING); + kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSING); commitPendingKeys(kvContainer); kvContainer.close(); // make sure the the container open keys from BlockMap gets removed - openContainerBlockMap.removeContainer( - request.getCloseContainer().getContainerID()); + openContainerBlockMap.removeContainer(kvData.getContainerID()); } catch (StorageContainerException ex) { return ContainerUtils.logAndReturnError(LOG, ex, request); } catch (IOException ex) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java index 8f067d9f5c..30fe1134ad 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java @@ -196,7 +196,7 @@ private long getContainerIdForCmd( ContainerProtos.Type type = request.getCmdType(); switch (type) { case CloseContainer: - return request.getCloseContainer().getContainerID(); + return request.getContainerID(); // Right now, we handle only closeContainer via queuing it over the // over the XceiVerServer. For all other commands we throw Illegal // argument exception here. Will need to extend the switch cases diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index 42ec54f4ec..8e8a1befe4 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; +import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; @@ -55,6 +56,7 @@ public class TestKeyValueHandler { private final String baseDir = MiniDFSCluster.getBaseDirectory(); private final String volume = baseDir + "disk1"; + private static final long DUMMY_CONTAINER_ID = 9999; @Test /** @@ -74,8 +76,13 @@ public void testHandlerCommandHandling() throws Exception{ // Test Create Container Request handling ContainerCommandRequestProto createContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.CreateContainer); - + ContainerProtos.ContainerCommandRequestProto.newBuilder() + .setCmdType(ContainerProtos.Type.CreateContainer) + .setContainerID(DUMMY_CONTAINER_ID) + .setDatanodeUuid(DATANODE_UUID) + .setCreateContainer(ContainerProtos.CreateContainerRequestProto + .getDefaultInstance()) + .build(); dispatcher.dispatch(createContainerRequest); Mockito.verify(handler, times(1)).handleCreateContainer( any(ContainerCommandRequestProto.class), any()); @@ -191,6 +198,7 @@ private ContainerCommandRequestProto getDummyCommandRequestProto( ContainerCommandRequestProto request = ContainerProtos.ContainerCommandRequestProto.newBuilder() .setCmdType(cmdType) + .setContainerID(DUMMY_CONTAINER_ID) .setDatanodeUuid(DATANODE_UUID) .build(); diff --git a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java index 3716aceaa4..89215fa527 100644 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoContainerHandler.java @@ -81,7 +81,6 @@ public void execute(CommandLine cmd) throws IOException { containerData.getState() == ContainerLifeCycleState.OPEN ? "OPEN" : "CLOSED"; logOut("Container State: %s", openStatus); - logOut("Container DB Path: %s", containerData.getDbPath()); logOut("Container Path: %s", containerData.getContainerPath()); // Output meta data. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index d25b73ee21..f3980a50ae 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -220,6 +220,7 @@ public static ContainerCommandRequestProto getWriteChunkRequest( ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.WriteChunk); + request.setContainerID(blockID.getContainerID()); request.setWriteChunk(writeRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -261,6 +262,7 @@ public static ContainerCommandRequestProto getWriteSmallFileRequest( ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.PutSmallFile); + request.setContainerID(blockID.getContainerID()); request.setPutSmallFile(smallFileRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -279,6 +281,7 @@ public static ContainerCommandRequestProto getReadSmallFileRequest( ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.GetSmallFile); + request.setContainerID(getKey.getGetKey().getBlockID().getContainerID()); request.setGetSmallFile(smallFileRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -308,6 +311,7 @@ public static ContainerCommandRequestProto getReadChunkRequest( ContainerCommandRequestProto.Builder newRequest = ContainerCommandRequestProto.newBuilder(); newRequest.setCmdType(ContainerProtos.Type.ReadChunk); + newRequest.setContainerID(readRequest.getBlockID().getContainerID()); newRequest.setReadChunk(readRequest); newRequest.setTraceID(UUID.randomUUID().toString()); newRequest.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -340,6 +344,7 @@ public static ContainerCommandRequestProto getDeleteChunkRequest( ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.DeleteChunk); + request.setContainerID(writeRequest.getBlockID().getContainerID()); request.setDeleteChunk(deleteRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -356,15 +361,12 @@ public static ContainerCommandRequestProto getCreateContainerRequest( long containerID, Pipeline pipeline) throws IOException { LOG.trace("addContainer: {}", containerID); - ContainerProtos.CreateContainerRequestProto.Builder createRequest = - ContainerProtos.CreateContainerRequestProto - .newBuilder(); - createRequest.setContainerID(containerID); - ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.CreateContainer); - request.setCreateContainer(createRequest); + request.setContainerID(containerID); + request.setCreateContainer( + ContainerProtos.CreateContainerRequestProto.getDefaultInstance()); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -385,7 +387,6 @@ public static ContainerCommandRequestProto getUpdateContainerRequest( long containerID, Map metaData) throws IOException { ContainerProtos.UpdateContainerRequestProto.Builder updateRequestBuilder = ContainerProtos.UpdateContainerRequestProto.newBuilder(); - updateRequestBuilder.setContainerID(containerID); String[] keys = metaData.keySet().toArray(new String[]{}); for(int i=0; i writeChunkBuilder(BlockID blockID, Pipeline pipeline, ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.WriteChunk); + request.setContainerID(blockID.getContainerID()); request.setWriteChunk(writeRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -177,6 +176,7 @@ public void testPutKeyWithMultipleChunks() ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.PutKey); + request.setContainerID(blockID.getContainerID()); request.setPutKey(putKeyRequestProto); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -213,6 +213,7 @@ public void testDeleteChunk() throws Exception { ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.DeleteChunk); + request.setContainerID(blockID.getContainerID()); request.setDeleteChunk(deleteChunkProto); request.setWriteChunk(writeRequest); request.setTraceID(UUID.randomUUID().toString()); @@ -242,13 +243,12 @@ public void testCloseContainer() throws Exception { .get(blockID.getLocalID())); Assert.assertTrue( keyData.getChunks().size() == chunkList.size()); - ContainerProtos.CloseContainerRequestProto.Builder closeContainerProto = - ContainerProtos.CloseContainerRequestProto.newBuilder(); - closeContainerProto.setContainerID(blockID.getContainerID()); ContainerProtos.ContainerCommandRequestProto.Builder request = ContainerProtos.ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.CloseContainer); - request.setCloseContainer(closeContainerProto); + request.setContainerID(blockID.getContainerID()); + request.setCloseContainer( + ContainerProtos.CloseContainerRequestProto.getDefaultInstance()); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); dispatcher.dispatch(request.build()); diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java index 93e7ef1168..e757a7f22b 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java @@ -50,8 +50,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .CreateContainerRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ReadChunkRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos @@ -156,15 +154,14 @@ public void cleanup() throws IOException { FileUtils.deleteDirectory(new File(baseDir)); } - private ContainerCommandRequestProto getCreateContainerCommand(long containerID) { - CreateContainerRequestProto.Builder createRequest = - CreateContainerRequestProto.newBuilder(); - createRequest.setContainerID(containerID).build(); - + private ContainerCommandRequestProto getCreateContainerCommand( + long containerID) { ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.CreateContainer); - request.setCreateContainer(createRequest); + request.setContainerID(containerID); + request.setCreateContainer( + ContainerProtos.CreateContainerRequestProto.getDefaultInstance()); request.setDatanodeUuid(datanodeUuid); request.setTraceID(containerID + "-trace"); return request.build(); @@ -181,6 +178,7 @@ private ContainerCommandRequestProto getWriteChunkCommand( ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto .newBuilder(); request.setCmdType(ContainerProtos.Type.WriteChunk) + .setContainerID(blockID.getContainerID()) .setTraceID(getBlockTraceID(blockID)) .setDatanodeUuid(datanodeUuid) .setWriteChunk(writeChunkRequest); @@ -193,9 +191,11 @@ private ContainerCommandRequestProto getReadChunkCommand( .newBuilder() .setBlockID(blockID.getDatanodeBlockIDProtobuf()) .setChunkData(getChunkInfo(blockID, chunkName)); + ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto .newBuilder(); request.setCmdType(ContainerProtos.Type.ReadChunk) + .setContainerID(blockID.getContainerID()) .setTraceID(getBlockTraceID(blockID)) .setDatanodeUuid(datanodeUuid) .setReadChunk(readChunkRequest); @@ -219,9 +219,11 @@ private ContainerCommandRequestProto getPutKeyCommand( PutKeyRequestProto.Builder putKeyRequest = PutKeyRequestProto .newBuilder() .setKeyData(getKeyData(blockID, chunkKey)); + ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto .newBuilder(); request.setCmdType(ContainerProtos.Type.PutKey) + .setContainerID(blockID.getContainerID()) .setTraceID(getBlockTraceID(blockID)) .setDatanodeUuid(datanodeUuid) .setPutKey(putKeyRequest); @@ -234,6 +236,7 @@ private ContainerCommandRequestProto getGetKeyCommand(BlockID blockID) { ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto .newBuilder() .setCmdType(ContainerProtos.Type.GetKey) + .setContainerID(blockID.getContainerID()) .setTraceID(getBlockTraceID(blockID)) .setDatanodeUuid(datanodeUuid) .setGetKey(readKeyRequest);