diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index 02572a8fa2..e81f1c6c72 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -1111,5 +1111,10 @@ public long getNumKeys(long containerId) { return cData.getKeyCount(); } - + @Override + public void updateDeleteTransactionId(long containerId, + long deleteTransactionId) { + containerMap.get(containerId) + .updateDeleteTransactionId(deleteTransactionId); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java index 49b68dc2a0..cf68b08a23 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java @@ -264,4 +264,6 @@ void closeContainer(long containerID) */ long getNumKeys(long containerId); + void updateDeleteTransactionId(long containerId, long deleteTransactionId); + } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java index f954d98f93..d215da9865 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java @@ -158,6 +158,13 @@ private void deleteContainerBlocks(DeletedBlocksTransaction delTX, containerInfo.getDBPath()); } + if (delTX.getTxID() < containerInfo.getDeleteTransactionId()) { + LOG.debug(String.format("Ignoring delete blocks for containerId: %d." + + " Outdated delete transactionId %d < %d", containerId, + delTX.getTxID(), containerInfo.getDeleteTransactionId())); + return; + } + int newDeletionBlocks = 0; MetadataStore containerDB = KeyUtils.getDB(containerInfo, config); for (Long blk : delTX.getLocalIDList()) { @@ -165,10 +172,20 @@ private void deleteContainerBlocks(DeletedBlocksTransaction delTX, byte[] blkBytes = Longs.toByteArray(blk); byte[] blkInfo = containerDB.get(blkBytes); if (blkInfo != null) { + byte[] deletingKeyBytes = + DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk); + byte[] deletedKeyBytes = + DFSUtil.string2Bytes(OzoneConsts.DELETED_KEY_PREFIX + blk); + if (containerDB.get(deletingKeyBytes) != null + || containerDB.get(deletedKeyBytes) != null) { + LOG.debug(String.format( + "Ignoring delete for block %d in container %d." + + " Entry already added.", blk, containerId)); + continue; + } // Found the block in container db, // use an atomic update to change its state to deleting. - batch.put(DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk), - blkInfo); + batch.put(deletingKeyBytes, blkInfo); batch.delete(blkBytes); try { containerDB.writeBatch(batch); @@ -186,11 +203,13 @@ private void deleteContainerBlocks(DeletedBlocksTransaction delTX, LOG.debug("Block {} not found or already under deletion in" + " container {}, skip deleting it.", blk, containerId); } - containerDB.put(DFSUtil.string2Bytes( - OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()), - Longs.toByteArray(delTX.getTxID())); } + containerDB.put(DFSUtil.string2Bytes( + OzoneConsts.DELETE_TRANSACTION_KEY_PREFIX + delTX.getContainerID()), + Longs.toByteArray(delTX.getTxID())); + containerManager + .updateDeleteTransactionId(delTX.getContainerID(), delTX.getTxID()); // update pending deletion blocks count in in-memory container status containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java new file mode 100644 index 0000000000..43e3f5095f --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.common.statemachine.commandhandler; + +import com.google.common.primitives.Longs; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.server.StorageContainerManager; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.container.common.helpers.ContainerData; +import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl; +import org.apache.hadoop.ozone.ksm.KeySpaceManager; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.ozShell.TestOzoneShell; +import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.utils.MetadataStore; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; + +public class TestBlockDeletion { + private static OzoneConfiguration conf = null; + private static ObjectStore store; + private static ContainerManagerImpl dnContainerManager = null; + private static StorageContainerManager scm = null; + private static KeySpaceManager ksm = null; + private static Set containerIdsWithDeletedBlocks; + + @BeforeClass + public static void init() throws Exception { + conf = new OzoneConfiguration(); + + String path = + GenericTestUtils.getTempPath(TestOzoneShell.class.getSimpleName()); + File baseDir = new File(path); + baseDir.mkdirs(); + + path += conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, + OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT); + + conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path); + conf.setQuietMode(false); + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, + TimeUnit.MILLISECONDS); + + MiniOzoneCluster cluster = + MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build(); + cluster.waitForClusterToBeReady(); + store = OzoneClientFactory.getRpcClient(conf).getObjectStore(); + dnContainerManager = + (ContainerManagerImpl) cluster.getHddsDatanodes().get(0) + .getDatanodeStateMachine().getContainer().getContainerManager(); + ksm = cluster.getKeySpaceManager(); + scm = cluster.getStorageContainerManager(); + containerIdsWithDeletedBlocks = new HashSet<>(); + } + + @Test(timeout = 60000) + public void testBlockDeletion() + throws IOException, InterruptedException { + String volumeName = UUID.randomUUID().toString(); + String bucketName = UUID.randomUUID().toString(); + + String value = RandomStringUtils.random(1000000); + store.createVolume(volumeName); + OzoneVolume volume = store.getVolume(volumeName); + volume.createBucket(bucketName); + OzoneBucket bucket = volume.getBucket(bucketName); + + String keyName = UUID.randomUUID().toString(); + + OzoneOutputStream out = bucket.createKey(keyName, value.getBytes().length, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE); + out.write(value.getBytes()); + out.close(); + + KsmKeyArgs keyArgs = new KsmKeyArgs.Builder().setVolumeName(volumeName) + .setBucketName(bucketName).setKeyName(keyName).setDataSize(0) + .setType(HddsProtos.ReplicationType.STAND_ALONE) + .setFactor(HddsProtos.ReplicationFactor.ONE).build(); + List ksmKeyLocationInfoGroupList = + ksm.lookupKey(keyArgs).getKeyLocationVersions(); + + // verify key blocks were created in DN. + Assert.assertTrue(verifyBlocksCreated(ksmKeyLocationInfoGroupList)); + // No containers with deleted blocks + Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty()); + // Delete transactionIds for the containers should be 0 + matchContainerTransactionIds(); + ksm.deleteKey(keyArgs); + Thread.sleep(5000); + // The blocks should be deleted in the DN. + Assert.assertTrue(verifyBlocksDeleted(ksmKeyLocationInfoGroupList)); + + // Few containers with deleted blocks + Assert.assertTrue(!containerIdsWithDeletedBlocks.isEmpty()); + // Containers in the DN and SCM should have same delete transactionIds + matchContainerTransactionIds(); + } + + private void matchContainerTransactionIds() throws IOException { + List containerDataList = new ArrayList<>(); + dnContainerManager.listContainer(0, 10000, containerDataList); + for (ContainerData containerData : containerDataList) { + long containerId = containerData.getContainerID(); + if (containerIdsWithDeletedBlocks.contains(containerId)) { + Assert.assertTrue( + scm.getContainerInfo(containerId).getDeleteTransactionId() > 0); + } else { + Assert.assertEquals( + scm.getContainerInfo(containerId).getDeleteTransactionId(), 0); + } + Assert.assertEquals(dnContainerManager.readContainer(containerId) + .getDeleteTransactionId(), + scm.getContainerInfo(containerId).getDeleteTransactionId()); + } + } + + private boolean verifyBlocksCreated( + List ksmKeyLocationInfoGroups) + throws IOException { + return performOperationOnKeyContainers((blockID) -> { + try { + MetadataStore db = KeyUtils.getDB( + dnContainerManager.getContainerMap().get(blockID.getContainerID()), + conf); + Assert.assertNotNull(db.get(Longs.toByteArray(blockID.getLocalID()))); + } catch (IOException e) { + e.printStackTrace(); + } + }, ksmKeyLocationInfoGroups); + } + + private boolean verifyBlocksDeleted( + List ksmKeyLocationInfoGroups) + throws IOException { + return performOperationOnKeyContainers((blockID) -> { + try { + MetadataStore db = KeyUtils.getDB( + dnContainerManager.getContainerMap().get(blockID.getContainerID()), + conf); + Assert.assertNull(db.get(Longs.toByteArray(blockID.getLocalID()))); + Assert.assertNull(db.get(DFSUtil.string2Bytes( + OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID()))); + Assert.assertNotNull(DFSUtil.string2Bytes( + OzoneConsts.DELETED_KEY_PREFIX + blockID.getLocalID())); + containerIdsWithDeletedBlocks.add(blockID.getContainerID()); + } catch (IOException e) { + e.printStackTrace(); + } + }, ksmKeyLocationInfoGroups); + } + + private boolean performOperationOnKeyContainers(Consumer consumer, + List ksmKeyLocationInfoGroups) + throws IOException { + + try { + for (KsmKeyLocationInfoGroup ksmKeyLocationInfoGroup : ksmKeyLocationInfoGroups) { + List ksmKeyLocationInfos = + ksmKeyLocationInfoGroup.getLocationList(); + for (KsmKeyLocationInfo ksmKeyLocationInfo : ksmKeyLocationInfos) { + BlockID blockID = ksmKeyLocationInfo.getBlockID(); + consumer.accept(blockID); + } + } + } catch (Error e) { + e.printStackTrace(); + return false; + } + return true; + } +} \ No newline at end of file