big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-355. Disable OpenKeyDeleteService and DeleteKeysService.

Browse Source 
Contributed by Anu Engineer.
This commit is contained in:
Anu Engineer 2018-08-17 11:50:46 -07:00
parent 60ffec9f79
commit ab37423ad8
4 changed files with 36 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
View File

@ -52,6 +52,7 @@
import org.apache.hadoop.utils.MetadataStore;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
@ -102,6 +103,7 @@ public static void init() throws Exception {
}
@Test(timeout = 60000)
@Ignore("Until delete background service is fixed.")
public void testBlockDeletion()
throws IOException, InterruptedException {
String volumeName = UUID.randomUUID().toString();

11
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
View File

@ -56,12 +56,12 @@
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BackgroundService;
import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataStore;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
@ -1188,10 +1188,11 @@ public void testGetScmInfo() throws IOException {
}
@Test
//Disabling this test
@Ignore("Disabling this test until Open Key is fixed.")
public void testExpiredOpenKey() throws Exception {
BackgroundService openKeyCleanUpService = ((KeyManagerImpl)cluster
.getOzoneManager().getKeyManager()).getOpenKeyCleanupService();
// BackgroundService openKeyCleanUpService = ((KeyManagerImpl)cluster
// .getOzoneManager().getKeyManager()).getOpenKeyCleanupService();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
@ -1252,7 +1253,7 @@ public void testExpiredOpenKey() throws Exception {
KeyArgs keyArgs5 = new KeyArgs("testKey5", bucketArgs);
storageHandler.newKeyWriter(keyArgs5);
openKeyCleanUpService.triggerBackgroundTaskForTesting();
//openKeyCleanUpService.triggerBackgroundTaskForTesting();
Thread.sleep(2000);
// now all k1-k4 should have been removed by the clean-up task, only k5
// should be present in ExpiredOpenKeys.

22
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/package-info.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/**
* Ozone Manager Tests.
*/
package org.apache.hadoop.ozone.om;

37
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View File

@ -16,7 +16,6 @@
*/
package org.apache.hadoop.ozone.om;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdfs.DFSUtil;
@ -34,7 +33,6 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BackgroundService;
import org.apache.hadoop.utils.BatchOperation;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
@ -91,15 +89,14 @@ public class KeyManagerImpl implements KeyManager {
private final OMMetadataManager metadataManager;
private final long scmBlockSize;
private final boolean useRatis;
private final BackgroundService keyDeletingService;
private final BackgroundService openKeyCleanupService;
private final long preallocateMax;
private final Random random;
private final String omId;
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
OMMetadataManager metadataManager, OzoneConfiguration conf,
OMMetadataManager metadataManager,
OzoneConfiguration conf,
String omId) {
this.scmBlockClient = scmBlockClient;
this.metadataManager = metadataManager;
@ -107,43 +104,20 @@ public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
long blockDeleteInterval = conf.getTimeDuration(
OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
TimeUnit.MILLISECONDS);
long serviceTimeout = conf.getTimeDuration(
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
this.preallocateMax = conf.getLong(
OZONE_KEY_PREALLOCATION_MAXSIZE,
OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT);
keyDeletingService = new KeyDeletingService(
scmBlockClient, this, blockDeleteInterval, serviceTimeout, conf);
int openkeyCheckInterval = conf.getInt(
OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS,
OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT);
openKeyCleanupService = new OpenKeyCleanupService(
scmBlockClient, this, openkeyCheckInterval, serviceTimeout);
random = new Random();
this.omId = omId;
}
@VisibleForTesting
public BackgroundService getOpenKeyCleanupService() {
return openKeyCleanupService;
}
@Override
public void start() {
keyDeletingService.start();
openKeyCleanupService.start();
}
@Override
public void stop() throws IOException {
keyDeletingService.shutdown();
openKeyCleanupService.shutdown();
}
private void validateBucket(String volumeName, String bucketName)
@ -398,7 +372,7 @@ public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
String bucketName = args.getBucketName();
String fromKeyName = args.getKeyName();
if (toKeyName.length() == 0 || fromKeyName.length() == 0) {
LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}",
volumeName, bucketName, fromKeyName, toKeyName);
throw new OMException("Key name is empty",
ResultCodes.FAILED_INVALID_KEY_NAME);
@ -446,7 +420,7 @@ public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
batch.put(toKey, newKeyInfo.getProtobuf().toByteArray());
metadataManager.writeBatch(batch);
} catch (DBException ex) {
LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}.",
LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} toKey:{}",
volumeName, bucketName, fromKeyName, toKeyName, ex);
throw new OMException(ex.getMessage(),
ResultCodes.FAILED_KEY_RENAME);
@ -487,7 +461,8 @@ public void deleteKey(OmKeyArgs args) throws IOException {
@Override
public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix, int maxKeys) throws IOException {
String startKey, String keyPrefix,
int maxKeys) throws IOException {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);