HADOOP-13254. Create framework for configurable disk checkers (yufeigu via rkanter)

This commit is contained in:
Robert Kanter 2016-07-15 13:41:26 -07:00
parent 0881ed3fc3
commit 442162048a
6 changed files with 285 additions and 6 deletions

View File

@ -0,0 +1,34 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import java.io.File;
/**
* BasicDiskValidator is a wrapper around {@link DiskChecker}.
*/
public class BasicDiskValidator implements DiskValidator {
public static final String NAME = "basic";
@Override
public void checkStatus(File dir) throws DiskErrorException {
DiskChecker.checkDir(dir);
}
}

View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import java.io.File;
/**
* A interface for disk validators.
*
* The {@link #checkStatus(File)} operation checks status of a file/dir.
*
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public interface DiskValidator {
/**
* Check the status of a file/dir.
* @param dir a file/dir
* @throws DiskErrorException if any disk error
*/
void checkStatus(File dir) throws DiskErrorException;
}

View File

@ -0,0 +1,88 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import java.util.concurrent.ConcurrentHashMap;
/**
* The factory class to create instance of {@link DiskValidator}.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public final class DiskValidatorFactory {
@VisibleForTesting
static final ConcurrentHashMap<Class<? extends DiskValidator>, DiskValidator>
INSTANCES = new ConcurrentHashMap<>();
private DiskValidatorFactory() {
}
/**
* Returns a {@link DiskValidator} instance corresponding to the passed clazz.
* @param clazz a class extends {@link DiskValidator}
*/
public static DiskValidator
getInstance(Class<? extends DiskValidator> clazz) {
DiskValidator diskValidator;
if (INSTANCES.containsKey(clazz)) {
diskValidator = INSTANCES.get(clazz);
} else {
diskValidator = ReflectionUtils.newInstance(clazz, null);
// check the return of putIfAbsent() to see if any other thread have put
// the instance with the same key into INSTANCES
DiskValidator diskValidatorRet =
INSTANCES.putIfAbsent(clazz, diskValidator);
if (diskValidatorRet != null) {
diskValidator = diskValidatorRet;
}
}
return diskValidator;
}
/**
* Returns {@link DiskValidator} instance corresponding to its name.
* The diskValidator parameter can be "basic" for {@link BasicDiskValidator}.
* @param diskValidator canonical class name, for example, "basic"
* @throws DiskErrorException if the class cannot be located
*/
@SuppressWarnings("unchecked")
public static DiskValidator getInstance(String diskValidator)
throws DiskErrorException {
@SuppressWarnings("rawtypes")
Class clazz;
if (diskValidator.equalsIgnoreCase(BasicDiskValidator.NAME)) {
clazz = BasicDiskValidator.class;
} else {
try {
clazz = Class.forName(diskValidator);
} catch (ClassNotFoundException cnfe) {
throw new DiskErrorException(diskValidator
+ " DiskValidator class not found.", cnfe);
}
}
return getInstance(clazz);
}
}

View File

@ -0,0 +1,57 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import java.io.File;
/**
* The class to test BasicDiskValidator.
*/
public class TestBasicDiskValidator extends TestDiskChecker {
@Override
protected void checkDirs(boolean isDir, String perm, boolean success)
throws Throwable {
File localDir = File.createTempFile("test", "tmp");
try {
if (isDir) {
// reuse the file path generated by File#createTempFile to create a dir
localDir.delete();
localDir.mkdir();
}
Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
localDir.getAbsolutePath()));
DiskValidatorFactory.getInstance(BasicDiskValidator.NAME).
checkStatus(localDir);
assertTrue("call to checkDir() succeeded.", success);
} catch (DiskErrorException e) {
// call to checkDir() succeeded even though it was expected to fail
// if success is false, otherwise throw the exception
if (success) {
throw e;
}
} finally {
localDir.delete();
}
}
}

View File

@ -138,30 +138,30 @@ private void _checkDirs(boolean isDir, FsPermission perm, boolean success)
@Test (timeout = 30000)
public void testCheckDir_normal_local() throws Throwable {
_checkDirs(true, "755", true);
checkDirs(true, "755", true);
}
@Test (timeout = 30000)
public void testCheckDir_notDir_local() throws Throwable {
_checkDirs(false, "000", false);
checkDirs(false, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notReadable_local() throws Throwable {
_checkDirs(true, "000", false);
checkDirs(true, "000", false);
}
@Test (timeout = 30000)
public void testCheckDir_notWritable_local() throws Throwable {
_checkDirs(true, "444", false);
checkDirs(true, "444", false);
}
@Test (timeout = 30000)
public void testCheckDir_notListable_local() throws Throwable {
_checkDirs(true, "666", false);
checkDirs(true, "666", false);
}
private void _checkDirs(boolean isDir, String perm, boolean success)
protected void checkDirs(boolean isDir, String perm, boolean success)
throws Throwable {
File localDir = File.createTempFile("test", "tmp");
if (isDir) {

View File

@ -0,0 +1,59 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.util;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
/**
* The class to test DiskValidatorFactory.
*/
public class TestDiskValidatorFactory {
/**
* Trivial tests that make sure
* {@link DiskValidatorFactory#getInstance(String)} works as expected.
*
* @throws DiskErrorException if fail to get the instance.
*/
@Test
public void testGetInstance() throws DiskErrorException {
DiskValidator diskValidator = DiskValidatorFactory.getInstance("basic");
assertNotNull("Fail to get the instance.", diskValidator);
assertEquals("Fail to create the correct instance.",
diskValidator.getClass(), BasicDiskValidator.class);
assertNotNull("Fail to cache the object", DiskValidatorFactory.INSTANCES.
get(BasicDiskValidator.class));
}
/**
* To test whether an exception is threw out as expected if trying to create
* a non-exist class.
* @throws DiskErrorException if fail to get the instance.
*/
@Test(expected = DiskErrorException.class)
public void testGetInstanceOfNonExistClass() throws DiskErrorException {
DiskValidatorFactory.getInstance("non-exist");
}
}