big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-270. Move generic container util functions to ContianerUtils.

Browse Source 
Contributed by Hanisha Koneru.
This commit is contained in:
Anu Engineer 2018-07-27 07:12:21 -07:00
parent 64e739e344
commit 3cc7ce816e
7 changed files with 62 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
View File

@ -198,20 +198,6 @@ public static String getContainerDbFileName(String containerName) {
return containerName + OzoneConsts.DN_CONTAINER_DB;
}
/**
* Returns container file location.
*
* @param containerData - Data
* @param location - Root path
* @return Path
*/
public static File getContainerFile(ContainerData containerData,
Path location) {
return location.resolve(Long.toString(containerData
.getContainerID()).concat(CONTAINER_EXTENSION))
.toFile();
}
/**
* Persistent a {@link DatanodeDetails} to a local file.
*
@ -300,4 +286,24 @@ public static String getChecksum(String containerDataYamlStr)
}
}
/**
* Get the .container file from the containerBaseDir
* @param containerBaseDir container base directory. The name of this
* directory is same as the containerID
* @return the .container file
*/
public static File getContainerFile(File containerBaseDir) {
// Container file layout is
// .../<<containerID>>/metadata/<<containerID>>.container
String containerFilePath = OzoneConsts.CONTAINER_META_PATH + File.separator
+ getContainerID(containerBaseDir) + OzoneConsts.CONTAINER_EXTENSION;
return new File(containerBaseDir, containerFilePath);
}
/**
* ContainerID can be decoded from the container base directory name
*/
public static long getContainerID(File containerBaseDir) {
return Long.parseLong(containerBaseDir.getName());
}
}

6
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java
View File

@ -28,6 +28,7 @@
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import java.io.File;
import java.util.Map;
@ -91,6 +92,11 @@ void update(Map<String, String> metaData, boolean forceUpdate)
*/
ContainerProtos.ContainerType getContainerType();
/**
* Returns containerFile.
*/
File getContainerFile();
/**
* updates the DeleteTransactionId.
* @param deleteTransactionId

22
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java
View File

@ -114,18 +114,16 @@ public void create(VolumeSet volumeSet, VolumeChoosingPolicy
containerMetaDataPath = KeyValueContainerLocationUtil
.getContainerMetaDataPath(hddsVolumeDir, scmId, containerID);
containerData.setMetadataPath(containerMetaDataPath.getPath());
File chunksPath = KeyValueContainerLocationUtil.getChunksLocationPath(
hddsVolumeDir, scmId, containerID);
File containerFile = KeyValueContainerLocationUtil.getContainerFile(
containerMetaDataPath, containerID);
File dbFile = KeyValueContainerLocationUtil.getContainerDBFile(
containerMetaDataPath, containerID);
// Check if it is new Container.
ContainerUtils.verifyIsNewContainer(containerMetaDataPath);
//Create Metadata path chunks path and metadata db
File dbFile = getContainerDBFile();
KeyValueContainerUtil.createContainerMetaData(containerMetaDataPath,
chunksPath, dbFile, config);
@ -133,13 +131,13 @@ public void create(VolumeSet volumeSet, VolumeChoosingPolicy
OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
//Set containerData for the KeyValueContainer.
containerData.setMetadataPath(containerMetaDataPath.getPath());
containerData.setChunksPath(chunksPath.getPath());
containerData.setContainerDBType(impl);
containerData.setDbFile(dbFile);
containerData.setVolume(containerVolume);
// Create .container file
File containerFile = getContainerFile();
writeToContainerFile(containerFile, true);
} catch (StorageContainerException ex) {
@ -415,11 +413,21 @@ public void writeLockInterruptibly() throws InterruptedException {
* Returns containerFile.
* @return .container File name
*/
private File getContainerFile() {
@Override
public File getContainerFile() {
return new File(containerData.getMetadataPath(), containerData
.getContainerID() + OzoneConsts.CONTAINER_EXTENSION);
}
/**
* Returns container DB file
* @return
*/
public File getContainerDBFile() {
return new File(containerData.getMetadataPath(), containerData
.getContainerID() + OzoneConsts.DN_CONTAINER_DB);
}
/**
* Creates a temporary file.
* @param file

17
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java
View File

@ -98,28 +98,11 @@ private static String getContainerSubDirectory(long containerId){
return Storage.CONTAINER_DIR + directory;
}
/**
* Returns containerFile.
* @param containerMetaDataPath
* @param containerID
* @return .container File name
*/
public static File getContainerFile(File containerMetaDataPath,
long containerID) {
Preconditions.checkNotNull(containerMetaDataPath);
return new File(containerMetaDataPath, containerID +
OzoneConsts.CONTAINER_EXTENSION);
}
/**
* Return containerDB File.
* @param containerMetaDataPath
* @param containerID
* @return containerDB File name
*/
public static File getContainerDBFile(File containerMetaDataPath,
long containerID) {
Preconditions.checkNotNull(containerMetaDataPath);
return new File(containerMetaDataPath, containerID + OzoneConsts
.DN_CONTAINER_DB);
}

27
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java
View File

@ -23,8 +23,8 @@
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.container.common.helpers
.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Storage;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
@ -32,7 +32,6 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -119,8 +118,7 @@ public boolean accept(File pathname) {
}
for (File scmLoc : scmDir) {
File currentDir = null;
currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT);
File currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT);
File[] containerTopDirs = currentDir.listFiles();
if (containerTopDirs != null) {
for (File containerTopDir : containerTopDirs) {
@ -128,21 +126,14 @@ public boolean accept(File pathname) {
File[] containerDirs = containerTopDir.listFiles();
if (containerDirs != null) {
for (File containerDir : containerDirs) {
File metadataPath = new File(containerDir + File.separator +
OzoneConsts.CONTAINER_META_PATH);
long containerID = Long.parseLong(containerDir.getName());
if (metadataPath.exists()) {
File containerFile = KeyValueContainerLocationUtil
.getContainerFile(metadataPath, containerID);
if (containerFile.exists()) {
verifyContainerFile(containerID, containerFile);
} else {
LOG.error("Missing .container file for ContainerID: {}",
containerID);
}
File containerFile = ContainerUtils.getContainerFile(
containerDir);
long containerID = ContainerUtils.getContainerID(containerDir);
if (containerFile.exists()) {
verifyContainerFile(containerID, containerFile);
} else {
LOG.error("Missing container metadata directory for " +
"ContainerID: {}", containerID);
LOG.error("Missing .container file for ContainerID: {}",
containerDir.getName());
}
}
}

24
hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
View File

@ -111,11 +111,9 @@ public void testCreateContainer() throws Exception {
File containerMetaDataLoc = new File(containerMetaDataPath);
//Check whether container file and container db file exists or not.
assertTrue(KeyValueContainerLocationUtil.getContainerFile(
containerMetaDataLoc, containerID).exists(), ".Container File does" +
" not exist");
assertTrue(KeyValueContainerLocationUtil.getContainerDBFile(
containerMetaDataLoc, containerID).exists(), "Container DB does " +
assertTrue(keyValueContainer.getContainerFile().exists(),
".Container File does not exist");
assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container DB does " +
"not exist");
}
@ -166,11 +164,9 @@ public void testDeleteContainer() throws Exception {
.getParentFile().exists());
assertFalse("Container File still exists",
KeyValueContainerLocationUtil.getContainerFile(containerMetaDataLoc,
containerID).exists());
keyValueContainer.getContainerFile().exists());
assertFalse("Container DB file still exists",
KeyValueContainerLocationUtil.getContainerDBFile(containerMetaDataLoc,
containerID).exists());
keyValueContainer.getContainerDBFile().exists());
}
@ -188,9 +184,7 @@ public void testCloseContainer() throws Exception {
//Check state in the .container file
String containerMetaDataPath = keyValueContainerData
.getMetadataPath();
File containerMetaDataLoc = new File(containerMetaDataPath);
File containerFile = KeyValueContainerLocationUtil.getContainerFile(
containerMetaDataLoc, containerID);
File containerFile = keyValueContainer.getContainerFile();
keyValueContainerData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);
@ -227,11 +221,7 @@ public void testUpdateContainer() throws IOException {
assertEquals(2, keyValueContainerData.getMetadata().size());
//Check metadata in the .container file
String containerMetaDataPath = keyValueContainerData
.getMetadataPath();
File containerMetaDataLoc = new File(containerMetaDataPath);
File containerFile = KeyValueContainerLocationUtil.getContainerFile(
containerMetaDataLoc, containerID);
File containerFile = keyValueContainer.getContainerFile();
keyValueContainerData = (KeyValueContainerData) ContainerDataYaml
.readContainerFile(containerFile);

13
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
import org.apache.hadoop.ozone.container.common.volume
@ -40,8 +41,6 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils;
import org.apache.hadoop.ozone.container.keyvalue.helpers
.KeyValueContainerLocationUtil;
import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl;
import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
@ -714,9 +713,7 @@ public void testUpdateContainer() throws IOException {
KeyValueContainer container =
(KeyValueContainer) addContainer(containerSet, testContainerID);
File orgContainerFile = KeyValueContainerLocationUtil.getContainerFile(
new File(container.getContainerData().getMetadataPath()),
testContainerID);
File orgContainerFile = container.getContainerFile();
Assert.assertTrue(orgContainerFile.exists());
Map<String, String> newMetadata = Maps.newHashMap();
@ -738,9 +735,9 @@ public void testUpdateContainer() throws IOException {
actualNewData.getMetadata().get("owner"));
// Verify container data on disk
File newContainerFile = KeyValueContainerLocationUtil.getContainerFile(
new File(actualNewData.getMetadataPath()),
testContainerID);
File containerBaseDir = new File(actualNewData.getMetadataPath())
.getParentFile();
File newContainerFile = ContainerUtils.getContainerFile(containerBaseDir);
Assert.assertTrue("Container file should exist.",
newContainerFile.exists());
Assert.assertEquals("Container file should be in same location.",