big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-12780. During WASB atomic rename handle crash when one directory has been renamed but not file under it. Contributed by Madhumita Chakraborty.

Browse Source
This commit is contained in:
cnauroth 2016-02-12 15:50:10 -08:00
parent 1de1641f17
commit 91a96eaa53
5 changed files with 106 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -1704,6 +1704,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12795. KMS does not log detailed stack trace for unexpected errors.
(cnauroth)
HADOOP-12780. During atomic rename handle crash when one directory has been
renamed but not file under it. (Madhumita Chakraborty via cnauroth)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

30
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
View File

@ -2563,6 +2563,36 @@ private void waitForCopyToComplete(CloudBlobWrapper blob, OperationContext opCon
}
}
/**
* Checks whether an explicit file/folder exists.
* This is used by redo of atomic rename.
* There was a bug(apache jira HADOOP-12780) during atomic rename if
* process crashes after an inner directory has been renamed but still
* there are file under that directory to be renamed then after the
* process comes again it tries to redo the renames. It checks whether
* the directory exists or not by calling filesystem.exist.
* But filesystem.Exists will treat that directory as implicit directory
* and return true as file exists under that directory. So It will try
* try to rename that directory and will fail as the corresponding blob
* does not exist. So this method explicitly checks for the blob.
*/
@Override
public boolean explicitFileExists(String key) throws AzureException {
CloudBlobWrapper blob;
try {
blob = getBlobReference(key);
if (null != blob && blob.exists(getInstrumentedContext())) {
return true;
}
return false;
} catch (StorageException e) {
throw new AzureException(e);
} catch (URISyntaxException e) {
throw new AzureException(e);
}
}
/**
* Changes the permission status on the given key.
*/

8
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
View File

@ -558,13 +558,13 @@ private void finishSingleFileRename(String fileName)
throws IOException {
Path srcFile = fullPath(srcKey, fileName);
Path dstFile = fullPath(dstKey, fileName);
boolean srcExists = fs.exists(srcFile);
boolean dstExists = fs.exists(dstFile);
String srcName = fs.pathToKey(srcFile);
String dstName = fs.pathToKey(dstFile);
boolean srcExists = fs.getStoreInterface().explicitFileExists(srcName);
boolean dstExists = fs.getStoreInterface().explicitFileExists(dstName);
if(srcExists) {
// Rename gets exclusive access (via a lease) for HBase write-ahead log
// (WAL) file processing correctness. See the rename code for details.
String srcName = fs.pathToKey(srcFile);
String dstName = fs.pathToKey(dstFile);
fs.getStoreInterface().rename(srcName, dstName, true, null);
} else if (!srcExists && dstExists) {
// The rename already finished, so do nothing.

2
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeFileSystemStore.java
View File

@ -109,4 +109,6 @@ void updateFolderLastModifiedTime(String key, Date lastModified,
SelfRenewingLease acquireLease(String key) throws AzureException;
DataOutputStream retrieveAppendStream(String key, int bufferSize) throws IOException;
boolean explicitFileExists(String key) throws AzureException;
}

67
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/NativeAzureFileSystemBaseTest.java
View File

@ -845,6 +845,73 @@ public void testRedoRenameFolderInFolderListing() throws IOException {
assertTrue(fs.exists(new Path(inner2renamed, "file")));
}
/**
* There is a nested folder and file under the folder to be renamed
* and the process crashes after the nested folder has been renamed but not the file.
* then when you list the parent folder, pending renames should be redone
* Apache jira HADOOP-12780
*/
@Test
public void testRedoRenameFolderRenameInProgress() throws IOException {
// create original folder
String parent = "parent";
Path parentFolder = new Path(parent);
assertTrue(fs.mkdirs(parentFolder));
Path folderToBeRenamed = new Path(parentFolder, "folderToBeRenamed");
assertTrue(fs.mkdirs(folderToBeRenamed));
String innerFolderName = "innerFolder";
Path inner = new Path(folderToBeRenamed, innerFolderName);
assertTrue(fs.mkdirs(inner));
String innerFileName = "file";
Path innerFile = new Path(inner, innerFileName);
assertTrue(fs.createNewFile(innerFile));
Path renamedFolder = new Path(parentFolder, "renamedFolder");
// propose (but don't do) the rename of innerFolder2
Path home = fs.getHomeDirectory();
String relativeHomeDir = getRelativePath(home.toString());
NativeAzureFileSystem.FolderRenamePending pending =
new NativeAzureFileSystem.FolderRenamePending(
relativeHomeDir + "/" + folderToBeRenamed,
relativeHomeDir + "/" + renamedFolder, null,
(NativeAzureFileSystem) fs);
// Create a rename-pending file and write rename information to it.
final String renamePendingStr = folderToBeRenamed + FolderRenamePending.SUFFIX;
Path renamePendingFile = new Path(renamePendingStr);
FSDataOutputStream out = fs.create(renamePendingFile, true);
assertTrue(out != null);
writeString(out, pending.makeRenamePendingFileContents());
// Rename inner folder to simulate the scenario where rename has started and
// only one directory has been renamed but not the files under it
((NativeAzureFileSystem) fs).getStoreInterface().rename(
relativeHomeDir + "/" +inner, relativeHomeDir + "/" +renamedFolder + "/" + innerFolderName , true, null);
// Instead of using fs.exist use store.explicitFileExists because fs.exist will return true
// even if directory has been renamed, but there are still file under that directory
assertFalse(((NativeAzureFileSystem) fs).getStoreInterface().
explicitFileExists(relativeHomeDir + "/" + inner)); // verify the explicit inner folder is gone
assertTrue(((NativeAzureFileSystem) fs).getStoreInterface().
explicitFileExists(relativeHomeDir + "/" + innerFile)); // verify inner file is present
// Redo the rename operation based on the contents of the
// -RenamePending.json file. Trigger the redo by checking for existence of
// the original folder. It must appear to not exist.
FileStatus[] listed = fs.listStatus(parentFolder);
assertEquals(1, listed.length);
assertTrue(listed[0].isDirectory());
// The rename pending file is not a directory, so at this point we know the
// redo has been done.
assertFalse(fs.exists(inner)); // verify original folder is gone
assertFalse(fs.exists(innerFile)); // verify original file is gone
assertTrue(fs.exists(renamedFolder)); // verify the target is there
assertTrue(fs.exists(new Path(renamedFolder, innerFolderName + "/" + innerFileName)));
}
/**
* Test the situation when the rename metadata file is empty
* i.e. it is created but not written yet. In that case in next rename