HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. Contributed by Haohui Mai.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1603705 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
eb93f73ea8
commit
a4e0ff5e05
@ -454,6 +454,8 @@ Release 2.5.0 - UNRELEASED
|
|||||||
|
|
||||||
HDFS-6530. Fix Balancer documentation. (szetszwo)
|
HDFS-6530. Fix Balancer documentation. (szetszwo)
|
||||||
|
|
||||||
|
HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9)
|
||||||
|
|
||||||
OPTIMIZATIONS
|
OPTIMIZATIONS
|
||||||
|
|
||||||
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
|
||||||
|
@ -252,7 +252,7 @@ void doCheckpoint() throws IOException {
|
|||||||
|
|
||||||
backupNode.namesystem.writeLock();
|
backupNode.namesystem.writeLock();
|
||||||
try {
|
try {
|
||||||
backupNode.namesystem.dir.setReady();
|
backupNode.namesystem.setImageLoaded();
|
||||||
if(backupNode.namesystem.getBlocksTotal() > 0) {
|
if(backupNode.namesystem.getBlocksTotal() > 0) {
|
||||||
backupNode.namesystem.setBlockTotal();
|
backupNode.namesystem.setBlockTotal();
|
||||||
}
|
}
|
||||||
|
@ -26,11 +26,10 @@
|
|||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.EnumSet;
|
import java.util.EnumSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
|
||||||
import java.util.concurrent.locks.Condition;
|
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
|
|
||||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||||
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.ContentSummary;
|
import org.apache.hadoop.fs.ContentSummary;
|
||||||
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
import org.apache.hadoop.fs.FileAlreadyExistsException;
|
||||||
@ -83,15 +82,14 @@
|
|||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
|
||||||
/*************************************************
|
/**
|
||||||
* FSDirectory stores the filesystem directory state.
|
* Both FSDirectory and FSNamesystem manage the state of the namespace.
|
||||||
* It handles writing/loading values to disk, and logging
|
* FSDirectory is a pure in-memory data structure, all of whose operations
|
||||||
* changes as we go.
|
* happen entirely in memory. In contrast, FSNamesystem persists the operations
|
||||||
*
|
* to the disk.
|
||||||
* It keeps the filename->blockset mapping always-current
|
* @see org.apache.hadoop.hdfs.server.namenode.FSNamesystem
|
||||||
* and logged to disk.
|
**/
|
||||||
*
|
@InterfaceAudience.Private
|
||||||
*************************************************/
|
|
||||||
public class FSDirectory implements Closeable {
|
public class FSDirectory implements Closeable {
|
||||||
private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
|
private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
|
||||||
final INodeDirectory r = new INodeDirectory(
|
final INodeDirectory r = new INodeDirectory(
|
||||||
@ -120,7 +118,6 @@ private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
|
|||||||
INodeDirectory rootDir;
|
INodeDirectory rootDir;
|
||||||
FSImage fsImage;
|
FSImage fsImage;
|
||||||
private final FSNamesystem namesystem;
|
private final FSNamesystem namesystem;
|
||||||
private volatile boolean ready = false;
|
|
||||||
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
|
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
|
||||||
private final int maxComponentLength;
|
private final int maxComponentLength;
|
||||||
private final int maxDirItems;
|
private final int maxDirItems;
|
||||||
@ -132,7 +129,6 @@ private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
|
|||||||
|
|
||||||
// lock to protect the directory and BlockMap
|
// lock to protect the directory and BlockMap
|
||||||
private final ReentrantReadWriteLock dirLock;
|
private final ReentrantReadWriteLock dirLock;
|
||||||
private final Condition cond;
|
|
||||||
|
|
||||||
// utility methods to acquire and release read lock and write lock
|
// utility methods to acquire and release read lock and write lock
|
||||||
void readLock() {
|
void readLock() {
|
||||||
@ -175,7 +171,6 @@ public int getWriteHoldCount() {
|
|||||||
|
|
||||||
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
|
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
|
||||||
this.dirLock = new ReentrantReadWriteLock(true); // fair
|
this.dirLock = new ReentrantReadWriteLock(true); // fair
|
||||||
this.cond = dirLock.writeLock().newCondition();
|
|
||||||
rootDir = createRoot(ns);
|
rootDir = createRoot(ns);
|
||||||
inodeMap = INodeMap.newInstance(rootDir);
|
inodeMap = INodeMap.newInstance(rootDir);
|
||||||
this.fsImage = fsImage;
|
this.fsImage = fsImage;
|
||||||
@ -231,38 +226,6 @@ public INodeDirectory getRoot() {
|
|||||||
return rootDir;
|
return rootDir;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Notify that loading of this FSDirectory is complete, and
|
|
||||||
* it is ready for use
|
|
||||||
*/
|
|
||||||
void imageLoadComplete() {
|
|
||||||
Preconditions.checkState(!ready, "FSDirectory already loaded");
|
|
||||||
setReady();
|
|
||||||
}
|
|
||||||
|
|
||||||
void setReady() {
|
|
||||||
if(ready) return;
|
|
||||||
writeLock();
|
|
||||||
try {
|
|
||||||
setReady(true);
|
|
||||||
this.nameCache.initialized();
|
|
||||||
cond.signalAll();
|
|
||||||
} finally {
|
|
||||||
writeUnlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//This is for testing purposes only
|
|
||||||
@VisibleForTesting
|
|
||||||
boolean isReady() {
|
|
||||||
return ready;
|
|
||||||
}
|
|
||||||
|
|
||||||
// exposed for unit tests
|
|
||||||
protected void setReady(boolean flag) {
|
|
||||||
ready = flag;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shutdown the filestore
|
* Shutdown the filestore
|
||||||
*/
|
*/
|
||||||
@ -271,22 +234,12 @@ public void close() throws IOException {
|
|||||||
fsImage.close();
|
fsImage.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
void markNameCacheInitialized() {
|
||||||
* Block until the object is ready to be used.
|
writeLock();
|
||||||
*/
|
try {
|
||||||
void waitForReady() {
|
nameCache.initialized();
|
||||||
if (!ready) {
|
} finally {
|
||||||
writeLock();
|
writeUnlock();
|
||||||
try {
|
|
||||||
while (!ready) {
|
|
||||||
try {
|
|
||||||
cond.await(5000, TimeUnit.MILLISECONDS);
|
|
||||||
} catch (InterruptedException ignored) {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
writeUnlock();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -312,7 +265,6 @@ INodeFile addFile(String path, PermissionStatus permissions,
|
|||||||
String clientMachine, DatanodeDescriptor clientNode)
|
String clientMachine, DatanodeDescriptor clientNode)
|
||||||
throws FileAlreadyExistsException, QuotaExceededException,
|
throws FileAlreadyExistsException, QuotaExceededException,
|
||||||
UnresolvedLinkException, SnapshotAccessControlException, AclException {
|
UnresolvedLinkException, SnapshotAccessControlException, AclException {
|
||||||
waitForReady();
|
|
||||||
|
|
||||||
long modTime = now();
|
long modTime = now();
|
||||||
INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null,
|
INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null,
|
||||||
@ -385,8 +337,6 @@ INodeFile unprotectedAddFile( long id,
|
|||||||
*/
|
*/
|
||||||
BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
|
BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
|
||||||
DatanodeStorageInfo[] targets) throws IOException {
|
DatanodeStorageInfo[] targets) throws IOException {
|
||||||
waitForReady();
|
|
||||||
|
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
|
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
|
||||||
@ -424,8 +374,6 @@ BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
|
|||||||
boolean removeBlock(String path, INodeFile fileNode, Block block)
|
boolean removeBlock(String path, INodeFile fileNode, Block block)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Preconditions.checkArgument(fileNode.isUnderConstruction());
|
Preconditions.checkArgument(fileNode.isUnderConstruction());
|
||||||
waitForReady();
|
|
||||||
|
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
return unprotectedRemoveBlock(path, fileNode, block);
|
return unprotectedRemoveBlock(path, fileNode, block);
|
||||||
@ -469,7 +417,6 @@ boolean renameTo(String src, String dst, long mtime)
|
|||||||
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: "
|
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: "
|
||||||
+src+" to "+dst);
|
+src+" to "+dst);
|
||||||
}
|
}
|
||||||
waitForReady();
|
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
if (!unprotectedRenameTo(src, dst, mtime))
|
if (!unprotectedRenameTo(src, dst, mtime))
|
||||||
@ -492,7 +439,6 @@ void renameTo(String src, String dst, long mtime,
|
|||||||
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src
|
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src
|
||||||
+ " to " + dst);
|
+ " to " + dst);
|
||||||
}
|
}
|
||||||
waitForReady();
|
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
if (unprotectedRenameTo(src, dst, mtime, options)) {
|
if (unprotectedRenameTo(src, dst, mtime, options)) {
|
||||||
@ -1024,7 +970,6 @@ boolean unprotectedRenameTo(String src, String dst, long timestamp,
|
|||||||
Block[] setReplication(String src, short replication, short[] blockRepls)
|
Block[] setReplication(String src, short replication, short[] blockRepls)
|
||||||
throws QuotaExceededException, UnresolvedLinkException,
|
throws QuotaExceededException, UnresolvedLinkException,
|
||||||
SnapshotAccessControlException {
|
SnapshotAccessControlException {
|
||||||
waitForReady();
|
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
return unprotectedSetReplication(src, replication, blockRepls);
|
return unprotectedSetReplication(src, replication, blockRepls);
|
||||||
@ -1147,7 +1092,6 @@ void concat(String target, String[] srcs, long timestamp)
|
|||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
// actual move
|
// actual move
|
||||||
waitForReady();
|
|
||||||
unprotectedConcat(target, srcs, timestamp);
|
unprotectedConcat(target, srcs, timestamp);
|
||||||
} finally {
|
} finally {
|
||||||
writeUnlock();
|
writeUnlock();
|
||||||
@ -1230,7 +1174,6 @@ long delete(String src, BlocksMapUpdateInfo collectedBlocks,
|
|||||||
if (NameNode.stateChangeLog.isDebugEnabled()) {
|
if (NameNode.stateChangeLog.isDebugEnabled()) {
|
||||||
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
|
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
|
||||||
}
|
}
|
||||||
waitForReady();
|
|
||||||
final long filesRemoved;
|
final long filesRemoved;
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
@ -1703,7 +1646,7 @@ private void updateCount(INodesInPath iip, int numOfINodes,
|
|||||||
long nsDelta, long dsDelta, boolean checkQuota)
|
long nsDelta, long dsDelta, boolean checkQuota)
|
||||||
throws QuotaExceededException {
|
throws QuotaExceededException {
|
||||||
assert hasWriteLock();
|
assert hasWriteLock();
|
||||||
if (!ready) {
|
if (!namesystem.isImageLoaded()) {
|
||||||
//still initializing. do not check or update quotas.
|
//still initializing. do not check or update quotas.
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -1896,7 +1839,7 @@ private static void verifyQuota(INode[] inodes, int pos, long nsDelta,
|
|||||||
*/
|
*/
|
||||||
private void verifyQuotaForRename(INode[] src, INode[] dst)
|
private void verifyQuotaForRename(INode[] src, INode[] dst)
|
||||||
throws QuotaExceededException {
|
throws QuotaExceededException {
|
||||||
if (!ready || skipQuotaCheck) {
|
if (!namesystem.isImageLoaded() || skipQuotaCheck) {
|
||||||
// Do not check quota if edits log is still being processed
|
// Do not check quota if edits log is still being processed
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -1952,7 +1895,7 @@ void verifySnapshotName(String snapshotName, String path)
|
|||||||
void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
|
void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
|
||||||
if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
|
if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
|
||||||
String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
|
String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
|
||||||
if (!ready) {
|
if (!namesystem.isImageLoaded()) {
|
||||||
s += " Please rename it before upgrade.";
|
s += " Please rename it before upgrade.";
|
||||||
}
|
}
|
||||||
throw new HadoopIllegalArgumentException(s);
|
throw new HadoopIllegalArgumentException(s);
|
||||||
@ -1979,7 +1922,7 @@ private void verifyMaxComponentLength(byte[] childName, Object parentPath,
|
|||||||
getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
|
getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
|
||||||
final PathComponentTooLongException e = new PathComponentTooLongException(
|
final PathComponentTooLongException e = new PathComponentTooLongException(
|
||||||
maxComponentLength, length, p, DFSUtil.bytes2String(childName));
|
maxComponentLength, length, p, DFSUtil.bytes2String(childName));
|
||||||
if (ready) {
|
if (namesystem.isImageLoaded()) {
|
||||||
throw e;
|
throw e;
|
||||||
} else {
|
} else {
|
||||||
// Do not throw if edits log is still being processed
|
// Do not throw if edits log is still being processed
|
||||||
@ -2003,7 +1946,7 @@ private void verifyMaxDirItems(INode[] pathComponents, int pos)
|
|||||||
if (count >= maxDirItems) {
|
if (count >= maxDirItems) {
|
||||||
final MaxDirectoryItemsExceededException e
|
final MaxDirectoryItemsExceededException e
|
||||||
= new MaxDirectoryItemsExceededException(maxDirItems, count);
|
= new MaxDirectoryItemsExceededException(maxDirItems, count);
|
||||||
if (ready) {
|
if (namesystem.isImageLoaded()) {
|
||||||
e.setPathName(getFullPathName(pathComponents, pos - 1));
|
e.setPathName(getFullPathName(pathComponents, pos - 1));
|
||||||
throw e;
|
throw e;
|
||||||
} else {
|
} else {
|
||||||
@ -2339,7 +2282,6 @@ private boolean unprotectedSetTimes(INode inode, long mtime,
|
|||||||
void reset() {
|
void reset() {
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
setReady(false);
|
|
||||||
rootDir = createRoot(getFSNamesystem());
|
rootDir = createRoot(getFSNamesystem());
|
||||||
inodeMap.clear();
|
inodeMap.clear();
|
||||||
addToInodeMap(rootDir);
|
addToInodeMap(rootDir);
|
||||||
|
@ -103,6 +103,7 @@
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.locks.Condition;
|
||||||
import java.util.concurrent.locks.ReentrantLock;
|
import java.util.concurrent.locks.ReentrantLock;
|
||||||
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
import java.util.concurrent.locks.ReentrantReadWriteLock;
|
||||||
|
|
||||||
@ -514,6 +515,59 @@ private void logAuditEvent(boolean succeeded,
|
|||||||
|
|
||||||
private final NNConf nnConf;
|
private final NNConf nnConf;
|
||||||
|
|
||||||
|
private volatile boolean imageLoaded = false;
|
||||||
|
private final Condition cond;
|
||||||
|
/**
|
||||||
|
* Notify that loading of this FSDirectory is complete, and
|
||||||
|
* it is imageLoaded for use
|
||||||
|
*/
|
||||||
|
void imageLoadComplete() {
|
||||||
|
Preconditions.checkState(!imageLoaded, "FSDirectory already loaded");
|
||||||
|
setImageLoaded();
|
||||||
|
}
|
||||||
|
|
||||||
|
void setImageLoaded() {
|
||||||
|
if(imageLoaded) return;
|
||||||
|
writeLock();
|
||||||
|
try {
|
||||||
|
setImageLoaded(true);
|
||||||
|
dir.markNameCacheInitialized();
|
||||||
|
cond.signalAll();
|
||||||
|
} finally {
|
||||||
|
writeUnlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//This is for testing purposes only
|
||||||
|
@VisibleForTesting
|
||||||
|
boolean isImageLoaded() {
|
||||||
|
return imageLoaded;
|
||||||
|
}
|
||||||
|
|
||||||
|
// exposed for unit tests
|
||||||
|
protected void setImageLoaded(boolean flag) {
|
||||||
|
imageLoaded = flag;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Block until the object is imageLoaded to be used.
|
||||||
|
*/
|
||||||
|
void waitForLoadingFSImage() {
|
||||||
|
if (!imageLoaded) {
|
||||||
|
writeLock();
|
||||||
|
try {
|
||||||
|
while (!imageLoaded) {
|
||||||
|
try {
|
||||||
|
cond.await(5000, TimeUnit.MILLISECONDS);
|
||||||
|
} catch (InterruptedException ignored) {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
writeUnlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Set the last allocated inode id when fsimage or editlog is loaded.
|
* Set the last allocated inode id when fsimage or editlog is loaded.
|
||||||
*/
|
*/
|
||||||
@ -555,6 +609,7 @@ void clear() {
|
|||||||
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
|
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
|
||||||
snapshotManager.clearSnapshottableDirs();
|
snapshotManager.clearSnapshottableDirs();
|
||||||
cacheManager.clear();
|
cacheManager.clear();
|
||||||
|
setImageLoaded(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
@ -682,6 +737,7 @@ static FSNamesystem loadFromDisk(Configuration conf) throws IOException {
|
|||||||
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
|
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
|
||||||
LOG.info("fsLock is fair:" + fair);
|
LOG.info("fsLock is fair:" + fair);
|
||||||
fsLock = new FSNamesystemLock(fair);
|
fsLock = new FSNamesystemLock(fair);
|
||||||
|
cond = fsLock.writeLock().newCondition();
|
||||||
try {
|
try {
|
||||||
resourceRecheckInterval = conf.getLong(
|
resourceRecheckInterval = conf.getLong(
|
||||||
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
|
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
|
||||||
@ -921,7 +977,7 @@ private void loadFSImage(StartupOption startOpt) throws IOException {
|
|||||||
}
|
}
|
||||||
writeUnlock();
|
writeUnlock();
|
||||||
}
|
}
|
||||||
dir.imageLoadComplete();
|
imageLoadComplete();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void startSecretManager() {
|
private void startSecretManager() {
|
||||||
@ -1840,6 +1896,7 @@ private void concatInt(String target, String [] srcs,
|
|||||||
HdfsFileStatus resultingStat = null;
|
HdfsFileStatus resultingStat = null;
|
||||||
FSPermissionChecker pc = getPermissionChecker();
|
FSPermissionChecker pc = getPermissionChecker();
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -2115,6 +2172,7 @@ private boolean setReplicationInt(String src, final short replication)
|
|||||||
FSPermissionChecker pc = getPermissionChecker();
|
FSPermissionChecker pc = getPermissionChecker();
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -2242,6 +2300,8 @@ private HdfsFileStatus startFileInt(String src, PermissionStatus permissions,
|
|||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
boolean create = flag.contains(CreateFlag.CREATE);
|
boolean create = flag.contains(CreateFlag.CREATE);
|
||||||
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
|
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
|
||||||
|
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -2730,6 +2790,7 @@ LocatedBlock getAdditionalBlock(String src, long fileId, String clientName,
|
|||||||
Block newBlock = null;
|
Block newBlock = null;
|
||||||
long offset;
|
long offset;
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -2952,6 +3013,7 @@ boolean abandonBlock(ExtendedBlock b, long fileId, String src, String holder)
|
|||||||
}
|
}
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -3050,6 +3112,7 @@ boolean completeFile(String src, String holder,
|
|||||||
boolean success = false;
|
boolean success = false;
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -3249,6 +3312,7 @@ private boolean renameToInt(String src, String dst, boolean logRetryCache)
|
|||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
checkNameNodeSafeMode("Cannot rename " + src);
|
checkNameNodeSafeMode("Cannot rename " + src);
|
||||||
|
waitForLoadingFSImage();
|
||||||
src = FSDirectory.resolvePath(src, srcComponents, dir);
|
src = FSDirectory.resolvePath(src, srcComponents, dir);
|
||||||
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
|
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -3356,6 +3420,7 @@ private void renameToInternal(FSPermissionChecker pc, String src, String dst,
|
|||||||
false);
|
false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
waitForLoadingFSImage();
|
||||||
long mtime = now();
|
long mtime = now();
|
||||||
dir.renameTo(src, dst, mtime, options);
|
dir.renameTo(src, dst, mtime, options);
|
||||||
getEditLog().logRename(src, dst, mtime, logRetryCache, options);
|
getEditLog().logRename(src, dst, mtime, logRetryCache, options);
|
||||||
@ -3429,6 +3494,8 @@ private boolean deleteInternal(String src, boolean recursive,
|
|||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
boolean ret = false;
|
boolean ret = false;
|
||||||
|
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -3902,6 +3969,8 @@ void fsync(String src, long fileId, String clientName, long lastBlockLength)
|
|||||||
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
|
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
|
||||||
|
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -4103,6 +4172,7 @@ private void finalizeINodeFileUnderConstruction(String src,
|
|||||||
INodeFile pendingFile, int latestSnapshot) throws IOException,
|
INodeFile pendingFile, int latestSnapshot) throws IOException,
|
||||||
UnresolvedLinkException {
|
UnresolvedLinkException {
|
||||||
assert hasWriteLock();
|
assert hasWriteLock();
|
||||||
|
|
||||||
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
|
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
|
||||||
Preconditions.checkArgument(uc != null);
|
Preconditions.checkArgument(uc != null);
|
||||||
leaseManager.removeLease(uc.getClientName(), src);
|
leaseManager.removeLease(uc.getClientName(), src);
|
||||||
@ -4114,6 +4184,7 @@ private void finalizeINodeFileUnderConstruction(String src,
|
|||||||
// since we just remove the uc feature from pendingFile
|
// since we just remove the uc feature from pendingFile
|
||||||
final INodeFile newFile = pendingFile.toCompleteFile(now());
|
final INodeFile newFile = pendingFile.toCompleteFile(now());
|
||||||
|
|
||||||
|
waitForLoadingFSImage();
|
||||||
// close file and persist block allocations for this file
|
// close file and persist block allocations for this file
|
||||||
closeFile(src, newFile);
|
closeFile(src, newFile);
|
||||||
|
|
||||||
@ -4172,6 +4243,7 @@ void commitBlockSynchronization(ExtendedBlock lastblock,
|
|||||||
+ ")");
|
+ ")");
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
String src = "";
|
String src = "";
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
try {
|
try {
|
||||||
checkOperation(OperationCategory.WRITE);
|
checkOperation(OperationCategory.WRITE);
|
||||||
@ -4517,7 +4589,7 @@ void incrDeletedFileCount(long count) {
|
|||||||
*/
|
*/
|
||||||
private void closeFile(String path, INodeFile file) {
|
private void closeFile(String path, INodeFile file) {
|
||||||
assert hasWriteLock();
|
assert hasWriteLock();
|
||||||
dir.waitForReady();
|
waitForLoadingFSImage();
|
||||||
// file is closed
|
// file is closed
|
||||||
getEditLog().logCloseFile(path, file);
|
getEditLog().logCloseFile(path, file);
|
||||||
if (NameNode.stateChangeLog.isDebugEnabled()) {
|
if (NameNode.stateChangeLog.isDebugEnabled()) {
|
||||||
@ -4541,7 +4613,7 @@ private INodeSymlink addSymlink(String path, String target,
|
|||||||
boolean createParent, boolean logRetryCache)
|
boolean createParent, boolean logRetryCache)
|
||||||
throws UnresolvedLinkException, FileAlreadyExistsException,
|
throws UnresolvedLinkException, FileAlreadyExistsException,
|
||||||
QuotaExceededException, SnapshotAccessControlException, AclException {
|
QuotaExceededException, SnapshotAccessControlException, AclException {
|
||||||
dir.waitForReady();
|
waitForLoadingFSImage();
|
||||||
|
|
||||||
final long modTime = now();
|
final long modTime = now();
|
||||||
if (createParent) {
|
if (createParent) {
|
||||||
@ -5804,7 +5876,7 @@ private void checkPermission(FSPermissionChecker pc,
|
|||||||
boolean ignoreEmptyDir, boolean resolveLink)
|
boolean ignoreEmptyDir, boolean resolveLink)
|
||||||
throws AccessControlException, UnresolvedLinkException {
|
throws AccessControlException, UnresolvedLinkException {
|
||||||
if (!pc.isSuperUser()) {
|
if (!pc.isSuperUser()) {
|
||||||
dir.waitForReady();
|
waitForLoadingFSImage();
|
||||||
readLock();
|
readLock();
|
||||||
try {
|
try {
|
||||||
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
|
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
|
||||||
@ -6271,6 +6343,7 @@ void updatePipeline(String clientName, ExtendedBlock oldBlock,
|
|||||||
+ ", newNodes=" + Arrays.asList(newNodes)
|
+ ", newNodes=" + Arrays.asList(newNodes)
|
||||||
+ ", clientName=" + clientName
|
+ ", clientName=" + clientName
|
||||||
+ ")");
|
+ ")");
|
||||||
|
waitForLoadingFSImage();
|
||||||
writeLock();
|
writeLock();
|
||||||
boolean success = false;
|
boolean success = false;
|
||||||
try {
|
try {
|
||||||
|
@ -1064,7 +1064,7 @@ static void doMerge(
|
|||||||
} finally {
|
} finally {
|
||||||
dstNamesystem.writeUnlock();
|
dstNamesystem.writeUnlock();
|
||||||
}
|
}
|
||||||
dstNamesystem.dir.imageLoadComplete();
|
dstNamesystem.imageLoadComplete();
|
||||||
}
|
}
|
||||||
// error simulation code for junit test
|
// error simulation code for junit test
|
||||||
CheckpointFaultInjector.getInstance().duringMerge();
|
CheckpointFaultInjector.getInstance().duringMerge();
|
||||||
|
@ -32,7 +32,6 @@
|
|||||||
|
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
import static org.mockito.Matchers.any;
|
import static org.mockito.Matchers.any;
|
||||||
import static org.mockito.Matchers.anyBoolean;
|
|
||||||
import static org.mockito.Mockito.*;
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -50,6 +49,7 @@ private FSNamesystem makeNameSystemSpy(Block block, INodeFile file)
|
|||||||
final DatanodeStorageInfo[] targets = {};
|
final DatanodeStorageInfo[] targets = {};
|
||||||
|
|
||||||
FSNamesystem namesystem = new FSNamesystem(conf, image);
|
FSNamesystem namesystem = new FSNamesystem(conf, image);
|
||||||
|
namesystem.setImageLoaded(true);
|
||||||
FSNamesystem namesystemSpy = spy(namesystem);
|
FSNamesystem namesystemSpy = spy(namesystem);
|
||||||
BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
|
BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
|
||||||
block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
|
block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
|
||||||
|
@ -36,7 +36,6 @@
|
|||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
|
|
||||||
import org.apache.hadoop.test.GenericTestUtils;
|
import org.apache.hadoop.test.GenericTestUtils;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
@ -127,16 +126,6 @@ public void testDumpTree() throws Exception {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testReset() throws Exception {
|
|
||||||
fsdir.reset();
|
|
||||||
Assert.assertFalse(fsdir.isReady());
|
|
||||||
final INodeDirectory root = (INodeDirectory) fsdir.getINode("/");
|
|
||||||
Assert.assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
|
|
||||||
fsdir.imageLoadComplete();
|
|
||||||
Assert.assertTrue(fsdir.isReady());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSkipQuotaCheck() throws Exception {
|
public void testSkipQuotaCheck() throws Exception {
|
||||||
try {
|
try {
|
||||||
|
@ -35,6 +35,7 @@
|
|||||||
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
|
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
|
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
|
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
|
||||||
|
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.mockito.Mockito;
|
import org.mockito.Mockito;
|
||||||
@ -194,4 +195,22 @@ public void testFSNamesystemLockCompatibility() {
|
|||||||
assertFalse(rwLock.isWriteLockedByCurrentThread());
|
assertFalse(rwLock.isWriteLockedByCurrentThread());
|
||||||
assertEquals(0, rwLock.getWriteHoldCount());
|
assertEquals(0, rwLock.getWriteHoldCount());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testReset() throws Exception {
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
|
||||||
|
FSImage fsImage = Mockito.mock(FSImage.class);
|
||||||
|
Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
|
||||||
|
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
|
||||||
|
fsn.imageLoadComplete();
|
||||||
|
assertTrue(fsn.isImageLoaded());
|
||||||
|
fsn.clear();
|
||||||
|
assertFalse(fsn.isImageLoaded());
|
||||||
|
final INodeDirectory root = (INodeDirectory) fsn.getFSDirectory()
|
||||||
|
.getINode("/");
|
||||||
|
assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
|
||||||
|
fsn.imageLoadComplete();
|
||||||
|
assertTrue(fsn.isImageLoaded());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -19,12 +19,9 @@
|
|||||||
package org.apache.hadoop.hdfs.server.namenode;
|
package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
|
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
|
||||||
import static org.apache.hadoop.util.Time.now;
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.mockito.Matchers.anyObject;
|
|
||||||
import static org.mockito.Mockito.doReturn;
|
import static org.mockito.Mockito.doReturn;
|
||||||
import static org.mockito.Mockito.mock;
|
import static org.mockito.Mockito.mock;
|
||||||
import static org.mockito.Mockito.when;
|
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@ -57,7 +54,7 @@ static private FSNamesystem getMockNamesystem() throws IOException {
|
|||||||
FSEditLog editLog = mock(FSEditLog.class);
|
FSEditLog editLog = mock(FSEditLog.class);
|
||||||
doReturn(editLog).when(fsImage).getEditLog();
|
doReturn(editLog).when(fsImage).getEditLog();
|
||||||
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
|
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
|
||||||
fsn.getFSDirectory().setReady(fsIsReady);
|
fsn.setImageLoaded(fsIsReady);
|
||||||
return fsn;
|
return fsn;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user