diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 445c50f841..e241460f10 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -914,6 +914,9 @@ Release 2.8.0 - UNRELEASED HDFS-8974. Convert docs in xdoc format to markdown. (Masatake Iwasaki via aajisaka) + HDFS-6763. Initialize file system-wide quota once on transitioning to active + (kihwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java index 8aee0bb037..c6ae0d56a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java @@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; @@ -94,9 +93,6 @@ static enum BNState { super(conf); storage.setDisablePreUpgradableLayoutCheck(true); bnState = BNState.DROP_UNTIL_NEXT_ROLL; - quotaInitThreads = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY, - DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT); } synchronized FSNamesystem getNamesystem() { @@ -222,9 +218,7 @@ private synchronized void applyEdits(long firstTxId, int numTxns, byte[] data) } lastAppliedTxId = logLoader.getLastAppliedTxId(); - FSImage.updateCountForQuota( - getNamesystem().dir.getBlockStoragePolicySuite(), - getNamesystem().dir.rootDir, quotaInitThreads); + getNamesystem().dir.updateCountForQuota(); } finally { backupInputStream.clear(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 8c74e48800..e25e0e086b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -57,8 +57,10 @@ import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo.UpdatedReplicationInfo; import org.apache.hadoop.hdfs.util.ByteArray; import org.apache.hadoop.hdfs.util.EnumCounters; +import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +70,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.RecursiveAction; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -138,6 +142,7 @@ private static INodeDirectory createRoot(FSNamesystem namesystem) { private final long contentSleepMicroSec; private final INodeMap inodeMap; // Synchronized by dirLock private long yieldCount = 0; // keep track of lock yield count. + private int quotaInitThreads; private final int inodeXAttrsLimit; //inode xattrs max limit @@ -312,6 +317,10 @@ public int getWriteHoldCount() { namesystem = ns; this.editLog = ns.getEditLog(); ezManager = new EncryptionZoneManager(this, conf); + + this.quotaInitThreads = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY, + DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT); } FSNamesystem getFSNamesystem() { @@ -503,6 +512,125 @@ void updateReplicationFactor(Collection blocks) { } } + /** + * Update the count of each directory with quota in the namespace. + * A directory's count is defined as the total number inodes in the tree + * rooted at the directory. + * + * This is an update of existing state of the filesystem and does not + * throw QuotaExceededException. + */ + void updateCountForQuota(int initThreads) { + writeLock(); + try { + int threads = (initThreads < 1) ? 1 : initThreads; + LOG.info("Initializing quota with " + threads + " thread(s)"); + long start = Time.now(); + QuotaCounts counts = new QuotaCounts.Builder().build(); + ForkJoinPool p = new ForkJoinPool(threads); + RecursiveAction task = new InitQuotaTask(getBlockStoragePolicySuite(), + rootDir.getStoragePolicyID(), rootDir, counts); + p.execute(task); + task.join(); + p.shutdown(); + LOG.info("Quota initialization completed in " + (Time.now() - start) + + " milliseconds\n" + counts); + } finally { + writeUnlock(); + } + } + + void updateCountForQuota() { + updateCountForQuota(quotaInitThreads); + } + + /** + * parallel initialization using fork-join. + */ + private static class InitQuotaTask extends RecursiveAction { + private final INodeDirectory dir; + private final QuotaCounts counts; + private final BlockStoragePolicySuite bsps; + private final byte blockStoragePolicyId; + + public InitQuotaTask(BlockStoragePolicySuite bsps, + byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) { + this.dir = dir; + this.counts = counts; + this.bsps = bsps; + this.blockStoragePolicyId = blockStoragePolicyId; + } + + public void compute() { + QuotaCounts myCounts = new QuotaCounts.Builder().build(); + dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, + myCounts); + + ReadOnlyList children = + dir.getChildrenList(CURRENT_STATE_ID); + + if (children.size() > 0) { + List subtasks = new ArrayList(); + for (INode child : children) { + final byte childPolicyId = + child.getStoragePolicyIDForQuota(blockStoragePolicyId); + if (child.isDirectory()) { + subtasks.add(new InitQuotaTask(bsps, childPolicyId, + child.asDirectory(), myCounts)); + } else { + // file or symlink. count using the local counts variable + myCounts.add(child.computeQuotaUsage(bsps, childPolicyId, false, + CURRENT_STATE_ID)); + } + } + // invoke and wait for completion + invokeAll(subtasks); + } + + if (dir.isQuotaSet()) { + // check if quota is violated. It indicates a software bug. + final QuotaCounts q = dir.getQuotaCounts(); + + final long nsConsumed = myCounts.getNameSpace(); + final long nsQuota = q.getNameSpace(); + if (Quota.isViolated(nsQuota, nsConsumed)) { + LOG.warn("Namespace quota violation in image for " + + dir.getFullPathName() + + " quota = " + nsQuota + " < consumed = " + nsConsumed); + } + + final long ssConsumed = myCounts.getStorageSpace(); + final long ssQuota = q.getStorageSpace(); + if (Quota.isViolated(ssQuota, ssConsumed)) { + LOG.warn("Storagespace quota violation in image for " + + dir.getFullPathName() + + " quota = " + ssQuota + " < consumed = " + ssConsumed); + } + + final EnumCounters tsConsumed = myCounts.getTypeSpaces(); + for (StorageType t : StorageType.getTypesSupportingQuota()) { + final long typeSpace = tsConsumed.get(t); + final long typeQuota = q.getTypeSpaces().get(t); + if (Quota.isViolated(typeQuota, typeSpace)) { + LOG.warn("Storage type quota violation in image for " + + dir.getFullPathName() + + " type = " + t.toString() + " quota = " + + typeQuota + " < consumed " + typeSpace); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("Setting quota for " + dir + "\n" + myCounts); + } + dir.getDirectoryWithQuotaFeature().setSpaceConsumed(nsConsumed, + ssConsumed, tsConsumed); + } + + synchronized(counts) { + counts.add(myCounts); + } + } + } + /** Updates namespace, storagespace and typespaces consumed for all * directories until the parent directory of file represented by path. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 0dd007d96e..93dc097bb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -27,8 +27,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.RecursiveAction; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -42,12 +40,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HAUtil; import org.apache.hadoop.hdfs.protocol.LayoutVersion; -import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption; @@ -61,7 +57,6 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageStorageInspector.FSImageFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; -import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand; @@ -70,9 +65,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.util.Canceler; -import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.MD5FileUtils; -import org.apache.hadoop.hdfs.util.ReadOnlyList; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.util.Time; @@ -147,12 +140,7 @@ protected FSImage(Configuration conf, storage.setRestoreFailedStorage(true); } - this.quotaInitThreads = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_KEY, - DFSConfigKeys.DFS_NAMENODE_QUOTA_INIT_THREADS_DEFAULT); - this.editLog = new FSEditLog(conf, storage, editsDirs); - archivalManager = new NNStorageRetentionManager(conf, storage, editLog); } @@ -853,125 +841,11 @@ private long loadEdits(Iterable editStreams, } } finally { FSEditLog.closeAllStreams(editStreams); - // update the counts - updateCountForQuota(target.getBlockManager().getStoragePolicySuite(), - target.dir.rootDir, quotaInitThreads); } prog.endPhase(Phase.LOADING_EDITS); return lastAppliedTxId - prevLastAppliedTxId; } - /** - * Update the count of each directory with quota in the namespace. - * A directory's count is defined as the total number inodes in the tree - * rooted at the directory. - * - * This is an update of existing state of the filesystem and does not - * throw QuotaExceededException. - */ - static void updateCountForQuota(BlockStoragePolicySuite bsps, - INodeDirectory root, int threads) { - threads = (threads < 1) ? 1 : threads; - LOG.info("Initializing quota with " + threads + " thread(s)"); - long start = Time.now(); - QuotaCounts counts = new QuotaCounts.Builder().build(); - ForkJoinPool p = new ForkJoinPool(threads); - RecursiveAction task = new InitQuotaTask(bsps, root.getStoragePolicyID(), - root, counts); - p.execute(task); - task.join(); - p.shutdown(); - LOG.info("Quota initialization completed in " + (Time.now() - start) + - " milliseconds\n" + counts); - } - - /** - * parallel initialization using fork-join. - */ - private static class InitQuotaTask extends RecursiveAction { - private final INodeDirectory dir; - private final QuotaCounts counts; - private final BlockStoragePolicySuite bsps; - private final byte blockStoragePolicyId; - - public InitQuotaTask(BlockStoragePolicySuite bsps, - byte blockStoragePolicyId, INodeDirectory dir, QuotaCounts counts) { - this.dir = dir; - this.counts = counts; - this.bsps = bsps; - this.blockStoragePolicyId = blockStoragePolicyId; - } - - public void compute() { - QuotaCounts myCounts = new QuotaCounts.Builder().build(); - dir.computeQuotaUsage4CurrentDirectory(bsps, blockStoragePolicyId, - myCounts); - - ReadOnlyList children = - dir.getChildrenList(Snapshot.CURRENT_STATE_ID); - - if (children.size() > 0) { - List subtasks = new ArrayList(); - for (INode child : children) { - final byte childPolicyId = - child.getStoragePolicyIDForQuota(blockStoragePolicyId); - if (child.isDirectory()) { - subtasks.add(new InitQuotaTask(bsps, childPolicyId, - child.asDirectory(), myCounts)); - } else { - // file or symlink. count using the local counts variable - myCounts.add(child.computeQuotaUsage(bsps, childPolicyId, false, - Snapshot.CURRENT_STATE_ID)); - } - } - // invoke and wait for completion - invokeAll(subtasks); - } - - if (dir.isQuotaSet()) { - // check if quota is violated. It indicates a software bug. - final QuotaCounts q = dir.getQuotaCounts(); - - final long nsConsumed = myCounts.getNameSpace(); - final long nsQuota = q.getNameSpace(); - if (Quota.isViolated(nsQuota, nsConsumed)) { - LOG.warn("Namespace quota violation in image for " - + dir.getFullPathName() - + " quota = " + nsQuota + " < consumed = " + nsConsumed); - } - - final long ssConsumed = myCounts.getStorageSpace(); - final long ssQuota = q.getStorageSpace(); - if (Quota.isViolated(ssQuota, ssConsumed)) { - LOG.warn("Storagespace quota violation in image for " - + dir.getFullPathName() - + " quota = " + ssQuota + " < consumed = " + ssConsumed); - } - - final EnumCounters tsConsumed = myCounts.getTypeSpaces(); - for (StorageType t : StorageType.getTypesSupportingQuota()) { - final long typeSpace = tsConsumed.get(t); - final long typeQuota = q.getTypeSpaces().get(t); - if (Quota.isViolated(typeQuota, typeSpace)) { - LOG.warn("Storage type quota violation in image for " - + dir.getFullPathName() - + " type = " + t.toString() + " quota = " - + typeQuota + " < consumed " + typeSpace); - } - } - if (LOG.isDebugEnabled()) { - LOG.debug("Setting quota for " + dir + "\n" + myCounts); - } - dir.getDirectoryWithQuotaFeature().setSpaceConsumed(nsConsumed, - ssConsumed, tsConsumed); - } - - synchronized(counts) { - counts.add(myCounts); - } - } - } - /** * Load the image namespace from the given image file, verifying * it against the MD5 sum stored in its associated .md5 file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7d766b98f9..328c29df0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1117,6 +1117,8 @@ void startActiveServices() throws IOException { getFSImage().editLog.openForWrite(getEffectiveLayoutVersion()); } + // Initialize the quota. + dir.updateCountForQuota(); // Enable quota checks. dir.enableQuotaChecks(); if (haEnabled) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java index 0765a22311..cf6463836e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java @@ -339,16 +339,13 @@ public void testQuotaInitialization() throws Exception { HashMap dsMap = new HashMap(); scanDirsWithQuota(root, nsMap, dsMap, false); - FSImage.updateCountForQuota( - fsdir.getBlockManager().getStoragePolicySuite(), root, 1); + fsdir.updateCountForQuota(1); scanDirsWithQuota(root, nsMap, dsMap, true); - FSImage.updateCountForQuota( - fsdir.getBlockManager().getStoragePolicySuite(), root, 2); + fsdir.updateCountForQuota(2); scanDirsWithQuota(root, nsMap, dsMap, true); - FSImage.updateCountForQuota( - fsdir.getBlockManager().getStoragePolicySuite(), root, 4); + fsdir.updateCountForQuota(4); scanDirsWithQuota(root, nsMap, dsMap, true); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java index 61b7f7ca5f..1ff18a0e5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java @@ -158,8 +158,7 @@ private void loadFSImageFromTempFile(File imageFile) throws IOException { fsn.getFSDirectory().writeLock(); try { loader.load(imageFile, false); - FSImage.updateCountForQuota(fsn.getBlockManager().getStoragePolicySuite(), - INodeDirectory.valueOf(fsn.getFSDirectory().getINode("/"), "/"), 4); + fsn.getFSDirectory().updateCountForQuota(); } finally { fsn.getFSDirectory().writeUnlock(); fsn.writeUnlock();