From ff91453227488902d8c730d69c90fb1a97b35d75 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 15 Feb 2012 22:00:08 +0000 Subject: [PATCH 1/5] HDFS-2938. Recursive delete of a large directory make namenode unresponsive. Contributed by Hari Mankude. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244752 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/FSNamesystem.java | 27 ++++++++++--------- .../namenode/TestLargeDirectoryDelete.java | 5 +++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0ab243cc28..6ce1558d04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -251,6 +251,9 @@ Release 0.23.2 - UNRELEASED HDFS-2950. Secondary NN HTTPS address should be listed as a NAMESERVICE_SPECIFIC_KEY. (todd) + HDFS-2938. Recursive delete of a large directory make namenode + unresponsive. (Hari Mankude via suresh) + Release 0.23.1 - 2012-02-08 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index e70ed27c2f..f846295f0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1980,15 +1980,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } finally { writeUnlock(); } - - getEditLog().logSync(); - - writeLock(); - try { - removeBlocks(collectedBlocks); // Incremental deletion of blocks - } finally { - writeUnlock(); - } + getEditLog().logSync(); + removeBlocks(collectedBlocks); // Incremental deletion of blocks collectedBlocks.clear(); if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* Namesystem.delete: " @@ -1997,16 +1990,24 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return true; } - /** From the given list, incrementally remove the blocks from blockManager */ + /** + * From the given list, incrementally remove the blocks from blockManager + * Writelock is dropped and reacquired every BLOCK_DELETION_INCREMENT to + * ensure that other waiters on the lock can get in. See HDFS-2938 + */ private void removeBlocks(List blocks) { - assert hasWriteLock(); int start = 0; int end = 0; while (start < blocks.size()) { end = BLOCK_DELETION_INCREMENT + start; end = end > blocks.size() ? blocks.size() : end; - for (int i=start; i 0) { - synchronized(mc.getNamesystem()) { + mc.getNamesystem().writeLock(); + try { lockOps++; + } finally { + mc.getNamesystem().writeUnlock(); } Thread.sleep(1); } From 1ceb0eeaf43d15e845e3fc6796539ff7bbaabd1b Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 15 Feb 2012 22:23:20 +0000 Subject: [PATCH 2/5] HADOOP-8074. Small bug in hadoop error message for unknown commands. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244766 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/fs/FsShell.java | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e6fe6e5c47..bbaee1f50b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -190,6 +190,9 @@ Release 0.23.2 - UNRELEASED Double.MAX_VALUE) to avoid making Ganglia's gmetad core. (Varun Kapoor via mattf) + HADOOP-8074. Small bug in hadoop error message for unknown commands. + (Colin Patrick McCabe via eli) + Release 0.23.1 - 2012-02-08 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java index 376ea79586..4da32789e5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java @@ -269,7 +269,15 @@ public class FsShell extends Configured implements Tool { private void displayError(String cmd, String message) { for (String line : message.split("\n")) { - System.err.println(cmd.substring(1) + ": " + line); + System.err.println(cmd + ": " + line); + if (cmd.charAt(0) != '-') { + Command instance = null; + instance = commandFactory.getInstance("-" + cmd); + if (instance != null) { + System.err.println("Did you mean -" + cmd + "? This command " + + "begins with a dash."); + } + } } } From 20816218f08c28cb1a1c2b33a8538eb1453236da Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 16 Feb 2012 05:53:25 +0000 Subject: [PATCH 3/5] MAPREDUCE-3864. Fix cluster setup docs for correct SecondaryNameNode HTTPS parameters. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1244852 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 22324a79f2..1def51b4ae 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -109,6 +109,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3736. Variable substitution depth too large for fs.default.name causes jobs to fail (ahmed via tucu). + + MAPREDUCE-3864. Fix cluster setup docs for correct SecondaryNameNode + HTTPS parameters. (todd) Release 0.23.1 - 2012-02-08 diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm index eca68a234b..ab97dc5a8e 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm @@ -738,9 +738,9 @@ KVNO Timestamp Principal *-------------------------+-------------------------+------------------------+ || Parameter || Value || Notes | *-------------------------+-------------------------+------------------------+ -| <<>> | | | +| <<>> | | | *-------------------------+-------------------------+------------------------+ -| <<>> | <50090> | | +| <<>> | <50470> | | *-------------------------+-------------------------+------------------------+ | <<>> | | | | | | | From cfc4ad76a325381119351092ac9e40544141b74a Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 16 Feb 2012 17:49:20 +0000 Subject: [PATCH 4/5] MAPREDUCE-3849. Change TokenCache's reading of the binary token file (Daryn Sharp via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1245099 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapreduce/security/TokenCache.java | 39 ++++---- .../mapreduce/security/TestTokenCache.java | 89 +++++++++++++++++++ 3 files changed, 109 insertions(+), 22 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 1def51b4ae..0c402a69b9 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -94,6 +94,9 @@ Release 0.23.2 - UNRELEASED IMPROVEMENTS + MAPREDUCE-3849. Change TokenCache's reading of the binary token file + (Daryn Sharp via bobby) + MAPREDUCE-3854. Fixed and reenabled tests related to MR child JVM's environmental variables in TestMiniMRChildTask. (Tom White via vinodkv) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 63cbef73c8..ef25939ebd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -114,31 +114,10 @@ public class TokenCache { throw new IOException( "Can't get Master Kerberos principal for use as renewer"); } - boolean readFile = true; + mergeBinaryTokens(credentials, conf); String fsName = fs.getCanonicalServiceName(); if (TokenCache.getDelegationToken(credentials, fsName) == null) { - //TODO: Need to come up with a better place to put - //this block of code to do with reading the file - if (readFile) { - readFile = false; - String binaryTokenFilename = - conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); - if (binaryTokenFilename != null) { - Credentials binary; - try { - binary = Credentials.readTokenStorageFile( - new Path("file:///" + binaryTokenFilename), conf); - } catch (IOException e) { - throw new RuntimeException(e); - } - credentials.addAll(binary); - } - if (TokenCache.getDelegationToken(credentials, fsName) != null) { - LOG.debug("DT for " + fsName + " is already present"); - return; - } - } List> tokens = fs.getDelegationTokens(delegTokenRenewer, credentials); if (tokens != null) { @@ -161,6 +140,22 @@ public class TokenCache { } } + private static void mergeBinaryTokens(Credentials creds, Configuration conf) { + String binaryTokenFilename = + conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY); + if (binaryTokenFilename != null) { + Credentials binary; + try { + binary = Credentials.readTokenStorageFile( + new Path("file:///" + binaryTokenFilename), conf); + } catch (IOException e) { + throw new RuntimeException(e); + } + // supplement existing tokens with the tokens in the binary file + creds.mergeAll(binary); + } + } + /** * file name used on HDFS for generated job token */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 5efc33ddc4..b0e9350dbe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -21,23 +21,27 @@ package org.apache.hadoop.mapreduce.security; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.URI; import java.util.LinkedList; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Master; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; @@ -162,6 +166,91 @@ public class TestTokenCache { return mockFs; } + @Test + @SuppressWarnings("deprecation") + public void testBinaryCredentials() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + String renewer = Master.getMasterPrincipal(conf); + + Path TEST_ROOT_DIR = + new Path(System.getProperty("test.build.data","test/build/data")); + // ick, but need fq path minus file:/ + String binaryTokenFile = FileSystem.getLocal(conf).makeQualified( + new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath(); + + FileSystem fs1 = createFileSystemForService("service1"); + FileSystem fs2 = createFileSystemForService("service2"); + FileSystem fs3 = createFileSystemForService("service3"); + + // get the tokens for fs1 & fs2 and write out to binary creds file + Credentials creds = new Credentials(); + Token token1 = fs1.getDelegationToken(renewer); + Token token2 = fs2.getDelegationToken(renewer); + creds.addToken(token1.getService(), token1); + creds.addToken(token2.getService(), token2); + // wait to set, else the obtain tokens call above will fail with FNF + conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, binaryTokenFile); + creds.writeTokenStorageFile(new Path(binaryTokenFile), conf); + + // re-init creds and add a newer token for fs1 + creds = new Credentials(); + Token newerToken1 = fs1.getDelegationToken(renewer); + assertFalse(newerToken1.equals(token1)); + creds.addToken(newerToken1.getService(), newerToken1); + checkToken(creds, newerToken1); + + // get token for fs1, see that fs2's token was loaded + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + checkToken(creds, newerToken1, token2); + + // get token for fs2, nothing should change since already present + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + checkToken(creds, newerToken1, token2); + + // get token for fs3, should only add token for fs3 + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + Token token3 = creds.getToken(new Text(fs3.getCanonicalServiceName())); + assertTrue(token3 != null); + checkToken(creds, newerToken1, token2, token3); + + // be paranoid, check one last time that nothing changes + TokenCache.obtainTokensForNamenodesInternal(fs1, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs2, creds, conf); + TokenCache.obtainTokensForNamenodesInternal(fs3, creds, conf); + checkToken(creds, newerToken1, token2, token3); + } + + private void checkToken(Credentials creds, Token ... tokens) { + assertEquals(tokens.length, creds.getAllTokens().size()); + for (Token token : tokens) { + Token credsToken = creds.getToken(token.getService()); + assertTrue(credsToken != null); + assertEquals(token, credsToken); + } + } + + @SuppressWarnings("deprecation") + private FileSystem createFileSystemForService(final String service) + throws IOException { + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.getCanonicalServiceName()).thenReturn(service); + when(mockFs.getDelegationToken(any(String.class))).thenAnswer( + new Answer>() { + int unique = 0; + @Override + public Token answer(InvocationOnMock invocation) throws Throwable { + Token token = new Token(); + token.setService(new Text(service)); + // use unique value so when we restore from token storage, we can + // tell if it's really the same token + token.setKind(new Text("token" + unique++)); + return token; + } + }); + return mockFs; + } + @Test public void testCleanUpTokenReferral() throws Exception { Configuration conf = new Configuration(); From eb80104c19073ebf8913263cea59128b7ae0041b Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 16 Feb 2012 18:13:23 +0000 Subject: [PATCH 5/5] HADOOP-8082 add hadoop-client and hadoop-minicluster to the dependency-management section. (tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1245110 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ hadoop-project/pom.xml | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index bbaee1f50b..85fc976c66 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -193,6 +193,9 @@ Release 0.23.2 - UNRELEASED HADOOP-8074. Small bug in hadoop error message for unknown commands. (Colin Patrick McCabe via eli) + HADOOP-8082 add hadoop-client and hadoop-minicluster to the + dependency-management section. (tucu) + Release 0.23.1 - 2012-02-08 INCOMPATIBLE CHANGES diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e4bd5044bf..6d19eca221 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -239,6 +239,17 @@ ${project.version} + + org.apache.hadoop + hadoop-client + ${project.version} + + + org.apache.hadoop + hadoop-minicluster + ${project.version} + + com.google.guava guava