From 91a75ad34f9c306d4a750b8e9b6334cf077a843e Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 28 Nov 2011 20:47:38 +0000 Subject: [PATCH 01/11] Fix WebHDFS capitalization in CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207585 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 50 ++++++++++----------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 30347fdf53..6dec35c841 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -98,7 +98,7 @@ Trunk (unreleased changes) HDFS-46. Change default namespace quota of root directory from Integer.MAX_VALUE to Long.MAX_VALUE. (Uma Maheswara Rao G via szetszwo) - HDFS-2373. Commands using webhdfs and hftp print unnecessary debug + HDFS-2373. Commands using WebHDFS and hftp print unnecessary debug info on the console with security enabled. (Arpit Gupta via suresh) HDFS-2349. Corruption detected during block transfers between DNs @@ -124,7 +124,7 @@ Release 0.23.1 - UNRELEASED NEW FEATURES - HDFS-2316. [umbrella] webhdfs: a complete FileSystem implementation for + HDFS-2316. [umbrella] WebHDFS: a complete FileSystem implementation for accessing HDFS over HTTP (szetszwo) IMPROVEMENTS @@ -500,24 +500,24 @@ Release 0.23.0 - 2011-11-01 HDFS-2284. Add a new FileSystem, webhdfs://, for supporting write Http access to HDFS. (szetszwo) - HDFS-2317. Support read access to HDFS in webhdfs. (szetszwo) + HDFS-2317. Support read access to HDFS in WebHDFS. (szetszwo) - HDFS-2338. Add configuration option to enable/disable webhdfs. + HDFS-2338. Add configuration option to enable/disable WebHDFS. (jitendra via szetszwo) - HDFS-2318. Provide authentication to webhdfs using SPNEGO and delegation + HDFS-2318. Provide authentication to WebHDFS using SPNEGO and delegation tokens. (szetszwo) - HDFS-2340. Support getFileBlockLocations and getDelegationToken in webhdfs. + HDFS-2340. Support getFileBlockLocations and getDelegationToken in WebHDFS. (szetszwo) - HDFS-2348. Support getContentSummary and getFileChecksum in webhdfs. + HDFS-2348. Support getContentSummary and getFileChecksum in WebHDFS. (szetszwo) - HDFS-2385. Support renew and cancel delegation tokens in webhdfs. + HDFS-2385. Support renew and cancel delegation tokens in WebHDFS. (szetszwo) - HDFS-2539. Support doAs and GETHOMEDIRECTORY in webhdfs. + HDFS-2539. Support doAs and GETHOMEDIRECTORY in WebHDFS. (szetszwo) IMPROVEMENTS @@ -915,19 +915,19 @@ Release 0.23.0 - 2011-11-01 HDFS-2485. Improve code layout and constants in UnderReplicatedBlocks (stevel) - HDFS-2356. Support case insensitive query parameter names in webhdfs. + HDFS-2356. Support case insensitive query parameter names in WebHDFS. (szetszwo) HDFS-2368. Move SPNEGO conf properties from hdfs-default.xml to hdfs-site.xml. (szetszwo) - HDFS-2395. Add a root element in the JSON responses of webhdfs. + HDFS-2395. Add a root element in the JSON responses of WebHDFS. (szetszwo) - HDFS-2427. Change the default permission in webhdfs to 755 and add range + HDFS-2427. Change the default permission in WebHDFS to 755 and add range check/validation for all parameters. (szetszwo) - HDFS-2501. Add version prefix and root methods to webhdfs. (szetszwo) + HDFS-2501. Add version prefix and root methods to WebHDFS. (szetszwo) HDFS-1869. mkdirs should use the supplied permission for all of the created directories. (Daryn Sharp via szetszwo) @@ -1310,30 +1310,30 @@ Release 0.23.0 - 2011-11-01 HDFS-2409. _HOST in dfs.web.authentication.kerberos.principal. (jitendra) - HDFS-2404. webhdfs liststatus json response is not correct. (suresh) + HDFS-2404. WebHDFS liststatus json response is not correct. (suresh) HDFS-2441. Remove the Content-Type set by HttpServer.QuotingInputFilter in - webhdfs responses. (szetszwo) + WebHDFS responses. (szetszwo) HDFS-2428. Convert com.sun.jersey.api.ParamException$QueryParamException - to IllegalArgumentException and response it as http BAD_REQUEST in webhdfs. + to IllegalArgumentException and response it as http BAD_REQUEST in WebHDFS. (szetszwo) HDFS-2424. Added a root element "HdfsFileStatuses" for the response - of webhdfs listStatus. (szetszwo) + of WebHDFS listStatus. (szetszwo) MAPREDUCE-2764. Fix renewal of dfs delegation tokens. (Owen via jitendra) - HDFS-2439. Fix NullPointerException in webhdfs when opening a non-existing + HDFS-2439. Fix NullPointerException in WebHDFS when opening a non-existing file or creating a file without specifying the replication parameter. (szetszwo) - HDFS-2453. Fix http response code for partial content in webhdfs, added + HDFS-2453. Fix http response code for partial content in WebHDFS, added getDefaultBlockSize() and getDefaultReplication() in WebHdfsFileSystem and cleared content type in ExceptionHandler. (szetszwo) HDFS-2411. The the auth to local mappings are not being respected, with - webhdfs enabled. (jitendra) + WebHDFS enabled. (jitendra) HDFS-2494. Close the streams and DFSClient in DatanodeWebHdfsMethods. (Uma Maheswara Rao G via szetszwo) @@ -1341,7 +1341,7 @@ Release 0.23.0 - 2011-11-01 HDFS-2298. Fix TestDfsOverAvroRpc by changing ClientProtocol to not include multiple methods of the same name. (cutting) - HDFS-2432. Webhdfs: response FORBIDDEN when setReplication on non-files; + HDFS-2432. WebHDFS: response FORBIDDEN when setReplication on non-files; clear umask before creating a flie; throw IllegalArgumentException if setOwner with both owner and group empty; throw FileNotFoundException if getFileStatus on non-existing files; fix bugs in getBlockLocations; and @@ -1350,19 +1350,19 @@ Release 0.23.0 - 2011-11-01 HDFS-2065. Add null checks in DFSClient.getFileChecksum(..). (Uma Maheswara Rao G via szetszwo) - HDFS-2416. distcp with a webhdfs uri on a secure cluster fails. (jitendra) + HDFS-2416. distcp with a WebHDFS uri on a secure cluster fails. (jitendra) - HDFS-2527. WebHdfs: remove the use of "Range" header in Open; use ugi + HDFS-2527. WebHDFS: remove the use of "Range" header in Open; use ugi username if renewer parameter is null in GetDelegationToken; response OK when setting replication for non-files; rename GETFILEBLOCKLOCATIONS to GET_BLOCK_LOCATIONS and state that it is a private unstable API; replace isDirectory and isSymlink with enum {FILE, DIRECTORY, SYMLINK} in HdfsFileStatus JSON object. (szetszwo) - HDFS-2528. Webhdfs: set delegation kind to WEBHDFS and add a HDFS token + HDFS-2528. WebHDFS: set delegation kind to WEBHDFS and add a HDFS token when http requests are redirected to datanode. (szetszwo) - HDFS-2540. Webhdfs: change "Expect: 100-continue" to two-step write; change + HDFS-2540. WebHDFS: change "Expect: 100-continue" to two-step write; change "HdfsFileStatus" and "localName" respectively to "FileStatus" and "pathSuffix" in JSON response. (szetszwo) From 18d99428555f32c5852827e0edaa64a7413aa152 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 28 Nov 2011 21:18:25 +0000 Subject: [PATCH 02/11] HADOOP-7863: apply HADOOP-7424 to 0.23.1 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207614 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 54829b2a9f..9a9cc21c4f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -54,9 +54,6 @@ Trunk (unreleased changes) HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya) - HADOOP-7424. Log an error if the topology script doesn't handle multiple args. - (Uma Maheswara Rao G via eli) - HADOOP-7792. Add verifyToken method to AbstractDelegationTokenSecretManager. (jitendra) @@ -126,6 +123,9 @@ Release 0.23.1 - Unreleased HADOOP-7858. Drop some info logging to DEBUG level in IPC, metrics, and HTTP. (todd via eli) + HADOOP-7424. Log an error if the topology script doesn't handle multiple args. + (Uma Maheswara Rao G via eli) + OPTIMIZATIONS BUG FIXES From 3c602e0ee056a4bf1f374da0ab65d247794b7875 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 28 Nov 2011 21:34:03 +0000 Subject: [PATCH 03/11] Apply HADOOP-7777 to 0.23.1 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207629 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9a9cc21c4f..8ddfb2c4bb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -6,9 +6,6 @@ Trunk (unreleased changes) NEW FEATURES - HADOOP-7777 Implement a base class for DNSToSwitchMapping implementations - that can offer extra topology information. (stevel) - IMPROVEMENTS HADOOP-7595. Upgrade dependency to Avro 1.5.3. (Alejandro Abdelnur via atm) @@ -111,7 +108,10 @@ Release 0.23.1 - Unreleased INCOMPATIBLE CHANGES - NEW FEATURES + NEW FEATURES + + HADOOP-7777 Implement a base class for DNSToSwitchMapping implementations + that can offer extra topology information. (stevel) IMPROVEMENTS From 49c44117f3ec02f629c4472de3021c2e9568519c Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Mon, 28 Nov 2011 22:51:45 +0000 Subject: [PATCH 04/11] Fix typo in HDFS-993 commit. Was -933. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207676 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6dec35c841..7ac1ec93e5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2590,7 +2590,7 @@ Release 0.21.0 - 2010-08-13 HDFS-1012. hdfsproxy: Support for fully qualified HDFS path in addition to simple unqualified path. (Srikanth Sundarrajan via szetszwo) - HDFS-933. Namenode should issue a delegation token only for kerberos + HDFS-993. Namenode should issue a delegation token only for kerberos authenticated clients.(jnp via boryas) HDFS-1087. Modify audit log to use a StringBuilder rather than a Formatter. From b32543d9ae8146e931a2ced6c0bd2727b4a57154 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 28 Nov 2011 23:20:16 +0000 Subject: [PATCH 05/11] HDFS-2596. TestDirectoryScanner doesn't test parallel scans. (eli) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207694 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hadoop/hdfs/server/datanode/TestDirectoryScanner.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7ac1ec93e5..9ab43c29f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -190,6 +190,8 @@ Release 0.23.1 - UNRELEASED HDFS-2588. hdfs jsp pages missing DOCTYPE. (Dave Vronay via mattf) HDFS-2590. Fix the missing links in the WebHDFS forrest doc. (szetszwo) + + HDFS-2596. TestDirectoryScanner doesn't test parallel scans. (eli) Release 0.23.0 - 2011-11-01 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 85eb8fe4a7..7410d42679 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -218,7 +218,7 @@ private void scan(long totalBlocks, int diffsize, long missingMetaFile, long mis public void testDirectoryScanner() throws Exception { // Run the test with and without parallel scanning - for (int parallelism = 1; parallelism < 2; parallelism++) { + for (int parallelism = 1; parallelism < 3; parallelism++) { runTest(parallelism); } } From 32935d3e6715f77df1ba10bbceab64f8e81166ab Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Mon, 28 Nov 2011 23:40:24 +0000 Subject: [PATCH 06/11] Fix CHANGES.txt for HDFS-2346 in trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207700 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9ab43c29f9..9aca401a66 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2124,6 +2124,9 @@ Release 0.21.1 - Unreleased HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() part 2 fails in two different ways. (Matt Foley via eli) + HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon + execution order of test methods (Laxman, Uma Maheswara Rao G via shv) + Release 0.21.0 - 2010-08-13 INCOMPATIBLE CHANGES From 82da8fb4901c8b389b79ae29b1e8f84f5cc10e3d Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Mon, 28 Nov 2011 23:53:49 +0000 Subject: [PATCH 07/11] Fix CHANGES.txt. Some changes have been committed under 0.21.1 section rather than 0.22.0. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207706 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9aca401a66..a6bff0bf31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2065,6 +2065,15 @@ Release 0.21.1 - Unreleased HDFS-1823. start-dfs.sh script fails if HADOOP_HOME is not set. (tomwhite via eli) + HDFS-1786. Some cli test cases expect a "null" message + (Uma Maheswara Rao G via todd) + + HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() + part 2 fails in two different ways. (Matt Foley via eli) + + HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon + execution order of test methods (Laxman, Uma Maheswara Rao G via shv) + Release 0.21.1 - Unreleased HDFS-1411. Correct backup node startup command in hdfs user guide. @@ -2114,19 +2123,9 @@ Release 0.21.1 - Unreleased block placement and checkpoint/backup node features. (Joe Crobak via szetszwo) - HDFS-1596. Replace fs.checkpoint.* with dfs.namenode.checkpoint.* in documentations. (Harsh J Chouraria via szetszwo) - HDFS-1786. Some cli test cases expect a "null" message - (Uma Maheswara Rao G via todd) - - HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() - part 2 fails in two different ways. (Matt Foley via eli) - - HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon - execution order of test methods (Laxman, Uma Maheswara Rao G via shv) - Release 0.21.0 - 2010-08-13 INCOMPATIBLE CHANGES From 3083d7d7db77446b04bb0f195f7924477df417b8 Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Tue, 29 Nov 2011 00:00:20 +0000 Subject: [PATCH 08/11] Fix CHANGES.txt. Moving changes from 0.21.1 section to 0.22.0. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207709 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 22 ++++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a6bff0bf31..043c1ff23c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -80,8 +80,6 @@ Trunk (unreleased changes) (Tomasz Nykiel via todd) BUG FIXES - HDFS-2287. TestParallelRead has a small off-by-one bug. (todd) - HDFS-2299. TestOfflineEditsViewer is failing on trunk. (Uma Maheswara Rao G via atm) HDFS-2310. TestBackupNode fails since HADOOP-7524 went in. @@ -1966,10 +1964,19 @@ Release 0.22.0 - Unreleased HDFS-2002. Incorrect computation of needed blocks in getTurnOffTip(). (Plamen Jeliazkov via shv) + HDFS-2573. TestFiDataXceiverServer is failing, not testing OOME (cos) + HDFS-2514. Link resolution bug for intermediate symlinks with relative targets. (eli) - HDFS-2573. TestFiDataXceiverServer is failing, not testing OOME (cos) + HDFS-1786. Some cli test cases expect a "null" message + (Uma Maheswara Rao G via todd) + + HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() + part 2 fails in two different ways. (Matt Foley via eli) + + HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon + execution order of test methods (Laxman, Uma Maheswara Rao G via shv) Release 0.21.1 - Unreleased @@ -2065,15 +2072,6 @@ Release 0.21.1 - Unreleased HDFS-1823. start-dfs.sh script fails if HADOOP_HOME is not set. (tomwhite via eli) - HDFS-1786. Some cli test cases expect a "null" message - (Uma Maheswara Rao G via todd) - - HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() - part 2 fails in two different ways. (Matt Foley via eli) - - HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon - execution order of test methods (Laxman, Uma Maheswara Rao G via shv) - Release 0.21.1 - Unreleased HDFS-1411. Correct backup node startup command in hdfs user guide. From 68f56ca906a845417705beef53bfb0f86117c260 Mon Sep 17 00:00:00 2001 From: Konstantin Shvachko Date: Tue, 29 Nov 2011 00:03:49 +0000 Subject: [PATCH 09/11] HDFS-2287. Fix second assert. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207710 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../test/java/org/apache/hadoop/hdfs/TestParallelRead.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 043c1ff23c..6e6c714acc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1978,6 +1978,8 @@ Release 0.22.0 - Unreleased HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon execution order of test methods (Laxman, Uma Maheswara Rao G via shv) + HDFS-2287. TestParallelRead has a small off-by-one bug. (todd) + Release 0.21.1 - Unreleased HDFS-1466. TestFcHdfsSymlink relies on /tmp/test not existing. (eli) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java index c5aec722fc..bbd001236f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java @@ -156,8 +156,8 @@ private void read(int start, int len) throws Exception { */ private void pRead(int start, int len) throws Exception { assertTrue( - "Bad args: " + start + " + " + len + " should be < " + fileSize, - start + len < fileSize); + "Bad args: " + start + " + " + len + " should be <= " + fileSize, + start + len <= fileSize); DFSInputStream dis = testInfo.dis; byte buf[] = new byte[len]; From 784a4a308437b5d98605a79bbd339426d8145ad8 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 29 Nov 2011 00:10:56 +0000 Subject: [PATCH 10/11] MAPREDUCE-3045. Fixed UI filters to not filter on hidden title-numeric sort fields. Contributed by Jonathan Eagles git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207714 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/webapps/static/yarn.dt.plugins.js | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 076004cd7d..ab117274f7 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -119,6 +119,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3413. RM web ui applications not sorted in any order by default. (Jonathan Eagles via mahadev) + MAPREDUCE-3045. Fixed UI filters to not filter on hidden title-numeric + sort fields. (Jonathan Eagles via sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js index 4d00706939..86661acb49 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js @@ -2,6 +2,11 @@ if (!jQuery.fn.dataTableExt.fnVersionCheck("1.7.5")) { alert("These plugins requires dataTables 1.7.5+"); } +// don't filter on hidden html elements for an sType of title-numeric +$.fn.dataTableExt.ofnSearch['title-numeric'] = function ( sData ) { + return sData.replace(/\n/g," ").replace( /<.*?>/g, "" ); +} + // 'title-numeric' sort type jQuery.fn.dataTableExt.oSort['title-numeric-asc'] = function(a,b) { var x = a.match(/title=["']?(-?\d+\.?\d*)/)[1]; From be9c46a4475f18ef33796ee63d73821c07c49fb5 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Tue, 29 Nov 2011 05:07:47 +0000 Subject: [PATCH 11/11] MAPREDUCE-3433. Finding counters by legacy group name returns empty counters. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1207755 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/counters/AbstractCounters.java | 11 ++++---- .../apache/hadoop/mapred/TestCounters.java | 28 +++++++++++++++++-- hadoop-tools/hadoop-streaming/pom.xml | 2 +- 4 files changed, 36 insertions(+), 8 deletions(-) rename hadoop-mapreduce-project/{src/test/mapred => hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java}/org/apache/hadoop/mapred/TestCounters.java (77%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ab117274f7..43bcf76ddc 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -189,6 +189,9 @@ Release 0.23.1 - Unreleased MAPREDUCE-3265. Removed debug logs during job submission to LOG.debug to cut down noise. (acmurthy) + MAPREDUCE-3433. Finding counters by legacy group name returns empty + counters. (tomwhite) + Release 0.23.0 - 2011-11-01 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java index 7102467a10..d7c12375b1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -194,15 +194,16 @@ public Iterator iterator() { * @return the group */ public synchronized G getGroup(String groupName) { - boolean isFGroup = isFrameworkGroup(groupName); - G group = isFGroup ? fgroups.get(groupName) : groups.get(groupName); + String newGroupName = filterGroupName(groupName); + boolean isFGroup = isFrameworkGroup(newGroupName); + G group = isFGroup ? fgroups.get(newGroupName) : groups.get(newGroupName); if (group == null) { - group = groupFactory.newGroup(filterGroupName(groupName), limits); + group = groupFactory.newGroup(newGroupName, limits); if (isFGroup) { - fgroups.put(groupName, group); + fgroups.put(newGroupName, group); } else { limits.checkGroups(groups.size() + 1); - groups.put(groupName, group); + groups.put(newGroupName, group); } } return group; diff --git a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java similarity index 77% rename from hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java index 41afa15f10..4ce1c8159d 100644 --- a/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java @@ -17,16 +17,19 @@ */ package org.apache.hadoop.mapred; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; + import java.io.IOException; import java.text.ParseException; +import org.apache.hadoop.mapreduce.JobCounter; import org.apache.hadoop.mapreduce.TaskCounter; +import org.junit.Test; /** * TestCounters checks the sanity and recoverability of {@code Counters} */ -public class TestCounters extends TestCase { +public class TestCounters { enum myCounters {TEST1, TEST2}; private static final long MAX_VALUE = 10; @@ -69,6 +72,7 @@ private void testCounter(Counters counter) throws ParseException { counter.hashCode(), recoveredCounter.hashCode()); } + @Test public void testCounters() throws IOException { Enum[] keysWithResource = {TaskCounter.MAP_INPUT_RECORDS, TaskCounter.MAP_OUTPUT_BYTES}; @@ -92,6 +96,26 @@ public void testCounters() throws IOException { } } + @SuppressWarnings("deprecation") + @Test + public void testLegacyNames() { + Counters counters = new Counters(); + counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1); + counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1); + + assertEquals("New name", 1, counters.findCounter( + TaskCounter.class.getName(), "MAP_INPUT_RECORDS").getValue()); + assertEquals("Legacy name", 1, counters.findCounter( + "org.apache.hadoop.mapred.Task$Counter", + "MAP_INPUT_RECORDS").getValue()); + + assertEquals("New name", 1, counters.findCounter( + JobCounter.class.getName(), "DATA_LOCAL_MAPS").getValue()); + assertEquals("Legacy name", 1, counters.findCounter( + "org.apache.hadoop.mapred.JobInProgress$Counter", + "DATA_LOCAL_MAPS").getValue()); + } + public static void main(String[] args) throws IOException { new TestCounters().testCounters(); } diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 3e6ae3f438..50a120531c 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -29,7 +29,7 @@ ${project.build.directory}/log - %regex[.*(TestStreamingBadRecords|TestStreamingCombiner|TestStreamingStatus|TestUlimit).*] + %regex[.*(TestStreamingBadRecords|TestStreamingStatus|TestUlimit).*]