From ef0118b91e384b9a6d96c2ae64480d9acf5aa6fb Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Sat, 9 Jun 2018 15:33:30 +0100 Subject: [PATCH] HADOOP-15520. Add tests for various org.apache.hadoop.util classes. Contributed by Arash Nabili --- .../util/TestCloseableReferenceCount.java | 91 +++++++++ .../hadoop/util/TestIntrusiveCollection.java | 193 ++++++++++++++++++ .../hadoop/util/TestLimitInputStream.java | 74 +++++++ .../org/apache/hadoop/util/TestShell.java | 8 + .../apache/hadoop/util/TestStringUtils.java | 27 +++ .../hadoop/util/TestUTF8ByteArrayUtils.java | 57 ++++++ 6 files changed, 450 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java new file mode 100644 index 0000000000..31e1899421 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestCloseableReferenceCount.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.nio.channels.ClosedChannelException; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestCloseableReferenceCount extends HadoopTestBase { + @Test + public void testReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testUnreference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.reference(); + assertFalse("New reference count should not equal STATUS_CLOSED_MASK", + clr.unreference()); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testUnreferenceCheckClosed() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.reference(); + clr.unreferenceCheckClosed(); + assertEquals("Incorrect reference count", 1, clr.getReferenceCount()); + } + + @Test + public void testSetClosed() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + assertTrue("Reference count should be open", clr.isOpen()); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + } + + @Test(expected = ClosedChannelException.class) + public void testReferenceClosedReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.reference(); + } + + @Test(expected = ClosedChannelException.class) + public void testUnreferenceClosedReference() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + clr.reference(); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.unreferenceCheckClosed(); + } + + @Test(expected = ClosedChannelException.class) + public void testDoubleClose() throws ClosedChannelException { + CloseableReferenceCount clr = new CloseableReferenceCount(); + assertTrue("Reference count should be open", clr.isOpen()); + clr.setClosed(); + assertFalse("Reference count should be closed", clr.isOpen()); + clr.setClosed(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java new file mode 100644 index 0000000000..03bbf7b12f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestIntrusiveCollection.java @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * 
+ * Story 1
+ * As a software developer,
+ *  I want to use the IntrusiveCollection class;
+ * So that I can save on memory usage during execution.
+ *
+ */ +package org.apache.hadoop.util; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; +import org.apache.hadoop.util.IntrusiveCollection.Element; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class TestIntrusiveCollection extends HadoopTestBase { + static class SimpleElement implements IntrusiveCollection.Element { + private Map, Element> + prevMap, nextMap; + private Map, Boolean> isMemberMap; + + public SimpleElement() { + prevMap = new HashMap<>(); + nextMap = new HashMap<>(); + isMemberMap = new HashMap<>(); + } + + @Override + public void insertInternal(IntrusiveCollection list, + Element prev, Element next) { + isMemberMap.put(list, true); + prevMap.put(list, prev); + nextMap.put(list, next); + } + + @Override + public void setPrev(IntrusiveCollection list, + Element prev) { + prevMap.put(list, prev); + } + + @Override + public void setNext(IntrusiveCollection list, + Element next) { + nextMap.put(list, next); + } + + @Override + public void removeInternal(IntrusiveCollection list) { + prevMap.remove(list); + nextMap.remove(list); + isMemberMap.remove(list); + } + + @Override + public Element getPrev(IntrusiveCollection list) { + return prevMap.getOrDefault(list, null); + } + + @Override + public Element getNext(IntrusiveCollection list) { + return nextMap.getOrDefault(list, null); + } + + @Override + public boolean isInList(IntrusiveCollection list) { + return isMemberMap.getOrDefault(list, false); + } + } + + /** + * 
+   * Scenario S1.1: Adding an element
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection is empty
+   * When    I insert an element
+   * Then    the IntrusiveCollection contains the newly added element.
+   *
+ */ + @Test + public void testShouldAddElement() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + + SimpleElement element = new SimpleElement(); + intrusiveCollection.add(element); + + assertFalse("Collection should not be empty", + intrusiveCollection.isEmpty()); + assertTrue("Collection should contain added element", + intrusiveCollection.contains(element)); + } + + /** + * 
+   * Scenario S1.2: Removing an element
+   * Given  an IntrusiveCollection has been created
+   *  and    the InstrusiveCollection contains a single element
+   * When    I remove the element
+   * Then    the IntrusiveCollection is empty.
+   *
+ */ + @Test + public void testShouldRemoveElement() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + SimpleElement element = new SimpleElement(); + intrusiveCollection.add(element); + + intrusiveCollection.remove(element); + + assertTrue("Collection should be empty", intrusiveCollection.isEmpty()); + assertFalse("Collection should not contain removed element", + intrusiveCollection.contains(element)); + } + + /** + * 
+   * Scenario S1.3: Removing all elements
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection contains multiple elements
+   * When    I remove all elements
+   * Then    the IntrusiveCollection is empty.
+   *
+ */ + @Test + public void testShouldRemoveAllElements() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + intrusiveCollection.add(new SimpleElement()); + intrusiveCollection.add(new SimpleElement()); + intrusiveCollection.add(new SimpleElement()); + + intrusiveCollection.clear(); + + assertTrue("Collection should be empty", intrusiveCollection.isEmpty()); + } + + /** + * 
+   * Scenario S1.4: Iterating through elements
+   * Given  an IntrusiveCollection has been created
+   *  and    the IntrusiveCollection contains multiple elements
+   * When    I iterate through the IntrusiveCollection
+   * Then    I get each element in the collection, successively.
+   *
+ */ + @Test + public void testIterateShouldReturnAllElements() { + IntrusiveCollection intrusiveCollection = + new IntrusiveCollection<>(); + SimpleElement elem1 = new SimpleElement(); + SimpleElement elem2 = new SimpleElement(); + SimpleElement elem3 = new SimpleElement(); + intrusiveCollection.add(elem1); + intrusiveCollection.add(elem2); + intrusiveCollection.add(elem3); + + Iterator iterator = intrusiveCollection.iterator(); + + assertEquals("First element returned is incorrect", elem1, iterator.next()); + assertEquals("Second element returned is incorrect", elem2, + iterator.next()); + assertEquals("Third element returned is incorrect", elem3, iterator.next()); + assertFalse("Iterator should not have next element", iterator.hasNext()); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java new file mode 100644 index 0000000000..368fa37b7b --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLimitInputStream.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Random; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +public class TestLimitInputStream extends HadoopTestBase { + static class RandomInputStream extends InputStream { + private Random rn = new Random(0); + + @Override + public int read() { return rn.nextInt(); } + } + + @Test + public void testRead() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 0)) { + assertEquals("Reading byte after reaching limit should return -1", -1, + limitInputStream.read()); + } + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 4)) { + assertEquals("Incorrect byte returned", new Random(0).nextInt(), + limitInputStream.read()); + } + } + + @Test(expected = IOException.class) + public void testResetWithoutMark() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 128)) { + limitInputStream.reset(); + } + } + + @Test + public void testReadBytes() throws IOException { + try (LimitInputStream limitInputStream = + new LimitInputStream(new RandomInputStream(), 128)) { + Random r = new Random(0); + byte[] data = new byte[4]; + byte[] expected = { (byte) r.nextInt(), (byte) r.nextInt(), + (byte) r.nextInt(), (byte) r.nextInt() }; + limitInputStream.read(data, 0, 4); + assertArrayEquals("Incorrect bytes returned", expected, data); + } + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index d0ebc2b83f..578d267114 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -27,6 +27,7 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; @@ -38,6 +39,8 @@ import org.apache.hadoop.test.GenericTestUtils; import static org.apache.hadoop.util.Shell.*; +import static org.junit.Assume.assumeTrue; + import org.junit.Assume; import org.junit.Before; import org.junit.Rule; @@ -528,4 +531,9 @@ public Boolean get() { public void testIsJavaVersionAtLeast() { assertTrue(Shell.isJavaVersionAtLeast(8)); } + + @Test + public void testIsBashSupported() throws InterruptedIOException { + assumeTrue("Bash is not supported", Shell.checkIsBashSupported()); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 96a6482363..3fdc1bb8f8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; import java.util.ArrayList; @@ -476,6 +477,32 @@ public void run() { executorService.awaitTermination(50, TimeUnit.SECONDS); } + @Test + public void testFormatTimeSortable() { + long timeDiff = 523452311; + String timeDiffStr = "99hrs, 59mins, 59sec"; + + assertEquals("Incorrect time diff string returned", timeDiffStr, + StringUtils.formatTimeSortable(timeDiff)); + } + + @Test + public void testIsAlpha() { + assertTrue("Reported hello as non-alpha string", + StringUtils.isAlpha("hello")); + assertFalse("Reported hello1 as alpha string", + StringUtils.isAlpha("hello1")); + } + + @Test + public void testEscapeHTML() { + String htmlStr = "

Hello. How are you?

"; + String escapedStr = "<p>Hello. How are you?</p>"; + + assertEquals("Incorrect escaped HTML string returned", + escapedStr, StringUtils.escapeHTML(htmlStr)); + } + // Benchmark for StringUtils split public static void main(String []args) { final String TO_SPLIT = "foo,bar,baz,blah,blah"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java new file mode 100644 index 0000000000..3aa549a4ca --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestUTF8ByteArrayUtils.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import org.junit.Test; + +import org.apache.hadoop.test.HadoopTestBase; + +import static org.junit.Assert.assertEquals; + +public class TestUTF8ByteArrayUtils extends HadoopTestBase { + @Test + public void testFindByte() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Character 'a' does not exist in string", -1, + UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'a')); + assertEquals("Did not find first occurrence of character 'o'", 4, + UTF8ByteArrayUtils.findByte(data, 0, data.length, (byte) 'o')); + } + + @Test + public void testFindBytes() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Did not find first occurrence of pattern 'ello'", 1, + UTF8ByteArrayUtils.findBytes(data, 0, data.length, "ello".getBytes())); + assertEquals( + "Substring starting at position 2 does not contain pattern 'ello'", -1, + UTF8ByteArrayUtils.findBytes(data, 2, data.length, "ello".getBytes())); + } + + @Test + public void testFindNthByte() { + byte[] data = "Hello, world!".getBytes(); + assertEquals("Did not find 2nd occurrence of character 'l'", 3, + UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 2)); + assertEquals("4th occurrence of character 'l' does not exist", -1, + UTF8ByteArrayUtils.findNthByte(data, 0, data.length, (byte) 'l', 4)); + assertEquals("Did not find 3rd occurrence of character 'l'", 10, + UTF8ByteArrayUtils.findNthByte(data, (byte) 'l', 3)); + } +}