From e4062ad027d84b4d7192ae77757a0373c4558390 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 20 May 2021 21:17:04 +0530 Subject: [PATCH] HADOOP-17115. Replace Guava Sets usage by Hadoop's own Sets in hadoop-common and hadoop-tools (#2985) Signed-off-by: Sean Busbey --- hadoop-common-project/hadoop-common/pom.xml | 32 ++ .../hadoop/metrics2/lib/MutableRates.java | 2 +- .../lib/MutableRatesWithAggregation.java | 2 +- .../apache/hadoop/util/JvmPauseMonitor.java | 2 - .../java/org/apache/hadoop/util/Sets.java | 377 ++++++++++++++++++ .../hadoop/crypto/key/TestValueQueue.java | 2 +- .../TestLoadBalancingKMSClientProvider.java | 2 +- .../apache/hadoop/test/GenericTestUtils.java | 2 +- hadoop-common-project/hadoop-kms/pom.xml | 32 ++ .../crypto/key/kms/server/KMSAudit.java | 2 +- hadoop-tools/hadoop-aws/pom.xml | 32 ++ .../auth/AssumedRoleCredentialProvider.java | 2 +- .../hadoop/fs/s3a/commit/staging/Paths.java | 4 +- .../fs/s3a/TestS3AAWSCredentialsProvider.java | 2 +- .../integration/ITestS3ACommitterMRJob.java | 2 +- .../commit/staging/TestStagingCommitter.java | 2 +- .../TestStagingPartitionedTaskCommit.java | 2 +- hadoop-tools/hadoop-distcp/pom.xml | 32 ++ .../org/apache/hadoop/tools/CopyListing.java | 2 +- .../dynamometer/TestDynamometerInfra.java | 2 +- hadoop-tools/hadoop-dynamometer/pom.xml | 32 ++ 21 files changed, 552 insertions(+), 17 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 0a2232dae9..e48f6e5294 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -634,6 +634,38 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-common provided Sets rather than Guava provided Sets + + org.apache.hadoop.thirdparty.com.google.common.collect.Sets + com.google.common.collect.Sets + + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java index c31c2e67f8..a83b68bab7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRates.java @@ -22,7 +22,7 @@ import java.util.Set; import static org.apache.hadoop.thirdparty.com.google.common.base.Preconditions.*; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java index 7795343de3..dc37f96f4f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java @@ -18,7 +18,7 @@ package org.apache.hadoop.metrics2.lib; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import java.lang.ref.WeakReference; import java.lang.reflect.Method; import java.util.Iterator; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java index 9c9953fb14..a792993493 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java @@ -29,10 +29,8 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java new file mode 100644 index 0000000000..bddcbeb21f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Sets.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import org.apache.hadoop.classification.InterfaceAudience; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Static utility methods pertaining to {@link Set} instances. + * This class is Hadoop's internal use alternative to Guava's Sets + * utility class. + * Javadocs for majority of APIs in this class are taken from Guava's Sets + * class from Guava release version 27.0-jre. + */ +@InterfaceAudience.Private +public final class Sets { + + private static final int MAX_POWER_OF_TWO = 1 << (Integer.SIZE - 2); + + private Sets() { + // empty + } + + /** + * Creates a mutable, initially empty {@code HashSet} instance. + * + *

Note: if mutability is not required, use ImmutableSet#of() + * instead. If {@code E} is an {@link Enum} type, use {@link EnumSet#noneOf} + * instead. Otherwise, strongly consider using a {@code LinkedHashSet} + * instead, at the cost of increased memory footprint, to get + * deterministic iteration behavior. + */ + public static HashSet newHashSet() { + return new HashSet(); + } + + /** + * Creates a mutable, empty {@code TreeSet} instance sorted by the + * natural sort ordering of its elements. + * + *

Note: if mutability is not required, use ImmutableSortedSet#of() + * instead. + * + * @return a new, empty {@code TreeSet} + */ + public static TreeSet newTreeSet() { + return new TreeSet(); + } + + /** + * Creates a mutable {@code HashSet} instance initially containing + * the given elements. + * + *

Note: if elements are non-null and won't be added or removed + * after this point, use ImmutableSet#of() or ImmutableSet#copyOf(Object[]) + * instead. If {@code E} is an {@link Enum} type, use + * {@link EnumSet#of(Enum, Enum[])} instead. Otherwise, strongly consider + * using a {@code LinkedHashSet} instead, at the cost of increased memory + * footprint, to get deterministic iteration behavior. + * + *

This method is just a small convenience, either for + * {@code newHashSet(}{@link Arrays#asList}{@code (...))}, or for creating an + * empty set then calling {@link Collections#addAll}. + */ + @SafeVarargs + public static HashSet newHashSet(E... elements) { + HashSet set = newHashSetWithExpectedSize(elements.length); + Collections.addAll(set, elements); + return set; + } + + /** + * Creates a mutable {@code HashSet} instance containing the given + * elements. A very thin convenience for creating an empty set then calling + * {@link Collection#addAll} or Iterables#addAll. + * + *

Note: if mutability is not required and the elements are + * non-null, use ImmutableSet#copyOf(Iterable) instead. (Or, change + * {@code elements} to be a FluentIterable and call {@code elements.toSet()}.) + * + *

Note: if {@code E} is an {@link Enum} type, use + * newEnumSet(Iterable, Class) instead. + */ + public static HashSet newHashSet(Iterable elements) { + return (elements instanceof Collection) + ? new HashSet(cast(elements)) + : newHashSet(elements.iterator()); + } + + /** + * Creates a mutable {@code TreeSet} instance containing the given + * elements sorted by their natural ordering. + * + *

Note: if mutability is not required, use + * ImmutableSortedSet#copyOf(Iterable) instead. + * + *

Note: If {@code elements} is a {@code SortedSet} with an + * explicit comparator, this method has different behavior than + * {@link TreeSet#TreeSet(SortedSet)}, which returns a {@code TreeSet} + * with that comparator. + * + *

Note for Java 7 and later: this method is now unnecessary and + * should be treated as deprecated. Instead, use the {@code TreeSet} + * constructor directly, taking advantage of the new + * "diamond" syntax. + * + *

This method is just a small convenience for creating an empty set and + * then calling Iterables#addAll. This method is not very useful and will + * likely be deprecated in the future. + * + * @param elements the elements that the set should contain + * @return a new {@code TreeSet} containing those elements (minus duplicates) + */ + public static TreeSet newTreeSet( + Iterable elements) { + TreeSet set = newTreeSet(); + addAll(set, elements); + return set; + } + + private static boolean addAll(TreeSet addTo, + Iterable elementsToAdd) { + if (elementsToAdd instanceof Collection) { + Collection c = cast(elementsToAdd); + return addTo.addAll(c); + } + if (elementsToAdd == null) { + throw new NullPointerException(); + } + return addAll(addTo, elementsToAdd.iterator()); + } + + /** + * Creates a mutable {@code HashSet} instance containing the given + * elements. A very thin convenience for creating an empty set and then + * calling Iterators#addAll. + * + *

Note: if mutability is not required and the elements are + * non-null, use ImmutableSet#copyOf(Iterator) instead. + * + *

Note: if {@code E} is an {@link Enum} type, you should create + * an {@link EnumSet} instead. + * + *

Overall, this method is not very useful and will likely be deprecated + * in the future. + */ + public static HashSet newHashSet(Iterator elements) { + HashSet set = newHashSet(); + addAll(set, elements); + return set; + } + + /** + * Returns a new hash set using the smallest initial table size that can hold + * {@code expectedSize} elements without resizing. Note that this is not what + * {@link HashSet#HashSet(int)} does, but it is what most users want and + * expect it to do. + * + *

This behavior can't be broadly guaranteed, but has been tested with + * OpenJDK 1.7 and 1.8. + * + * @param expectedSize the number of elements you expect to add to the + * returned set + * @return a new, empty hash set with enough capacity to hold + * {@code expectedSize} elements without resizing + * @throws IllegalArgumentException if {@code expectedSize} is negative + */ + public static HashSet newHashSetWithExpectedSize(int expectedSize) { + return new HashSet(capacity(expectedSize)); + } + + private static Collection cast(Iterable iterable) { + return (Collection) iterable; + } + + private static boolean addAll(Collection addTo, + Iterator iterator) { + if (addTo == null) { + throw new NullPointerException(); + } + if (iterator == null) { + throw new NullPointerException(); + } + boolean wasModified = false; + while (iterator.hasNext()) { + wasModified |= addTo.add(iterator.next()); + } + return wasModified; + } + + /** + * Returns the intersection of two sets as an unmodifiable set. + * The returned set contains all elements that are contained by both backing + * sets. + * + *

Results are undefined if {@code set1} and {@code set2} are sets based + * on different equivalence relations (as {@code HashSet}, {@code TreeSet}, + * and the keySet of an {@code IdentityHashMap} all are). + */ + public static Set intersection(final Set set1, + final Set set2) { + if (set1 == null) { + throw new NullPointerException("set1"); + } + if (set2 == null) { + throw new NullPointerException("set2"); + } + Set newSet = new HashSet<>(set1); + newSet.retainAll(set2); + return Collections.unmodifiableSet(newSet); + } + + /** + * Returns the union of two sets as an unmodifiable set. + * The returned set contains all elements that are contained in either + * backing set. + * + *

Results are undefined if {@code set1} and {@code set2} are sets + * based on different equivalence relations (as {@link HashSet}, + * {@link TreeSet}, and the {@link Map#keySet} of an + * {@code IdentityHashMap} all are). + */ + public static Set union( + final Set set1, final Set set2) { + if (set1 == null) { + throw new NullPointerException("set1"); + } + if (set2 == null) { + throw new NullPointerException("set2"); + } + Set newSet = new HashSet<>(set1); + newSet.addAll(set2); + return Collections.unmodifiableSet(newSet); + } + + /** + * Returns the difference of two sets as an unmodifiable set. + * The returned set contains all elements that are contained by {@code set1} + * and not contained by {@code set2}. + * + *

Results are undefined if {@code set1} and {@code set2} are sets based + * on different equivalence relations (as {@code HashSet}, {@code TreeSet}, + * and the keySet of an {@code IdentityHashMap} all are). + * + * This method is used to find difference for HashSets. For TreeSets with + * strict order requirement, recommended method is + * {@link #differenceInTreeSets(Set, Set)}. + */ + public static Set difference( + final Set set1, final Set set2) { + if (set1 == null) { + throw new NullPointerException("set1"); + } + if (set2 == null) { + throw new NullPointerException("set2"); + } + Set newSet = new HashSet<>(set1); + newSet.removeAll(set2); + return Collections.unmodifiableSet(newSet); + } + + /** + * Returns the difference of two sets as an unmodifiable set. + * The returned set contains all elements that are contained by {@code set1} + * and not contained by {@code set2}. + * + *

Results are undefined if {@code set1} and {@code set2} are sets based + * on different equivalence relations (as {@code HashSet}, {@code TreeSet}, + * and the keySet of an {@code IdentityHashMap} all are). + * + * This method is used to find difference for TreeSets. For HashSets, + * recommended method is {@link #difference(Set, Set)}. + */ + public static Set differenceInTreeSets( + final Set set1, final Set set2) { + if (set1 == null) { + throw new NullPointerException("set1"); + } + if (set2 == null) { + throw new NullPointerException("set2"); + } + Set newSet = new TreeSet<>(set1); + newSet.removeAll(set2); + return Collections.unmodifiableSet(newSet); + } + + /** + * Returns the symmetric difference of two sets as an unmodifiable set. + * The returned set contains all elements that are contained in either + * {@code set1} or {@code set2} but not in both. The iteration order of the + * returned set is undefined. + * + *

Results are undefined if {@code set1} and {@code set2} are sets based + * on different equivalence relations (as {@code HashSet}, {@code TreeSet}, + * and the keySet of an {@code IdentityHashMap} all are). + */ + public static Set symmetricDifference( + final Set set1, final Set set2) { + if (set1 == null) { + throw new NullPointerException("set1"); + } + if (set2 == null) { + throw new NullPointerException("set2"); + } + Set intersection = new HashSet<>(set1); + intersection.retainAll(set2); + Set symmetricDifference = new HashSet<>(set1); + symmetricDifference.addAll(set2); + symmetricDifference.removeAll(intersection); + return Collections.unmodifiableSet(symmetricDifference); + } + + /** + * Creates a thread-safe set backed by a hash map. The set is backed by a + * {@link ConcurrentHashMap} instance, and thus carries the same concurrency + * guarantees. + * + *

Unlike {@code HashSet}, this class does NOT allow {@code null} to be + * used as an element. The set is serializable. + * + * @return a new, empty thread-safe {@code Set} + */ + public static Set newConcurrentHashSet() { + return Collections.newSetFromMap(new ConcurrentHashMap()); + } + + /** + * Returns a capacity that is sufficient to keep the map from being resized + * as long as it grows no larger than expectedSize and the load factor + * is ≥ its default (0.75). + * The implementation of this method is adapted from Guava version 27.0-jre. + */ + private static int capacity(int expectedSize) { + if (expectedSize < 3) { + if (expectedSize < 0) { + throw new IllegalArgumentException( + "expectedSize cannot be negative but was: " + expectedSize); + } + return expectedSize + 1; + } + if (expectedSize < MAX_POWER_OF_TWO) { + // This is the calculation used in JDK8 to resize when a putAll + // happens; it seems to be the most conservative calculation we + // can make. 0.75 is the default load factor. + return (int) ((float) expectedSize / 0.75F + 1.0F); + } + return Integer.MAX_VALUE; // any large value + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java index 9b8638faa4..5da973c6a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestValueQueue.java @@ -32,7 +32,7 @@ import org.junit.Assert; import org.junit.Test; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; public class TestValueQueue { Logger LOG = LoggerFactory.getLogger(TestValueQueue.class); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java index 3684a1d0db..9122389bd6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/kms/TestLoadBalancingKMSClientProvider.java @@ -63,7 +63,7 @@ import org.junit.rules.Timeout; import org.mockito.Mockito; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; public class TestLoadBalancingKMSClientProvider { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index 8b1750def7..effb2608a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -72,7 +72,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.thirdparty.com.google.common.base.Joiner; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; import static org.apache.hadoop.util.functional.CommonCallableSupplier.submit; diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 9de8b9caf6..c932bae2fb 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -244,6 +244,38 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-common provided Sets rather than Guava provided Sets + + org.apache.hadoop.thirdparty.com.google.common.collect.Sets + com.google.common.collect.Sets + + + + + + + diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java index 4c64a37fea..3a11ec134f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAudit.java @@ -36,7 +36,7 @@ import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener; import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.HashSet; diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 0cab5ada21..1d82e44aec 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -467,6 +467,38 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-common provided Sets rather than Guava provided Sets + + org.apache.hadoop.thirdparty.com.google.common.collect.Sets + com.google.common.collect.Sets + + + + + + + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index c467702575..53ac4494b7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -33,7 +33,7 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException; import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java index 5f9e6e2136..c32a06a418 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.s3a.commit.staging; import java.io.IOException; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -27,7 +28,6 @@ import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.UncheckedExecutionException; import org.apache.commons.lang3.StringUtils; @@ -290,7 +290,7 @@ public static Set getPartitions(Path attemptPath, List taskOutput) throws IOException { // get a list of partition directories - Set partitions = Sets.newLinkedHashSet(); + Set partitions = new LinkedHashSet<>(); for (FileStatus fileStatus : taskOutput) { // sanity check the output paths Path outputFile = fileStatus.getPath(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 08d5e0db0d..f273e68371 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -30,7 +30,7 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.EnvironmentVariableCredentialsProvider; import com.amazonaws.auth.InstanceProfileCredentialsProvider; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java index 79838d6f00..aa068c08d6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/integration/ITestS3ACommitterMRJob.java @@ -34,7 +34,7 @@ import java.util.UUID; import java.util.stream.Collectors; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; import org.junit.FixMethodOrder; import org.junit.Rule; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java index 15ab89286d..94c0b29a3c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java @@ -35,7 +35,7 @@ import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; import org.hamcrest.core.StringStartsWith; import org.junit.After; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java index 2f46b72fba..9e98a226ef 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java @@ -26,7 +26,7 @@ import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; import org.apache.hadoop.thirdparty.com.google.common.collect.Lists; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; import org.junit.BeforeClass; import org.junit.Test; diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 7e5aaebc08..da598fc341 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -223,6 +223,38 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-common provided Sets rather than Guava provided Sets + + org.apache.hadoop.thirdparty.com.google.common.collect.Sets + com.google.common.collect.Sets + + + + + + + diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java index 164d014876..9f2f62d44a 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListing.java @@ -35,7 +35,7 @@ import java.net.URI; import java.util.Set; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; /** * The CopyListing abstraction is responsible for how the list of diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java index 092a54b78a..316a2bd276 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/test/java/org/apache/hadoop/tools/dynamometer/TestDynamometerInfra.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.tools.dynamometer; -import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import org.apache.hadoop.util.Sets; import java.util.Optional; import java.util.concurrent.TimeoutException; import java.util.concurrent.TimeUnit; diff --git a/hadoop-tools/hadoop-dynamometer/pom.xml b/hadoop-tools/hadoop-dynamometer/pom.xml index 30e28b6bf6..4811e695c3 100644 --- a/hadoop-tools/hadoop-dynamometer/pom.xml +++ b/hadoop-tools/hadoop-dynamometer/pom.xml @@ -50,6 +50,38 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + de.skuzzle.enforcer + restrict-imports-enforcer-rule + ${restrict-imports.enforcer.version} + + + + + banned-illegal-imports + process-sources + + enforce + + + + + true + Use hadoop-common provided Sets rather than Guava provided Sets + + org.apache.hadoop.thirdparty.com.google.common.collect.Sets + com.google.common.collect.Sets + + + + + + +