From 1e59bf7394da98d57f61c26d34c19f1394eadd23 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Thu, 26 Nov 2020 12:50:33 +0000 Subject: [PATCH] HADOOP-17385. ITestS3ADeleteCost.testDirMarkersFileCreation failure (#2473). Contributed by Steve Loughran The addition of deprecated S3A configuration options in HADOOP-17318 triggered a reload of default (xml resource) configurations, which breaks tests which fail if there's a per-bucket setting inconsistent with test setup. Creating an S3AFS instance before creating the Configuration() instance for test runs gets that reload out the way before test setup takes place. Along with the fix, extra changes in the failing test suite to fail fast when marker policy isn't as expected, and to log FS state better. Rather than create and discard an instance, add a new static method to S3AFS and invoke it in test setup. This forces the load Change-Id: Id52b1c46912c6fedd2ae270e2b1eb2222a360329 --- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 9 +++++++++ .../org/apache/hadoop/fs/contract/s3a/S3AContract.java | 4 ++++ .../org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java | 3 +++ .../hadoop/fs/s3a/performance/AbstractS3ACostTest.java | 9 +++++++++ .../hadoop/fs/s3a/performance/ITestS3ADeleteCost.java | 1 + 5 files changed, 26 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index cc37df766f..9d4479e444 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -4954,6 +4954,15 @@ public MarkerToolOperations createMarkerToolOperations() { return new MarkerToolOperationsImpl(operationCallbacks); } + /** + * This is purely for testing, as it force initializes all static + * initializers. See HADOOP-17385 for details. + */ + @InterfaceAudience.Private + public static void initializeClass() { + LOG.debug("Initialize S3A class"); + } + /** * The implementation of context accessors. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java index 13110201b7..0d3dd4c2f6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; /** @@ -53,6 +54,9 @@ public S3AContract(Configuration conf) { */ public S3AContract(Configuration conf, boolean addContractResource) { super(conf); + // Force deprecated key load through the + // static initializers. See: HADOOP-17385 + S3AFileSystem.initializeClass(); //insert the base features if (addContractResource) { addConfResource(CONTRACT_XML); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index da679cdeca..73a503ace4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -60,6 +60,9 @@ public void setup() throws Exception { // filesystems which add default configuration resources to do it before // our tests start adding/removing options. See HADOOP-16626. FileSystem.getLocal(new Configuration()); + // Force deprecated key load through the + // static initializers. See: HADOOP-17385 + S3AFileSystem.initializeClass(); super.setup(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java index d7e277fdc9..2b2fb7c9ca 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.Tristate; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -141,6 +142,14 @@ public void setup() throws Exception { isDeleting = !isKeeping; + // check that the FS has the expected state + DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy(); + Assertions.assertThat(markerPolicy.getMarkerPolicy()) + .describedAs("Marker policy for filesystem %s", fs) + .isEqualTo(isKeepingMarkers() + ? DirectoryPolicy.MarkerPolicy.Keep + : DirectoryPolicy.MarkerPolicy.Delete); + // insert new metrics so as to keep the list sorted costValidator = OperationCostValidator.builder(getFileSystem()) .withMetrics( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index 618b491920..0b5afc6178 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -267,6 +267,7 @@ public void testDirMarkersFileCreation() throws Throwable { verifyMetrics(() -> { file(new Path(srcDir, "source.txt")); + LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem()); return "after touch(fs, srcFilePath) " + getMetricSummary(); }, with(DIRECTORIES_CREATED, 0),