diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index f2cd2cd517..a02947b015 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -4954,6 +4954,15 @@ public MarkerToolOperations createMarkerToolOperations() { return new MarkerToolOperationsImpl(operationCallbacks); } + /** + * This is purely for testing, as it force initializes all static + * initializers. See HADOOP-17385 for details. + */ + @InterfaceAudience.Private + public static void initializeClass() { + LOG.debug("Initialize S3A class"); + } + /** * The implementation of context accessors. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java index 13110201b7..0d3dd4c2f6 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/S3AContract.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; /** @@ -53,6 +54,9 @@ public S3AContract(Configuration conf) { */ public S3AContract(Configuration conf, boolean addContractResource) { super(conf); + // Force deprecated key load through the + // static initializers. See: HADOOP-17385 + S3AFileSystem.initializeClass(); //insert the base features if (addContractResource) { addConfResource(CONTRACT_XML); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java index da679cdeca..73a503ace4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java @@ -60,6 +60,9 @@ public void setup() throws Exception { // filesystems which add default configuration resources to do it before // our tests start adding/removing options. See HADOOP-16626. FileSystem.getLocal(new Configuration()); + // Force deprecated key load through the + // static initializers. See: HADOOP-17385 + S3AFileSystem.initializeClass(); super.setup(); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java index d7e277fdc9..2b2fb7c9ca 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.Tristate; +import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -141,6 +142,14 @@ public void setup() throws Exception { isDeleting = !isKeeping; + // check that the FS has the expected state + DirectoryPolicy markerPolicy = fs.getDirectoryMarkerPolicy(); + Assertions.assertThat(markerPolicy.getMarkerPolicy()) + .describedAs("Marker policy for filesystem %s", fs) + .isEqualTo(isKeepingMarkers() + ? DirectoryPolicy.MarkerPolicy.Keep + : DirectoryPolicy.MarkerPolicy.Delete); + // insert new metrics so as to keep the list sorted costValidator = OperationCostValidator.builder(getFileSystem()) .withMetrics( diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index 618b491920..0b5afc6178 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -267,6 +267,7 @@ public void testDirMarkersFileCreation() throws Throwable { verifyMetrics(() -> { file(new Path(srcDir, "source.txt")); + LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem()); return "after touch(fs, srcFilePath) " + getMetricSummary(); }, with(DIRECTORIES_CREATED, 0),