From e199da3fae1c82e87f88c8c50f6a96c6515e2edd Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 17 Jun 2022 19:11:35 +0100 Subject: [PATCH] HADOOP-17833. Improve Magic Committer performance (#3289) Speed up the magic committer with key changes being * Writes under __magic always retain directory markers * File creation under __magic skips all overwrite checks, including the LIST call intended to stop files being created over dirs. * mkdirs under __magic probes the path for existence but does not look any further. Extra parallelism in task and job commit directory scanning Use of createFile and openFile with parameters which all for HEAD checks to be skipped. The committer can write the summary _SUCCESS file to the path `fs.s3a.committer.summary.report.directory`, which can be in a different file system/bucket if desired, using the job id as the filename. Also: HADOOP-15460. S3A FS to add `fs.s3a.create.performance` Application code can set the createFile() option fs.s3a.create.performance to true to disable the same safety checks when writing under magic directories. Use with care. The createFile option prefix `fs.s3a.create.header.` can be used to add custom headers to S3 objects when created. Contributed by Steve Loughran. --- .../hadoop/fs/audit/AuditConstants.java | 5 + .../hadoop/fs/audit/CommonAuditContext.java | 8 +- .../fs/statistics/StoreStatisticNames.java | 3 + .../apache/hadoop/util/JsonSerialization.java | 5 +- .../CloseableTaskPoolSubmitter.java | 2 +- .../filesystem/fsdataoutputstreambuilder.md | 59 +- .../hadoop/fs/contract/ContractTestUtils.java | 14 +- .../manifest/files/AbstractManifestData.java | 1 - hadoop-tools/hadoop-aws/pom.xml | 33 + .../org/apache/hadoop/fs/s3a/Constants.java | 18 + .../hadoop/fs/s3a/S3ABlockOutputStream.java | 59 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 378 +++++++---- .../hadoop/fs/s3a/S3AInstrumentation.java | 18 +- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 14 +- .../org/apache/hadoop/fs/s3a/Statistic.java | 13 + .../hadoop/fs/s3a/WriteOperationHelper.java | 88 +-- .../apache/hadoop/fs/s3a/WriteOperations.java | 35 +- .../hadoop/fs/s3a/api/RequestFactory.java | 10 +- .../fs/s3a/commit/AbstractS3ACommitter.java | 604 ++++++++++-------- .../hadoop/fs/s3a/commit/CommitConstants.java | 34 +- .../s3a/commit/CommitterStatisticNames.java | 211 ++++++ .../commit/InternalCommitterConstants.java | 6 + .../fs/s3a/commit/MagicCommitIntegration.java | 22 +- .../apache/hadoop/fs/s3a/commit/Tasks.java | 423 ------------ .../fs/s3a/commit/files/PendingSet.java | 44 +- .../commit/files/PersistentCommitData.java | 115 +++- .../s3a/commit/files/SinglePendingCommit.java | 36 +- .../fs/s3a/commit/files/SuccessData.java | 119 +++- .../fs/s3a/commit/files/package-info.java | 7 +- .../s3a/commit/impl/AuditContextUpdater.java | 90 +++ .../fs/s3a/commit/impl/CommitContext.java | 401 ++++++++++++ .../commit/{ => impl}/CommitOperations.java | 247 ++++--- .../commit/{ => impl}/CommitUtilsWithMR.java | 68 +- .../fs/s3a/commit/impl/package-info.java | 31 + .../s3a/commit/magic/MagicCommitTracker.java | 55 +- .../commit/magic/MagicS3GuardCommitter.java | 150 +++-- .../staging/DirectoryStagingCommitter.java | 8 +- .../staging/PartitionedStagingCommitter.java | 41 +- .../s3a/commit/staging/StagingCommitter.java | 134 ++-- .../hadoop/fs/s3a/impl/CreateFileBuilder.java | 263 ++++++++ .../hadoop/fs/s3a/impl/InternalConstants.java | 10 + .../hadoop/fs/s3a/impl/MkdirOperation.java | 68 +- .../hadoop/fs/s3a/impl/PutObjectOptions.java | 104 +++ .../fs/s3a/impl/RequestFactoryImpl.java | 37 +- .../fs/s3a/impl/S3AMultipartUploader.java | 3 +- .../BlockOutputStreamStatistics.java | 3 +- .../s3a/statistics/CommitterStatistics.java | 8 + .../s3a/statistics/PutTrackerStatistics.java | 29 + .../impl/EmptyS3AStatisticsContext.java | 7 + .../markdown/tools/hadoop-aws/auditing.md | 11 +- .../markdown/tools/hadoop-aws/committers.md | 11 +- .../fs/s3a/ITestS3AFileOperationCost.java | 63 -- .../hadoop/fs/s3a/ITestS3AMiscOperations.java | 16 +- .../hadoop/fs/s3a/MockS3AFileSystem.java | 30 +- .../hadoop/fs/s3a/MultipartTestUtils.java | 3 +- .../hadoop/fs/s3a/S3ATestConstants.java | 6 + .../apache/hadoop/fs/s3a/S3ATestUtils.java | 27 +- .../fs/s3a/TestS3ABlockOutputStream.java | 4 +- .../fs/s3a/audit/ITestAuditAccessChecks.java | 4 +- .../hadoop/fs/s3a/auth/ITestAssumeRole.java | 39 +- .../fs/s3a/commit/AbstractCommitITest.java | 101 ++- .../s3a/commit/AbstractITCommitProtocol.java | 86 +-- .../fs/s3a/commit/CommitterTestHelper.java | 148 +++++ .../s3a/commit/ITestCommitOperationCost.java | 325 ++++++++++ .../fs/s3a/commit/ITestCommitOperations.java | 288 +++++---- .../s3a/commit/ITestS3ACommitterFactory.java | 2 +- .../hadoop/fs/s3a/commit/TestTasks.java | 569 ----------------- .../magic/ITestMagicCommitProtocol.java | 27 +- .../magic/ITestS3AHugeMagicCommits.java | 29 +- .../staging/MockedStagingCommitter.java | 4 +- .../s3a/commit/staging/StagingTestBase.java | 39 +- .../staging/TestDirectoryCommitterScale.java | 50 +- .../commit/staging/TestStagingCommitter.java | 21 +- .../TestStagingDirectoryOutputCommitter.java | 47 +- .../TestStagingPartitionedFileListing.java | 4 +- .../TestStagingPartitionedJobCommit.java | 17 +- .../TestStagingPartitionedTaskCommit.java | 10 +- .../ITestStagingCommitProtocol.java | 2 +- .../commit/terasort/ITestTerasortOnS3A.java | 2 +- .../fs/s3a/impl/StubContextAccessor.java | 82 +++ .../fs/s3a/impl/TestCreateFileBuilder.java | 173 +++++ .../fs/s3a/impl/TestRequestFactory.java | 11 +- .../s3a/performance/AbstractS3ACostTest.java | 24 +- .../s3a/performance/ITestCreateFileCost.java | 248 +++++++ .../s3a/performance/ITestS3ADeleteCost.java | 11 +- .../fs/s3a/performance/OperationCost.java | 5 +- .../performance/OperationCostValidator.java | 18 +- .../scale/ITestS3ADirectoryPerformance.java | 5 +- .../ITestS3AMultipartUploadSizeLimits.java | 2 +- 89 files changed, 4371 insertions(+), 2334 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java delete mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/{ => impl}/CommitOperations.java (82%) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/{ => impl}/CommitUtilsWithMR.java (74%) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java delete mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index d9629e388b..0929c2be03 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -90,6 +90,11 @@ private AuditConstants() { */ public static final String PARAM_PROCESS = "ps"; + /** + * Task Attempt ID query header: {@value}. + */ + public static final String PARAM_TASK_ATTEMPT_ID = "ta"; + /** * Thread 0: the thread which created a span {@value}. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java index e188e168e5..2dcd4f8b3f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/CommonAuditContext.java @@ -124,11 +124,15 @@ private CommonAuditContext() { /** * Put a context entry. * @param key key - * @param value new value + * @param value new value., If null, triggers removal. * @return old value or null */ public Supplier put(String key, String value) { - return evaluatedEntries.put(key, () -> value); + if (value != null) { + return evaluatedEntries.put(key, () -> value); + } else { + return evaluatedEntries.remove(key); + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java index c458269c35..c04c1bb47f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java @@ -53,6 +53,9 @@ public final class StoreStatisticNames { /** {@value}. */ public static final String OP_CREATE = "op_create"; + /** {@value}. */ + public static final String OP_CREATE_FILE = "op_createfile"; + /** {@value}. */ public static final String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java index 0bba79fd77..52c6c45052 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java @@ -297,11 +297,12 @@ public void save(FileSystem fs, Path path, T instance, } /** - * Write the JSON as bytes, then close the file. + * Write the JSON as bytes, then close the stream. + * @param instance instance to write * @param dataOutputStream an output stream that will always be closed * @throws IOException on any failure */ - private void writeJsonAsBytes(T instance, + public void writeJsonAsBytes(T instance, OutputStream dataOutputStream) throws IOException { try { dataOutputStream.write(toBytes(instance)); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java index 26b687a3c5..695da7e932 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/functional/CloseableTaskPoolSubmitter.java @@ -34,7 +34,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Unstable -public final class CloseableTaskPoolSubmitter implements TaskPool.Submitter, +public class CloseableTaskPoolSubmitter implements TaskPool.Submitter, Closeable { /** Executors. */ diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md index 64dda2df8c..59a93c5887 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md @@ -26,7 +26,7 @@ create a new file or open an existing file on `FileSystem` for write. ## Invariants The `FSDataOutputStreamBuilder` interface does not validate parameters -and modify the state of `FileSystem` until [`build()`](#Builder.build) is +and modify the state of `FileSystem` until `build()` is invoked. ## Implementation-agnostic parameters. @@ -110,7 +110,7 @@ of `FileSystem`. #### Implementation Notes The concrete `FileSystem` and/or `FSDataOutputStreamBuilder` implementation -MUST verify that implementation-agnostic parameters (i.e., "syncable") or +MUST verify that implementation-agnostic parameters (i.e., "syncable`) or implementation-specific parameters (i.e., "foofs:cache") are supported. `FileSystem` will satisfy optional parameters (via `opt(key, ...)`) on best effort. If the mandatory parameters (via `must(key, ...)`) can not be satisfied @@ -182,3 +182,58 @@ see `FileSystem#create(path, ...)` and `FileSystem#append()`. result = FSDataOutputStream The result is `FSDataOutputStream` to be used to write data to filesystem. + + +## S3A-specific options + +Here are the custom options which the S3A Connector supports. + +| Name | Type | Meaning | +|-----------------------------|-----------|----------------------------------------| +| `fs.s3a.create.performance` | `boolean` | create a file with maximum performance | +| `fs.s3a.create.header` | `string` | prefix for user supplied headers | + +### `fs.s3a.create.performance` + +Prioritize file creation performance over safety checks for filesystem consistency. + +This: +1. Skips the `LIST` call which makes sure a file is being created over a directory. + Risk: a file is created over a directory. +1. Ignores the overwrite flag. +1. Never issues a `DELETE` call to delete parent directory markers. + +It is possible to probe an S3A Filesystem instance for this capability through +the `hasPathCapability(path, "fs.s3a.create.performance")` check. + +Creating files with this option over existing directories is likely +to make S3A filesystem clients behave inconsistently. + +Operations optimized for directories (e.g. listing calls) are likely +to see the directory tree not the file; operations optimized for +files (`getFileStatus()`, `isFile()`) more likely to see the file. +The exact form of the inconsistencies, and which operations/parameters +trigger this are undefined and may change between even minor releases. + +Using this option is the equivalent of pressing and holding down the +"Electronic Stability Control" +button on a rear-wheel drive car for five seconds: the safety checks are off. +Things wil be faster if the driver knew what they were doing. +If they didn't, the fact they had held the button down will +be used as evidence at the inquest as proof that they made a +conscious decision to choose speed over safety and +that the outcome was their own fault. + +Accordingly: *Use if and only if you are confident that the conditions are met.* + +### `fs.s3a.create.header` User-supplied header support + +Options with the prefix `fs.s3a.create.header.` will be added to to the +S3 object metadata as "user defined metadata". +This metadata is visible to all applications. It can also be retrieved through the +FileSystem/FileContext `listXAttrs()` and `getXAttrs()` API calls with the prefix `header.` + +When an object is renamed, the metadata is propagated the copy created. + +It is possible to probe an S3A Filesystem instance for this capability through +the `hasPathCapability(path, "fs.s3a.create.header")` check. \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java index eb56d957d9..90e3649c1b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java @@ -29,6 +29,8 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.functional.RemoteIterators; + import org.junit.Assert; import org.junit.AssumptionViolatedException; import org.slf4j.Logger; @@ -1446,11 +1448,7 @@ public static TreeScanResults treeWalk(FileSystem fs, Path path) */ public static List toList( RemoteIterator iterator) throws IOException { - ArrayList list = new ArrayList<>(); - while (iterator.hasNext()) { - list.add(iterator.next()); - } - return list; + return RemoteIterators.toList(iterator); } /** @@ -1464,11 +1462,7 @@ public static List toList( */ public static List iteratorToList( RemoteIterator iterator) throws IOException { - List list = new ArrayList<>(); - while (iterator.hasNext()) { - list.add(iterator.next()); - } - return list; + return RemoteIterators.toList(iterator); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java index 7020d5ca2d..bf67281785 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/files/AbstractManifestData.java @@ -70,7 +70,6 @@ public static Path unmarshallPath(String path) { throw new RuntimeException( "Failed to parse \"" + path + "\" : " + e, e); - } } diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 5583bb7ad0..6d9085b51a 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -405,6 +405,39 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + banned-illegal-imports + process-sources + + enforce + + + + + false + Restrict mapreduce imports to committer code + + org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter + org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory + org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory + org.apache.hadoop.fs.s3a.commit.impl.* + org.apache.hadoop.fs.s3a.commit.magic.* + org.apache.hadoop.fs.s3a.commit.staging.* + + + org.apache.hadoop.mapreduce.** + org.apache.hadoop.mapred.** + + + + + + + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 2e4f1e8361..48d17aa402 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1159,4 +1159,22 @@ private Constants() { * Require that all S3 access is made through Access Points. */ public static final String AWS_S3_ACCESSPOINT_REQUIRED = "fs.s3a.accesspoint.required"; + + /** + * Flag for create performance. + * This is *not* a configuration option; it is for use in the + * {code createFile()} builder. + * Value {@value}. + */ + public static final String FS_S3A_CREATE_PERFORMANCE = "fs.s3a.create.performance"; + + /** + * Prefix for adding a header to the object when created. + * The actual value must have a "." suffix and then the actual header. + * This is *not* a configuration option; it is only for use in the + * {code createFile()} builder. + * Value {@value}. + */ + public static final String FS_S3A_CREATE_HEADER = "fs.s3a.create.header"; + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java index 6e6871751d..8b1865c77c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java @@ -40,6 +40,7 @@ import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.UploadPartRequest; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture; @@ -68,6 +69,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Statistic.*; import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; @@ -103,6 +105,11 @@ class S3ABlockOutputStream extends OutputStream implements /** IO Statistics. */ private final IOStatistics iostatistics; + /** + * The options this instance was created with. + */ + private final BlockOutputStreamBuilder builder; + /** Total bytes for uploads submitted so far. */ private long bytesSubmitted; @@ -167,6 +174,7 @@ class S3ABlockOutputStream extends OutputStream implements S3ABlockOutputStream(BlockOutputStreamBuilder builder) throws IOException { builder.validate(); + this.builder = builder; this.key = builder.key; this.blockFactory = builder.blockFactory; this.blockSize = (int) builder.blockSize; @@ -332,6 +340,7 @@ public synchronized void write(byte[] source, int offset, int len) * initializing the upload, or if a previous operation * has failed. */ + @Retries.RetryTranslated private synchronized void uploadCurrentBlock(boolean isLast) throws IOException { Preconditions.checkState(hasActiveBlock(), "No active block"); @@ -353,6 +362,7 @@ private synchronized void uploadCurrentBlock(boolean isLast) * can take time and potentially fail. * @throws IOException failure to initialize the upload */ + @Retries.RetryTranslated private void initMultipartUpload() throws IOException { if (multiPartUpload == null) { LOG.debug("Initiating Multipart upload"); @@ -546,9 +556,15 @@ private int putObject() throws IOException { int size = block.dataSize(); final S3ADataBlocks.BlockUploadData uploadData = block.startUpload(); final PutObjectRequest putObjectRequest = uploadData.hasFile() ? - writeOperationHelper.createPutObjectRequest(key, uploadData.getFile()) - : writeOperationHelper.createPutObjectRequest(key, - uploadData.getUploadStream(), size, null); + writeOperationHelper.createPutObjectRequest( + key, + uploadData.getFile(), + builder.putOptions) + : writeOperationHelper.createPutObjectRequest( + key, + uploadData.getUploadStream(), + size, + builder.putOptions); BlockUploadProgress callback = new BlockUploadProgress( block, progressListener, now()); @@ -559,7 +575,7 @@ private int putObject() throws IOException { try { // the putObject call automatically closes the input // stream afterwards. - return writeOperationHelper.putObject(putObjectRequest); + return writeOperationHelper.putObject(putObjectRequest, builder.putOptions); } finally { cleanupWithLogger(LOG, uploadData, block); } @@ -702,8 +718,21 @@ private class MultiPartUpload { */ private IOException blockUploadFailure; + /** + * Constructor. + * Initiates the MPU request against S3. + * @param key upload destination + * @throws IOException failure + */ + + @Retries.RetryTranslated MultiPartUpload(String key) throws IOException { - this.uploadId = writeOperationHelper.initiateMultiPartUpload(key); + this.uploadId = trackDuration(statistics, + OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(), + () -> writeOperationHelper.initiateMultiPartUpload( + key, + builder.putOptions)); + this.partETagsFutures = new ArrayList<>(2); LOG.debug("Initiated multi-part upload for {} with " + "id '{}'", writeOperationHelper, uploadId); @@ -887,7 +916,8 @@ private void complete(List partETags) uploadId, partETags, bytesSubmitted, - errorCount); + errorCount, + builder.putOptions); }); } finally { statistics.exceptionInMultipartComplete(errorCount.get()); @@ -1057,6 +1087,11 @@ public static final class BlockOutputStreamBuilder { /** is Client side Encryption enabled? */ private boolean isCSEEnabled; + /** + * Put object options. + */ + private PutObjectOptions putOptions; + private BlockOutputStreamBuilder() { } @@ -1070,6 +1105,7 @@ public void validate() { requireNonNull(statistics, "null statistics"); requireNonNull(writeOperations, "null writeOperationHelper"); requireNonNull(putTracker, "null putTracker"); + requireNonNull(putOptions, "null putOptions"); Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE, "Block size is too small: %s", blockSize); } @@ -1182,5 +1218,16 @@ public BlockOutputStreamBuilder withCSEEnabled(boolean value) { isCSEEnabled = value; return this; } + + /** + * Set builder value. + * @param value new value + * @return the builder + */ + public BlockOutputStreamBuilder withPutOptions( + final PutObjectOptions value) { + putOptions = value; + return this; + } } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 831770f4a3..bb23373bb4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -22,6 +22,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; +import java.io.UncheckedIOException; import java.net.URI; import java.nio.file.AccessDeniedException; import java.text.DateFormat; @@ -91,6 +92,7 @@ import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.Globber; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.impl.OpenFileParameters; @@ -104,6 +106,7 @@ import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.CopyFromLocalOperation; import org.apache.hadoop.fs.s3a.impl.CopyOutcome; +import org.apache.hadoop.fs.s3a.impl.CreateFileBuilder; import org.apache.hadoop.fs.s3a.impl.DeleteOperation; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicy; import org.apache.hadoop.fs.s3a.impl.DirectoryPolicyImpl; @@ -114,6 +117,7 @@ import org.apache.hadoop.fs.s3a.impl.MkdirOperation; import org.apache.hadoop.fs.s3a.impl.OpenFileSupport; import org.apache.hadoop.fs.s3a.impl.OperationCallbacks; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.impl.RenameOperation; import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl; import org.apache.hadoop.fs.s3a.impl.S3AMultipartUploaderBuilder; @@ -207,7 +211,8 @@ import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_ABORT_PENDING_UPLOADS; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS; import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.submit; -import static org.apache.hadoop.fs.s3a.impl.CallableSupplier.waitForCompletionIgnoringExceptions; +import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_NO_OVERWRITE; +import static org.apache.hadoop.fs.s3a.impl.CreateFileBuilder.OPTIONS_CREATE_FILE_OVERWRITE; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isObjectNotFound; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.AP_INACCESSIBLE; @@ -1614,10 +1619,15 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { final Path path = qualify(f); + // the span will be picked up inside the output stream return trackDurationAndSpan(INVOCATION_CREATE, path, () -> - innerCreateFile(path, permission, overwrite, bufferSize, replication, - blockSize, progress)); + innerCreateFile(path, + progress, + getActiveAuditSpan(), + overwrite + ? OPTIONS_CREATE_FILE_OVERWRITE + : OPTIONS_CREATE_FILE_NO_OVERWRITE)); } /** @@ -1625,58 +1635,68 @@ public FSDataOutputStream create(Path f, FsPermission permission, * reporting; in the active span. * Retry policy: retrying, translated on the getFileStatus() probe. * No data is uploaded to S3 in this call, so no retry issues related to that. + * The "performance" flag disables safety checks for the path being a file, + * parent directory existing, and doesn't attempt to delete + * dir markers, irrespective of FS settings. + * If true, this method call does no IO at all. * @param path the file name to open - * @param permission the permission to set. - * @param overwrite if a file with this name already exists, then if true, - * the file will be overwritten, and if false an error will be thrown. - * @param bufferSize the size of the buffer to be used. - * @param replication required block replication for the file. - * @param blockSize the requested block size. * @param progress the progress reporter. + * @param auditSpan audit span + * @param options options for the file * @throws IOException in the event of IO related errors. - * @see #setPermission(Path, FsPermission) */ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed") @Retries.RetryTranslated - private FSDataOutputStream innerCreateFile(Path path, - FsPermission permission, - boolean overwrite, - int bufferSize, - short replication, - long blockSize, - Progressable progress) throws IOException { + private FSDataOutputStream innerCreateFile( + final Path path, + final Progressable progress, + final AuditSpan auditSpan, + final CreateFileBuilder.CreateFileOptions options) throws IOException { + auditSpan.activate(); String key = pathToKey(path); - FileStatus status = null; - try { - // get the status or throw an FNFE. - // when overwriting, there is no need to look for any existing file, - // and attempting to do so can poison the load balancers with 404 - // entries. - status = innerGetFileStatus(path, false, - overwrite - ? StatusProbeEnum.DIRECTORIES - : StatusProbeEnum.ALL); + EnumSet flags = options.getFlags(); + boolean overwrite = flags.contains(CreateFlag.OVERWRITE); + boolean performance = options.isPerformance(); + boolean skipProbes = performance || isUnderMagicCommitPath(path); + if (skipProbes) { + LOG.debug("Skipping existence/overwrite checks"); + } else { + try { + // get the status or throw an FNFE. + // when overwriting, there is no need to look for any existing file, + // just a directory (for safety) + FileStatus status = innerGetFileStatus(path, false, + overwrite + ? StatusProbeEnum.DIRECTORIES + : StatusProbeEnum.ALL); - // if the thread reaches here, there is something at the path - if (status.isDirectory()) { - // path references a directory: automatic error - throw new FileAlreadyExistsException(path + " is a directory"); + // if the thread reaches here, there is something at the path + if (status.isDirectory()) { + // path references a directory: automatic error + throw new FileAlreadyExistsException(path + " is a directory"); + } + if (!overwrite) { + // path references a file and overwrite is disabled + throw new FileAlreadyExistsException(path + " already exists"); + } + LOG.debug("Overwriting file {}", path); + } catch (FileNotFoundException e) { + // this means there is nothing at the path; all good. } - if (!overwrite) { - // path references a file and overwrite is disabled - throw new FileAlreadyExistsException(path + " already exists"); - } - LOG.debug("Overwriting file {}", path); - } catch (FileNotFoundException e) { - // this means the file is not found - } instrumentation.fileCreated(); - PutTracker putTracker = - committerIntegration.createTracker(path, key); - String destKey = putTracker.getDestKey(); final BlockOutputStreamStatistics outputStreamStatistics = statisticsContext.newOutputStreamStatistics(); + PutTracker putTracker = + committerIntegration.createTracker(path, key, outputStreamStatistics); + String destKey = putTracker.getDestKey(); + + // put options are derived from the path and the + // option builder. + boolean keep = performance || keepDirectoryMarkers(path); + final PutObjectOptions putOptions = + new PutObjectOptions(keep, null, options.getHeaders()); + final S3ABlockOutputStream.BlockOutputStreamBuilder builder = S3ABlockOutputStream.builder() .withKey(destKey) @@ -1686,7 +1706,7 @@ private FSDataOutputStream innerCreateFile(Path path, .withProgress(progress) .withPutTracker(putTracker) .withWriteOperations( - createWriteOperationHelper(getActiveAuditSpan())) + createWriteOperationHelper(auditSpan)) .withExecutorService( new SemaphoredDelegatingExecutor( boundedThreadPool, @@ -1697,12 +1717,12 @@ private FSDataOutputStream innerCreateFile(Path path, getConf().getBoolean( DOWNGRADE_SYNCABLE_EXCEPTIONS, DOWNGRADE_SYNCABLE_EXCEPTIONS_DEFAULT)) - .withCSEEnabled(isCSEEnabled); + .withCSEEnabled(isCSEEnabled) + .withPutOptions(putOptions); return new FSDataOutputStream( new S3ABlockOutputStream(builder), null); } - /** * Create a Write Operation Helper with the current active span. * All operations made through this helper will activate the @@ -1735,10 +1755,65 @@ public WriteOperationHelper createWriteOperationHelper(AuditSpan auditSpan) { auditSpan); } + /** + * Create instance of an FSDataOutputStreamBuilder for + * creating a file at the given path. + * @param path path to create + * @return a builder. + * @throws UncheckedIOException for problems creating the audit span + */ + @Override + @AuditEntryPoint + public FSDataOutputStreamBuilder createFile(final Path path) { + try { + final Path qualified = qualify(path); + final AuditSpan span = entryPoint(INVOCATION_CREATE_FILE, + pathToKey(qualified), + null); + return new CreateFileBuilder(this, + qualified, + new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_FILE, span)) + .create() + .overwrite(true); + } catch (IOException e) { + // catch any IOEs raised in span creation and convert to + // an UncheckedIOException + throw new UncheckedIOException(e); + } + } + + /** + * Callback for create file operations. + */ + private final class CreateFileBuilderCallbacksImpl implements + CreateFileBuilder.CreateFileBuilderCallbacks { + + private final Statistic statistic; + /** span for operations. */ + private final AuditSpan span; + + private CreateFileBuilderCallbacksImpl( + final Statistic statistic, + final AuditSpan span) { + this.statistic = statistic; + this.span = span; + } + + @Override + public FSDataOutputStream createFileFromBuilder( + final Path path, + final Progressable progress, + final CreateFileBuilder.CreateFileOptions options) throws IOException { + // the span will be picked up inside the output stream + return trackDuration(getDurationTrackerFactory(), statistic.getSymbol(), () -> + innerCreateFile(path, progress, span, options)); + } + } + /** * {@inheritDoc} - * @throws FileNotFoundException if the parent directory is not present -or - * is not a directory. + * The S3A implementations downgrades to the recursive creation, to avoid + * any race conditions with parent entries "disappearing". */ @Override @AuditEntryPoint @@ -1750,30 +1825,23 @@ public FSDataOutputStream createNonRecursive(Path p, long blockSize, Progressable progress) throws IOException { final Path path = makeQualified(p); - // this span is passed into the stream. - try (AuditSpan span = entryPoint(INVOCATION_CREATE_NON_RECURSIVE, path)) { - Path parent = path.getParent(); - // expect this to raise an exception if there is no parent dir - if (parent != null && !parent.isRoot()) { - S3AFileStatus status; - try { - // optimize for the directory existing: Call list first - status = innerGetFileStatus(parent, false, - StatusProbeEnum.DIRECTORIES); - } catch (FileNotFoundException e) { - // no dir, fall back to looking for a file - // (failure condition if true) - status = innerGetFileStatus(parent, false, - StatusProbeEnum.HEAD_ONLY); - } - if (!status.isDirectory()) { - throw new FileAlreadyExistsException("Not a directory: " + parent); - } - } - return innerCreateFile(path, permission, - flags.contains(CreateFlag.OVERWRITE), bufferSize, - replication, blockSize, progress); + + // span is created and passed in to the callbacks. + final AuditSpan span = entryPoint(INVOCATION_CREATE_NON_RECURSIVE, + pathToKey(path), + null); + // uses the CreateFileBuilder, filling it in with the relevant arguments. + final CreateFileBuilder builder = new CreateFileBuilder(this, + path, + new CreateFileBuilderCallbacksImpl(INVOCATION_CREATE_NON_RECURSIVE, span)) + .create() + .withFlags(flags) + .blockSize(blockSize) + .bufferSize(bufferSize); + if (progress != null) { + builder.progress(progress); } + return builder.build(); } /** @@ -2671,7 +2739,7 @@ private DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteRequest) */ public PutObjectRequest newPutObjectRequest(String key, ObjectMetadata metadata, File srcfile) { - return requestFactory.newPutObjectRequest(key, metadata, srcfile); + return requestFactory.newPutObjectRequest(key, metadata, null, srcfile); } /** @@ -2721,12 +2789,14 @@ public UploadInfo putObject(PutObjectRequest putObjectRequest) { * Auditing: must be inside an audit span. * Important: this call will close any input stream in the request. * @param putObjectRequest the request + * @param putOptions put object options * @return the upload initiated * @throws AmazonClientException on problems */ @VisibleForTesting - @Retries.OnceRaw("For PUT; post-PUT actions are RetryTranslated") - PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest) + @Retries.OnceRaw("For PUT; post-PUT actions are RetryExceptionsSwallowed") + PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions) throws AmazonClientException { long len = getPutRequestLength(putObjectRequest); LOG.debug("PUT {} bytes to {}", len, putObjectRequest.getKey()); @@ -2737,9 +2807,10 @@ PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest) OBJECT_PUT_REQUESTS.getSymbol(), () -> s3.putObject(putObjectRequest)); incrementPutCompletedStatistics(true, len); - // update metadata + // apply any post-write actions. finishedWrite(putObjectRequest.getKey(), len, - result.getETag(), result.getVersionId()); + result.getETag(), result.getVersionId(), + putOptions); return result; } catch (SdkBaseException e) { incrementPutCompletedStatistics(false, len); @@ -3011,7 +3082,7 @@ private void createFakeDirectoryIfNecessary(Path f) // is mostly harmless to create a new one. if (!key.isEmpty() && !s3Exists(f, StatusProbeEnum.DIRECTORIES)) { LOG.debug("Creating new fake directory at {}", f); - createFakeDirectory(key); + createFakeDirectory(key, putOptionsForPath(f)); } } @@ -3026,7 +3097,7 @@ private void createFakeDirectoryIfNecessary(Path f) protected void maybeCreateFakeParentDirectory(Path path) throws IOException, AmazonClientException { Path parent = path.getParent(); - if (parent != null && !parent.isRoot()) { + if (parent != null && !parent.isRoot() && !isUnderMagicCommitPath(parent)) { createFakeDirectoryIfNecessary(parent); } } @@ -3197,6 +3268,11 @@ public UserGroupInformation getOwner() { * Make the given path and all non-existent parents into * directories. Has the semantics of Unix {@code 'mkdir -p'}. * Existence of the directory hierarchy is not an error. + * Parent elements are scanned to see if any are a file, + * except under __magic paths. + * There the FS assumes that the destination directory creation + * did that scan and that paths in job/task attempts are all + * "well formed" * @param p path to create * @param permission to apply to path * @return true if a directory was created or already existed @@ -3214,7 +3290,8 @@ public boolean mkdirs(Path p, FsPermission permission) throws IOException, new MkdirOperation( createStoreContext(), path, - createMkdirOperationCallbacks())); + createMkdirOperationCallbacks(), + isMagicCommitPath(path))); } /** @@ -3240,9 +3317,13 @@ public S3AFileStatus probePathStatus(final Path path, } @Override - public void createFakeDirectory(final String key) + public void createFakeDirectory(final Path dir, final boolean keepMarkers) throws IOException { - S3AFileSystem.this.createEmptyObject(key); + S3AFileSystem.this.createFakeDirectory( + pathToKey(dir), + keepMarkers + ? PutObjectOptions.keepingDirs() + : putOptionsForPath(dir)); } } @@ -3608,7 +3689,7 @@ public void copyLocalFileFromTo(File file, Path from, Path to) throws IOExceptio S3AFileSystem.this.invoker.retry( "putObject(" + "" + ")", to.toString(), true, - () -> executePut(putObjectRequest, progress)); + () -> executePut(putObjectRequest, progress, putOptionsForPath(to))); return null; }); @@ -3627,7 +3708,7 @@ public boolean createEmptyDir(Path path, StoreContext storeContext) new MkdirOperation( storeContext, path, - createMkdirOperationCallbacks())); + createMkdirOperationCallbacks(), false)); } } @@ -3637,14 +3718,18 @@ public boolean createEmptyDir(Path path, StoreContext storeContext) * aborted before an {@code InterruptedIOException} is thrown. * @param putObjectRequest request * @param progress optional progress callback + * @param putOptions put object options * @return the upload result * @throws InterruptedIOException if the blocking was interrupted. */ - @Retries.OnceRaw("For PUT; post-PUT actions are RetryTranslated") - UploadResult executePut(PutObjectRequest putObjectRequest, - Progressable progress) + @Retries.OnceRaw("For PUT; post-PUT actions are RetrySwallowed") + UploadResult executePut( + final PutObjectRequest putObjectRequest, + final Progressable progress, + final PutObjectOptions putOptions) throws InterruptedIOException { String key = putObjectRequest.getKey(); + long len = getPutRequestLength(putObjectRequest); UploadInfo info = putObject(putObjectRequest); Upload upload = info.getUpload(); ProgressableProgressListener listener = new ProgressableProgressListener( @@ -3652,9 +3737,10 @@ UploadResult executePut(PutObjectRequest putObjectRequest, upload.addProgressListener(listener); UploadResult result = waitForUploadCompletion(key, info); listener.uploadCompleted(); + // post-write actions - finishedWrite(key, info.getLength(), - result.getETag(), result.getVersionId()); + finishedWrite(key, len, + result.getETag(), result.getVersionId(), putOptions); return result; } @@ -3663,7 +3749,9 @@ UploadResult executePut(PutObjectRequest putObjectRequest, * If the waiting for completion is interrupted, the upload will be * aborted before an {@code InterruptedIOException} is thrown. * If the upload (or its result collection) failed, this is where - * the failure is raised as an AWS exception + * the failure is raised as an AWS exception. + * Calls {@link #incrementPutCompletedStatistics(boolean, long)} + * to update the statistics. * @param key destination key * @param uploadInfo upload to wait for * @return the upload result @@ -3985,63 +4073,64 @@ InitiateMultipartUploadResult initiateMultipartUpload( /** * Perform post-write actions. - *

+ *

* This operation MUST be called after any PUT/multipart PUT completes * successfully. - *

- * The actions include: - *
    - *
  1. - * Calling - * {@link #deleteUnnecessaryFakeDirectories(Path)} - * if directory markers are not being retained. - *
    2. - *
  2. - * Updating any metadata store with details on the newly created - * object. - *
    3. - *
+ *

+ * The actions include calling + * {@link #deleteUnnecessaryFakeDirectories(Path)} + * if directory markers are not being retained. * @param key key written to * @param length total length of file written * @param eTag eTag of the written object * @param versionId S3 object versionId of the written object + * @param putOptions put object options */ @InterfaceAudience.Private - @Retries.RetryTranslated("Except if failOnMetadataWriteError=false, in which" - + " case RetryExceptionsSwallowed") - void finishedWrite(String key, long length, String eTag, String versionId) { + @Retries.RetryExceptionsSwallowed + void finishedWrite( + String key, + long length, + String eTag, + String versionId, + PutObjectOptions putOptions) { LOG.debug("Finished write to {}, len {}. etag {}, version {}", key, length, eTag, versionId); - Path p = keyToQualifiedPath(key); Preconditions.checkArgument(length >= 0, "content length is negative"); - // kick off an async delete - CompletableFuture deletion; - if (!keepDirectoryMarkers(p)) { - deletion = submit( - unboundedThreadPool, getActiveAuditSpan(), - () -> { - deleteUnnecessaryFakeDirectories( - p.getParent() - ); - return null; - }); - } else { - deletion = null; + if (!putOptions.isKeepMarkers()) { + Path p = keyToQualifiedPath(key); + deleteUnnecessaryFakeDirectories(p.getParent()); } - - // catch up with any delete operation. - waitForCompletionIgnoringExceptions(deletion); } /** * Should we keep directory markers under the path being created * by mkdir/file creation/rename? + * This is done if marker retention is enabled for the path, + * or it is under a magic path where we are saving IOPs + * knowing that all committers are on the same code version and + * therefore marker aware. * @param path path to probe * @return true if the markers MAY be retained, * false if they MUST be deleted */ private boolean keepDirectoryMarkers(Path path) { - return directoryPolicy.keepDirectoryMarkers(path); + return directoryPolicy.keepDirectoryMarkers(path) + || isUnderMagicCommitPath(path); + } + + /** + * Should we keep directory markers under the path being created + * by mkdir/file creation/rename? + * See {@link #keepDirectoryMarkers(Path)} for the policy. + * + * @param path path to probe + * @return the options to use with the put request + */ + private PutObjectOptions putOptionsForPath(Path path) { + return keepDirectoryMarkers(path) + ? PutObjectOptions.keepingDirs() + : PutObjectOptions.deletingDirs(); } /** @@ -4078,27 +4167,32 @@ private void deleteUnnecessaryFakeDirectories(Path path) { * Create a fake directory, always ending in "/". * Retry policy: retrying; translated. * @param objectName name of directory object. + * @param putOptions put object options * @throws IOException IO failure */ @Retries.RetryTranslated - private void createFakeDirectory(final String objectName) + private void createFakeDirectory(final String objectName, + final PutObjectOptions putOptions) throws IOException { - createEmptyObject(objectName); + createEmptyObject(objectName, putOptions); } /** * Used to create an empty file that represents an empty directory. + * The policy for deleting parent dirs depends on the path, dir + * status and the putOptions value. * Retry policy: retrying; translated. * @param objectName object to create + * @param putOptions put object options * @throws IOException IO failure */ @Retries.RetryTranslated - private void createEmptyObject(final String objectName) + private void createEmptyObject(final String objectName, PutObjectOptions putOptions) throws IOException { invoker.retry("PUT 0-byte object ", objectName, true, () -> putObjectDirect(getRequestFactory() - .newDirectoryMarkerRequest(objectName))); + .newDirectoryMarkerRequest(objectName), putOptions)); incrementPutProgressStatistics(objectName, 0); instrumentation.directoryCreated(); } @@ -4207,14 +4301,26 @@ public boolean isMagicCommitEnabled() { /** * Predicate: is a path a magic commit path? - * True if magic commit is enabled and the path qualifies as special. + * True if magic commit is enabled and the path qualifies as special, + * and is not a a .pending or .pendingset file, * @param path path to examine - * @return true if the path is or is under a magic directory + * @return true if writing a file to the path triggers a "magic" write. */ public boolean isMagicCommitPath(Path path) { return committerIntegration.isMagicCommitPath(path); } + /** + * Predicate: is a path under a magic commit path? + * True if magic commit is enabled and the path is under __magic, + * irrespective of file type. + * @param path path to examine + * @return true if the path is in a magic dir and the FS has magic writes enabled. + */ + private boolean isUnderMagicCommitPath(Path path) { + return committerIntegration.isUnderMagicPath(path); + } + /** * Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}. * Override superclass so as to disable symlink resolution as symlinks @@ -4766,10 +4872,20 @@ public boolean hasPathCapability(final Path path, final String capability) case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_KEEP: case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_DELETE: case STORE_CAPABILITY_DIRECTORY_MARKER_POLICY_AUTHORITATIVE: - case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP: - case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE: return getDirectoryMarkerPolicy().hasPathCapability(path, cap); + // keep for a magic path or if the policy retains it + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_KEEP: + return keepDirectoryMarkers(path); + // delete is the opposite of keep + case STORE_CAPABILITY_DIRECTORY_MARKER_ACTION_DELETE: + return !keepDirectoryMarkers(path); + + // create file options + case FS_S3A_CREATE_PERFORMANCE: + case FS_S3A_CREATE_HEADER: + return true; + default: return super.hasPathCapability(p, cap); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java index eec6366720..67734b7502 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java @@ -1308,8 +1308,18 @@ private void mergeOutputStreamStatistics( incrementCounter(STREAM_WRITE_EXCEPTIONS, source.lookupCounterValue( StreamStatisticNames.STREAM_WRITE_EXCEPTIONS)); + // merge in all the IOStatistics - this.getIOStatistics().aggregate(source.getIOStatistics()); + final IOStatisticsStore sourceIOStatistics = source.getIOStatistics(); + this.getIOStatistics().aggregate(sourceIOStatistics); + + // propagate any extra values into the FS-level stats. + incrementMutableCounter(OBJECT_PUT_REQUESTS.getSymbol(), + sourceIOStatistics.counters().get(OBJECT_PUT_REQUESTS.getSymbol())); + incrementMutableCounter( + COMMITTER_MAGIC_MARKER_PUT.getSymbol(), + sourceIOStatistics.counters().get(COMMITTER_MAGIC_MARKER_PUT.getSymbol())); + } /** @@ -1366,9 +1376,12 @@ private OutputStreamStatistics( STREAM_WRITE_BLOCK_UPLOADS_BYTES_PENDING.getSymbol()) .withDurationTracking( ACTION_EXECUTOR_ACQUIRED, + COMMITTER_MAGIC_MARKER_PUT.getSymbol(), INVOCATION_ABORT.getSymbol(), + MULTIPART_UPLOAD_COMPLETED.getSymbol(), OBJECT_MULTIPART_UPLOAD_ABORTED.getSymbol(), - MULTIPART_UPLOAD_COMPLETED.getSymbol()) + OBJECT_MULTIPART_UPLOAD_INITIATED.getSymbol(), + OBJECT_PUT_REQUESTS.getSymbol()) .build(); setIOStatistics(st); // these are extracted to avoid lookups on heavily used counters. @@ -1630,6 +1643,7 @@ private CommitterStatisticsImpl() { COMMITTER_TASKS_SUCCEEDED.getSymbol()) .withDurationTracking( COMMITTER_COMMIT_JOB.getSymbol(), + COMMITTER_LOAD_SINGLE_PENDING_FILE.getSymbol(), COMMITTER_MATERIALIZE_FILE.getSymbol(), COMMITTER_STAGE_FILE_UPLOAD.getSymbol()) .build(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index d644d3f476..e7e741d42c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -88,6 +88,7 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; +import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; /** * Utility methods for S3A code. @@ -1467,17 +1468,22 @@ public static List flatmapLocatedFiles( /** * List located files and filter them as a classic listFiles(path, filter) * would do. + * This will be incremental, fetching pages async. + * While it is rare for job to have many thousands of files, jobs + * against versioned buckets may return earlier if there are many + * non-visible objects. * @param fileSystem filesystem * @param path path to list * @param recursive recursive listing? * @param filter filter for the filename - * @return the filtered list of entries + * @return interator over the entries. * @throws IOException IO failure. */ - public static List listAndFilter(FileSystem fileSystem, + public static RemoteIterator listAndFilter(FileSystem fileSystem, Path path, boolean recursive, PathFilter filter) throws IOException { - return flatmapLocatedFiles(fileSystem.listFiles(path, recursive), - status -> maybe(filter.accept(status.getPath()), status)); + return filteringRemoteIterator( + fileSystem.listFiles(path, recursive), + status -> filter.accept(status.getPath())); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java index 86cb18076c..dfe9fdf2d8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java @@ -111,6 +111,10 @@ public enum Statistic { StoreStatisticNames.OP_CREATE, "Calls of create()", TYPE_DURATION), + INVOCATION_CREATE_FILE( + StoreStatisticNames.OP_CREATE_FILE, + "Calls of createFile()", + TYPE_DURATION), INVOCATION_CREATE_NON_RECURSIVE( StoreStatisticNames.OP_CREATE_NON_RECURSIVE, "Calls of createNonRecursive()", @@ -459,10 +463,19 @@ public enum Statistic { "committer_commits_reverted", "Count of commits reverted", TYPE_COUNTER), + COMMITTER_LOAD_SINGLE_PENDING_FILE( + "committer_load_single_pending_file", + "Duration to load a single pending file in task commit", + TYPE_DURATION), COMMITTER_MAGIC_FILES_CREATED( "committer_magic_files_created", "Count of files created under 'magic' paths", TYPE_COUNTER), + + COMMITTER_MAGIC_MARKER_PUT( + "committer_magic_marker_put", + "Duration Tracking of marker files created under 'magic' paths", + TYPE_DURATION), COMMITTER_MATERIALIZE_FILE( "committer_materialize_file", "Duration Tracking of time to materialize a file in job commit", diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index ee91bacfb0..ce50f0b85e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import com.amazonaws.services.s3.model.AmazonS3Exception; @@ -40,8 +39,6 @@ import com.amazonaws.services.s3.model.SelectObjectContentResult; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; -import com.amazonaws.services.s3.transfer.model.UploadResult; -import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +48,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.s3a.select.SelectBinding; @@ -58,6 +56,7 @@ import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.functional.CallableRaisingIOE; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.util.Preconditions.checkNotNull; import static org.apache.hadoop.fs.s3a.Invoker.*; @@ -234,22 +233,20 @@ private void deactivateAuditSpan() { * @param destKey destination key * @param inputStream source data. * @param length size, if known. Use -1 for not known - * @param headers optional map of custom headers. + * @param options options for the request * @return the request */ @Retries.OnceRaw public PutObjectRequest createPutObjectRequest(String destKey, InputStream inputStream, long length, - final Map headers) { + final PutObjectOptions options) { activateAuditSpan(); ObjectMetadata objectMetadata = newObjectMetadata(length); - if (headers != null) { - objectMetadata.setUserMetadata(headers); - } return getRequestFactory().newPutObjectRequest( destKey, objectMetadata, + options, inputStream); } @@ -257,18 +254,26 @@ public PutObjectRequest createPutObjectRequest(String destKey, * Create a {@link PutObjectRequest} request to upload a file. * @param dest key to PUT to. * @param sourceFile source file + * @param options options for the request * @return the request */ @Retries.OnceRaw - public PutObjectRequest createPutObjectRequest(String dest, - File sourceFile) { + public PutObjectRequest createPutObjectRequest( + String dest, + File sourceFile, + final PutObjectOptions options) { Preconditions.checkState(sourceFile.length() < Integer.MAX_VALUE, "File length is too big for a single PUT upload"); activateAuditSpan(); - return getRequestFactory(). + final ObjectMetadata objectMetadata = + newObjectMetadata((int) sourceFile.length()); + + PutObjectRequest putObjectRequest = getRequestFactory(). newPutObjectRequest(dest, - newObjectMetadata((int) sourceFile.length()), + objectMetadata, + options, sourceFile); + return putObjectRequest; } /** @@ -298,21 +303,20 @@ public ObjectMetadata newObjectMetadata(long length) { } /** - * Start the multipart upload process. - * Retry policy: retrying, translated. - * @param destKey destination of upload - * @return the upload result containing the ID - * @throws IOException IO problem + * {@inheritDoc} */ @Retries.RetryTranslated - public String initiateMultiPartUpload(String destKey) throws IOException { + public String initiateMultiPartUpload( + final String destKey, + final PutObjectOptions options) + throws IOException { LOG.debug("Initiating Multipart upload to {}", destKey); try (AuditSpan span = activateAuditSpan()) { return retry("initiate MultiPartUpload", destKey, true, () -> { final InitiateMultipartUploadRequest initiateMPURequest = getRequestFactory().newMultipartUploadRequest( - destKey); + destKey, options); return owner.initiateMultipartUpload(initiateMPURequest) .getUploadId(); }); @@ -322,13 +326,14 @@ public String initiateMultiPartUpload(String destKey) throws IOException { /** * Finalize a multipart PUT operation. * This completes the upload, and, if that works, calls - * {@link S3AFileSystem#finishedWrite(String, long, String, String)} + * {@link S3AFileSystem#finishedWrite(String, long, String, String, org.apache.hadoop.fs.s3a.impl.PutObjectOptions)} * to update the filesystem. * Retry policy: retrying, translated. * @param destKey destination of the commit * @param uploadId multipart operation Id * @param partETags list of partial uploads * @param length length of the upload + * @param putOptions put object options * @param retrying retrying callback * @return the result of the operation. * @throws IOException on problems. @@ -339,6 +344,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( String uploadId, List partETags, long length, + PutObjectOptions putOptions, Retried retrying) throws IOException { if (partETags.isEmpty()) { throw new PathIOException(destKey, @@ -357,7 +363,8 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( request); }); owner.finishedWrite(destKey, length, uploadResult.getETag(), - uploadResult.getVersionId()); + uploadResult.getVersionId(), + putOptions); return uploadResult; } } @@ -373,6 +380,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( * @param length length of the upload * @param errorCount a counter incremented by 1 on every error; for * use in statistics + * @param putOptions put object options * @return the result of the operation. * @throws IOException if problems arose which could not be retried, or * the retry count was exceeded @@ -383,7 +391,8 @@ public CompleteMultipartUploadResult completeMPUwithRetries( String uploadId, List partETags, long length, - AtomicInteger errorCount) + AtomicInteger errorCount, + PutObjectOptions putOptions) throws IOException { checkNotNull(uploadId); checkNotNull(partETags); @@ -393,8 +402,8 @@ public CompleteMultipartUploadResult completeMPUwithRetries( uploadId, partETags, length, - (text, e, r, i) -> errorCount.incrementAndGet() - ); + putOptions, + (text, e, r, i) -> errorCount.incrementAndGet()); } /** @@ -550,37 +559,43 @@ public String toString() { * Byte length is calculated from the file length, or, if there is no * file, from the content length of the header. * @param putObjectRequest the request + * @param putOptions put object options * @return the upload initiated * @throws IOException on problems */ @Retries.RetryTranslated - public PutObjectResult putObject(PutObjectRequest putObjectRequest) + public PutObjectResult putObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions) throws IOException { return retry("Writing Object", putObjectRequest.getKey(), true, withinAuditSpan(getAuditSpan(), () -> - owner.putObjectDirect(putObjectRequest))); + owner.putObjectDirect(putObjectRequest, putOptions))); } /** - * PUT an object via the transfer manager. + * PUT an object. + * * @param putObjectRequest the request - * @return the result of the operation + * @param putOptions put object options + * * @throws IOException on problems */ @Retries.RetryTranslated - public UploadResult uploadObject(PutObjectRequest putObjectRequest) + public void uploadObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions) throws IOException { - // no retry; rely on xfer manager logic - return retry("Writing Object", + + retry("Writing Object", putObjectRequest.getKey(), true, withinAuditSpan(getAuditSpan(), () -> - owner.executePut(putObjectRequest, null))); + owner.putObjectDirect(putObjectRequest, putOptions))); } /** * Revert a commit by deleting the file. - * Relies on retry code in filesystem + * Relies on retry code in filesystem. + * Does not attempt to recreate the parent directory * @throws IOException on problems * @param destKey destination key */ @@ -591,13 +606,14 @@ public void revertCommit(String destKey) throws IOException { Path destPath = owner.keyToQualifiedPath(destKey); owner.deleteObjectAtPath(destPath, destKey, true); - owner.maybeCreateFakeParentDirectory(destPath); })); } /** * This completes a multipart upload to the destination key via * {@code finalizeMultipartUpload()}. + * Markers are never deleted on commit; this avoids having to + * issue many duplicate deletions. * Retry policy: retrying, translated. * Retries increment the {@code errorCount} counter. * @param destKey destination @@ -623,8 +639,8 @@ public CompleteMultipartUploadResult commitUpload( uploadId, partETags, length, - Invoker.NO_OP - ); + PutObjectOptions.keepingDirs(), + Invoker.NO_OP); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java index 7604bbe46b..321390446f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperations.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.List; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; @@ -38,11 +37,11 @@ import com.amazonaws.services.s3.model.SelectObjectContentResult; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; -import com.amazonaws.services.s3.transfer.model.UploadResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.store.audit.AuditSpanSource; import org.apache.hadoop.util.functional.CallableRaisingIOE; @@ -79,22 +78,25 @@ T retry(String action, * @param destKey destination key * @param inputStream source data. * @param length size, if known. Use -1 for not known - * @param headers optional map of custom headers. + * @param options options for the request * @return the request */ PutObjectRequest createPutObjectRequest(String destKey, InputStream inputStream, long length, - @Nullable Map headers); + @Nullable PutObjectOptions options); /** * Create a {@link PutObjectRequest} request to upload a file. * @param dest key to PUT to. * @param sourceFile source file + * @param options options for the request * @return the request */ - PutObjectRequest createPutObjectRequest(String dest, - File sourceFile); + PutObjectRequest createPutObjectRequest( + String dest, + File sourceFile, + @Nullable PutObjectOptions options); /** * Callback on a successful write. @@ -121,11 +123,12 @@ PutObjectRequest createPutObjectRequest(String dest, * Start the multipart upload process. * Retry policy: retrying, translated. * @param destKey destination of upload + * @param options options for the put request * @return the upload result containing the ID * @throws IOException IO problem */ @Retries.RetryTranslated - String initiateMultiPartUpload(String destKey) throws IOException; + String initiateMultiPartUpload(String destKey, PutObjectOptions options) throws IOException; /** * This completes a multipart upload to the destination key via @@ -138,6 +141,7 @@ PutObjectRequest createPutObjectRequest(String dest, * @param length length of the upload * @param errorCount a counter incremented by 1 on every error; for * use in statistics + * @param putOptions put object options * @return the result of the operation. * @throws IOException if problems arose which could not be retried, or * the retry count was exceeded @@ -148,7 +152,8 @@ CompleteMultipartUploadResult completeMPUwithRetries( String uploadId, List partETags, long length, - AtomicInteger errorCount) + AtomicInteger errorCount, + PutObjectOptions putOptions) throws IOException; /** @@ -238,26 +243,32 @@ UploadPartRequest newUploadPartRequest( * Byte length is calculated from the file length, or, if there is no * file, from the content length of the header. * @param putObjectRequest the request + * @param putOptions put object options * @return the upload initiated * @throws IOException on problems */ @Retries.RetryTranslated - PutObjectResult putObject(PutObjectRequest putObjectRequest) + PutObjectResult putObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions) throws IOException; /** * PUT an object via the transfer manager. + * * @param putObjectRequest the request - * @return the result of the operation + * @param putOptions put object options + * * @throws IOException on problems */ @Retries.RetryTranslated - UploadResult uploadObject(PutObjectRequest putObjectRequest) + void uploadObject(PutObjectRequest putObjectRequest, + PutObjectOptions putOptions) throws IOException; /** * Revert a commit by deleting the file. - * Relies on retry code in filesystem + * No attempt is made to probe for/recreate a parent dir marker + * Relies on retry code in filesystem. * @throws IOException on problems * @param destKey destination key */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java index 20cd27225a..cae4d3ef03 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java @@ -50,6 +50,7 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; /** * Factory for S3 objects. @@ -141,11 +142,12 @@ CopyObjectRequest newCopyObjectRequest(String srcKey, * Adds the ACL and metadata * @param key key of object * @param metadata metadata header + * @param options options for the request * @param srcfile source file * @return the request */ PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, File srcfile); + ObjectMetadata metadata, PutObjectOptions options, File srcfile); /** * Create a {@link PutObjectRequest} request. @@ -153,11 +155,13 @@ PutObjectRequest newPutObjectRequest(String key, * operation. * @param key key of object * @param metadata metadata header + * @param options options for the request * @param inputStream source data. * @return the request */ PutObjectRequest newPutObjectRequest(String key, ObjectMetadata metadata, + PutObjectOptions options, InputStream inputStream); /** @@ -190,10 +194,12 @@ AbortMultipartUploadRequest newAbortMultipartUploadRequest( /** * Start a multipart upload. * @param destKey destination object key + * @param options options for the request * @return the request. */ InitiateMultipartUploadRequest newMultipartUploadRequest( - String destKey); + String destKey, + @Nullable PutObjectOptions options); /** * Complete a multipart upload. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java index f08d6448e4..78b687cc6f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java @@ -26,32 +26,30 @@ import java.util.Date; import java.util.List; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; import com.amazonaws.services.s3.model.MultipartUpload; - -import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.util.Preconditions; -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.audit.AuditConstants; -import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.audit.AuditSpanSource; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; +import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; import org.apache.hadoop.fs.s3a.commit.files.SuccessData; +import org.apache.hadoop.fs.s3a.commit.impl.AuditContextUpdater; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; @@ -62,24 +60,33 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperations; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestStoreOperationsThroughFileSystem; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.DurationInfo; -import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.hadoop.util.functional.InvocationRaisingIOE; +import org.apache.hadoop.util.functional.TaskPool; -import static org.apache.hadoop.fs.s3a.Constants.THREAD_POOL_SHUTDOWN_DELAY_SECONDS; +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_MAX_TOTAL_TASKS; +import static org.apache.hadoop.fs.s3a.Constants.MAXIMUM_CONNECTIONS; +import static org.apache.hadoop.fs.s3a.Constants.MAX_TOTAL_TASKS; import static org.apache.hadoop.fs.s3a.Invoker.ignoreIOExceptions; import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_COMMIT_JOB; import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*; -import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.E_NO_SPARK_UUID; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.FS_S3A_COMMITTER_UUID_SOURCE; import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.SPARK_WRITE_UUID; +import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation; +import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.DiagnosticKeys.STAGE; +import static org.apache.hadoop.mapreduce.lib.output.committer.manifest.impl.ManifestCommitterSupport.createJobSummaryFilename; +import static org.apache.hadoop.util.functional.RemoteIterators.toList; /** * Abstract base class for S3A committers; allows for any commonality @@ -136,8 +143,6 @@ public abstract class AbstractS3ACommitter extends PathOutputCommitter */ private final JobUUIDSource uuidSource; - private final CommonAuditContext commonAuditContext; - /** * Has this instance been used for job setup? * If so then it is safe for a locally generated @@ -145,11 +150,6 @@ public abstract class AbstractS3ACommitter extends PathOutputCommitter */ private boolean jobSetup; - /** - * Thread pool for task execution. - */ - private ExecutorService threadPool; - /** Underlying commit operations. */ private final CommitOperations commitOperations; @@ -203,9 +203,8 @@ protected AbstractS3ACommitter( Path outputPath, TaskAttemptContext context) throws IOException { super(outputPath, context); - Preconditions.checkArgument(outputPath != null, "null output path"); - Preconditions.checkArgument(context != null, "null job context"); - this.jobContext = context; + setOutputPath(outputPath); + this.jobContext = requireNonNull(context, "null job context"); this.role = "Task committer " + context.getTaskAttemptID(); setConf(context.getConfiguration()); Pair id = buildJobUUID( @@ -219,17 +218,20 @@ protected AbstractS3ACommitter( S3AFileSystem fs = getDestS3AFS(); // set this thread's context with the job ID. // audit spans created in this thread will pick - // up this value. - this.commonAuditContext = currentAuditContext(); - updateCommonContext(); + // up this value., including the commit operations instance + // soon to be created. + new AuditContextUpdater(jobContext) + .updateCurrentAuditContext(); + // the filesystem is the span source, always. - auditSpanSource = fs.getAuditSpanSource(); + this.auditSpanSource = fs.getAuditSpanSource(); this.createJobMarker = context.getConfiguration().getBoolean( CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER); // the statistics are shared between this committer and its operations. this.committerStatistics = fs.newCommitterStatistics(); - this.commitOperations = new CommitOperations(fs, committerStatistics); + this.commitOperations = new CommitOperations(fs, committerStatistics, + outputPath.toString()); } /** @@ -267,8 +269,7 @@ public final Path getOutputPath() { * @param outputPath new value */ protected final void setOutputPath(Path outputPath) { - Preconditions.checkNotNull(outputPath, "Null output path"); - this.outputPath = outputPath; + this.outputPath = requireNonNull(outputPath, "Null output path"); } /** @@ -338,6 +339,12 @@ public Path getJobAttemptPath(JobContext context) { return getJobAttemptPath(getAppAttemptId(context)); } + /** + * Compute the path under which all job attempts will be placed. + * @return the path to store job attempt data. + */ + protected abstract Path getJobPath(); + /** * Compute the path where the output of a given job attempt will be placed. * @param appAttemptId the ID of the application attempt for this job. @@ -440,6 +447,7 @@ protected FileSystem getDestinationFS(Path out, Configuration config) protected boolean requiresDelayedCommitOutputInFileSystem() { return false; } + /** * Task recovery considered Unsupported: Warn and fail. * @param taskContext Context of the task whose output is being recovered @@ -450,7 +458,7 @@ public void recoverTask(TaskAttemptContext taskContext) throws IOException { LOG.warn("Cannot recover task {}", taskContext.getTaskAttemptID()); throw new PathCommitException(outputPath, String.format("Unable to recover task %s", - taskContext.getTaskAttemptID())); + taskContext.getTaskAttemptID())); } /** @@ -459,11 +467,16 @@ public void recoverTask(TaskAttemptContext taskContext) throws IOException { * * While the classic committers create a 0-byte file, the S3A committers * PUT up a the contents of a {@link SuccessData} file. + * * @param context job context * @param pending the pending commits + * + * @return the success data, even if the marker wasn't created + * * @throws IOException IO failure */ - protected void maybeCreateSuccessMarkerFromCommits(JobContext context, + protected SuccessData maybeCreateSuccessMarkerFromCommits( + JobContext context, ActiveCommit pending) throws IOException { List filenames = new ArrayList<>(pending.size()); // The list of committed objects in pending is size limited in @@ -472,41 +485,86 @@ protected void maybeCreateSuccessMarkerFromCommits(JobContext context, // load in all the pending statistics IOStatisticsSnapshot snapshot = new IOStatisticsSnapshot( pending.getIOStatistics()); + // and the current statistics snapshot.aggregate(getIOStatistics()); - maybeCreateSuccessMarker(context, filenames, snapshot); + return maybeCreateSuccessMarker(context, filenames, snapshot); } /** * if the job requires a success marker on a successful job, - * create the file {@link CommitConstants#_SUCCESS}. + * create the {@code _SUCCESS} file. * * While the classic committers create a 0-byte file, the S3A committers * PUT up a the contents of a {@link SuccessData} file. + * The file is returned, even if no marker is created. + * This is so it can be saved to a report directory. * @param context job context * @param filenames list of filenames. * @param ioStatistics any IO Statistics to include * @throws IOException IO failure + * @return the success data. */ - protected void maybeCreateSuccessMarker(JobContext context, - List filenames, + protected SuccessData maybeCreateSuccessMarker( + final JobContext context, + final List filenames, final IOStatisticsSnapshot ioStatistics) throws IOException { + + SuccessData successData = + createSuccessData(context, filenames, ioStatistics, + getDestFS().getConf()); if (createJobMarker) { - // create a success data structure and then save it - SuccessData successData = new SuccessData(); - successData.setCommitter(getName()); - successData.setJobId(uuid); - successData.setJobIdSource(uuidSource.getText()); - successData.setDescription(getRole()); - successData.setHostname(NetUtils.getLocalHostname()); - Date now = new Date(); - successData.setTimestamp(now.getTime()); - successData.setDate(now.toString()); - successData.setFilenames(filenames); - successData.getIOStatistics().aggregate(ioStatistics); + // save it to the job dest dir commitOperations.createSuccessMarker(getOutputPath(), successData, true); } + return successData; + } + + /** + * Create the success data structure from a job context. + * @param context job context. + * @param filenames short list of filenames; nullable + * @param ioStatistics IOStatistics snapshot + * @param destConf config of the dest fs, can be null + * @return the structure + * + */ + private SuccessData createSuccessData(final JobContext context, + final List filenames, + final IOStatisticsSnapshot ioStatistics, + final Configuration destConf) { + // create a success data structure + SuccessData successData = new SuccessData(); + successData.setCommitter(getName()); + successData.setJobId(uuid); + successData.setJobIdSource(uuidSource.getText()); + successData.setDescription(getRole()); + successData.setHostname(NetUtils.getLocalHostname()); + Date now = new Date(); + successData.setTimestamp(now.getTime()); + successData.setDate(now.toString()); + if (filenames != null) { + successData.setFilenames(filenames); + } + successData.getIOStatistics().aggregate(ioStatistics); + // attach some config options as diagnostics to assist + // in debugging performance issues. + + // commit thread pool size + successData.addDiagnostic(FS_S3A_COMMITTER_THREADS, + Integer.toString(getJobCommitThreadCount(context))); + + // and filesystem http connection and thread pool sizes + if (destConf != null) { + successData.addDiagnostic(MAXIMUM_CONNECTIONS, + destConf.get(MAXIMUM_CONNECTIONS, + Integer.toString(DEFAULT_MAXIMUM_CONNECTIONS))); + successData.addDiagnostic(MAX_TOTAL_TASKS, + destConf.get(MAX_TOTAL_TASKS, + Integer.toString(DEFAULT_MAX_TOTAL_TASKS))); + } + return successData; } /** @@ -556,7 +614,11 @@ public void setupJob(JobContext context) throws IOException { @Override public void setupTask(TaskAttemptContext context) throws IOException { TaskAttemptID attemptID = context.getTaskAttemptID(); - updateCommonContext(); + + // update the context so that task IO in the same thread has + // the relevant values. + new AuditContextUpdater(context) + .updateCurrentAuditContext(); try (DurationInfo d = new DurationInfo(LOG, "Setup Task %s", attemptID)) { @@ -572,6 +634,9 @@ && getUUIDSource() == JobUUIDSource.GeneratedLocally) { } Path taskAttemptPath = getTaskAttemptPath(context); FileSystem fs = taskAttemptPath.getFileSystem(getConf()); + // delete that ta path if somehow it was there + fs.delete(taskAttemptPath, true); + // create an empty directory fs.mkdirs(taskAttemptPath); } } @@ -596,25 +661,23 @@ protected FileSystem getTaskAttemptFilesystem(TaskAttemptContext context) * On a failure or abort of a single file's commit, all its uploads are * aborted. * The revert operation lists the files already committed and deletes them. - * @param context job context + * @param commitContext commit context * @param pending pending uploads * @throws IOException on any failure */ protected void commitPendingUploads( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending) throws IOException { if (pending.isEmpty()) { LOG.warn("{}: No pending uploads to commit", getRole()); } try (DurationInfo ignored = new DurationInfo(LOG, - "committing the output of %s task(s)", pending.size()); - CommitOperations.CommitContext commitContext - = initiateCommitOperation()) { + "committing the output of %s task(s)", pending.size())) { - Tasks.foreach(pending.getSourceFiles()) + TaskPool.foreach(pending.getSourceFiles()) .stopOnFailure() .suppressExceptions(false) - .executeWith(buildSubmitter(context)) + .executeWith(commitContext.getOuterSubmitter()) .abortWith(status -> loadAndAbort(commitContext, pending, status, true, false)) .revertWith(status -> @@ -629,35 +692,39 @@ protected void commitPendingUploads( * This check avoids the situation where the inability to read * a file only surfaces partway through the job commit, so * results in the destination being tainted. - * @param context job context + * @param commitContext commit context * @param pending the pending operations * @throws IOException any failure */ protected void precommitCheckPendingFiles( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending) throws IOException { FileSystem sourceFS = pending.getSourceFS(); try (DurationInfo ignored = new DurationInfo(LOG, "Preflight Load of pending files")) { - Tasks.foreach(pending.getSourceFiles()) + TaskPool.foreach(pending.getSourceFiles()) .stopOnFailure() .suppressExceptions(false) - .executeWith(buildSubmitter(context)) - .run(status -> PendingSet.load(sourceFS, status)); + .executeWith(commitContext.getOuterSubmitter()) + .run(status -> PersistentCommitData.load(sourceFS, status, + commitContext.getPendingSetSerializer())); } } /** * Load a pendingset file and commit all of its contents. + * Invoked within a parallel run; the commitContext thread + * pool is already busy/possibly full, so do not + * execute work through the same submitter. * @param commitContext context to commit through * @param activeCommit commit state * @param status file to load * @throws IOException failure */ private void loadAndCommit( - final CommitOperations.CommitContext commitContext, + final CommitContext commitContext, final ActiveCommit activeCommit, final FileStatus status) throws IOException { @@ -665,18 +732,20 @@ private void loadAndCommit( try (DurationInfo ignored = new DurationInfo(LOG, "Loading and committing files in pendingset %s", path)) { - PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(), - status); + PendingSet pendingSet = PersistentCommitData.load( + activeCommit.getSourceFS(), + status, + commitContext.getPendingSetSerializer()); String jobId = pendingSet.getJobId(); if (!StringUtils.isEmpty(jobId) && !getUUID().equals(jobId)) { throw new PathCommitException(path, String.format("Mismatch in Job ID (%s) and commit job ID (%s)", getUUID(), jobId)); } - Tasks.foreach(pendingSet.getCommits()) + TaskPool.foreach(pendingSet.getCommits()) .stopOnFailure() .suppressExceptions(false) - .executeWith(singleThreadSubmitter()) + .executeWith(commitContext.getInnerSubmitter()) .onFailure((commit, exception) -> commitContext.abortSingleCommit(commit)) .abortWith(commitContext::abortSingleCommit) @@ -692,22 +761,27 @@ private void loadAndCommit( /** * Load a pendingset file and revert all of its contents. + * Invoked within a parallel run; the commitContext thread + * pool is already busy/possibly full, so do not + * execute work through the same submitter. * @param commitContext context to commit through * @param activeCommit commit state * @param status status of file to load * @throws IOException failure */ private void loadAndRevert( - final CommitOperations.CommitContext commitContext, + final CommitContext commitContext, final ActiveCommit activeCommit, final FileStatus status) throws IOException { final Path path = status.getPath(); try (DurationInfo ignored = new DurationInfo(LOG, false, "Committing %s", path)) { - PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(), - status); - Tasks.foreach(pendingSet.getCommits()) + PendingSet pendingSet = PersistentCommitData.load( + activeCommit.getSourceFS(), + status, + commitContext.getPendingSetSerializer()); + TaskPool.foreach(pendingSet.getCommits()) .suppressExceptions(true) .run(commitContext::revertCommit); } @@ -715,6 +789,9 @@ private void loadAndRevert( /** * Load a pendingset file and abort all of its contents. + * Invoked within a parallel run; the commitContext thread + * pool is already busy/possibly full, so do not + * execute work through the same submitter. * @param commitContext context to commit through * @param activeCommit commit state * @param status status of file to load @@ -722,7 +799,7 @@ private void loadAndRevert( * @throws IOException failure */ private void loadAndAbort( - final CommitOperations.CommitContext commitContext, + final CommitContext commitContext, final ActiveCommit activeCommit, final FileStatus status, final boolean suppressExceptions, @@ -731,11 +808,13 @@ private void loadAndAbort( final Path path = status.getPath(); try (DurationInfo ignored = new DurationInfo(LOG, false, "Aborting %s", path)) { - PendingSet pendingSet = PendingSet.load(activeCommit.getSourceFS(), - status); + PendingSet pendingSet = PersistentCommitData.load( + activeCommit.getSourceFS(), + status, + commitContext.getPendingSetSerializer()); FileSystem fs = getDestFS(); - Tasks.foreach(pendingSet.getCommits()) - .executeWith(singleThreadSubmitter()) + TaskPool.foreach(pendingSet.getCommits()) + .executeWith(commitContext.getInnerSubmitter()) .suppressExceptions(suppressExceptions) .run(commit -> { try { @@ -752,28 +831,51 @@ private void loadAndAbort( } /** - * Start the final commit/abort commit operations. + * Start the final job commit/abort commit operations. + * @param context job context * @return a commit context through which the operations can be invoked. * @throws IOException failure. */ - protected CommitOperations.CommitContext initiateCommitOperation() + protected CommitContext initiateJobOperation( + final JobContext context) throws IOException { - return getCommitOperations().initiateCommitOperation(getOutputPath()); + + return getCommitOperations().createCommitContext( + context, + getOutputPath(), + getJobCommitThreadCount(context)); + } + /** + * Start a ask commit/abort commit operations. + * This may have a different thread count. + * @param context job or task context + * @return a commit context through which the operations can be invoked. + * @throws IOException failure. + */ + protected CommitContext initiateTaskOperation( + final JobContext context) + throws IOException { + + return getCommitOperations().createCommitContext( + context, + getOutputPath(), + getTaskCommitThreadCount(context)); } /** * Internal Job commit operation: where the S3 requests are made * (potentially in parallel). - * @param context job context + * @param commitContext commit context * @param pending pending commits * @throws IOException any failure */ - protected void commitJobInternal(JobContext context, - ActiveCommit pending) + protected void commitJobInternal( + final CommitContext commitContext, + final ActiveCommit pending) throws IOException { trackDurationOfInvocation(committerStatistics, COMMITTER_COMMIT_JOB.getSymbol(), - () -> commitPendingUploads(context, pending)); + () -> commitPendingUploads(commitContext, pending)); } @Override @@ -782,7 +884,9 @@ public void abortJob(JobContext context, JobStatus.State state) LOG.info("{}: aborting job {} in state {}", getRole(), jobIdString(context), state); // final cleanup operations - abortJobInternal(context, false); + try (CommitContext commitContext = initiateJobOperation(context)){ + abortJobInternal(commitContext, false); + } } @@ -790,30 +894,33 @@ public void abortJob(JobContext context, JobStatus.State state) * The internal job abort operation; can be overridden in tests. * This must clean up operations; it is called when a commit fails, as * well as in an {@link #abortJob(JobContext, JobStatus.State)} call. - * The base implementation calls {@link #cleanup(JobContext, boolean)} + * The base implementation calls {@link #cleanup(CommitContext, boolean)} * so cleans up the filesystems and destroys the thread pool. * Subclasses must always invoke this superclass method after their * own operations. - * @param context job context + * Creates and closes its own commit context. + * + * @param commitContext commit context * @param suppressExceptions should exceptions be suppressed? * @throws IOException any IO problem raised when suppressExceptions is false. */ - protected void abortJobInternal(JobContext context, + protected void abortJobInternal(CommitContext commitContext, boolean suppressExceptions) throws IOException { - cleanup(context, suppressExceptions); + cleanup(commitContext, suppressExceptions); } /** * Abort all pending uploads to the destination directory during * job cleanup operations. * Note: this instantiates the thread pool if required -so - * {@link #destroyThreadPool()} must be called after this. * @param suppressExceptions should exceptions be suppressed + * @param commitContext commit context * @throws IOException IO problem */ protected void abortPendingUploadsInCleanup( - boolean suppressExceptions) throws IOException { + boolean suppressExceptions, + CommitContext commitContext) throws IOException { // return early if aborting is disabled. if (!shouldAbortUploadsInCleanup()) { LOG.debug("Not cleanup up pending uploads to {} as {} is false ", @@ -824,9 +931,7 @@ protected void abortPendingUploadsInCleanup( Path dest = getOutputPath(); try (DurationInfo ignored = new DurationInfo(LOG, "Aborting all pending commits under %s", - dest); - CommitOperations.CommitContext commitContext - = initiateCommitOperation()) { + dest)) { CommitOperations ops = getCommitOperations(); List pending; try { @@ -840,8 +945,8 @@ protected void abortPendingUploadsInCleanup( LOG.warn("{} pending uploads were found -aborting", pending.size()); LOG.warn("If other tasks/jobs are writing to {}," + "this action may cause them to fail", dest); - Tasks.foreach(pending) - .executeWith(buildSubmitter(getJobContext())) + TaskPool.foreach(pending) + .executeWith(commitContext.getOuterSubmitter()) .suppressExceptions(suppressExceptions) .run(u -> commitContext.abortMultipartCommit( u.getKey(), u.getUploadId())); @@ -863,12 +968,12 @@ private boolean shouldAbortUploadsInCleanup() { * they can be loaded. * The Magic committer does not, because of the overhead of reading files * from S3 makes it too expensive. - * @param context job context + * @param commitContext commit context * @param pending the pending operations * @throws IOException any failure */ @VisibleForTesting - public void preCommitJob(JobContext context, + public void preCommitJob(CommitContext commitContext, ActiveCommit pending) throws IOException { } @@ -890,23 +995,55 @@ public void preCommitJob(JobContext context, @Override public void commitJob(JobContext context) throws IOException { String id = jobIdString(context); + // the commit context is created outside a try-with-resources block + // so it can be used in exception handling. + CommitContext commitContext = null; + + SuccessData successData = null; + IOException failure = null; + String stage = "preparing"; try (DurationInfo d = new DurationInfo(LOG, "%s: commitJob(%s)", getRole(), id)) { + commitContext = initiateJobOperation(context); ActiveCommit pending - = listPendingUploadsToCommit(context); - preCommitJob(context, pending); - commitJobInternal(context, pending); + = listPendingUploadsToCommit(commitContext); + stage = "precommit"; + preCommitJob(commitContext, pending); + stage = "commit"; + commitJobInternal(commitContext, pending); + stage = "completed"; jobCompleted(true); - maybeCreateSuccessMarkerFromCommits(context, pending); - cleanup(context, false); + stage = "marker"; + successData = maybeCreateSuccessMarkerFromCommits(context, pending); + stage = "cleanup"; + cleanup(commitContext, false); } catch (IOException e) { + // failure. record it for the summary + failure = e; LOG.warn("Commit failure for job {}", id, e); jobCompleted(false); - abortJobInternal(context, true); + abortJobInternal(commitContext, true); throw e; } finally { - resetCommonContext(); + // save the report summary, even on failure + if (commitContext != null) { + if (successData == null) { + // if the commit did not get as far as creating success data, create one. + successData = createSuccessData(context, null, null, + getDestFS().getConf()); + } + // save quietly, so no exceptions are raised + maybeSaveSummary(stage, + commitContext, + successData, + failure, + true, + true); + // and close that commit context + commitContext.close(); + } } + } /** @@ -925,30 +1062,28 @@ protected void jobCompleted(boolean success) { /** * Get the list of pending uploads for this job attempt. - * @param context job context + * @param commitContext commit context * @return a list of pending uploads. * @throws IOException Any IO failure */ protected abstract ActiveCommit listPendingUploadsToCommit( - JobContext context) + CommitContext commitContext) throws IOException; /** * Cleanup the job context, including aborting anything pending * and destroying the thread pool. - * @param context job context + * @param commitContext commit context * @param suppressExceptions should exceptions be suppressed? * @throws IOException any failure if exceptions were not suppressed. */ - protected void cleanup(JobContext context, + protected void cleanup(CommitContext commitContext, boolean suppressExceptions) throws IOException { try (DurationInfo d = new DurationInfo(LOG, - "Cleanup job %s", jobIdString(context))) { - abortPendingUploadsInCleanup(suppressExceptions); + "Cleanup job %s", jobIdString(commitContext.getJobContext()))) { + abortPendingUploadsInCleanup(suppressExceptions, commitContext); } finally { - destroyThreadPool(); cleanupStagingDirs(); - resetCommonContext(); } } @@ -958,8 +1093,9 @@ public void cleanupJob(JobContext context) throws IOException { String r = getRole(); String id = jobIdString(context); LOG.warn("{}: using deprecated cleanupJob call for {}", r, id); - try (DurationInfo d = new DurationInfo(LOG, "%s: cleanup Job %s", r, id)) { - cleanup(context, true); + try (DurationInfo d = new DurationInfo(LOG, "%s: cleanup Job %s", r, id); + CommitContext commitContext = initiateJobOperation(context)) { + cleanup(commitContext, true); } } @@ -1016,137 +1152,26 @@ protected String getRole() { return role; } - /** - * Returns an {@link Tasks.Submitter} for parallel tasks. The number of - * threads in the thread-pool is set by fs.s3a.committer.threads. - * If num-threads is 0, this will return null; - * this is used in Tasks as a cue - * to switch to single-threaded execution. - * - * @param context the JobContext for this commit - * @return a submitter or null - */ - protected Tasks.Submitter buildSubmitter( - JobContext context) { - if (getThreadCount(context) > 0) { - return new PoolSubmitter(context); - } else { - return null; - } - } - - /** - * Returns an {@link ExecutorService} for parallel tasks. The number of - * threads in the thread-pool is set by fs.s3a.committer.threads. - * If num-threads is 0, this will raise an exception. - * - * @param context the JobContext for this commit - * @param numThreads threads - * @return an {@link ExecutorService} for the number of threads - */ - private synchronized ExecutorService buildThreadPool( - JobContext context, int numThreads) { - Preconditions.checkArgument(numThreads > 0, - "Cannot create a thread pool with no threads"); - if (threadPool == null) { - LOG.debug("{}: creating thread pool of size {}", getRole(), numThreads); - threadPool = HadoopExecutors.newFixedThreadPool(numThreads, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat(THREAD_PREFIX + context.getJobID() + "-%d") - .build()); - } - return threadPool; - } - /** * Get the thread count for this job's commit operations. * @param context the JobContext for this commit * @return a possibly zero thread count. */ - private int getThreadCount(final JobContext context) { + private int getJobCommitThreadCount(final JobContext context) { return context.getConfiguration().getInt( FS_S3A_COMMITTER_THREADS, DEFAULT_COMMITTER_THREADS); } /** - * Submit a runnable. - * This will demand-create the thread pool if needed. - *

- * This is synchronized to ensure the thread pool is always valid when - * work is synchronized. See HADOOP-16798. + * Get the thread count for this task's commit operations. * @param context the JobContext for this commit - * @param task task to execute - * @return the future of the submitted task. + * @return a possibly zero thread count. */ - private synchronized Future submitRunnable( - final JobContext context, - final Runnable task) { - return buildThreadPool(context, getThreadCount(context)).submit(task); - } - - /** - * The real task submitter, which hands off the work to - * the current thread pool. - */ - private final class PoolSubmitter implements Tasks.Submitter { - - private final JobContext context; - - private final int numThreads; - - private PoolSubmitter(final JobContext context) { - this.numThreads = getThreadCount(context); - Preconditions.checkArgument(numThreads > 0, - "Cannot create a thread pool with no threads"); - this.context = context; - } - - @Override - public Future submit(final Runnable task) { - return submitRunnable(context, task); - } - - } - - /** - * Destroy any thread pools; wait for that to finish, - * but don't overreact if it doesn't finish in time. - */ - protected void destroyThreadPool() { - ExecutorService pool; - // reset the thread pool in a sync block, then shut it down - // afterwards. This allows for other threads to create a - // new thread pool on demand. - synchronized(this) { - pool = this.threadPool; - threadPool = null; - } - if (pool != null) { - LOG.debug("Destroying thread pool"); - HadoopExecutors.shutdown(pool, LOG, - THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS); - } - } - - /** - * Get the thread pool for executing the single file commit/revert - * within the commit of all uploads of a single task. - * This is currently null; it is here to allow the Tasks class to - * provide the logic for execute/revert. - * @return null. always. - */ - protected final synchronized Tasks.Submitter singleThreadSubmitter() { - return null; - } - - /** - * Does this committer have a thread pool? - * @return true if a thread pool exists. - */ - public synchronized boolean hasThreadPool() { - return threadPool != null; + private int getTaskCommitThreadCount(final JobContext context) { + return context.getConfiguration().getInt( + FS_S3A_COMMITTER_THREADS, + DEFAULT_COMMITTER_THREADS); } /** @@ -1164,24 +1189,23 @@ protected void deleteTaskAttemptPathQuietly(TaskAttemptContext context) { * Abort all pending uploads in the list. * This operation is used by the magic committer as part of its * rollback after a failure during task commit. - * @param context job context + * @param commitContext commit context * @param pending pending uploads * @param suppressExceptions should exceptions be suppressed * @throws IOException any exception raised */ - protected void abortPendingUploads(JobContext context, - List pending, - boolean suppressExceptions) + protected void abortPendingUploads( + final CommitContext commitContext, + final List pending, + final boolean suppressExceptions) throws IOException { if (pending == null || pending.isEmpty()) { LOG.info("{}: no pending commits to abort", getRole()); } else { try (DurationInfo d = new DurationInfo(LOG, - "Aborting %s uploads", pending.size()); - CommitOperations.CommitContext commitContext - = initiateCommitOperation()) { - Tasks.foreach(pending) - .executeWith(buildSubmitter(context)) + "Aborting %s uploads", pending.size())) { + TaskPool.foreach(pending) + .executeWith(commitContext.getOuterSubmitter()) .suppressExceptions(suppressExceptions) .run(commitContext::abortSingleCommit); } @@ -1190,27 +1214,24 @@ protected void abortPendingUploads(JobContext context, /** * Abort all pending uploads in the list. - * @param context job context + * @param commitContext commit context * @param pending pending uploads * @param suppressExceptions should exceptions be suppressed? * @param deleteRemoteFiles should remote files be deleted? * @throws IOException any exception raised */ protected void abortPendingUploads( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending, final boolean suppressExceptions, final boolean deleteRemoteFiles) throws IOException { - if (pending.isEmpty()) { LOG.info("{}: no pending commits to abort", getRole()); } else { try (DurationInfo d = new DurationInfo(LOG, - "Aborting %s uploads", pending.size()); - CommitOperations.CommitContext commitContext - = initiateCommitOperation()) { - Tasks.foreach(pending.getSourceFiles()) - .executeWith(buildSubmitter(context)) + "Aborting %s uploads", pending.size())) { + TaskPool.foreach(pending.getSourceFiles()) + .executeWith(commitContext.getOuterSubmitter()) .suppressExceptions(suppressExceptions) .run(path -> loadAndAbort(commitContext, @@ -1392,13 +1413,7 @@ public String toString() { */ protected final void updateCommonContext() { currentAuditContext().put(AuditConstants.PARAM_JOB_ID, uuid); - } - /** - * Remove JobID from the current thread's context. - */ - protected final void resetCommonContext() { - currentAuditContext().remove(AuditConstants.PARAM_JOB_ID); } protected AuditSpanSource getAuditSpanSource() { @@ -1424,6 +1439,77 @@ protected AuditSpan startOperation(String name, return getAuditSpanSource().createSpan(name, path1, path2); } + + /** + * Save a summary to the report dir if the config option + * is set. + * The report will be updated with the current active stage, + * and if {@code thrown} is non-null, it will be added to the + * diagnostics (and the job tagged as a failure). + * Static for testability. + * @param activeStage active stage + * @param context commit context. + * @param report summary file. + * @param thrown any exception indicting failure. + * @param quiet should exceptions be swallowed. + * @param overwrite should the existing file be overwritten + * @return the path of a file, if successfully saved + * @throws IOException if a failure occured and quiet==false + */ + private static Path maybeSaveSummary( + String activeStage, + CommitContext context, + SuccessData report, + Throwable thrown, + boolean quiet, + boolean overwrite) throws IOException { + Configuration conf = context.getConf(); + String reportDir = conf.getTrimmed(OPT_SUMMARY_REPORT_DIR, ""); + if (reportDir.isEmpty()) { + LOG.debug("No summary directory set in " + OPT_SUMMARY_REPORT_DIR); + return null; + } + LOG.debug("Summary directory set to {}", reportDir); + + Path reportDirPath = new Path(reportDir); + Path path = new Path(reportDirPath, + createJobSummaryFilename(context.getJobId())); + + if (thrown != null) { + report.recordJobFailure(thrown); + } + report.putDiagnostic(STAGE, activeStage); + // the store operations here is explicitly created for the FS where + // the reports go, which may not be the target FS of the job. + + final FileSystem fs = path.getFileSystem(conf); + try (ManifestStoreOperations operations = new ManifestStoreOperationsThroughFileSystem( + fs)) { + if (!overwrite) { + // check for file existence so there is no need to worry about + // precisely what exception is raised when overwrite=false and dest file + // exists + try { + FileStatus st = operations.getFileStatus(path); + // get here and the file exists + LOG.debug("Report already exists: {}", st); + return null; + } catch (FileNotFoundException ignored) { + } + } + report.save(fs, path, SuccessData.serializer()); + LOG.info("Job summary saved to {}", path); + return path; + } catch (IOException e) { + LOG.debug("Failed to save summary to {}", path, e); + if (quiet) { + return null; + } else { + throw e; + } + } + } + /** * State of the active commit operation. * @@ -1445,7 +1531,7 @@ protected AuditSpan startOperation(String name, * * */ - public static class ActiveCommit { + public static final class ActiveCommit { private static final AbstractS3ACommitter.ActiveCommit EMPTY = new ActiveCommit(null, new ArrayList<>()); @@ -1486,6 +1572,7 @@ public static class ActiveCommit { * @param sourceFS filesystem containing the list of pending files * @param sourceFiles .pendingset files to load and commit. */ + @SuppressWarnings("unchecked") public ActiveCommit( final FileSystem sourceFS, final List sourceFiles) { @@ -1496,13 +1583,14 @@ public ActiveCommit( /** * Create an active commit of the given pending files. * @param pendingFS source filesystem. - * @param statuses list of file status or subclass to use. + * @param statuses iterator of file status or subclass to use. * @return the commit + * @throws IOException if the iterator raises one. */ - public static ActiveCommit fromStatusList( + public static ActiveCommit fromStatusIterator( final FileSystem pendingFS, - final List statuses) { - return new ActiveCommit(pendingFS, statuses); + final RemoteIterator statuses) throws IOException { + return new ActiveCommit(pendingFS, toList(statuses)); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java index bbc59f168f..b85ce27650 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java @@ -22,7 +22,6 @@ import org.apache.hadoop.classification.InterfaceStability; import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; -import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.COMMITTER_FACTORY_SCHEME_PATTERN; /** * Constants for working with committers. @@ -57,6 +56,12 @@ private CommitConstants() { */ public static final String PENDINGSET_SUFFIX = ".pendingset"; + + /** + * Prefix to use for config options: {@value}. + */ + public static final String OPT_PREFIX = "fs.s3a.committer."; + /** * Flag to indicate whether support for the Magic committer is enabled * in the filesystem. @@ -121,9 +126,8 @@ private CommitConstants() { /** * Temp data which is not auto-committed: {@value}. - * Uses a different name from normal just to make clear it is different. */ - public static final String TEMP_DATA = "__temp-data"; + public static final String TEMP_DATA = TEMPORARY; /** @@ -144,7 +148,7 @@ private CommitConstants() { * Key to set for the S3A schema to use the specific committer. */ public static final String S3A_COMMITTER_FACTORY_KEY = String.format( - COMMITTER_FACTORY_SCHEME_PATTERN, "s3a"); + "mapreduce.outputcommitter.factory.scheme.s3a"); /** * S3 Committer factory: {@value}. @@ -222,13 +226,19 @@ private CommitConstants() { /** * Number of threads in committers for parallel operations on files * (upload, commit, abort, delete...): {@value}. + * Two thread pools this size are created, one for the outer + * task-level parallelism, and one for parallel execution + * within tasks (POSTs to commit individual uploads) + * If the value is negative, it is inverted and then multiplied + * by the number of cores in the CPU. */ public static final String FS_S3A_COMMITTER_THREADS = "fs.s3a.committer.threads"; + /** * Default value for {@link #FS_S3A_COMMITTER_THREADS}: {@value}. */ - public static final int DEFAULT_COMMITTER_THREADS = 8; + public static final int DEFAULT_COMMITTER_THREADS = 32; /** * Path in the cluster filesystem for temporary data: {@value}. @@ -330,4 +340,18 @@ private CommitConstants() { public static final String XA_MAGIC_MARKER = XA_HEADER_PREFIX + X_HEADER_MAGIC_MARKER; + /** + * Task Attempt ID query header: {@value}. + */ + public static final String PARAM_TASK_ATTEMPT_ID = "ta"; + + /** + * Directory for saving job summary reports. + * These are the _SUCCESS files, but are saved even on + * job failures. + * Value: {@value}. + */ + public static final String OPT_SUMMARY_REPORT_DIR = + OPT_PREFIX + "summary.report.directory"; + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java new file mode 100644 index 0000000000..20af292b3e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitterStatisticNames.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.commit; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.statistics.StoreStatisticNames; + +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OP_RENAME; + +/** + * Statistic names for committers. + * Please keep in sync with org.apache.hadoop.fs.s3a.Statistic + * so that S3A and manifest committers are in sync. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public final class CommitterStatisticNames { + + + /** Amount of data committed: {@value}. */ + public static final String COMMITTER_BYTES_COMMITTED_COUNT = + "committer_bytes_committed"; + + /** Duration Tracking of time to commit an entire job: {@value}. */ + public static final String COMMITTER_COMMIT_JOB = + "committer_commit_job"; + + /** Number of files committed: {@value}. */ + public static final String COMMITTER_FILES_COMMITTED_COUNT = + "committer_files_committed"; + /** "Count of successful tasks:: {@value}. */ + public static final String COMMITTER_TASKS_COMPLETED_COUNT = + "committer_tasks_completed"; + + /** Count of failed tasks: {@value}. */ + public static final String COMMITTER_TASKS_FAILED_COUNT = + "committer_tasks_failed"; + + /** Count of commits aborted: {@value}. */ + public static final String COMMITTER_COMMITS_ABORTED_COUNT = + "committer_commits_aborted"; + + /** Count of commits reverted: {@value}. */ + public static final String COMMITTER_COMMITS_REVERTED_COUNT = + "committer_commits_reverted"; + + /** Count of commits failed: {@value}. */ + public static final String COMMITTER_COMMITS_FAILED = + "committer_commits" + StoreStatisticNames.SUFFIX_FAILURES; + + /** + * The number of files in a task. This will be a MeanStatistic. + */ + public static final String COMMITTER_FILE_COUNT_MEAN = + "committer_task_file_count"; + + /** + * File Size. + */ + public static final String COMMITTER_FILE_SIZE_MEAN = + "committer_task_file_size"; + + /** + * What is a task attempt's directory count. + */ + public static final String COMMITTER_TASK_DIRECTORY_COUNT_MEAN = + "committer_task_directory_count"; + + /** + * What is the depth of a task attempt's directory tree. + */ + public static final String COMMITTER_TASK_DIRECTORY_DEPTH_MEAN = + "committer_task_directory_depth"; + + /** + * The number of files in a task. This will be a MeanStatistic. + */ + public static final String COMMITTER_TASK_FILE_COUNT_MEAN = + "committer_task_file_count"; + + /** + * The number of files in a task. This will be a MeanStatistic. + */ + public static final String COMMITTER_TASK_FILE_SIZE_MEAN = + "committer_task_file_size"; + + /** Directory creation {@value}. */ + public static final String OP_CREATE_DIRECTORIES = "op_create_directories"; + + /** Creating a single directory {@value}. */ + public static final String OP_CREATE_ONE_DIRECTORY = + "op_create_one_directory"; + + /** Directory scan {@value}. */ + public static final String OP_DIRECTORY_SCAN = "op_directory_scan"; + + /** + * Overall job commit {@value}. + */ + public static final String OP_STAGE_JOB_COMMIT = COMMITTER_COMMIT_JOB; + + /** {@value}. */ + public static final String OP_LOAD_ALL_MANIFESTS = "op_load_all_manifests"; + + /** + * Load a task manifest: {@value}. + */ + public static final String OP_LOAD_MANIFEST = "op_load_manifest"; + + /** Rename a file: {@value}. */ + public static final String OP_RENAME_FILE = OP_RENAME; + + /** + * Save a task manifest: {@value}. + */ + public static final String OP_SAVE_TASK_MANIFEST = + "op_task_stage_save_task_manifest"; + + /** + * Task abort: {@value}. + */ + public static final String OP_STAGE_TASK_ABORT_TASK + = "op_task_stage_abort_task"; + + /** + * Job abort: {@value}. + */ + public static final String OP_STAGE_JOB_ABORT = "op_job_stage_abort"; + + /** + * Job cleanup: {@value}. + */ + public static final String OP_STAGE_JOB_CLEANUP = "op_job_stage_cleanup"; + + /** + * Prepare Directories Stage: {@value}. + */ + public static final String OP_STAGE_JOB_CREATE_TARGET_DIRS = + "op_job_stage_create_target_dirs"; + + /** + * Load Manifest Stage: {@value}. + */ + public static final String OP_STAGE_JOB_LOAD_MANIFESTS = + "op_job_stage_load_manifests"; + + /** + * Rename files stage duration: {@value}. + */ + public static final String OP_STAGE_JOB_RENAME_FILES = + "op_job_stage_rename_files"; + + + /** + * Job Setup Stage: {@value}. + */ + public static final String OP_STAGE_JOB_SETUP = "op_job_stage_setup"; + + /** + * Job saving _SUCCESS marker Stage: {@value}. + */ + public static final String OP_STAGE_JOB_SAVE_SUCCESS = + "op_job_stage_save_success_marker"; + + /** + * Output Validation (within job commit) Stage: {@value}. + */ + public static final String OP_STAGE_JOB_VALIDATE_OUTPUT = + + "op_job_stage_optional_validate_output"; + /** + * Task saving manifest file Stage: {@value}. + */ + public static final String OP_STAGE_TASK_SAVE_MANIFEST = + "op_task_stage_save_manifest"; + + /** + * Task Setup Stage: {@value}. + */ + public static final String OP_STAGE_TASK_SETUP = "op_task_stage_setup"; + + + /** + * Task Commit Stage: {@value}. + */ + public static final String OP_STAGE_TASK_COMMIT = "op_stage_task_commit"; + + /** Task Scan directory Stage: {@value}. */ + public static final String OP_STAGE_TASK_SCAN_DIRECTORY + = "op_stage_task_scan_directory"; + + private CommitterStatisticNames() { + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java index fcafdd1ed1..b2d4bfaeea 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java @@ -35,6 +35,12 @@ @InterfaceStability.Unstable public final class InternalCommitterConstants { + /** + * How long threads in the thread pool stay alive when + * idle. Value in seconds: {@value}. + */ + public static final long THREAD_KEEP_ALIVE_TIME = 60L; + private InternalCommitterConstants() { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java index 41d36b2a8d..e6524c9196 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker; import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation; +import org.apache.hadoop.fs.s3a.statistics.PutTrackerStatistics; import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*; @@ -38,8 +39,9 @@ * in this case: *
    *
  1. {@link #isMagicCommitPath(Path)} will always return false.
    2. - *
  2. {@link #createTracker(Path, String)} will always return an instance - * of {@link PutTracker}.
    3. + *
  3. {@link #isUnderMagicPath(Path)} will always return false.
    4. + *
  4. {@link #createTracker(Path, String, PutTrackerStatistics)} will always + * return an instance of {@link PutTracker}.
    5. *
* *

Important

: must not directly or indirectly import a class which @@ -88,9 +90,11 @@ public String keyOfFinalDestination(List elements, String key) { * for the commit tracker. * @param path path of nominal write * @param key key of path of nominal write + * @param trackerStatistics tracker statistics * @return the tracker for this operation. */ - public PutTracker createTracker(Path path, String key) { + public PutTracker createTracker(Path path, String key, + PutTrackerStatistics trackerStatistics) { final List elements = splitPathToElements(path); PutTracker tracker; @@ -106,7 +110,8 @@ public PutTracker createTracker(Path path, String key) { key, destKey, pendingsetPath, - owner.getWriteOperationHelper()); + owner.getWriteOperationHelper(), + trackerStatistics); LOG.debug("Created {}", tracker); } else { LOG.warn("File being created has a \"magic\" path, but the filesystem" @@ -184,4 +189,13 @@ private boolean isCommitMetadataFile(List elements) { || last.endsWith(CommitConstants.PENDINGSET_SUFFIX); } + /** + * Is this path in/under a magic path...regardless of file type. + * This is used to optimize create() operations. + * @param path path to check + * @return true if the path is one a magic file write expects. + */ + public boolean isUnderMagicPath(Path path) { + return magicCommitEnabled && isMagicPath(splitPathToElements(path)); + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java deleted file mode 100644 index c318e86605..0000000000 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a.commit; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utility class for parallel execution, takes closures for the various - * actions. - * There is no retry logic: it is expected to be handled by the closures. - */ -public final class Tasks { - private static final Logger LOG = LoggerFactory.getLogger(Tasks.class); - - private Tasks() { - } - - /** - * Callback invoked to process an item. - * @param item type being processed - * @param exception class which may be raised - */ - @FunctionalInterface - public interface Task { - void run(I item) throws E; - } - - /** - * Callback invoked on a failure. - * @param item type being processed - * @param exception class which may be raised - */ - @FunctionalInterface - public interface FailureTask { - - /** - * process a failure. - * @param item item the task is processing - * @param exception the exception which was raised. - * @throws E Exception of type E - */ - void run(I item, Exception exception) throws E; - } - - /** - * Builder for task execution. - * @param item type - */ - public static class Builder { - private final Iterable items; - private Submitter service = null; - private FailureTask onFailure = null; - private boolean stopOnFailure = false; - private boolean suppressExceptions = false; - private Task revertTask = null; - private boolean stopRevertsOnFailure = false; - private Task abortTask = null; - private boolean stopAbortsOnFailure = false; - - /** - * Create the builder. - * @param items items to process - */ - Builder(Iterable items) { - this.items = items; - } - - /** - * Declare executor service: if null, the tasks are executed in a single - * thread. - * @param submitter service to schedule tasks with. - * @return this builder. - */ - public Builder executeWith(Submitter submitter) { - this.service = submitter; - return this; - } - - public Builder onFailure(FailureTask task) { - this.onFailure = task; - return this; - } - - public Builder stopOnFailure() { - this.stopOnFailure = true; - return this; - } - - public Builder suppressExceptions() { - return suppressExceptions(true); - } - - public Builder suppressExceptions(boolean suppress) { - this.suppressExceptions = suppress; - return this; - } - - public Builder revertWith(Task task) { - this.revertTask = task; - return this; - } - - public Builder stopRevertsOnFailure() { - this.stopRevertsOnFailure = true; - return this; - } - - public Builder abortWith(Task task) { - this.abortTask = task; - return this; - } - - public Builder stopAbortsOnFailure() { - this.stopAbortsOnFailure = true; - return this; - } - - public boolean run(Task task) throws E { - if (service != null) { - return runParallel(task); - } else { - return runSingleThreaded(task); - } - } - - private boolean runSingleThreaded(Task task) - throws E { - List succeeded = new ArrayList<>(); - List exceptions = new ArrayList<>(); - - Iterator iterator = items.iterator(); - boolean threw = true; - try { - while (iterator.hasNext()) { - I item = iterator.next(); - try { - task.run(item); - succeeded.add(item); - - } catch (Exception e) { - exceptions.add(e); - - if (onFailure != null) { - try { - onFailure.run(item, e); - } catch (Exception failException) { - LOG.error("Failed to clean up on failure", e); - // keep going - } - } - - if (stopOnFailure) { - break; - } - } - } - - threw = false; - - } finally { - // threw handles exceptions that were *not* caught by the catch block, - // and exceptions that were caught and possibly handled by onFailure - // are kept in exceptions. - if (threw || !exceptions.isEmpty()) { - if (revertTask != null) { - boolean failed = false; - for (I item : succeeded) { - try { - revertTask.run(item); - } catch (Exception e) { - LOG.error("Failed to revert task", e); - failed = true; - // keep going - } - if (stopRevertsOnFailure && failed) { - break; - } - } - } - - if (abortTask != null) { - boolean failed = false; - while (iterator.hasNext()) { - try { - abortTask.run(iterator.next()); - } catch (Exception e) { - failed = true; - LOG.error("Failed to abort task", e); - // keep going - } - if (stopAbortsOnFailure && failed) { - break; - } - } - } - } - } - - if (!suppressExceptions && !exceptions.isEmpty()) { - Tasks.throwOne(exceptions); - } - - return !threw && exceptions.isEmpty(); - } - - private boolean runParallel(final Task task) - throws E { - final Queue succeeded = new ConcurrentLinkedQueue<>(); - final Queue exceptions = new ConcurrentLinkedQueue<>(); - final AtomicBoolean taskFailed = new AtomicBoolean(false); - final AtomicBoolean abortFailed = new AtomicBoolean(false); - final AtomicBoolean revertFailed = new AtomicBoolean(false); - - List> futures = new ArrayList<>(); - - for (final I item : items) { - // submit a task for each item that will either run or abort the task - futures.add(service.submit(new Runnable() { - @Override - public void run() { - if (!(stopOnFailure && taskFailed.get())) { - // run the task - boolean threw = true; - try { - LOG.debug("Executing task"); - task.run(item); - succeeded.add(item); - LOG.debug("Task succeeded"); - - threw = false; - - } catch (Exception e) { - taskFailed.set(true); - exceptions.add(e); - LOG.info("Task failed", e); - - if (onFailure != null) { - try { - onFailure.run(item, e); - } catch (Exception failException) { - LOG.error("Failed to clean up on failure", e); - // swallow the exception - } - } - } finally { - if (threw) { - taskFailed.set(true); - } - } - - } else if (abortTask != null) { - // abort the task instead of running it - if (stopAbortsOnFailure && abortFailed.get()) { - return; - } - - boolean failed = true; - try { - LOG.info("Aborting task"); - abortTask.run(item); - failed = false; - } catch (Exception e) { - LOG.error("Failed to abort task", e); - // swallow the exception - } finally { - if (failed) { - abortFailed.set(true); - } - } - } - } - })); - } - - // let the above tasks complete (or abort) - waitFor(futures); - int futureCount = futures.size(); - futures.clear(); - - if (taskFailed.get() && revertTask != null) { - // at least one task failed, revert any that succeeded - LOG.info("Reverting all {} succeeded tasks from {} futures", - succeeded.size(), futureCount); - for (final I item : succeeded) { - futures.add(service.submit(() -> { - if (stopRevertsOnFailure && revertFailed.get()) { - return; - } - - boolean failed = true; - try { - revertTask.run(item); - failed = false; - } catch (Exception e) { - LOG.error("Failed to revert task", e); - // swallow the exception - } finally { - if (failed) { - revertFailed.set(true); - } - } - })); - } - - // let the revert tasks complete - waitFor(futures); - } - - if (!suppressExceptions && !exceptions.isEmpty()) { - Tasks.throwOne(exceptions); - } - - return !taskFailed.get(); - } - } - - /** - * Wait for all the futures to complete; there's a small sleep between - * each iteration; enough to yield the CPU. - * @param futures futures. - */ - private static void waitFor(Collection> futures) { - int size = futures.size(); - LOG.debug("Waiting for {} tasks to complete", size); - int oldNumFinished = 0; - while (true) { - int numFinished = (int) futures.stream().filter(Future::isDone).count(); - - if (oldNumFinished != numFinished) { - LOG.debug("Finished count -> {}/{}", numFinished, size); - oldNumFinished = numFinished; - } - - if (numFinished == size) { - // all of the futures are done, stop looping - break; - } else { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - futures.forEach(future -> future.cancel(true)); - Thread.currentThread().interrupt(); - break; - } - } - } - } - - public static Builder foreach(Iterable items) { - return new Builder<>(items); - } - - public static Builder foreach(I[] items) { - return new Builder<>(Arrays.asList(items)); - } - - @SuppressWarnings("unchecked") - private static void throwOne( - Collection exceptions) - throws E { - Iterator iter = exceptions.iterator(); - Exception e = iter.next(); - Class exceptionClass = e.getClass(); - - while (iter.hasNext()) { - Exception other = iter.next(); - if (!exceptionClass.isInstance(other)) { - e.addSuppressed(other); - } - } - - Tasks.castAndThrow(e); - } - - @SuppressWarnings("unchecked") - private static void castAndThrow(Exception e) throws E { - if (e instanceof RuntimeException) { - throw (RuntimeException) e; - } - throw (E) e; - } - - /** - * Interface to whatever lets us submit tasks. - */ - public interface Submitter { - - /** - * Submit work. - * @param task task to execute - * @return the future of the submitted task. - */ - Future submit(Runnable task); - } - -} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java index 318896e236..108e6d45bc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.commit.ValidationFailure; +import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.util.JsonSerialization; @@ -63,8 +64,7 @@ @SuppressWarnings("unused") @InterfaceAudience.Private @InterfaceStability.Unstable -public class PendingSet extends PersistentCommitData - implements IOStatisticsSource { +public class PendingSet extends PersistentCommitData { private static final Logger LOG = LoggerFactory.getLogger(PendingSet.class); /** @@ -112,38 +112,13 @@ public PendingSet(int size) { } /** - * Get a JSON serializer for this class. + * Get a shared JSON serializer for this class. * @return a serializer. */ public static JsonSerialization serializer() { - return new JsonSerialization<>(PendingSet.class, false, true); + return new JsonSerialization<>(PendingSet.class, false, false); } - /** - * Load an instance from a file, then validate it. - * @param fs filesystem - * @param path path - * @return the loaded instance - * @throws IOException IO failure - * @throws ValidationFailure if the data is invalid - */ - public static PendingSet load(FileSystem fs, Path path) - throws IOException { - return load(fs, path, null); - } - - /** - * Load an instance from a file, then validate it. - * @param fs filesystem - * @param status status of file to load - * @return the loaded instance - * @throws IOException IO failure - * @throws ValidationFailure if the data is invalid - */ - public static PendingSet load(FileSystem fs, FileStatus status) - throws IOException { - return load(fs, status.getPath(), status); - } /** * Load an instance from a file, then validate it. @@ -211,8 +186,8 @@ public void validate() throws ValidationFailure { } @Override - public byte[] toBytes() throws IOException { - return serializer().toBytes(this); + public byte[] toBytes(JsonSerialization serializer) throws IOException { + return serializer.toBytes(this); } /** @@ -224,9 +199,10 @@ public int size() { } @Override - public void save(FileSystem fs, Path path, boolean overwrite) - throws IOException { - serializer().save(fs, path, this, overwrite); + public IOStatistics save(final FileSystem fs, + final Path path, + final JsonSerialization serializer) throws IOException { + return saveFile(fs, path, this, serializer, true); } /** @return the version marker. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java index dba44b9a01..be2de6cf89 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java @@ -21,19 +21,33 @@ import java.io.IOException; import java.io.Serializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.commit.ValidationFailure; +import org.apache.hadoop.fs.statistics.IOStatistics; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.util.JsonSerialization; + +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; /** * Class for single/multiple commit data structures. + * The mapreduce hierarchy {@code AbstractManifestData} is a fork + * of this; the Success data JSON format must stay compatible */ -@SuppressWarnings("serial") @InterfaceAudience.Private @InterfaceStability.Unstable -public abstract class PersistentCommitData implements Serializable { +public abstract class PersistentCommitData + implements Serializable, IOStatisticsSource { + private static final Logger LOG = LoggerFactory.getLogger(PersistentCommitData.class); /** * Supported version value: {@value}. @@ -52,18 +66,109 @@ public abstract class PersistentCommitData implements Serializable { * Serialize to JSON and then to a byte array, after performing a * preflight validation of the data to be saved. * @return the data in a persistable form. + * @param serializer serializer to use * @throws IOException serialization problem or validation failure. */ - public abstract byte[] toBytes() throws IOException; + public abstract byte[] toBytes(JsonSerialization serializer) throws IOException; /** * Save to a hadoop filesystem. + * The destination file is overwritten, and on s3a stores the + * performance flag is set to turn off all existence checks and + * parent dir cleanup. + * The assumption here is: the job knows what it is doing. + * * @param fs filesystem * @param path path - * @param overwrite should any existing file be overwritten + * @param serializer serializer to use + * @return IOStats from the output stream. + * * @throws IOException IO exception */ - public abstract void save(FileSystem fs, Path path, boolean overwrite) + public abstract IOStatistics save(FileSystem fs, Path path, JsonSerialization serializer) throws IOException; + /** + * Load an instance from a status, then validate it. + * This uses the openFile() API, which S3A supports for + * faster load and declaring sequential access, always + * @param type of persistent format + * @param fs filesystem + * @param status status of file to load + * @param serializer serializer to use + * @return the loaded instance + * @throws IOException IO failure + * @throws ValidationFailure if the data is invalid + */ + public static T load(FileSystem fs, + FileStatus status, + JsonSerialization serializer) + throws IOException { + Path path = status.getPath(); + LOG.debug("Reading commit data from file {}", path); + T result = serializer.load(fs, path, status); + result.validate(); + return result; + } + + /** + * Save to a file. + * This uses the createFile() API, which S3A supports for + * faster load and declaring sequential access, always + * + * @param type of persistent format + * @param fs filesystem + * @param path path to save to + * @param instance data to save + * @param serializer serializer to use + * @param performance skip all safety check on the write + * + * @return any IOStatistics from the output stream, or null + * + * @throws IOException IO failure + */ + public static IOStatistics saveFile( + final FileSystem fs, + final Path path, + final T instance, + final JsonSerialization serializer, + final boolean performance) + throws IOException { + + FSDataOutputStreamBuilder builder = fs.createFile(path) + .create() + .recursive() + .overwrite(true); + // switch to performance mode + builder.opt(FS_S3A_CREATE_PERFORMANCE, performance); + return saveToStream(path, instance, builder, serializer); + } + + /** + * Save to a file. + * This uses the createFile() API, which S3A supports for + * faster load and declaring sequential access, always + * @param type of persistent format + * @param path path to save to (used for logging) + * @param instance data to save + * @param builder builder already prepared for the write + * @param serializer serializer to use + * @return any IOStatistics from the output stream, or null + * @throws IOException IO failure + */ + public static IOStatistics saveToStream( + final Path path, + final T instance, + final FSDataOutputStreamBuilder builder, + final JsonSerialization serializer) throws IOException { + LOG.debug("saving commit data to file {}", path); + FSDataOutputStream dataOutputStream = builder.build(); + try { + dataOutputStream.write(serializer.toBytes(instance)); + } finally { + dataOutputStream.close(); + } + return dataOutputStream.getIOStatistics(); + } + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java index b53ef75d82..77c3fed11f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java @@ -34,8 +34,6 @@ import com.amazonaws.services.s3.model.PartETag; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.hadoop.util.Preconditions; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -43,9 +41,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.commit.ValidationFailure; +import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.util.JsonSerialization; +import org.apache.hadoop.util.Preconditions; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.validateCollectionClass; import static org.apache.hadoop.fs.s3a.commit.ValidationFailure.verify; @@ -69,8 +69,8 @@ @SuppressWarnings("unused") @InterfaceAudience.Private @InterfaceStability.Unstable -public class SinglePendingCommit extends PersistentCommitData - implements Iterable, IOStatisticsSource { +public class SinglePendingCommit extends PersistentCommitData + implements Iterable { /** * Serialization ID: {@value}. @@ -141,26 +141,32 @@ public SinglePendingCommit() { * @return a serializer. */ public static JsonSerialization serializer() { - return new JsonSerialization<>(SinglePendingCommit.class, false, true); + return new JsonSerialization<>(SinglePendingCommit.class, false, false); } /** * Load an instance from a file, then validate it. * @param fs filesystem * @param path path + * @param status nullable status of file to load + * @param serDeser serializer; if null use the shared static one. * @return the loaded instance * @throws IOException IO failure * @throws ValidationFailure if the data is invalid */ - public static SinglePendingCommit load(FileSystem fs, Path path) + public static SinglePendingCommit load(FileSystem fs, + Path path, + FileStatus status, + JsonSerialization serDeser) throws IOException { - return load(fs, path, null); + return load(fs, path, serDeser, null); } /** * Load an instance from a file, then validate it. * @param fs filesystem * @param path path + * @param serDeser deserializer * @param status status of file to load or null * @return the loaded instance * @throws IOException IO failure @@ -168,9 +174,12 @@ public static SinglePendingCommit load(FileSystem fs, Path path) */ public static SinglePendingCommit load(FileSystem fs, Path path, + JsonSerialization serDeser, @Nullable FileStatus status) throws IOException { - SinglePendingCommit instance = serializer().load(fs, path, status); + JsonSerialization jsonSerialization = + serDeser != null ? serDeser : serializer(); + SinglePendingCommit instance = jsonSerialization.load(fs, path, status); instance.filename = path.toString(); instance.validate(); return instance; @@ -264,15 +273,16 @@ public String toString() { } @Override - public byte[] toBytes() throws IOException { + public byte[] toBytes(JsonSerialization serializer) throws IOException { validate(); - return serializer().toBytes(this); + return serializer.toBytes(this); } @Override - public void save(FileSystem fs, Path path, boolean overwrite) - throws IOException { - serializer().save(fs, path, this, overwrite); + public IOStatistics save(final FileSystem fs, + final Path path, + final JsonSerialization serializer) throws IOException { + return saveFile(fs, path, this, serializer, true); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java index 00f196a0b4..0e0a03f587 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java @@ -30,13 +30,14 @@ import org.slf4j.LoggerFactory; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.commit.ValidationFailure; +import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; -import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.util.JsonSerialization; /** @@ -51,10 +52,24 @@ *
  • Not loadable? Something else.
    • * * - * This is an unstable structure intended for diagnostics and testing. - * Applications reading this data should use/check the {@link #name} field - * to differentiate from any other JSON-based manifest and to identify - * changes in the output format. + * This should be considered public, and MUST stay compatible + * at the JSON format level with that of + * {@code org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData} + *

    + * The JSON format SHOULD be considered public and evolving + * with compatibility across versions. + *

    + * All the Java serialization data is different and may change + * across versions with no stability guarantees other than + * "manifest summaries MAY be serialized between processes with + * the exact same version of this binary on their classpaths." + * That is sufficient for testing in Spark. + *

    + * To aid with Java serialization, the maps and lists are + * exclusively those which serialize well. + * IOStatisticsSnapshot has a lot of complexity in marshalling + * there; this class doesn't worry about concurrent access + * so is simpler. * * Note: to deal with scale issues, the S3A committers do not include any * more than the number of objects listed in @@ -65,8 +80,7 @@ @SuppressWarnings("unused") @InterfaceAudience.Private @InterfaceStability.Unstable -public class SuccessData extends PersistentCommitData - implements IOStatisticsSource { +public class SuccessData extends PersistentCommitData { private static final Logger LOG = LoggerFactory.getLogger(SuccessData.class); @@ -80,7 +94,7 @@ public class SuccessData extends PersistentCommitData /** * Serialization ID: {@value}. */ - private static final long serialVersionUID = 507133045258460083L + VERSION; + private static final long serialVersionUID = 507133045258460084L + VERSION; /** * Name to include in persisted data, so as to differentiate from @@ -92,7 +106,14 @@ public class SuccessData extends PersistentCommitData /** * Name of file; includes version marker. */ - private String name; + private String name = NAME; + + /** + * Did this succeed? + * It is implicitly true in a _SUCCESS file, but if the file + * is also saved to a log dir, then it depends on the outcome + */ + private boolean success = true; /** Timestamp of creation. */ private long timestamp; @@ -142,7 +163,17 @@ public class SuccessData extends PersistentCommitData * IOStatistics. */ @JsonProperty("iostatistics") - private IOStatisticsSnapshot iostats = new IOStatisticsSnapshot(); + private IOStatisticsSnapshot iostatistics = new IOStatisticsSnapshot(); + + /** + * State (committed, aborted). + */ + private String state; + + /** + * Stage: last stage executed. + */ + private String stage; @Override public void validate() throws ValidationFailure { @@ -153,16 +184,17 @@ public void validate() throws ValidationFailure { } @Override - public byte[] toBytes() throws IOException { - return serializer().toBytes(this); + public byte[] toBytes(JsonSerialization serializer) throws IOException { + return serializer.toBytes(this); } @Override - public void save(FileSystem fs, Path path, boolean overwrite) - throws IOException { + public IOStatistics save(final FileSystem fs, + final Path path, + final JsonSerialization serializer) throws IOException { // always set the name field before being saved. name = NAME; - serializer().save(fs, path, this, overwrite); + return saveFile(fs, path, this, serializer, true); } @Override @@ -250,8 +282,8 @@ public static SuccessData load(FileSystem fs, Path path) * Get a JSON serializer for this class. * @return a serializer. */ - private static JsonSerialization serializer() { - return new JsonSerialization<>(SuccessData.class, false, true); + public static JsonSerialization serializer() { + return new JsonSerialization<>(SuccessData.class, false, false); } public String getName() { @@ -371,10 +403,59 @@ public void setJobIdSource(final String jobIdSource) { @Override public IOStatisticsSnapshot getIOStatistics() { - return iostats; + return iostatistics; } public void setIOStatistics(final IOStatisticsSnapshot ioStatistics) { - this.iostats = ioStatistics; + this.iostatistics = ioStatistics; + } + + /** + * Set the success flag. + * @param success did the job succeed? + */ + public void setSuccess(boolean success) { + this.success = success; + } + + /** + * Get the success flag. + * @return did the job succeed? + */ + public boolean getSuccess() { + return success; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getStage() { + return stage; + } + + /** + * Add a diagnostics entry. + * @param key name + * @param value value + */ + public void putDiagnostic(String key, String value) { + diagnostics.put(key, value); + } + + /** + * Note a failure by setting success flag to false, + * then add the exception to the diagnostics. + * @param thrown throwable + */ + public void recordJobFailure(Throwable thrown) { + setSuccess(false); + String stacktrace = ExceptionUtils.getStackTrace(thrown); + diagnostics.put("exception", thrown.toString()); + diagnostics.put("stacktrace", stacktrace); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java index 0d574948e9..36eedba83b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java @@ -29,12 +29,11 @@ *

  • The summary information saved in the {@code _SUCCESS} file.
    • * * - * There are no guarantees of stability between versions; these are internal - * structures. * * The {@link org.apache.hadoop.fs.s3a.commit.files.SuccessData} file is - * the one visible to callers after a job completes; it is an unstable - * manifest intended for testing only. + * the one visible to callers after a job completes; it is compatible with + * the manifest committer format persisted in + * {@code org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData} * */ @InterfaceAudience.Private diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java new file mode 100644 index 0000000000..20024ba601 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/AuditContextUpdater.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.commit.impl; + +import org.apache.hadoop.fs.audit.AuditConstants; +import org.apache.hadoop.fs.audit.CommonAuditContext; +import org.apache.hadoop.fs.s3a.commit.CommitConstants; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.TaskAttemptID; + +import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext; + +/** + * Class to track/update context information to set + * in threads. + */ +public final class AuditContextUpdater { + + /** + * Job ID. + */ + private final String jobId; + + /** + * Task attempt ID for auditing. + */ + private final String taskAttemptId; + + /** + * Construct. Stores job information + * to attach to thread contexts. + * @param jobContext job/task context. + */ + public AuditContextUpdater(final JobContext jobContext) { + this.jobId = jobContext.getJobID().toString(); + + if (jobContext instanceof TaskAttemptContext) { + // it's a task, extract info for auditing + final TaskAttemptID tid = ((TaskAttemptContext) jobContext).getTaskAttemptID(); + this.taskAttemptId = tid.toString(); + } else { + this.taskAttemptId = null; + } + } + + public AuditContextUpdater(String jobId) { + this.jobId = jobId; + this.taskAttemptId = null; + } + + /** + * Add job/task info to current audit context. + */ + public void updateCurrentAuditContext() { + final CommonAuditContext auditCtx = currentAuditContext(); + auditCtx.put(AuditConstants.PARAM_JOB_ID, jobId); + if (taskAttemptId != null) { + auditCtx.put(AuditConstants.PARAM_TASK_ATTEMPT_ID, taskAttemptId); + } else { + currentAuditContext().remove(CommitConstants.PARAM_TASK_ATTEMPT_ID); + } + + } + + /** + * Remove job/task info from the current audit context. + */ + public void resetCurrentAuditContext() { + currentAuditContext().remove(AuditConstants.PARAM_JOB_ID); + currentAuditContext().remove(CommitConstants.PARAM_TASK_ATTEMPT_ID); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java new file mode 100644 index 0000000000..8bff165f2e --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitContext.java @@ -0,0 +1,401 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.commit.impl; + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.impl.WeakReferenceThreadMap; +import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; +import org.apache.hadoop.fs.s3a.commit.files.PendingSet; +import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.hadoop.util.JsonSerialization; +import org.apache.hadoop.util.Preconditions; +import org.apache.hadoop.util.concurrent.HadoopExecutors; +import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor; +import org.apache.hadoop.util.functional.TaskPool; + +import static org.apache.hadoop.fs.s3a.Constants.THREAD_POOL_SHUTDOWN_DELAY_SECONDS; +import static org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter.THREAD_PREFIX; +import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.THREAD_KEEP_ALIVE_TIME; + +/** + * Commit context. + * + * It is used to manage the final commit sequence where files become + * visible. + * + * Once the commit operation has completed, it must be closed. + * It MUST NOT be reused. + * + * Audit integration: job and task attributes are added to the thread local context + * on create, removed on close(). + * + * JSON Serializers are created on demand, on a per thread basis. + * A {@link WeakReferenceThreadMap} is used here; a GC may lose the + * references, but they will recreated as needed. + */ +public final class CommitContext implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger( + CommitContext.class); + + /** + * The actual commit operations. + */ + private final CommitOperations commitOperations; + + /** + * Job Context. + */ + private final JobContext jobContext; + + /** + * Serializer pool. + */ + + private final WeakReferenceThreadMap> + pendingSetSerializer = + new WeakReferenceThreadMap<>((k) -> PendingSet.serializer(), null); + + private final WeakReferenceThreadMap> + singleCommitSerializer = + new WeakReferenceThreadMap<>((k) -> SinglePendingCommit.serializer(), null); + + /** + * Submitter for per task operations, e.g loading manifests. + */ + private PoolSubmitter outerSubmitter; + + /** + * Submitter for operations within the tasks, + * such as POSTing the final commit operations. + */ + private PoolSubmitter innerSubmitter; + + /** + * Job Configuration. + */ + private final Configuration conf; + + /** + * Job ID. + */ + private final String jobId; + + /** + * Audit context; will be reset when this is closed. + */ + private final AuditContextUpdater auditContextUpdater; + + /** + * Number of committer threads. + */ + private final int committerThreads; + + /** + * Create. + * @param commitOperations commit callbacks + * @param jobContext job context + * @param committerThreads number of commit threads + */ + public CommitContext( + final CommitOperations commitOperations, + final JobContext jobContext, + final int committerThreads) { + this.commitOperations = commitOperations; + this.jobContext = jobContext; + this.conf = jobContext.getConfiguration(); + this.jobId = jobContext.getJobID().toString(); + this.auditContextUpdater = new AuditContextUpdater(jobContext); + this.auditContextUpdater.updateCurrentAuditContext(); + this.committerThreads = committerThreads; + + buildSubmitters(); + } + + /** + * Create for testing. + * This has no job context; instead the values + * are set explicitly. + * @param commitOperations commit callbacks + * @param conf job conf + * @param jobId ID + * @param committerThreads number of commit threads + */ + public CommitContext(final CommitOperations commitOperations, + final Configuration conf, + final String jobId, + final int committerThreads) { + this.commitOperations = commitOperations; + this.jobContext = null; + this.conf = conf; + this.jobId = jobId; + this.auditContextUpdater = new AuditContextUpdater(jobId); + this.auditContextUpdater.updateCurrentAuditContext(); + this.committerThreads = committerThreads; + buildSubmitters(); + } + + /** + * Build the submitters and thread pools if the number of committerThreads + * is greater than zero. + * This should only be called in constructors; it is synchronized to keep + * SpotBugs happy. + */ + private synchronized void buildSubmitters() { + if (committerThreads != 0) { + outerSubmitter = new PoolSubmitter(buildThreadPool(committerThreads)); + } + } + + /** + * Returns an {@link ExecutorService} for parallel tasks. The number of + * threads in the thread-pool is set by fs.s3a.committer.threads. + * If num-threads is 0, this will raise an exception. + * The threads have a lifespan set by + * {@link InternalCommitterConstants#THREAD_KEEP_ALIVE_TIME}. + * When the thread pool is full, the caller runs + * policy takes over. + * @param numThreads thread count, may be negative. + * @return an {@link ExecutorService} for the number of threads + */ + private ExecutorService buildThreadPool( + int numThreads) { + if (numThreads < 0) { + // a negative number means "multiple of available processors" + numThreads = numThreads * -Runtime.getRuntime().availableProcessors(); + } + Preconditions.checkArgument(numThreads > 0, + "Cannot create a thread pool with no threads"); + LOG.debug("creating thread pool of size {}", numThreads); + final ThreadFactory factory = new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(THREAD_PREFIX + jobId + "-%d") + .build(); + return new HadoopThreadPoolExecutor(0, numThreads, + THREAD_KEEP_ALIVE_TIME, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + factory, + new ThreadPoolExecutor.CallerRunsPolicy()); + } + + /** + * Commit the operation, throwing an exception on any failure. + * See {@code CommitOperations#commitOrFail(SinglePendingCommit)}. + * @param commit commit to execute + * @throws IOException on a failure + */ + public void commitOrFail(SinglePendingCommit commit) throws IOException { + commitOperations.commitOrFail(commit); + } + + /** + * Commit a single pending commit; exceptions are caught + * and converted to an outcome. + * See {@link CommitOperations#commit(SinglePendingCommit, String)}. + * @param commit entry to commit + * @param origin origin path/string for outcome text + * @return the outcome + */ + public CommitOperations.MaybeIOE commit(SinglePendingCommit commit, + String origin) { + return commitOperations.commit(commit, origin); + } + + /** + * See {@link CommitOperations#abortSingleCommit(SinglePendingCommit)}. + * @param commit pending commit to abort + * @throws FileNotFoundException if the abort ID is unknown + * @throws IOException on any failure + */ + public void abortSingleCommit(final SinglePendingCommit commit) + throws IOException { + commitOperations.abortSingleCommit(commit); + } + + /** + * See {@link CommitOperations#revertCommit(SinglePendingCommit)}. + * @param commit pending commit + * @throws IOException failure + */ + public void revertCommit(final SinglePendingCommit commit) + throws IOException { + commitOperations.revertCommit(commit); + } + + /** + * See {@link CommitOperations#abortMultipartCommit(String, String)}.. + * @param destKey destination key + * @param uploadId upload to cancel + * @throws FileNotFoundException if the abort ID is unknown + * @throws IOException on any failure + */ + public void abortMultipartCommit( + final String destKey, + final String uploadId) + throws IOException { + commitOperations.abortMultipartCommit(destKey, uploadId); + } + + @Override + public synchronized void close() throws IOException { + + destroyThreadPools(); + auditContextUpdater.resetCurrentAuditContext(); + } + + @Override + public String toString() { + return "CommitContext{}"; + } + + /** + * Job Context. + * @return job context. + */ + public JobContext getJobContext() { + return jobContext; + } + + /** + * Return a submitter. + * If created with 0 threads, this returns null so + * TaskPool knows to run it in the current thread. + * @return a submitter or null + */ + public synchronized TaskPool.Submitter getOuterSubmitter() { + return outerSubmitter; + } + + /** + * Return a submitter. As this pool is used less often, + * create it on demand. + * If created with 0 threads, this returns null so + * TaskPool knows to run it in the current thread. + * @return a submitter or null + */ + public synchronized TaskPool.Submitter getInnerSubmitter() { + if (innerSubmitter == null && committerThreads > 0) { + innerSubmitter = new PoolSubmitter(buildThreadPool(committerThreads)); + } + return innerSubmitter; + } + + /** + * Get a serializer for .pending files. + * @return a serializer. + */ + public JsonSerialization getSinglePendingFileSerializer() { + return singleCommitSerializer.getForCurrentThread(); + } + + /** + * Get a serializer for .pendingset files. + * @return a serializer. + */ + public JsonSerialization getPendingSetSerializer() { + return pendingSetSerializer.getForCurrentThread(); + } + + /** + * Destroy any thread pools; wait for that to finish, + * but don't overreact if it doesn't finish in time. + */ + private synchronized void destroyThreadPools() { + try { + IOUtils.cleanupWithLogger(LOG, outerSubmitter, innerSubmitter); + } finally { + outerSubmitter = null; + innerSubmitter = null; + } + } + + /** + * Job configuration. + * @return configuration (never null) + */ + public Configuration getConf() { + return conf; + } + + /** + * Get the job ID. + * @return job ID. + */ + public String getJobId() { + return jobId; + } + + /** + * Submitter for a given thread pool. + */ + private final class PoolSubmitter implements TaskPool.Submitter, Closeable { + + private ExecutorService executor; + + private PoolSubmitter(ExecutorService executor) { + this.executor = executor; + } + + @Override + public synchronized void close() throws IOException { + if (executor != null) { + HadoopExecutors.shutdown(executor, LOG, + THREAD_POOL_SHUTDOWN_DELAY_SECONDS, TimeUnit.SECONDS); + } + executor = null; + } + + /** + * Forward to the submitter, wrapping in task + * context setting, so as to ensure that all operations + * have job/task attributes. + * @param task task to execute + * @return the future. + */ + @Override + public Future submit(Runnable task) { + return executor.submit(() -> { + auditContextUpdater.updateCurrentAuditContext(); + try { + task.run(); + } finally { + auditContextUpdater.resetCurrentAuditContext(); + } + }); + } + + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java similarity index 82% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java index 840cf8e0f2..0772e143f6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitOperations.java @@ -16,24 +16,26 @@ * limitations under the License. */ -package org.apache.hadoop.fs.s3a.commit; +package org.apache.hadoop.fs.s3a.commit.impl; -import java.io.Closeable; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; +import javax.annotation.Nullable; + import com.amazonaws.services.s3.model.MultipartUpload; import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; -import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,25 +49,33 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.WriteOperations; +import org.apache.hadoop.fs.s3a.commit.CommitConstants; +import org.apache.hadoop.fs.s3a.commit.PathCommitException; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; import org.apache.hadoop.fs.s3a.commit.files.SuccessData; import org.apache.hadoop.fs.s3a.impl.AbstractStoreOperation; import org.apache.hadoop.fs.s3a.impl.HeaderProcessing; import org.apache.hadoop.fs.s3a.impl.InternalConstants; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.util.functional.TaskPool; import static java.util.Objects.requireNonNull; -import static org.apache.hadoop.fs.s3a.S3AUtils.*; +import static org.apache.hadoop.fs.s3a.S3AUtils.listAndFilter; import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_COMMIT_JOB; import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MATERIALIZE_FILE; +import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_LOAD_SINGLE_PENDING_FILE; import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_STAGE_FILE_UPLOAD; -import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.XA_MAGIC_MARKER; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants._SUCCESS; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration; import static org.apache.hadoop.util.functional.RemoteIterators.cleanupRemoteIterator; @@ -115,7 +125,7 @@ public class CommitOperations extends AbstractStoreOperation * @throws IOException failure to bind. */ public CommitOperations(S3AFileSystem fs) throws IOException { - this(requireNonNull(fs), fs.newCommitterStatistics()); + this(requireNonNull(fs), fs.newCommitterStatistics(), "/"); } /** @@ -123,10 +133,12 @@ public CommitOperations(S3AFileSystem fs) throws IOException { * the commit operations. * @param fs FS to bind to * @param committerStatistics committer statistics + * @param outputPath destination of work. * @throws IOException failure to bind. */ public CommitOperations(S3AFileSystem fs, - CommitterStatistics committerStatistics) throws IOException { + CommitterStatistics committerStatistics, + String outputPath) throws IOException { super(requireNonNull(fs).createStoreContext()); this.fs = fs; statistics = requireNonNull(committerStatistics); @@ -134,7 +146,7 @@ public CommitOperations(S3AFileSystem fs, writeOperations = fs.createWriteOperationHelper( fs.getAuditSpanSource().createSpan( COMMITTER_COMMIT_JOB.getSymbol(), - "/", null)); + outputPath, null)); } /** @@ -168,7 +180,7 @@ public IOStatistics getIOStatistics() { * @param commit commit to execute * @throws IOException on a failure */ - private void commitOrFail( + public void commitOrFail( final SinglePendingCommit commit) throws IOException { commit(commit, commit.getFilename()).maybeRethrow(); } @@ -180,7 +192,7 @@ private void commitOrFail( * @param origin origin path/string for outcome text * @return the outcome */ - private MaybeIOE commit( + public MaybeIOE commit( final SinglePendingCommit commit, final String origin) { LOG.debug("Committing single commit {}", commit); @@ -225,10 +237,9 @@ private long innerCommit( // finalize the commit writeOperations.commitUpload( commit.getDestinationKey(), - commit.getUploadId(), - toPartEtags(commit.getEtags()), - commit.getLength() - ); + commit.getUploadId(), + toPartEtags(commit.getEtags()), + commit.getLength()); return commit.getLength(); } @@ -236,43 +247,69 @@ private long innerCommit( * Locate all files with the pending suffix under a directory. * @param pendingDir directory * @param recursive recursive listing? - * @return the list of all located entries + * @return iterator of all located entries * @throws IOException if there is a problem listing the path. */ - public List locateAllSinglePendingCommits( + public RemoteIterator locateAllSinglePendingCommits( Path pendingDir, boolean recursive) throws IOException { return listAndFilter(fs, pendingDir, recursive, PENDING_FILTER); } /** - * Load all single pending commits in the directory. + * Load all single pending commits in the directory, using the + * outer submitter. * All load failures are logged and then added to list of files which would * not load. + * * @param pendingDir directory containing commits * @param recursive do a recursive scan? + * @param commitContext commit context + * * @return tuple of loaded entries and those pending files which would * not load/validate. + * * @throws IOException on a failure to list the files. */ public Pair>> - loadSinglePendingCommits(Path pendingDir, boolean recursive) + loadSinglePendingCommits(Path pendingDir, + boolean recursive, + CommitContext commitContext) throws IOException { - List statusList = locateAllSinglePendingCommits( - pendingDir, recursive); - PendingSet commits = new PendingSet( - statusList.size()); - List> failures = new ArrayList<>(1); - for (LocatedFileStatus status : statusList) { - try { - commits.add(SinglePendingCommit.load(fs, status.getPath(), status)); - } catch (IOException e) { - LOG.warn("Failed to load commit file {}", status.getPath(), e); - failures.add(Pair.of(status, e)); - } - } + PendingSet commits = new PendingSet(); + List pendingFiles = Collections.synchronizedList( + new ArrayList<>(1)); + List> failures = Collections.synchronizedList( + new ArrayList<>(1)); + + TaskPool.foreach(locateAllSinglePendingCommits(pendingDir, recursive)) + //. stopOnFailure() + .suppressExceptions(false) + .executeWith(commitContext.getOuterSubmitter()) + .run(status -> { + Path path = status.getPath(); + try { + // load the file + SinglePendingCommit singleCommit = trackDuration(statistics, + COMMITTER_LOAD_SINGLE_PENDING_FILE.getSymbol(), () -> + SinglePendingCommit.load(fs, + path, + status, + commitContext.getSinglePendingFileSerializer())); + // aggregate stats + commits.getIOStatistics() + .aggregate(singleCommit.getIOStatistics()); + // then clear so they aren't marshalled again. + singleCommit.getIOStatistics().clear(); + pendingFiles.add(singleCommit); + } catch (IOException e) { + LOG.warn("Failed to load commit file {}", path, e); + failures.add(Pair.of(status, e)); + } + }); + commits.setCommits(pendingFiles); return Pair.of(commits, failures); } @@ -296,7 +333,7 @@ public IOException makeIOE(String key, Exception ex) { * @throws FileNotFoundException if the abort ID is unknown * @throws IOException on any failure */ - private void abortSingleCommit(SinglePendingCommit commit) + public void abortSingleCommit(SinglePendingCommit commit) throws IOException { String destKey = commit.getDestinationKey(); String origin = commit.getFilename() != null @@ -315,7 +352,7 @@ private void abortSingleCommit(SinglePendingCommit commit) * @throws FileNotFoundException if the abort ID is unknown * @throws IOException on any failure */ - private void abortMultipartCommit(String destKey, String uploadId) + public void abortMultipartCommit(String destKey, String uploadId) throws IOException { try (DurationInfo d = new DurationInfo(LOG, "Aborting commit ID %s to path %s", uploadId, destKey)) { @@ -328,11 +365,13 @@ private void abortMultipartCommit(String destKey, String uploadId) /** * Enumerate all pending files in a dir/tree, abort. * @param pendingDir directory of pending operations + * @param commitContext commit context * @param recursive recurse? * @return the outcome of all the abort operations * @throws IOException if there is a problem listing the path. */ public MaybeIOE abortAllSinglePendingCommits(Path pendingDir, + CommitContext commitContext, boolean recursive) throws IOException { Preconditions.checkArgument(pendingDir != null, "null pendingDir"); @@ -350,12 +389,14 @@ public MaybeIOE abortAllSinglePendingCommits(Path pendingDir, LOG.debug("No files to abort under {}", pendingDir); } while (pendingFiles.hasNext()) { - LocatedFileStatus status = pendingFiles.next(); + final LocatedFileStatus status = pendingFiles.next(); Path pendingFile = status.getPath(); if (pendingFile.getName().endsWith(CommitConstants.PENDING_SUFFIX)) { try { - abortSingleCommit(SinglePendingCommit.load(fs, pendingFile, - status)); + abortSingleCommit(SinglePendingCommit.load(fs, + pendingFile, + status, + commitContext.getSinglePendingFileSerializer())); } catch (FileNotFoundException e) { LOG.debug("listed file already deleted: {}", pendingFile); } catch (IOException | IllegalArgumentException e) { @@ -437,7 +478,7 @@ public void createSuccessMarker(Path outputPath, successData); try (DurationInfo ignored = new DurationInfo(LOG, "Writing success file %s", markerPath)) { - successData.save(fs, markerPath, true); + successData.save(fs, markerPath, SuccessData.serializer()); } } @@ -466,7 +507,7 @@ public void revertCommit(SinglePendingCommit commit) throws IOException { * @return a pending upload entry * @throws IOException failure */ - public SinglePendingCommit uploadFileToPendingCommit(File localFile, + public SinglePendingCommit uploadFileToPendingCommit(File localFile, Path destPath, String partition, long uploadPartSize, @@ -494,7 +535,8 @@ public SinglePendingCommit uploadFileToPendingCommit(File localFile, destPath)) { statistics.commitCreated(); - uploadId = writeOperations.initiateMultiPartUpload(destKey); + uploadId = writeOperations.initiateMultiPartUpload(destKey, + PutObjectOptions.keepingDirs()); long length = localFile.length(); SinglePendingCommit commitData = new SinglePendingCommit(); @@ -592,13 +634,41 @@ public void jobCompleted(boolean success) { } /** - * Begin the final commit. + * Create a commit context for a job or task. + * + * @param context job context * @param path path for all work. + * @param committerThreads thread pool size * @return the commit context to pass in. * @throws IOException failure. */ - public CommitContext initiateCommitOperation(Path path) throws IOException { - return new CommitContext(); + public CommitContext createCommitContext( + JobContext context, + Path path, + int committerThreads) throws IOException { + return new CommitContext(this, context, + committerThreads); + } + + /** + * Create a stub commit context for tests. + * There's no job context. + * @param path path for all work. + * @param jobId job ID; if null a random UUID is generated. + * @param committerThreads number of committer threads. + * @return the commit context to pass in. + * @throws IOException failure. + */ + public CommitContext createCommitContextForTesting( + Path path, @Nullable String jobId, int committerThreads) throws IOException { + final String id = jobId != null + ? jobId + : UUID.randomUUID().toString(); + + return new CommitContext(this, + getStoreContext().getConfiguration(), + id, + committerThreads); } /** @@ -624,98 +694,6 @@ public static Optional extractMagicFileLength(FileSystem fs, Path path) return HeaderProcessing.extractXAttrLongValue(bytes); } - /** - * Commit context. - * - * It is used to manage the final commit sequence where files become - * visible. - * - * This can only be created through {@link #initiateCommitOperation(Path)}. - * - * Once the commit operation has completed, it must be closed. - * It must not be reused. - */ - public final class CommitContext implements Closeable { - - - /** - * Create. - */ - private CommitContext() { - } - - /** - * Commit the operation, throwing an exception on any failure. - * See {@link CommitOperations#commitOrFail(SinglePendingCommit)}. - * @param commit commit to execute - * @throws IOException on a failure - */ - public void commitOrFail(SinglePendingCommit commit) throws IOException { - CommitOperations.this.commitOrFail(commit); - } - - /** - * Commit a single pending commit; exceptions are caught - * and converted to an outcome. - * See {@link CommitOperations#commit(SinglePendingCommit, String)}. - * @param commit entry to commit - * @param origin origin path/string for outcome text - * @return the outcome - */ - public MaybeIOE commit(SinglePendingCommit commit, - String origin) { - return CommitOperations.this.commit(commit, origin); - } - - /** - * See {@link CommitOperations#abortSingleCommit(SinglePendingCommit)}. - * @param commit pending commit to abort - * @throws FileNotFoundException if the abort ID is unknown - * @throws IOException on any failure - */ - public void abortSingleCommit(final SinglePendingCommit commit) - throws IOException { - CommitOperations.this.abortSingleCommit(commit); - } - - /** - * See {@link CommitOperations#revertCommit(SinglePendingCommit)}. - * @param commit pending commit - * @throws IOException failure - */ - public void revertCommit(final SinglePendingCommit commit) - throws IOException { - CommitOperations.this.revertCommit(commit); - } - - /** - * See {@link CommitOperations#abortMultipartCommit(String, String)}.. - * @param destKey destination key - * @param uploadId upload to cancel - * @throws FileNotFoundException if the abort ID is unknown - * @throws IOException on any failure - */ - public void abortMultipartCommit( - final String destKey, - final String uploadId) - throws IOException { - CommitOperations.this.abortMultipartCommit(destKey, uploadId); - } - - @Override - public void close() throws IOException { - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder( - "CommitContext{"); - sb.append('}'); - return sb.toString(); - } - - } - /** * A holder for a possible IOException; the call {@link #maybeRethrow()} * will throw any exception passed into the constructor, and be a no-op @@ -788,5 +766,4 @@ public static MaybeIOE of(IOException ex) { } } - } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java similarity index 74% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java index 9e5ee860e8..c38ab2e9ba 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/CommitUtilsWithMR.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hadoop.fs.s3a.commit; +package org.apache.hadoop.fs.s3a.commit.impl; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -68,31 +68,67 @@ public static int getAppAttemptId(JobContext context) { /** * Compute the "magic" path for a job attempt. * @param jobUUID unique Job ID. + * @param appAttemptId the ID of the application attempt for this job. * @param dest the final output directory * @return the path to store job attempt data. */ - public static Path getMagicJobAttemptPath(String jobUUID, Path dest) { - return new Path(getMagicJobAttemptsPath(dest), - formatAppAttemptDir(jobUUID)); + public static Path getMagicJobAttemptPath(String jobUUID, + int appAttemptId, + Path dest) { + return new Path( + getMagicJobAttemptsPath(dest), + formatAppAttemptDir(jobUUID, appAttemptId)); } /** - * Format the application attempt directory. + * Compute the "magic" path for a job. * @param jobUUID unique Job ID. - * @return the directory name for the application attempt + * @param dest the final output directory + * @return the path to store job attempt data. */ - public static String formatAppAttemptDir(String jobUUID) { + public static Path getMagicJobPath(String jobUUID, + Path dest) { + return new Path( + getMagicJobAttemptsPath(dest), + formatJobDir(jobUUID)); + } + + /** + * Build the name of the job directory, without + * app attempt. + * This is the path to use for cleanup. + * @param jobUUID unique Job ID. + * @return the directory name for the job + */ + public static String formatJobDir( + String jobUUID) { return String.format("job-%s", jobUUID); } + /** + * Build the name of the job attempt directory. + * @param jobUUID unique Job ID. + * @param appAttemptId the ID of the application attempt for this job. + * @return the directory tree for the application attempt + */ + public static String formatAppAttemptDir( + String jobUUID, + int appAttemptId) { + return formatJobDir(jobUUID) + String.format("/%02d", appAttemptId); + } + /** * Compute the path where the output of magic task attempts are stored. * @param jobUUID unique Job ID. * @param dest destination of work + * @param appAttemptId the ID of the application attempt for this job. * @return the path where the output of magic task attempts are stored. */ - public static Path getMagicTaskAttemptsPath(String jobUUID, Path dest) { - return new Path(getMagicJobAttemptPath(jobUUID, dest), "tasks"); + public static Path getMagicTaskAttemptsPath( + String jobUUID, + Path dest, + int appAttemptId) { + return new Path(getMagicJobAttemptPath(jobUUID, appAttemptId, dest), "tasks"); } /** @@ -115,6 +151,8 @@ public static Path getMagicTaskAttemptPath(TaskAttemptContext context, /** * Get the base Magic attempt path, without any annotations to mark relative * references. + * If there is an app attempt property in the context configuration, that + * is included. * @param context task context. * @param jobUUID unique Job ID. * @param dest The output path to commit work into @@ -123,8 +161,9 @@ public static Path getMagicTaskAttemptPath(TaskAttemptContext context, public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context, String jobUUID, Path dest) { - return new Path(getMagicTaskAttemptsPath(jobUUID, dest), - String.valueOf(context.getTaskAttemptID())); + return new Path( + getMagicTaskAttemptsPath(jobUUID, dest, getAppAttemptId(context)), + String.valueOf(context.getTaskAttemptID())); } /** @@ -132,12 +171,13 @@ public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context, * This data is not magic * @param jobUUID unique Job ID. * @param out output directory of job + * @param appAttemptId the ID of the application attempt for this job. * @return the path to store temporary job attempt data. */ public static Path getTempJobAttemptPath(String jobUUID, - Path out) { + Path out, final int appAttemptId) { return new Path(new Path(out, TEMP_DATA), - formatAppAttemptDir(jobUUID)); + formatAppAttemptDir(jobUUID, appAttemptId)); } /** @@ -150,7 +190,7 @@ public static Path getTempJobAttemptPath(String jobUUID, public static Path getTempTaskAttemptPath(TaskAttemptContext context, final String jobUUID, Path out) { return new Path( - getTempJobAttemptPath(jobUUID, out), + getTempJobAttemptPath(jobUUID, out, getAppAttemptId(context)), String.valueOf(context.getTaskAttemptID())); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java new file mode 100644 index 0000000000..b4977f34d5 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/impl/package-info.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Internal classes which make use of mapreduce code. + * These MUST NOT be referred to in production code except + * in org.apache.hadoop.fs.s3a.commit classes which are only + * used within job/task committers. + */ + +@InterfaceAudience.Private +@InterfaceStability.Unstable +package org.apache.hadoop.fs.s3a.commit.impl; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java index 7ea4e88308..c85571a194 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java @@ -26,20 +26,26 @@ import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.PutObjectRequest; -import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.WriteOperationHelper; import org.apache.hadoop.fs.s3a.commit.PutTracker; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; +import org.apache.hadoop.fs.s3a.statistics.PutTrackerStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; +import org.apache.hadoop.util.Preconditions; +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_MARKER_PUT; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.X_HEADER_MAGIC_MARKER; +import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDurationOfInvocation; /** * Put tracker for Magic commits. @@ -57,6 +63,7 @@ public class MagicCommitTracker extends PutTracker { private final WriteOperationHelper writer; private final String bucket; private static final byte[] EMPTY = new byte[0]; + private final PutTrackerStatistics trackerStatistics; /** * Magic commit tracker. @@ -66,19 +73,22 @@ public class MagicCommitTracker extends PutTracker { * @param destKey key for the destination * @param pendingsetKey key of the pendingset file * @param writer writer instance to use for operations; includes audit span + * @param trackerStatistics tracker statistics */ public MagicCommitTracker(Path path, String bucket, String originalDestKey, String destKey, String pendingsetKey, - WriteOperationHelper writer) { + WriteOperationHelper writer, + PutTrackerStatistics trackerStatistics) { super(destKey); this.bucket = bucket; this.path = path; this.originalDestKey = originalDestKey; this.pendingPartKey = pendingsetKey; this.writer = writer; + this.trackerStatistics = requireNonNull(trackerStatistics); LOG.info("File {} is written as magic file to path {}", path, destKey); } @@ -126,6 +136,19 @@ public boolean aboutToComplete(String uploadId, Preconditions.checkArgument(!parts.isEmpty(), "No uploaded parts to save"); + // put a 0-byte file with the name of the original under-magic path + // Add the final file length as a header + // this is done before the task commit, so its duration can be + // included in the statistics + Map headers = new HashMap<>(); + headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten)); + PutObjectRequest originalDestPut = writer.createPutObjectRequest( + originalDestKey, + new ByteArrayInputStream(EMPTY), + 0, + new PutObjectOptions(true, null, headers)); + upload(originalDestPut); + // build the commit summary SinglePendingCommit commitData = new SinglePendingCommit(); commitData.touch(System.currentTimeMillis()); @@ -138,7 +161,8 @@ public boolean aboutToComplete(String uploadId, commitData.bindCommitData(parts); commitData.setIOStatistics( new IOStatisticsSnapshot(iostatistics)); - byte[] bytes = commitData.toBytes(); + + byte[] bytes = commitData.toBytes(SinglePendingCommit.serializer()); LOG.info("Uncommitted data pending to file {};" + " commit metadata for {} parts in {}. size: {} byte(s)", path.toUri(), parts.size(), pendingPartKey, bytesWritten); @@ -148,19 +172,20 @@ public boolean aboutToComplete(String uploadId, pendingPartKey, new ByteArrayInputStream(bytes), bytes.length, null); - writer.uploadObject(put); - - // Add the final file length as a header - Map headers = new HashMap<>(); - headers.put(X_HEADER_MAGIC_MARKER, Long.toString(bytesWritten)); - // now put a 0-byte file with the name of the original under-magic path - PutObjectRequest originalDestPut = writer.createPutObjectRequest( - originalDestKey, - new ByteArrayInputStream(EMPTY), - 0, - headers); - writer.uploadObject(originalDestPut); + upload(put); return false; + + } + /** + * PUT an object. + * @param request the request + * @throws IOException on problems + */ + @Retries.RetryTranslated + private void upload(PutObjectRequest request) throws IOException { + trackDurationOfInvocation(trackerStatistics, + COMMITTER_MAGIC_MARKER_PUT.getSymbol(), () -> + writer.uploadObject(request, PutObjectOptions.keepingDirs())); } @Override diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java index c1ecd7d6b9..007e9b3709 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java @@ -32,11 +32,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR; import org.apache.hadoop.fs.statistics.IOStatisticsLogging; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -45,9 +46,10 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TASK_ATTEMPT_ID; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TEMP_DATA; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*; import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*; -import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*; +import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.demandStringifyIOStatistics; /** @@ -100,25 +102,28 @@ public void setupJob(JobContext context) throws IOException { try (DurationInfo d = new DurationInfo(LOG, "Setup Job %s", jobIdString(context))) { super.setupJob(context); - Path jobAttemptPath = getJobAttemptPath(context); - getDestinationFS(jobAttemptPath, - context.getConfiguration()).mkdirs(jobAttemptPath); + Path jobPath = getJobPath(); + final FileSystem destFS = getDestinationFS(jobPath, + context.getConfiguration()); + destFS.delete(jobPath, true); + destFS.mkdirs(jobPath); } } /** * Get the list of pending uploads for this job attempt, by listing * all .pendingset files in the job attempt directory. - * @param context job context + * @param commitContext job context * @return a list of pending commits. * @throws IOException Any IO failure */ protected ActiveCommit listPendingUploadsToCommit( - JobContext context) + CommitContext commitContext) throws IOException { FileSystem fs = getDestFS(); - return ActiveCommit.fromStatusList(fs, - listAndFilter(fs, getJobAttemptPath(context), false, + return ActiveCommit.fromStatusIterator(fs, + listAndFilter(fs, getJobAttemptPath(commitContext.getJobContext()), + false, CommitOperations.PENDINGSET_FILTER)); } @@ -126,11 +131,16 @@ protected ActiveCommit listPendingUploadsToCommit( * Delete the magic directory. */ public void cleanupStagingDirs() { - Path path = magicSubdir(getOutputPath()); + final Path out = getOutputPath(); + Path path = magicSubdir(out); try(DurationInfo ignored = new DurationInfo(LOG, true, "Deleting magic directory %s", path)) { Invoker.ignoreIOExceptions(LOG, "cleanup magic directory", path.toString(), () -> deleteWithWarning(getDestFS(), path, true)); + // and the job temp directory with manifests + Invoker.ignoreIOExceptions(LOG, "cleanup job directory", path.toString(), + () -> deleteWithWarning(getDestFS(), + new Path(out, TEMP_DATA), true)); } } @@ -146,13 +156,8 @@ public void cleanupStagingDirs() { @Override public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { - Path taskAttemptPath = getTaskAttemptPath(context); - try (DurationInfo d = new DurationInfo(LOG, - "needsTaskCommit task %s", context.getTaskAttemptID())) { - return taskAttemptPath.getFileSystem( - context.getConfiguration()) - .exists(taskAttemptPath); - } + // return true as a dir was created here in setup; + return true; } @Override @@ -167,9 +172,9 @@ public void commitTask(TaskAttemptContext context) throws IOException { throw e; } finally { // delete the task attempt so there's no possibility of a second attempt + // incurs a LIST, a bulk DELETE and maybe a parent dir creation, however + // as it happens during task commit, it should be off the critical path. deleteTaskAttemptPathQuietly(context); - destroyThreadPool(); - resetCommonContext(); } getCommitOperations().taskCompleted(true); LOG.debug("aggregate statistics\n{}", @@ -191,43 +196,48 @@ private PendingSet innerCommitTask( Path taskAttemptPath = getTaskAttemptPath(context); // load in all pending commits. CommitOperations actions = getCommitOperations(); - Pair>> - loaded = actions.loadSinglePendingCommits( - taskAttemptPath, true); - PendingSet pendingSet = loaded.getKey(); - List> failures = loaded.getValue(); - if (!failures.isEmpty()) { - // At least one file failed to load - // revert all which did; report failure with first exception - LOG.error("At least one commit file could not be read: failing"); - abortPendingUploads(context, pendingSet.getCommits(), true); - throw failures.get(0).getValue(); - } - // patch in IDs - String jobId = getUUID(); - String taskId = String.valueOf(context.getTaskAttemptID()); - for (SinglePendingCommit commit : pendingSet.getCommits()) { - commit.setJobId(jobId); - commit.setTaskId(taskId); - } - pendingSet.putExtraData(TASK_ATTEMPT_ID, taskId); - pendingSet.setJobId(jobId); - Path jobAttemptPath = getJobAttemptPath(context); - TaskAttemptID taskAttemptID = context.getTaskAttemptID(); - Path taskOutcomePath = new Path(jobAttemptPath, - taskAttemptID.getTaskID().toString() + - CommitConstants.PENDINGSET_SUFFIX); - LOG.info("Saving work of {} to {}", taskAttemptID, taskOutcomePath); - LOG.debug("task statistics\n{}", - IOStatisticsLogging.demandStringifyIOStatisticsSource(pendingSet)); - try { - // We will overwrite if there exists a pendingSet file already - pendingSet.save(getDestFS(), taskOutcomePath, true); - } catch (IOException e) { - LOG.warn("Failed to save task commit data to {} ", - taskOutcomePath, e); - abortPendingUploads(context, pendingSet.getCommits(), true); - throw e; + PendingSet pendingSet; + try (CommitContext commitContext = initiateTaskOperation(context)) { + Pair>> + loaded = actions.loadSinglePendingCommits( + taskAttemptPath, true, commitContext); + pendingSet = loaded.getKey(); + List> failures = loaded.getValue(); + if (!failures.isEmpty()) { + // At least one file failed to load + // revert all which did; report failure with first exception + LOG.error("At least one commit file could not be read: failing"); + abortPendingUploads(commitContext, pendingSet.getCommits(), true); + throw failures.get(0).getValue(); + } + // patch in IDs + String jobId = getUUID(); + String taskId = String.valueOf(context.getTaskAttemptID()); + for (SinglePendingCommit commit : pendingSet.getCommits()) { + commit.setJobId(jobId); + commit.setTaskId(taskId); + } + pendingSet.putExtraData(TASK_ATTEMPT_ID, taskId); + pendingSet.setJobId(jobId); + Path jobAttemptPath = getJobAttemptPath(context); + TaskAttemptID taskAttemptID = context.getTaskAttemptID(); + Path taskOutcomePath = new Path(jobAttemptPath, + taskAttemptID.getTaskID().toString() + + CommitConstants.PENDINGSET_SUFFIX); + LOG.info("Saving work of {} to {}", taskAttemptID, taskOutcomePath); + LOG.debug("task statistics\n{}", + IOStatisticsLogging.demandStringifyIOStatisticsSource(pendingSet)); + try { + // We will overwrite if there exists a pendingSet file already + pendingSet.save(getDestFS(), + taskOutcomePath, + commitContext.getPendingSetSerializer()); + } catch (IOException e) { + LOG.warn("Failed to save task commit data to {} ", + taskOutcomePath, e); + abortPendingUploads(commitContext, pendingSet.getCommits(), true); + throw e; + } } return pendingSet; } @@ -246,25 +256,35 @@ private PendingSet innerCommitTask( public void abortTask(TaskAttemptContext context) throws IOException { Path attemptPath = getTaskAttemptPath(context); try (DurationInfo d = new DurationInfo(LOG, - "Abort task %s", context.getTaskAttemptID())) { - getCommitOperations().abortAllSinglePendingCommits(attemptPath, true); + "Abort task %s", context.getTaskAttemptID()); + CommitContext commitContext = initiateTaskOperation(context)) { + getCommitOperations().abortAllSinglePendingCommits(attemptPath, + commitContext, + true); } finally { deleteQuietly( attemptPath.getFileSystem(context.getConfiguration()), attemptPath, true); - destroyThreadPool(); - resetCommonContext(); } } + /** + * Compute the path under which all job attempts will be placed. + * @return the path to store job attempt data. + */ + @Override + protected Path getJobPath() { + return getMagicJobPath(getUUID(), getOutputPath()); + } + /** * Compute the path where the output of a given job attempt will be placed. * For the magic committer, the path includes the job UUID. * @param appAttemptId the ID of the application attempt for this job. * @return the path to store job attempt data. */ - protected Path getJobAttemptPath(int appAttemptId) { - return getMagicJobAttemptPath(getUUID(), getOutputPath()); + protected final Path getJobAttemptPath(int appAttemptId) { + return getMagicJobAttemptPath(getUUID(), appAttemptId, getOutputPath()); } /** @@ -274,12 +294,12 @@ protected Path getJobAttemptPath(int appAttemptId) { * @param context the context of the task attempt. * @return the path where a task attempt should be stored. */ - public Path getTaskAttemptPath(TaskAttemptContext context) { + public final Path getTaskAttemptPath(TaskAttemptContext context) { return getMagicTaskAttemptPath(context, getUUID(), getOutputPath()); } @Override - protected Path getBaseTaskAttemptPath(TaskAttemptContext context) { + protected final Path getBaseTaskAttemptPath(TaskAttemptContext context) { return getBaseMagicTaskAttemptPath(context, getUUID(), getOutputPath()); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java index 1a5a63c940..99683db984 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathExistsException; import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; @@ -98,17 +99,18 @@ public void setupJob(JobContext context) throws IOException { * Pre-commit actions for a job. * Here: look at the conflict resolution mode and choose * an action based on the current policy. - * @param context job context + * @param commitContext commit context * @param pending pending commits * @throws IOException any failure */ @Override public void preCommitJob( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending) throws IOException { + final JobContext context = commitContext.getJobContext(); // see if the files can be loaded. - super.preCommitJob(context, pending); + super.preCommitJob(commitContext, pending); Path outputPath = getOutputPath(); FileSystem fs = getDestFS(); Configuration fsConf = fs.getConf(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java index 214c7abdc7..5d1a20e424 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java @@ -32,12 +32,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.commit.PathCommitException; -import org.apache.hadoop.fs.s3a.commit.Tasks; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; +import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; -import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.functional.TaskPool; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.COMMITTER_NAME_PARTITIONED; @@ -89,7 +90,8 @@ public String toString() { @Override protected int commitTaskInternal(TaskAttemptContext context, - List taskOutput) throws IOException { + List taskOutput, + CommitContext commitContext) throws IOException { Path attemptPath = getTaskAttemptPath(context); Set partitions = Paths.getPartitions(attemptPath, taskOutput); @@ -109,7 +111,7 @@ protected int commitTaskInternal(TaskAttemptContext context, } } } - return super.commitTaskInternal(context, taskOutput); + return super.commitTaskInternal(context, taskOutput, commitContext); } /** @@ -121,13 +123,13 @@ protected int commitTaskInternal(TaskAttemptContext context, *
  • APPEND: allowed.; no need to check.
    • *
  • REPLACE deletes all existing partitions.
    • * - * @param context job context + * @param commitContext commit context * @param pending the pending operations * @throws IOException any failure */ @Override public void preCommitJob( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending) throws IOException { FileSystem fs = getDestFS(); @@ -135,7 +137,7 @@ public void preCommitJob( // enforce conflict resolution Configuration fsConf = fs.getConf(); boolean shouldPrecheckPendingFiles = true; - switch (getConflictResolutionMode(context, fsConf)) { + switch (getConflictResolutionMode(commitContext.getJobContext(), fsConf)) { case FAIL: // FAIL checking is done on the task side, so this does nothing break; @@ -144,17 +146,17 @@ public void preCommitJob( break; case REPLACE: // identify and replace the destination partitions - replacePartitions(context, pending); + replacePartitions(commitContext, pending); // and so there is no need to do another check. shouldPrecheckPendingFiles = false; break; default: throw new PathCommitException("", getRole() + ": unknown conflict resolution mode: " - + getConflictResolutionMode(context, fsConf)); + + getConflictResolutionMode(commitContext.getJobContext(), fsConf)); } if (shouldPrecheckPendingFiles) { - precommitCheckPendingFiles(context, pending); + precommitCheckPendingFiles(commitContext, pending); } } @@ -176,17 +178,16 @@ public void preCommitJob( * } * * - * @param context job context + * @param commitContext commit context * @param pending the pending operations * @throws IOException any failure */ private void replacePartitions( - final JobContext context, + final CommitContext commitContext, final ActiveCommit pending) throws IOException { Map partitions = new ConcurrentHashMap<>(); FileSystem sourceFS = pending.getSourceFS(); - Tasks.Submitter submitter = buildSubmitter(context); try (DurationInfo ignored = new DurationInfo(LOG, "Replacing partitions")) { @@ -194,13 +195,15 @@ private void replacePartitions( // for a marginal optimisation, the previous parent is tracked, so // if a task writes many files to the same dir, the synchronized map // is updated only once. - Tasks.foreach(pending.getSourceFiles()) + TaskPool.foreach(pending.getSourceFiles()) .stopOnFailure() .suppressExceptions(false) - .executeWith(submitter) + .executeWith(commitContext.getOuterSubmitter()) .run(status -> { - PendingSet pendingSet = PendingSet.load(sourceFS, - status); + PendingSet pendingSet = PersistentCommitData.load( + sourceFS, + status, + commitContext.getPendingSetSerializer()); Path lastParent = null; for (SinglePendingCommit commit : pendingSet.getCommits()) { Path parent = commit.destinationPath().getParent(); @@ -213,10 +216,10 @@ private void replacePartitions( } // now do the deletes FileSystem fs = getDestFS(); - Tasks.foreach(partitions.keySet()) + TaskPool.foreach(partitions.keySet()) .stopOnFailure() .suppressExceptions(false) - .executeWith(submitter) + .executeWith(commitContext.getOuterSubmitter()) .run(partitionPath -> { LOG.debug("{}: removing partition path to be replaced: " + getRole(), partitionPath); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java index 121ea7f851..36eae012de 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java @@ -39,24 +39,25 @@ import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; -import org.apache.hadoop.fs.s3a.commit.Tasks; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.functional.TaskPool; -import static org.apache.hadoop.util.Preconditions.*; +import static java.util.Objects.requireNonNull; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3AUtils.*; import static org.apache.hadoop.fs.s3a.Invoker.*; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*; -import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*; +import static org.apache.hadoop.fs.s3a.commit.impl.CommitUtilsWithMR.*; import static org.apache.hadoop.util.functional.RemoteIterators.cleanupRemoteIterator; +import static org.apache.hadoop.util.functional.RemoteIterators.toList; /** * Committer based on the contributed work of the @@ -111,7 +112,7 @@ public class StagingCommitter extends AbstractS3ACommitter { public StagingCommitter(Path outputPath, TaskAttemptContext context) throws IOException { super(outputPath, context); - this.constructorOutputPath = checkNotNull(getOutputPath(), "output path"); + this.constructorOutputPath = requireNonNull(getOutputPath(), "output path"); Configuration conf = getConf(); this.uploadPartSize = conf.getLongBytes( MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE); @@ -121,8 +122,8 @@ public StagingCommitter(Path outputPath, setWorkPath(buildWorkPath(context, getUUID())); this.wrappedCommitter = createWrappedCommitter(context, conf); setOutputPath(constructorOutputPath); - Path finalOutputPath = getOutputPath(); - checkNotNull(finalOutputPath, "Output path cannot be null"); + Path finalOutputPath = requireNonNull(getOutputPath(), + "Output path cannot be null"); S3AFileSystem fs = getS3AFileSystem(finalOutputPath, context.getConfiguration(), false); s3KeyPrefix = fs.pathToKey(finalOutputPath); @@ -243,10 +244,18 @@ private static Path getJobAttemptPath(int appAttemptId, Path out) { @Override protected Path getJobAttemptPath(int appAttemptId) { - return new Path(getPendingJobAttemptsPath(commitsDirectory), + return new Path(getJobPath(), String.valueOf(appAttemptId)); } + /** + * Compute the path under which all job attempts will be placed. + * @return the path to store job attempt data. + */ + protected Path getJobPath() { + return getPendingJobAttemptsPath(commitsDirectory); + } + /** * Compute the path where the output of pending task attempts are stored. * @param context the context of the job with pending tasks. @@ -275,7 +284,7 @@ public static Path getTaskAttemptPath(TaskAttemptContext context, Path out) { * @return the location of pending job attempts. */ private static Path getPendingJobAttemptsPath(Path out) { - checkNotNull(out, "Null 'out' path"); + requireNonNull(out, "Null 'out' path"); return new Path(out, TEMPORARY); } @@ -296,12 +305,12 @@ public Path getCommittedTaskPath(TaskAttemptContext context) { * @param context task context */ private static void validateContext(TaskAttemptContext context) { - checkNotNull(context, "null context"); - checkNotNull(context.getTaskAttemptID(), + requireNonNull(context, "null context"); + requireNonNull(context.getTaskAttemptID(), "null task attempt ID"); - checkNotNull(context.getTaskAttemptID().getTaskID(), + requireNonNull(context.getTaskAttemptID().getTaskID(), "null task ID"); - checkNotNull(context.getTaskAttemptID().getJobID(), + requireNonNull(context.getTaskAttemptID().getJobID(), "null job ID"); } @@ -342,14 +351,13 @@ protected List getTaskOutput(TaskAttemptContext context) throws IOException { // get files on the local FS in the attempt path - Path attemptPath = getTaskAttemptPath(context); - checkNotNull(attemptPath, - "No attemptPath path in {}", this); + Path attemptPath = requireNonNull(getTaskAttemptPath(context), + "No attemptPath path"); LOG.debug("Scanning {} for files to commit", attemptPath); - return listAndFilter(getTaskAttemptFilesystem(context), - attemptPath, true, HIDDEN_FILE_FILTER); + return toList(listAndFilter(getTaskAttemptFilesystem(context), + attemptPath, true, HIDDEN_FILE_FILTER)); } /** @@ -425,46 +433,46 @@ public void setupJob(JobContext context) throws IOException { /** * Get the list of pending uploads for this job attempt. - * @param context job context + * @param commitContext job context * @return a list of pending uploads. * @throws IOException Any IO failure */ @Override protected ActiveCommit listPendingUploadsToCommit( - JobContext context) + CommitContext commitContext) throws IOException { - return listPendingUploads(context, false); + return listPendingUploads(commitContext, false); } /** * Get the list of pending uploads for this job attempt, swallowing * exceptions. - * @param context job context + * @param commitContext commit context * @return a list of pending uploads. If an exception was swallowed, * then this may not match the actual set of pending operations * @throws IOException shouldn't be raised, but retained for the compiler */ protected ActiveCommit listPendingUploadsToAbort( - JobContext context) throws IOException { - return listPendingUploads(context, true); + CommitContext commitContext) throws IOException { + return listPendingUploads(commitContext, true); } /** * Get the list of pending uploads for this job attempt. - * @param context job context + * @param commitContext commit context * @param suppressExceptions should exceptions be swallowed? * @return a list of pending uploads. If exceptions are being swallowed, * then this may not match the actual set of pending operations * @throws IOException Any IO failure which wasn't swallowed. */ protected ActiveCommit listPendingUploads( - JobContext context, boolean suppressExceptions) throws IOException { + CommitContext commitContext, boolean suppressExceptions) throws IOException { try (DurationInfo ignored = new DurationInfo(LOG, "Listing pending uploads")) { - Path wrappedJobAttemptPath = getJobAttemptPath(context); + Path wrappedJobAttemptPath = getJobAttemptPath(commitContext.getJobContext()); final FileSystem attemptFS = wrappedJobAttemptPath.getFileSystem( - context.getConfiguration()); - return ActiveCommit.fromStatusList(attemptFS, + commitContext.getConf()); + return ActiveCommit.fromStatusIterator(attemptFS, listAndFilter(attemptFS, wrappedJobAttemptPath, false, HIDDEN_FILE_FILTER)); @@ -491,27 +499,39 @@ public void cleanupStagingDirs() { } } + /** + * Staging committer cleanup includes calling wrapped committer's + * cleanup method, and removing all destination paths in the final + * filesystem. + * @param commitContext commit context + * @param suppressExceptions should exceptions be suppressed? + * @throws IOException IO failures if exceptions are not suppressed. + */ @Override @SuppressWarnings("deprecation") - protected void cleanup(JobContext context, + protected void cleanup(CommitContext commitContext, boolean suppressExceptions) throws IOException { maybeIgnore(suppressExceptions, "Cleanup wrapped committer", - () -> wrappedCommitter.cleanupJob(context)); + () -> wrappedCommitter.cleanupJob( + commitContext.getJobContext())); maybeIgnore(suppressExceptions, "Delete destination paths", - () -> deleteDestinationPaths(context)); - super.cleanup(context, suppressExceptions); + () -> deleteDestinationPaths( + commitContext.getJobContext())); + super.cleanup(commitContext, suppressExceptions); } @Override - protected void abortJobInternal(JobContext context, + protected void abortJobInternal(CommitContext commitContext, boolean suppressExceptions) throws IOException { String r = getRole(); + JobContext context = commitContext.getJobContext(); boolean failed = false; try (DurationInfo d = new DurationInfo(LOG, "%s: aborting job in state %s ", r, jobIdString(context))) { - ActiveCommit pending = listPendingUploadsToAbort(context); - abortPendingUploads(context, pending, suppressExceptions, true); + ActiveCommit pending = listPendingUploadsToAbort(commitContext); + abortPendingUploads(commitContext, + pending, suppressExceptions, true); } catch (FileNotFoundException e) { // nothing to list LOG.debug("No job directory to read uploads from"); @@ -519,7 +539,7 @@ protected void abortJobInternal(JobContext context, failed = true; maybeIgnore(suppressExceptions, "aborting job", e); } finally { - super.abortJobInternal(context, failed || suppressExceptions); + super.abortJobInternal(commitContext, failed || suppressExceptions); } } @@ -590,17 +610,16 @@ public boolean needsTaskCommit(TaskAttemptContext context) @Override public void commitTask(TaskAttemptContext context) throws IOException { try (DurationInfo d = new DurationInfo(LOG, - "%s: commit task %s", getRole(), context.getTaskAttemptID())) { - int count = commitTaskInternal(context, getTaskOutput(context)); + "%s: commit task %s", getRole(), context.getTaskAttemptID()); + CommitContext commitContext + = initiateTaskOperation(context)) { + int count = commitTaskInternal(context, getTaskOutput(context), commitContext); LOG.info("{}: upload file count: {}", getRole(), count); } catch (IOException e) { LOG.error("{}: commit of task {} failed", getRole(), context.getTaskAttemptID(), e); getCommitOperations().taskCompleted(false); throw e; - } finally { - destroyThreadPool(); - resetCommonContext(); } getCommitOperations().taskCompleted(true); } @@ -610,11 +629,13 @@ public void commitTask(TaskAttemptContext context) throws IOException { * writing a pending entry for them. * @param context task context * @param taskOutput list of files from the output + * @param commitContext commit context * @return number of uploads committed. * @throws IOException IO Failures. */ protected int commitTaskInternal(final TaskAttemptContext context, - List taskOutput) + List taskOutput, + CommitContext commitContext) throws IOException { LOG.debug("{}: commitTaskInternal", getRole()); Configuration conf = context.getConfiguration(); @@ -649,10 +670,10 @@ protected int commitTaskInternal(final TaskAttemptContext context, pendingCommits.putExtraData(TASK_ATTEMPT_ID, context.getTaskAttemptID().toString()); try { - Tasks.foreach(taskOutput) + TaskPool.foreach(taskOutput) .stopOnFailure() .suppressExceptions(false) - .executeWith(buildSubmitter(context)) + .executeWith(commitContext.getOuterSubmitter()) .run(stat -> { Path path = stat.getPath(); File localFile = new File(path.toUri().getPath()); @@ -676,13 +697,14 @@ protected int commitTaskInternal(final TaskAttemptContext context, } // save the data - // although overwrite=false, there's still a risk of > 1 entry being - // committed if the FS doesn't have create-no-overwrite consistency. + // overwrite any existing file, so whichever task attempt + // committed last wins. LOG.debug("Saving {} pending commit(s)) to file {}", pendingCommits.size(), commitsAttemptPath); - pendingCommits.save(commitsFS, commitsAttemptPath, false); + pendingCommits.save(commitsFS, commitsAttemptPath, + commitContext.getPendingSetSerializer()); threw = false; } finally { @@ -690,12 +712,11 @@ protected int commitTaskInternal(final TaskAttemptContext context, LOG.error( "{}: Exception during commit process, aborting {} commit(s)", getRole(), commits.size()); - try(CommitOperations.CommitContext commitContext - = initiateCommitOperation(); - DurationInfo ignored = new DurationInfo(LOG, + try(DurationInfo ignored = new DurationInfo(LOG, "Aborting %s uploads", commits.size())) { - Tasks.foreach(commits) + TaskPool.foreach(commits) .suppressExceptions() + .executeWith(commitContext.getOuterSubmitter()) .run(commitContext::abortSingleCommit); } deleteTaskAttemptPathQuietly(context); @@ -738,9 +759,6 @@ public void abortTask(TaskAttemptContext context) throws IOException { LOG.error("{}: exception when aborting task {}", getRole(), context.getTaskAttemptID(), e); throw e; - } finally { - destroyThreadPool(); - resetCommonContext(); } } @@ -859,16 +877,16 @@ public static String getConfictModeOption(JobContext context, * Pre-commit actions for a job. * Loads all the pending files to verify they can be loaded * and parsed. - * @param context job context + * @param commitContext commit context * @param pending pending commits * @throws IOException any failure */ @Override public void preCommitJob( - final JobContext context, + CommitContext commitContext, final ActiveCommit pending) throws IOException { // see if the files can be loaded. - precommitCheckPendingFiles(context, pending); + precommitCheckPendingFiles(commitContext, pending); } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java new file mode 100644 index 0000000000..0392afac59 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/CreateFileBuilder.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIOException; +import org.apache.hadoop.fs.s3a.Constants; +import org.apache.hadoop.util.Progressable; + +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CREATE_FILE_KEYS; + +/** + * Builder used in create file; takes a callback to the operation + * to create the file. + * Is non-recursive unless explicitly changed. + */ +public class CreateFileBuilder extends + FSDataOutputStreamBuilder { + + /** + * Flag set to create with overwrite. + */ + public static final EnumSet CREATE_OVERWRITE_FLAGS = + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE); + + /** + * Flag set to create without overwrite. + */ + public static final EnumSet CREATE_NO_OVERWRITE_FLAGS = + EnumSet.of(CreateFlag.CREATE); + + /** + * Classic create file option set: overwriting. + */ + public static final CreateFileOptions OPTIONS_CREATE_FILE_OVERWRITE = + new CreateFileOptions(CREATE_OVERWRITE_FLAGS, true, false, null); + + /** + * Classic create file option set: no overwrite. + */ + public static final CreateFileOptions OPTIONS_CREATE_FILE_NO_OVERWRITE = + new CreateFileOptions(CREATE_NO_OVERWRITE_FLAGS, true, false, null); + + /** + * Callback interface. + */ + private final CreateFileBuilderCallbacks callbacks; + + /** + * Constructor. + * @param fileSystem fs; used by superclass. + * @param path qualified path to create + * @param callbacks callbacks. + */ + public CreateFileBuilder( + @Nonnull final FileSystem fileSystem, + @Nonnull final Path path, + @Nonnull final CreateFileBuilderCallbacks callbacks) { + + super(fileSystem, path); + this.callbacks = callbacks; + } + + @Override + public CreateFileBuilder getThisBuilder() { + return this; + } + + @Override + public FSDataOutputStream build() throws IOException { + Path path = getPath(); + + final Configuration options = getOptions(); + final Map headers = new HashMap<>(); + final Set mandatoryKeys = getMandatoryKeys(); + final Set keysToValidate = new HashSet<>(); + + // pick up all headers from the mandatory list and strip them before + // validating the keys + String headerPrefix = FS_S3A_CREATE_HEADER + "."; + final int prefixLen = headerPrefix.length(); + mandatoryKeys.stream().forEach(key -> { + if (key.startsWith(headerPrefix) && key.length() > prefixLen) { + headers.put(key.substring(prefixLen), options.get(key)); + } else { + keysToValidate.add(key); + } + }); + + rejectUnknownMandatoryKeys(keysToValidate, CREATE_FILE_KEYS, "for " + path); + + // and add any optional headers + getOptionalKeys().stream() + .filter(key -> key.startsWith(headerPrefix) && key.length() > prefixLen) + .forEach(key -> headers.put(key.substring(prefixLen), options.get(key))); + + + EnumSet flags = getFlags(); + if (flags.contains(CreateFlag.APPEND)) { + throw new UnsupportedOperationException("Append is not supported"); + } + if (!flags.contains(CreateFlag.CREATE) && + !flags.contains(CreateFlag.OVERWRITE)) { + throw new PathIOException(path.toString(), + "Must specify either create or overwrite"); + } + + final boolean performance = + options.getBoolean(Constants.FS_S3A_CREATE_PERFORMANCE, false); + return callbacks.createFileFromBuilder( + path, + getProgress(), + new CreateFileOptions(flags, isRecursive(), performance, headers)); + + } + + /** + * Pass flags down. + * @param flags input flags. + * @return this builder. + */ + public CreateFileBuilder withFlags(EnumSet flags) { + if (flags.contains(CreateFlag.CREATE)) { + create(); + } + if (flags.contains(CreateFlag.APPEND)) { + append(); + } + overwrite(flags.contains(CreateFlag.OVERWRITE)); + return this; + } + + /** + * make the flag getter public. + * @return creation flags. + */ + public EnumSet getFlags() { + return super.getFlags(); + } + + /** + * Callbacks for creating the file. + */ + public interface CreateFileBuilderCallbacks { + + /** + * Create a file from the builder. + * @param path path to file + * @param progress progress callback + * @param options options for the file + * @return the stream + * @throws IOException any IO problem + */ + FSDataOutputStream createFileFromBuilder( + Path path, + Progressable progress, + CreateFileOptions options) throws IOException; + } + + /** + * Create file options as built from the builder set or the classic + * entry point. + */ + public static final class CreateFileOptions { + + /** + * creation flags. + * create parent dirs? + * progress callback. + * performance flag. + */ + private final EnumSet flags; + + /** + * create parent dirs? + */ + private final boolean recursive; + + /** + * performance flag. + */ + private final boolean performance; + + /** + * Headers; may be null. + */ + private final Map headers; + + /** + * @param flags creation flags + * @param recursive create parent dirs? + * @param performance performance flag + * @param headers nullable header map. + */ + public CreateFileOptions( + final EnumSet flags, + final boolean recursive, + final boolean performance, + final Map headers) { + this.flags = flags; + this.recursive = recursive; + this.performance = performance; + this.headers = headers; + } + + @Override + public String toString() { + return "CreateFileOptions{" + + "flags=" + flags + + ", recursive=" + recursive + + ", performance=" + performance + + ", headers=" + headers + + '}'; + } + + public EnumSet getFlags() { + return flags; + } + + public boolean isRecursive() { + return recursive; + } + + public boolean isPerformance() { + return performance; + } + + public Map getHeaders() { + return headers; + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 49b9feeb6f..6e4946dfb5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -18,7 +18,9 @@ package org.apache.hadoop.fs.s3a.impl; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -160,4 +162,12 @@ private InternalConstants() { * will go through the AccessPoint. */ public static final String ARN_BUCKET_OPTION = "fs.s3a.bucket.%s.accesspoint.arn"; + + /** + * The known keys used in a createFile call. + */ + public static final Set CREATE_FILE_KEYS = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(Constants.FS_S3A_CREATE_PERFORMANCE))); + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java index 2b9a0e89b1..98a91b1881 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java @@ -41,6 +41,18 @@ * It performs the directory listing probe ahead of the simple object HEAD * call for this reason -the object is the failure mode which SHOULD NOT * be encountered on normal execution. + * + * Magic paths are handled specially + *
      + *
    • The only path check is for a directory already existing there.
      • + *
    • No ancestors are checked
      • + *
    • Parent markers are never deleted, irrespective of FS settings
      • + *
    + * As a result, irrespective of depth, the operations performed are only + *
      + *
    1. One LIST
      2. + *
    2. If needed, one PUT
      3. + *
    */ public class MkdirOperation extends ExecutingStoreOperation { @@ -51,13 +63,21 @@ public class MkdirOperation extends ExecutingStoreOperation { private final MkdirCallbacks callbacks; + /** + * Should checks for ancestors existing be skipped? + * This flag is set when working with magic directories. + */ + private final boolean isMagicPath; + public MkdirOperation( final StoreContext storeContext, final Path dir, - final MkdirCallbacks callbacks) { + final MkdirCallbacks callbacks, + final boolean isMagicPath) { super(storeContext); this.dir = dir; this.callbacks = callbacks; + this.isMagicPath = isMagicPath; } /** @@ -77,6 +97,14 @@ public Boolean execute() throws IOException { return true; } + // get the file status of the path. + // this is done even for a magic path, to avoid always issuing PUT + // requests. Doing that without a check wouild seem to be an + // optimization, but it is not because + // 1. PUT is slower than HEAD + // 2. Write capacity is less than read capacity on a shard + // 3. It adds needless entries in versioned buckets, slowing + // down subsequent operations. FileStatus fileStatus = getPathStatusExpectingDir(dir); if (fileStatus != null) { if (fileStatus.isDirectory()) { @@ -85,7 +113,17 @@ public Boolean execute() throws IOException { throw new FileAlreadyExistsException("Path is a file: " + dir); } } - // dir, walk up tree + // file status was null + + // is the path magic? + // If so, we declare success without looking any further + if (isMagicPath) { + // Create the marker file immediately, + // and don't delete markers + callbacks.createFakeDirectory(dir, true); + return true; + } + // Walk path to root, ensuring closest ancestor is a directory, not file Path fPart = dir.getParent(); try { @@ -110,14 +148,15 @@ public Boolean execute() throws IOException { LOG.info("mkdirs({}}: Access denied when looking" + " for parent directory {}; skipping checks", dir, fPart); - LOG.debug("{}", e.toString(), e); + LOG.debug("{}", e, e); } // if we get here there is no directory at the destination. // so create one. - String key = getStoreContext().pathToKey(dir); - // Create the marker file, maybe delete the parent entries - callbacks.createFakeDirectory(key); + + // Create the marker file, delete the parent entries + // if the filesystem isn't configured to retain them + callbacks.createFakeDirectory(dir, false); return true; } @@ -140,15 +179,21 @@ private S3AFileStatus probePathStatusOrNull(final Path path, /** * Get the status of a path -optimized for paths * where there is a directory marker or child entries. + * + * Under a magic path, there's no check for a file, + * just the listing. + * * @param path path to probe. + * * @return the status + * * @throws IOException failure */ private S3AFileStatus getPathStatusExpectingDir(final Path path) throws IOException { S3AFileStatus status = probePathStatusOrNull(path, StatusProbeEnum.DIRECTORIES); - if (status == null) { + if (status == null && !isMagicPath) { status = probePathStatusOrNull(path, StatusProbeEnum.FILE); } @@ -174,10 +219,15 @@ S3AFileStatus probePathStatus(Path path, /** * Create a fake directory, always ending in "/". * Retry policy: retrying; translated. - * @param key name of directory object. + * the keepMarkers flag controls whether or not markers + * are automatically kept (this is set when creating + * directories under a magic path, always) + * @param dir dir to create + * @param keepMarkers always keep markers + * * @throws IOException IO failure */ @Retries.RetryTranslated - void createFakeDirectory(String key) throws IOException; + void createFakeDirectory(Path dir, boolean keepMarkers) throws IOException; } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java new file mode 100644 index 0000000000..e14285a1ca --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/PutObjectOptions.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Extensible structure for options when putting/writing objects. + */ +public final class PutObjectOptions { + + /** + * Can the PUT operation skip marker deletion? + */ + private final boolean keepMarkers; + + /** + * Storage class, if not null. + */ + private final String storageClass; + + /** + * Headers; may be null. + */ + private final Map headers; + + /** + * Constructor. + * @param keepMarkers Can the PUT operation skip marker deletion? + * @param storageClass Storage class, if not null. + * @param headers Headers; may be null. + */ + public PutObjectOptions( + final boolean keepMarkers, + @Nullable final String storageClass, + @Nullable final Map headers) { + this.keepMarkers = keepMarkers; + this.storageClass = storageClass; + this.headers = headers; + } + + /** + * Get the marker retention flag. + * @return true if markers are to be retained. + */ + public boolean isKeepMarkers() { + return keepMarkers; + } + + /** + * Headers for the put/post request. + * @return headers or null. + */ + public Map getHeaders() { + return headers; + } + + @Override + public String toString() { + return "PutObjectOptions{" + + "keepMarkers=" + keepMarkers + + ", storageClass='" + storageClass + '\'' + + '}'; + } + + private static final PutObjectOptions KEEP_DIRS = new PutObjectOptions(true, + null, null); + private static final PutObjectOptions DELETE_DIRS = new PutObjectOptions(false, + null, null); + + /** + * Get the options to keep directories. + * @return an instance which keeps dirs + */ + public static PutObjectOptions keepingDirs() { + return KEEP_DIRS; + } + + /** + * Get the options to delete directory markers. + * @return an instance which deletes dirs + */ + public static PutObjectOptions deletingDirs() { + return DELETE_DIRS; + } + +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index 5a693c6c3d..1e6629f9c7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -23,7 +23,9 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; +import javax.annotation.Nullable; import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; @@ -362,15 +364,19 @@ protected void copyEncryptionParameters( * Adds the ACL, storage class and metadata * @param key key of object * @param metadata metadata header + * @param options options for the request, including headers * @param srcfile source file * @return the request */ @Override public PutObjectRequest newPutObjectRequest(String key, - ObjectMetadata metadata, File srcfile) { + ObjectMetadata metadata, + final PutObjectOptions options, + File srcfile) { Preconditions.checkNotNull(srcfile); PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, srcfile); + maybeSetMetadata(options, metadata); setOptionalPutRequestParameters(putObjectRequest); putObjectRequest.setCannedAcl(cannedACL); if (storageClass != null) { @@ -386,15 +392,18 @@ public PutObjectRequest newPutObjectRequest(String key, * operation. * @param key key of object * @param metadata metadata header + * @param options options for the request * @param inputStream source data. * @return the request */ @Override public PutObjectRequest newPutObjectRequest(String key, ObjectMetadata metadata, + @Nullable final PutObjectOptions options, InputStream inputStream) { Preconditions.checkNotNull(inputStream); Preconditions.checkArgument(isNotEmpty(key), "Null/empty key"); + maybeSetMetadata(options, metadata); PutObjectRequest putObjectRequest = new PutObjectRequest(getBucket(), key, inputStream, metadata); setOptionalPutRequestParameters(putObjectRequest); @@ -418,7 +427,7 @@ public int read() throws IOException { final ObjectMetadata md = createObjectMetadata(0L, true); md.setContentType(HeaderProcessing.CONTENT_TYPE_X_DIRECTORY); PutObjectRequest putObjectRequest = - newPutObjectRequest(key, md, im); + newPutObjectRequest(key, md, null, im); return putObjectRequest; } @@ -444,11 +453,14 @@ public AbortMultipartUploadRequest newAbortMultipartUploadRequest( @Override public InitiateMultipartUploadRequest newMultipartUploadRequest( - String destKey) { + final String destKey, + @Nullable final PutObjectOptions options) { + final ObjectMetadata objectMetadata = newObjectMetadata(-1); + maybeSetMetadata(options, objectMetadata); final InitiateMultipartUploadRequest initiateMPURequest = new InitiateMultipartUploadRequest(getBucket(), destKey, - newObjectMetadata(-1)); + objectMetadata); initiateMPURequest.setCannedACL(getCannedACL()); if (getStorageClass() != null) { initiateMPURequest.withStorageClass(getStorageClass()); @@ -601,6 +613,23 @@ public void setEncryptionSecrets(final EncryptionSecrets secrets) { encryptionSecrets = secrets; } + /** + * Set the metadata from the options if the options are not + * null and the metadata contains headers. + * @param options options for the request + * @param objectMetadata metadata to patch + */ + private void maybeSetMetadata( + @Nullable PutObjectOptions options, + final ObjectMetadata objectMetadata) { + if (options != null) { + Map headers = options.getHeaders(); + if (headers != null) { + objectMetadata.setUserMetadata(headers); + } + } + } + /** * Create a builder. * @return new builder. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java index db6beaff5b..3a6a04e5e7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AMultipartUploader.java @@ -123,7 +123,8 @@ public CompletableFuture startUpload( String key = context.pathToKey(dest); return context.submit(new CompletableFuture<>(), () -> { - String uploadId = writeOperations.initiateMultiPartUpload(key); + String uploadId = writeOperations.initiateMultiPartUpload(key, + PutObjectOptions.keepingDirs()); statistics.uploadStarted(); return BBUploadHandle.from(ByteBuffer.wrap( uploadId.getBytes(Charsets.UTF_8))); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java index 772b965d4f..bd1466b2a4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/BlockOutputStreamStatistics.java @@ -25,7 +25,8 @@ * Block output stream statistics. */ public interface BlockOutputStreamStatistics extends Closeable, - S3AStatisticInterface { + S3AStatisticInterface, + PutTrackerStatistics { /** * Block is queued for upload. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java index fd232a058d..53c25b0c65 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/CommitterStatistics.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.s3a.statistics; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; + /** * Statistics for S3A committers. */ @@ -63,4 +65,10 @@ public interface CommitterStatistics * @param success success flag */ void jobCompleted(boolean success); + + /** + * Return the writeable IOStatisticsStore. + * @return the statistics + */ + IOStatisticsStore getIOStatistics(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java new file mode 100644 index 0000000000..b422e0c9d5 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/PutTrackerStatistics.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.statistics; + +/** + * Interface for PUT tracking. + * It is subclassed by {@link BlockOutputStreamStatistics}, + * so that operations performed by the PutTracker update + * the stream statistics. + * Having a separate interface helps isolate operations. + */ +public interface PutTrackerStatistics extends S3AStatisticInterface { +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java index f618270798..5c0995e41b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/statistics/impl/EmptyS3AStatisticsContext.java @@ -33,6 +33,7 @@ import org.apache.hadoop.fs.s3a.statistics.StatisticsFromAwsSdk; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.DurationTracker; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.emptyStatistics; import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.stubDurationTracker; @@ -137,6 +138,7 @@ private static class EmptyS3AStatisticImpl implements public DurationTracker trackDuration(String key, long count) { return stubDurationTracker(); } + } /** @@ -381,6 +383,11 @@ public void taskCompleted(final boolean success) { @Override public void jobCompleted(final boolean success) { } + + @Override + public IOStatisticsStore getIOStatistics() { + return null; + } } private static final class EmptyBlockOutputStreamStatistics diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 7c00462735..8ccc36cf83 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -226,18 +226,25 @@ If any of the field values were `null`, the field is omitted. | `cm` | Command | `S3GuardTool$BucketInfo` | | `fs` | FileSystem ID | `af5943a9-b6f6-4eec-9c58-008982fc492a` | | `id` | Span ID | `3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3` | -| `ji` | Job ID | `(Generated by query engine)` | +| `ji` | Job ID (S3A committer)| `(Generated by query engine)` | | `op` | Filesystem API call | `op_rename` | | `p1` | Path 1 of operation | `s3a://alice-london/path1` | | `p2` | Path 2 of operation | `s3a://alice-london/path2` | | `pr` | Principal | `alice` | | `ps` | Unique process UUID | `235865a0-d399-4696-9978-64568db1b51c` | +| `ta` | Task Attempt ID (S3A committer) | | | `t0` | Thread 0: thread span was created in | `100` | | `t1` | Thread 1: thread this operation was executed in | `200` | | `ts` | Timestamp (UTC epoch millis) | `1617116985923` | +_Notes_ -Thread IDs are from the current thread in the JVM. +* Thread IDs are from the current thread in the JVM, so can be compared to those in````````` + Log4J logs. They are never unique. +* Task Attempt/Job IDs are only ever set during operations involving the S3A committers, specifically + all operations excecuted by the committer. + Operations executed in the same thread as the committer's instantiation _may_ also report the + IDs, even if they are unrelated to the actual task. Consider them "best effort". ```java Long.toString(Thread.currentThread().getId()) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md index b19f30f1a3..cfeff28d54 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md @@ -549,7 +549,7 @@ Conflict management is left to the execution engine itself. | `fs.s3a.buffer.dir` | Local filesystem directory for data being written and/or staged. | `${env.LOCAL_DIRS:-${hadoop.tmp.dir}}/s3a` | | `fs.s3a.committer.magic.enabled` | Enable "magic committer" support in the filesystem. | `true` | | `fs.s3a.committer.abort.pending.uploads` | list and abort all pending uploads under the destination path when the job is committed or aborted. | `true` | -| `fs.s3a.committer.threads` | Number of threads in committers for parallel operations on files. | 8 | +| `fs.s3a.committer.threads` | Number of threads in committers for parallel operations on files.| -4 | | `fs.s3a.committer.generate.uuid` | Generate a Job UUID if none is passed down from Spark | `false` | | `fs.s3a.committer.require.uuid` |Require the Job UUID to be passed down from Spark | `false` | @@ -587,10 +587,15 @@ Conflict management is left to the execution engine itself. fs.s3a.committer.threads - 8 + -4 Number of threads in committers for parallel operations on files - (upload, commit, abort, delete...) + (upload, commit, abort, delete...). + Two thread pools this size are created, one for the outer + task-level parallelism, and one for parallel execution + within tasks (POSTs to commit individual uploads) + If the value is negative, it is inverted and then multiplied + by the number of cores in the CPU. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java index 27c70b2b21..dae6312d48 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java @@ -18,8 +18,6 @@ package org.apache.hadoop.fs.s3a; - -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; @@ -369,67 +367,6 @@ public void testNeedEmptyDirectoryProbeRequiresList() throws Throwable { fs.s3GetFileStatus(new Path("/something"), "/something", StatusProbeEnum.HEAD_ONLY, true)); } - @Test - public void testCreateCost() throws Throwable { - describe("Test file creation cost"); - Path testFile = methodPath(); - // when overwrite is false, the path is checked for existence. - create(testFile, false, - CREATE_FILE_NO_OVERWRITE); - // but when true: only the directory checks take place. - create(testFile, true, CREATE_FILE_OVERWRITE); - } - - @Test - public void testCreateCostFileExists() throws Throwable { - describe("Test cost of create file failing with existing file"); - Path testFile = file(methodPath()); - - // now there is a file there, an attempt with overwrite == false will - // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - FILE_STATUS_FILE_PROBE, - () -> file(testFile, false)); - } - - @Test - public void testCreateCostDirExists() throws Throwable { - describe("Test cost of create file failing with existing dir"); - Path testFile = dir(methodPath()); - - // now there is a file there, an attempt with overwrite == false will - // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - GET_FILE_STATUS_ON_DIR_MARKER, - () -> file(testFile, false)); - } - - /** - * Use the builder API. - * This always looks for a parent unless the caller says otherwise. - */ - @Test - public void testCreateBuilder() throws Throwable { - describe("Test builder file creation cost"); - Path testFile = methodPath(); - dir(testFile.getParent()); - - // builder defaults to looking for parent existence (non-recursive) - buildFile(testFile, false, false, - GET_FILE_STATUS_FNFE // destination file - .plus(FILE_STATUS_DIR_PROBE)); // parent dir - // recursive = false and overwrite=true: - // only make sure the dest path isn't a directory. - buildFile(testFile, true, true, - FILE_STATUS_DIR_PROBE); - - // now there is a file there, an attempt with overwrite == false will - // fail on the first HEAD. - interceptOperation(FileAlreadyExistsException.class, "", - GET_FILE_STATUS_ON_FILE, - () -> buildFile(testFile, false, true, - GET_FILE_STATUS_ON_FILE)); - } @Test public void testCostOfGlobStatus() throws Throwable { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java index b4d2527a46..2d29282ad0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java @@ -36,8 +36,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.store.EtagChecksum; import org.apache.hadoop.test.LambdaTestUtils; @@ -102,18 +102,6 @@ public void testCreateNonRecursiveSuccess() throws IOException { assertIsFile(shouldWork); } - @Test(expected = FileNotFoundException.class) - public void testCreateNonRecursiveNoParent() throws IOException { - createNonRecursive(path("/recursive/node")); - } - - @Test(expected = FileAlreadyExistsException.class) - public void testCreateNonRecursiveParentIsFile() throws IOException { - Path parent = path("/file.txt"); - touch(getFileSystem(), parent); - createNonRecursive(new Path(parent, "fail")); - } - @Test public void testPutObjectDirect() throws Throwable { final S3AFileSystem fs = getFileSystem(); @@ -126,7 +114,7 @@ public void testPutObjectDirect() throws Throwable { new ByteArrayInputStream("PUT".getBytes()), metadata); LambdaTestUtils.intercept(IllegalStateException.class, - () -> fs.putObjectDirect(put)); + () -> fs.putObjectDirect(put, PutObjectOptions.keepingDirs())); assertPathDoesNotExist("put object was created", path); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index b597460bfd..2eb35daab4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -40,7 +40,11 @@ import org.apache.hadoop.fs.s3a.audit.AuditTestSupport; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.impl.RequestFactoryImpl; +import org.apache.hadoop.fs.s3a.impl.StoreContext; +import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; +import org.apache.hadoop.fs.s3a.impl.StubContextAccessor; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; @@ -211,7 +215,11 @@ public boolean exists(Path f) throws IOException { } @Override - void finishedWrite(String key, long length, String eTag, String versionId) { + void finishedWrite(String key, + long length, + String eTag, + String versionId, + final PutObjectOptions putOptions) { } @@ -377,11 +385,29 @@ public CommitterStatistics newCommitterStatistics() { @Override public void operationRetried(Exception ex) { - /** no-op */ + /* no-op */ } @Override protected DurationTrackerFactory getDurationTrackerFactory() { return stubDurationTrackerFactory(); } + + /** + * Build an immutable store context. + * If called while the FS is being initialized, + * some of the context will be incomplete. + * new store context instances should be created as appropriate. + * @return the store context of this FS. + */ + public StoreContext createStoreContext() { + return new StoreContextBuilder().setFsURI(getUri()) + .setBucket(getBucket()) + .setConfiguration(getConf()) + .setUsername(getUsername()) + .setAuditor(getAuditor()) + .setContextAccessors(new StubContextAccessor(getBucket())) + .build(); + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java index 04c2b2a09b..9ea33cf69c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MultipartTestUtils.java @@ -22,6 +22,7 @@ import com.amazonaws.services.s3.model.PartETag; import com.amazonaws.services.s3.model.UploadPartRequest; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.io.IOUtils; @@ -78,7 +79,7 @@ public static IdKey createPartUpload(S3AFileSystem fs, String key, int len, WriteOperationHelper writeHelper = fs.getWriteOperationHelper(); byte[] data = dataset(len, 'a', 'z'); InputStream in = new ByteArrayInputStream(data); - String uploadId = writeHelper.initiateMultiPartUpload(key); + String uploadId = writeHelper.initiateMultiPartUpload(key, PutObjectOptions.keepingDirs()); UploadPartRequest req = writeHelper.newUploadPartRequest(key, uploadId, partNo, len, in, null, 0L); PartETag partEtag = writeHelper.uploadPart(req).getPartETag(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java index 47ff2f326e..a6269c4376 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java @@ -245,4 +245,10 @@ public interface S3ATestConstants { * used. */ int KMS_KEY_GENERATION_REQUEST_PARAMS_BYTES_WRITTEN = 94; + + /** + * Build directory property. + * Value: {@value}. + */ + String PROJECT_BUILD_DIRECTORY_PROPERTY = "project.build.directory"; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 21ad7f87d6..48cb52c5ac 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -35,8 +35,6 @@ import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; -import org.apache.hadoop.fs.s3a.commit.CommitConstants; - import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy; import org.apache.hadoop.fs.s3a.impl.ContextAccessors; import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; @@ -577,6 +575,19 @@ public static Configuration prepareTestConfiguration(final Configuration conf) { return conf; } + /** + * build dir. + * @return the directory for the project's build, as set by maven, + * falling back to pwd + "target" if running from an IDE; + */ + public static File getProjectBuildDir() { + String propval = System.getProperty(PROJECT_BUILD_DIRECTORY_PROPERTY); + if (StringUtils.isEmpty(propval)) { + propval = "target"; + } + return new File(propval).getAbsoluteFile(); + } + /** * Clear any Hadoop credential provider path. * This is needed if people's test setups switch to credential providers, @@ -1301,18 +1312,6 @@ public static long lsR(FileSystem fileSystem, Path path, boolean recursive) public static final DateFormat LISTING_FORMAT = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); - /** - * Skip a test if the FS isn't marked as supporting magic commits. - * @param fs filesystem - */ - public static void assumeMagicCommitEnabled(S3AFileSystem fs) - throws IOException { - assume("Magic commit option disabled on " + fs, - fs.hasPathCapability( - fs.getWorkingDirectory(), - CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER)); - } - /** * Probe for the configuration containing a specific credential provider. * If the list is empty, there will be no match, even if the named provider diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index 21f268dfb2..08f4f8bc9d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.s3a.audit.AuditTestSupport; import org.apache.hadoop.fs.s3a.commit.PutTracker; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.util.Progressable; import org.junit.Before; @@ -65,7 +66,8 @@ private S3ABlockOutputStream.BlockOutputStreamBuilder mockS3ABuilder() { .withKey("") .withProgress(progressable) .withPutTracker(putTracker) - .withWriteOperations(oHelper); + .withWriteOperations(oHelper) + .withPutOptions(PutObjectOptions.keepingDirs()); return builder; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java index a5a1b454d0..4b0b65fd78 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/ITestAuditAccessChecks.java @@ -75,7 +75,9 @@ public Configuration createConfiguration() { @Override public void setup() throws Exception { super.setup(); - auditor = (AccessCheckingAuditor) getFileSystem().getAuditor(); + final S3AFileSystem fs = getFileSystem(); + auditor = (AccessCheckingAuditor) fs.getAuditor(); + setSpanSource(fs); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 86bfa2bb07..51eac7e8cc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -47,9 +47,10 @@ import org.apache.hadoop.fs.s3a.S3ATestConstants; import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; @@ -72,7 +73,7 @@ * Tests use of assumed roles. * Only run if an assumed role is provided. */ -@SuppressWarnings({"IOResourceOpenedButNotSafelyClosed", "ThrowableNotThrown"}) +@SuppressWarnings("ThrowableNotThrown") public class ITestAssumeRole extends AbstractS3ATestBase { private static final Logger LOG = @@ -563,9 +564,9 @@ public void testRestrictedCommitActions() throws Throwable { roleFS = (S3AFileSystem) writeableDir.getFileSystem(conf); CommitterStatistics committerStatistics = fs.newCommitterStatistics(); CommitOperations fullOperations = new CommitOperations(fs, - committerStatistics); + committerStatistics, "/"); CommitOperations operations = new CommitOperations(roleFS, - committerStatistics); + committerStatistics, "/"); File localSrc = File.createTempFile("source", ""); writeCSVData(localSrc); @@ -595,37 +596,37 @@ public void testRestrictedCommitActions() throws Throwable { SinglePendingCommit pending = fullOperations.uploadFileToPendingCommit(src, dest, "", uploadPartSize, progress); - pending.save(fs, new Path(readOnlyDir, - name + CommitConstants.PENDING_SUFFIX), true); + pending.save(fs, + new Path(readOnlyDir, name + CommitConstants.PENDING_SUFFIX), + SinglePendingCommit.serializer()); assertTrue(src.delete()); })); progress.assertCount("progress counter is not expected", range); - try { + try(CommitContext commitContext = + operations.createCommitContextForTesting(uploadDest, + null, 0)) { // we expect to be able to list all the files here Pair>> pendingCommits = operations.loadSinglePendingCommits(readOnlyDir, - true); + true, commitContext); // all those commits must fail List commits = pendingCommits.getLeft().getCommits(); assertEquals(range, commits.size()); - try(CommitOperations.CommitContext commitContext - = operations.initiateCommitOperation(uploadDest)) { - commits.parallelStream().forEach( - (c) -> { - CommitOperations.MaybeIOE maybeIOE = - commitContext.commit(c, "origin"); - Path path = c.destinationPath(); - assertCommitAccessDenied(path, maybeIOE); - }); - } + commits.parallelStream().forEach( + (c) -> { + CommitOperations.MaybeIOE maybeIOE = + commitContext.commit(c, "origin"); + Path path = c.destinationPath(); + assertCommitAccessDenied(path, maybeIOE); + }); // fail of all list and abort of .pending files. LOG.info("abortAllSinglePendingCommits({})", readOnlyDir); assertCommitAccessDenied(readOnlyDir, - operations.abortAllSinglePendingCommits(readOnlyDir, true)); + operations.abortAllSinglePendingCommits(readOnlyDir, commitContext, true)); // try writing a magic file Path magicDestPath = new Path(readOnlyDir, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java index 17ce7e4ddc..9d9000cafb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java @@ -18,9 +18,12 @@ package org.apache.hadoop.fs.s3a.commit; +import java.io.File; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import java.util.List; -import java.util.stream.Collectors; import org.assertj.core.api.Assertions; import org.slf4j.Logger; @@ -34,17 +37,20 @@ import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.WriteOperationHelper; -import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.fs.s3a.commit.files.SuccessData; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TypeConverter; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestPrinter; +import org.apache.hadoop.mapreduce.lib.output.committer.manifest.files.ManifestSuccessData; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; +import static java.time.temporal.ChronoField.DAY_OF_MONTH; +import static java.time.temporal.ChronoField.MONTH_OF_YEAR; +import static java.time.temporal.ChronoField.YEAR; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; @@ -60,6 +66,17 @@ public abstract class AbstractCommitITest extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(AbstractCommitITest.class); + /** + * Helper class for commit operations and assertions. + */ + private CommitterTestHelper testHelper; + + /** + * Directory for job summary reports. + * This should be set up in test suites testing against real object stores. + */ + private File reportDir; + /** * Creates a configuration for commit operations: commit is enabled in the FS * and output is multipart to on-heap arrays. @@ -81,6 +98,8 @@ protected Configuration createConfiguration() { conf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE); conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE); conf.set(FAST_UPLOAD_BUFFER, FAST_UPLOAD_BUFFER_ARRAY); + // and bind the report dir + conf.set(OPT_SUMMARY_REPORT_DIR, reportDir.toURI().toString()); return conf; } @@ -92,6 +111,36 @@ public Logger log() { return LOG; } + /** + * Get directory for reports; valid after + * setup. + * @return where success/failure reports go. + */ + protected File getReportDir() { + return reportDir; + } + + @Override + public void setup() throws Exception { + // set the manifest committer to a localfs path for reports across + // all threads. + // do this before superclass setup so reportDir is non-null there + // and can be used in creating the configuration. + reportDir = new File(getProjectBuildDir(), "reports"); + reportDir.mkdirs(); + + super.setup(); + testHelper = new CommitterTestHelper(getFileSystem()); + } + + /** + * Get helper class. + * @return helper; only valid after setup. + */ + public CommitterTestHelper getTestHelper() { + return testHelper; + } + /*** * Bind to the named committer. * @@ -117,12 +166,14 @@ public void rmdir(Path dir, Configuration conf) throws IOException { if (dir != null) { describe("deleting %s", dir); FileSystem fs = dir.getFileSystem(conf); + fs.delete(dir, true); + } } /** - * Create a random Job ID using the fork ID as part of the number. + * Create a random Job ID using the fork ID and the current time. * @return fork ID string in a format parseable by Jobs * @throws Exception failure */ @@ -132,7 +183,14 @@ public static String randomJobId() throws Exception { String trailingDigits = testUniqueForkId.substring(l - 4, l); try { int digitValue = Integer.valueOf(trailingDigits); - return String.format("20070712%04d_%04d", + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .appendValue(YEAR, 4) + .appendValue(MONTH_OF_YEAR, 2) + .appendValue(DAY_OF_MONTH, 2) + .toFormatter(); + return String.format("%s%04d_%04d", + LocalDateTime.now().format(formatter), (long)(Math.random() * 1000), digitValue); } catch (NumberFormatException e) { @@ -146,22 +204,9 @@ public static String randomJobId() throws Exception { * @return a count of aborts * @throws IOException trouble. */ - protected int abortMultipartUploadsUnderPath(Path path) throws IOException { - S3AFileSystem fs = getFileSystem(); - if (fs != null && path != null) { - String key = fs.pathToKey(path); - int count = 0; - try (AuditSpan span = span()) { - WriteOperationHelper writeOps = fs.getWriteOperationHelper(); - count = writeOps.abortMultipartUploadsUnderPath(key); - if (count > 0) { - log().info("Multipart uploads deleted: {}", count); - } - } - return count; - } else { - return 0; - } + protected void abortMultipartUploadsUnderPath(Path path) throws IOException { + getTestHelper() + .abortMultipartUploadsUnderPath(path); } /** @@ -183,10 +228,9 @@ protected void assertMultipartUploadsPending(Path path) throws IOException { protected void assertNoMultipartUploadsPending(Path path) throws IOException { List uploads = listMultipartUploads(getFileSystem(), pathToPrefix(path)); - if (!uploads.isEmpty()) { - String result = uploads.stream().collect(Collectors.joining("\n")); - fail("Multipart uploads in progress under " + path + " \n" + result); - } + Assertions.assertThat(uploads) + .describedAs("Multipart uploads in progress under " + path) + .isEmpty(); } /** @@ -341,6 +385,13 @@ public static SuccessData validateSuccessFile(final Path outputPath, .describedAs("JobID in " + commitDetails) .isEqualTo(jobId); } + // also load as a manifest success data file + // to verify consistency and that the CLI tool works. + Path success = new Path(outputPath, _SUCCESS); + final ManifestPrinter showManifest = new ManifestPrinter(); + ManifestSuccessData manifestSuccessData = + showManifest.loadAndPrintManifest(fs, success); + return successData; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java index bf99d49576..b193cca03d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java @@ -126,7 +126,7 @@ public abstract class AbstractITCommitProtocol extends AbstractCommitITest { private static final Text VAL_2 = new Text("val2"); /** A job to abort in test case teardown. */ - private List abortInTeardown = new ArrayList<>(1); + private final List abortInTeardown = new ArrayList<>(1); private final StandardCommitterFactory standardCommitterFactory = new StandardCommitterFactory(); @@ -562,7 +562,7 @@ protected void commit(AbstractS3ACommitter committer, describe("\ncommitting job"); committer.commitJob(jContext); describe("commit complete\n"); - verifyCommitterHasNoThreads(committer); + } } @@ -617,6 +617,9 @@ public void testRecoveryAndCleanup() throws Exception { assertNotNull("null outputPath in committer " + committer, committer.getOutputPath()); + // note the task attempt path. + Path job1TaskAttempt0Path = committer.getTaskAttemptPath(tContext); + // Commit the task. This will promote data and metadata to where // job commits will pick it up on commit or abort. commitTask(committer, tContext); @@ -636,6 +639,15 @@ public void testRecoveryAndCleanup() throws Exception { intercept(PathCommitException.class, "recover", () -> committer2.recoverTask(tContext2)); + // the new task attempt path is different from the first, because the + // job attempt counter is used in the path + final Path job2TaskAttempt0Path = committer2.getTaskAttemptPath(tContext2); + LOG.info("Job attempt 1 task attempt path {}; attempt 2 path {}", + job1TaskAttempt0Path, job2TaskAttempt0Path); + assertNotEquals("Task attempt paths must differ", + job1TaskAttempt0Path, + job2TaskAttempt0Path); + // at this point, task attempt 0 has failed to recover // it should be abortable though. This will be a no-op as it already // committed @@ -645,7 +657,7 @@ public void testRecoveryAndCleanup() throws Exception { committer2.abortJob(jContext2, JobStatus.State.KILLED); // now, state of system may still have pending data assertNoMultipartUploadsPending(outDir); - verifyCommitterHasNoThreads(committer2); + } protected void assertTaskAttemptPathDoesNotExist( @@ -747,7 +759,9 @@ private void validateMapFileOutputContent( assertPathExists("Map output", expectedMapDir); assertIsDirectory(expectedMapDir); FileStatus[] files = fs.listStatus(expectedMapDir); - assertTrue("No files found in " + expectedMapDir, files.length > 0); + Assertions.assertThat(files) + .describedAs("Files found in " + expectedMapDir) + .hasSizeGreaterThan(0); assertPathExists("index file in " + expectedMapDir, new Path(expectedMapDir, MapFile.INDEX_FILE_NAME)); assertPathExists("data file in " + expectedMapDir, @@ -795,9 +809,9 @@ public void testCommitLifecycle() throws Exception { try { applyLocatedFiles(getFileSystem().listFiles(outDir, false), - (status) -> - assertFalse("task committed file to dest :" + status, - status.getPath().toString().contains("part"))); + (status) -> Assertions.assertThat(status.getPath().toString()) + .describedAs("task committed file to dest :" + status) + .doesNotContain("part")); } catch (FileNotFoundException ignored) { log().info("Outdir {} is not created by task commit phase ", outDir); @@ -1071,27 +1085,34 @@ public void testMapFileOutputCommitter() throws Exception { // hidden filenames (_ or . prefixes) describe("listing"); FileStatus[] filtered = fs.listStatus(outDir, HIDDEN_FILE_FILTER); - assertEquals("listed children under " + ls, - 1, filtered.length); + Assertions.assertThat(filtered) + .describedAs("listed children under " + ls) + .hasSize(1); FileStatus fileStatus = filtered[0]; - assertTrue("Not the part file: " + fileStatus, - fileStatus.getPath().getName().startsWith(PART_00000)); + Assertions.assertThat(fileStatus.getPath().getName()) + .describedAs("Not a part file: " + fileStatus) + .startsWith(PART_00000); describe("getReaders()"); - assertEquals("Number of MapFile.Reader entries with shared FS " - + outDir + " : " + ls, - 1, getReaders(fs, outDir, conf).length); + Assertions.assertThat(getReaders(fs, outDir, conf)) + .describedAs("Number of MapFile.Reader entries with shared FS %s: %s", + outDir, ls) + .hasSize(1); describe("getReaders(new FS)"); FileSystem fs2 = FileSystem.get(outDir.toUri(), conf); - assertEquals("Number of MapFile.Reader entries with shared FS2 " - + outDir + " : " + ls, - 1, getReaders(fs2, outDir, conf).length); + Assertions.assertThat(getReaders(fs2, outDir, conf)) + .describedAs("Number of MapFile.Reader entries with shared FS2 %s: %s", + outDir, ls) + .hasSize(1); describe("MapFileOutputFormat.getReaders"); - assertEquals("Number of MapFile.Reader entries with new FS in " - + outDir + " : " + ls, - 1, MapFileOutputFormat.getReaders(outDir, conf).length); + + Assertions.assertThat(MapFileOutputFormat.getReaders(outDir, conf)) + .describedAs("Number of MapFile.Reader entries with new FS in %s: %s", + outDir, ls) + .hasSize(1); + } /** Open the output generated by this format. */ @@ -1165,7 +1186,7 @@ public void testAbortTaskThenJob() throws Exception { committer.abortJob(jobData.jContext, JobStatus.State.FAILED); assertJobAbortCleanedUp(jobData); - verifyCommitterHasNoThreads(committer); + } /** @@ -1219,7 +1240,7 @@ public void testFailAbort() throws Exception { // try again; expect abort to be idempotent. committer.abortJob(jContext, JobStatus.State.FAILED); assertNoMultipartUploadsPending(outDir); - verifyCommitterHasNoThreads(committer); + } public void assertPart0000DoesNotExist(Path dir) throws Exception { @@ -1433,7 +1454,7 @@ public void testAMWorkflow() throws Throwable { AbstractS3ACommitter committer2 = (AbstractS3ACommitter) outputFormat.getOutputCommitter(newAttempt); committer2.abortTask(tContext); - verifyCommitterHasNoThreads(committer2); + assertNoMultipartUploadsPending(getOutDir()); } @@ -1777,11 +1798,10 @@ public void testS3ACommitterFactoryBinding() throws Throwable { conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskAttempt0); - String name = getCommitterName(); S3ACommitterFactory factory = new S3ACommitterFactory(); - assertEquals("Wrong committer from factory", - createCommitter(outDir, tContext).getClass(), - factory.createOutputCommitter(outDir, tContext).getClass()); + Assertions.assertThat(factory.createOutputCommitter(outDir, tContext).getClass()) + .describedAs("Committer from factory with name %s", getCommitterName()) + .isEqualTo(createCommitter(outDir, tContext).getClass()); } /** @@ -1830,7 +1850,7 @@ protected void validateTaskAttemptWorkingDirectory( protected void commitTask(final AbstractS3ACommitter committer, final TaskAttemptContext tContext) throws IOException { committer.commitTask(tContext); - verifyCommitterHasNoThreads(committer); + } /** @@ -1842,15 +1862,7 @@ protected void commitTask(final AbstractS3ACommitter committer, protected void commitJob(final AbstractS3ACommitter committer, final JobContext jContext) throws IOException { committer.commitJob(jContext); - verifyCommitterHasNoThreads(committer); + } - /** - * Verify that the committer does not have a thread pool. - * @param committer committer to validate. - */ - protected void verifyCommitterHasNoThreads(AbstractS3ACommitter committer) { - assertFalse("Committer has an active thread pool", - committer.hasThreadPool()); - } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java new file mode 100644 index 0000000000..cd703f96da --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterTestHelper.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.commit; + +import java.io.IOException; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.MultipartTestUtils; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.contract.ContractTestUtils.verifyPathExists; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.XA_MAGIC_MARKER; +import static org.apache.hadoop.fs.s3a.commit.impl.CommitOperations.extractMagicFileLength; + +/** + * Helper for committer tests: extra assertions and the like. + */ +public class CommitterTestHelper { + + private static final Logger LOG = + LoggerFactory.getLogger(CommitterTestHelper.class); + + /** + * Filesystem under test. + */ + private final S3AFileSystem fileSystem; + + /** + * Constructor. + * @param fileSystem filesystem to work with. + */ + public CommitterTestHelper(S3AFileSystem fileSystem) { + this.fileSystem = requireNonNull(fileSystem); + } + + /** + * Get the filesystem. + * @return the filesystem. + */ + public S3AFileSystem getFileSystem() { + return fileSystem; + } + + /** + * Assert a path refers to a marker file of an expected length; + * the length is extracted from the custom header. + * @param path magic file. + * @param dataSize expected data size + * @throws IOException IO failure + */ + public void assertIsMarkerFile(Path path, long dataSize) throws IOException { + final S3AFileSystem fs = getFileSystem(); + FileStatus status = verifyPathExists(fs, + "uploaded file commit", path); + Assertions.assertThat(status.getLen()) + .describedAs("Marker File file %s: %s", path, status) + .isEqualTo(0); + Assertions.assertThat(extractMagicFileLength(fs, path)) + .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + path) + .isNotEmpty() + .hasValue(dataSize); + } + + /** + * Assert a file does not have the magic marker header. + * @param path magic file. + * @throws IOException IO failure + */ + public void assertFileLacksMarkerHeader(Path path) throws IOException { + Assertions.assertThat(extractMagicFileLength(getFileSystem(), + path)) + .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + path) + .isEmpty(); + } + + /** + * Create a new path which has the same filename as the dest file, but + * is in a magic directory under the destination dir. + * @param destFile final destination file + * @return magic path + */ + public static Path makeMagic(Path destFile) { + return new Path(destFile.getParent(), + MAGIC + '/' + BASE + "/" + destFile.getName()); + } + + /** + * Assert that an output stream is magic. + * @param stream stream to probe. + */ + public static void assertIsMagicStream(final FSDataOutputStream stream) { + Assertions.assertThat(stream.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT)) + .describedAs("Stream capability %s in stream %s", + STREAM_CAPABILITY_MAGIC_OUTPUT, stream) + .isTrue(); + } + + /** + * Abort all multipart uploads under a path. + * @param path path for uploads to abort; may be null + * @return a count of aborts + * @throws IOException trouble. + */ + public void abortMultipartUploadsUnderPath(Path path) { + + MultipartTestUtils.clearAnyUploads(getFileSystem(), path); + } + + /** + * Get a list of all pending uploads under a prefix, one which can be printed. + * @param prefix prefix to look under + * @return possibly empty list + * @throws IOException IO failure. + */ + public List listMultipartUploads( + String prefix) throws IOException { + + return MultipartTestUtils.listMultipartUploads(getFileSystem(), prefix); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java new file mode 100644 index 0000000000..d0c86b738c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperationCost.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.commit; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData; +import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; +import org.apache.hadoop.fs.s3a.performance.AbstractS3ACostTest; +import org.apache.hadoop.fs.statistics.IOStatisticsLogging; + +import static org.apache.hadoop.fs.s3a.Statistic.ACTION_HTTP_GET_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_FILES_CREATED; +import static org.apache.hadoop.fs.s3a.Statistic.COMMITTER_MAGIC_MARKER_PUT; +import static org.apache.hadoop.fs.s3a.Statistic.DIRECTORIES_CREATED; +import static org.apache.hadoop.fs.s3a.Statistic.FAKE_DIRECTORIES_DELETED; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_LIST_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_METADATA_REQUESTS; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_MULTIPART_UPLOAD_INITIATED; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_PUT_REQUESTS; +import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC; +import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.assertIsMagicStream; +import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.makeMagic; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_FILES_LIST_OP; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.LIST_OPERATION; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST; +import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString; +import static org.apache.hadoop.util.functional.RemoteIterators.toList; + +/** + * Assert cost of commit operations; + *
      + *
    1. Even on marker deleting filesystems, + * operations under magic dirs do not trigger marker deletion.
      2. + *
    2. Loading pending files from FileStatus entries skips HEAD checks.
      3. + *
    3. Mkdir under magic dirs doesn't check ancestor or dest type
      4. + *
    + */ +public class ITestCommitOperationCost extends AbstractS3ACostTest { + + private static final Logger LOG = + LoggerFactory.getLogger(ITestCommitOperationCost.class); + + /** + * Helper for the tests. + */ + private CommitterTestHelper testHelper; + + /** + * Create with markers kept, always. + */ + public ITestCommitOperationCost() { + super(false); + } + + @Override + public void setup() throws Exception { + super.setup(); + testHelper = new CommitterTestHelper(getFileSystem()); + } + + @Override + public void teardown() throws Exception { + try { + if (testHelper != null) { + testHelper.abortMultipartUploadsUnderPath(methodPath()); + } + } finally { + super.teardown(); + } + } + + /** + * Get a method-relative path. + * @param filename filename + * @return new path + * @throws IOException failure to create/parse the path. + */ + private Path methodSubPath(String filename) throws IOException { + return new Path(methodPath(), filename); + } + + /** + * Return the FS IOStats, prettified. + * @return string for assertions. + */ + protected String fileSystemIOStats() { + return ioStatisticsToPrettyString(getFileSystem().getIOStatistics()); + } + + @Test + public void testMagicMkdir() throws Throwable { + describe("Mkdirs __magic always skips dir marker deletion"); + S3AFileSystem fs = getFileSystem(); + Path baseDir = methodPath(); + // create dest dir marker, always + fs.mkdirs(baseDir); + Path magicDir = new Path(baseDir, MAGIC); + verifyMetrics(() -> { + fs.mkdirs(magicDir); + return fileSystemIOStats(); + }, + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0), + with(DIRECTORIES_CREATED, 1)); + verifyMetrics(() -> { + fs.delete(magicDir, true); + return fileSystemIOStats(); + }, + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 1), + with(DIRECTORIES_CREATED, 0)); + assertPathExists("parent", baseDir); + } + + /** + * When a magic subdir is deleted, parent dirs are not recreated. + */ + @Test + public void testMagicMkdirs() throws Throwable { + describe("Mkdirs __magic/subdir always skips dir marker deletion"); + S3AFileSystem fs = getFileSystem(); + Path baseDir = methodPath(); + Path magicDir = new Path(baseDir, MAGIC); + fs.delete(baseDir, true); + + Path magicSubdir = new Path(magicDir, "subdir"); + verifyMetrics(() -> { + fs.mkdirs(magicSubdir, FsPermission.getDirDefault()); + return "after mkdirs " + fileSystemIOStats(); + }, + always(LIST_OPERATION), + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0), + with(DIRECTORIES_CREATED, 1)); + assertPathExists("magicSubdir", magicSubdir); + + verifyMetrics(() -> { + fs.delete(magicSubdir, true); + return "after delete " + fileSystemIOStats(); + }, + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 1), + with(OBJECT_LIST_REQUEST, 1), + with(OBJECT_METADATA_REQUESTS, 1), + with(DIRECTORIES_CREATED, 0)); + // no marker dir creation + assertPathDoesNotExist("magicDir", magicDir); + assertPathDoesNotExist("baseDir", baseDir); + } + + /** + * Active stream; a field is used so closures can write to + * it. + */ + private FSDataOutputStream stream; + + /** + * Abort any active stream. + * @throws IOException failure + */ + private void abortActiveStream() throws IOException { + if (stream != null) { + stream.abort(); + stream.close(); + } + } + + @Test + public void testCostOfCreatingMagicFile() throws Throwable { + describe("Files created under magic paths skip existence checks and marker deletes"); + S3AFileSystem fs = getFileSystem(); + Path destFile = methodSubPath("file.txt"); + fs.delete(destFile.getParent(), true); + Path magicDest = makeMagic(destFile); + + // when the file is created, there is no check for overwrites + // or the dest being a directory, even if overwrite=false + try { + verifyMetrics(() -> { + stream = fs.create(magicDest, false); + return stream.toString(); + }, + always(NO_HEAD_OR_LIST), + with(COMMITTER_MAGIC_FILES_CREATED, 1), + with(COMMITTER_MAGIC_MARKER_PUT, 0), + with(OBJECT_MULTIPART_UPLOAD_INITIATED, 1)); + assertIsMagicStream(stream); + + stream.write("hello".getBytes(StandardCharsets.UTF_8)); + + // when closing, there will be no directories deleted + // we do expect two PUT requests, because the marker and manifests + // are both written + LOG.info("closing magic stream to {}", magicDest); + verifyMetrics(() -> { + stream.close(); + return stream.toString(); + }, + always(NO_HEAD_OR_LIST), + with(OBJECT_PUT_REQUESTS, 2), + with(COMMITTER_MAGIC_MARKER_PUT, 2), + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0)); + + } catch (Exception e) { + abortActiveStream(); + throw e; + } + // list the manifests + final CommitOperations commitOperations = new CommitOperations(fs); + List pending = verifyMetrics(() -> + toList(commitOperations. + locateAllSinglePendingCommits(magicDest.getParent(), false)), + always(LIST_FILES_LIST_OP)); + Assertions.assertThat(pending) + .describedAs("pending commits") + .hasSize(1); + + // load the only pending commit + SinglePendingCommit singleCommit = verifyMetrics(() -> + PersistentCommitData.load(fs, + pending.get(0), + SinglePendingCommit.serializer()), + always(NO_HEAD_OR_LIST), + with(ACTION_HTTP_GET_REQUEST, 1)); + + // commit it through the commit operations. + verifyMetrics(() -> { + commitOperations.commitOrFail(singleCommit); + return ioStatisticsToPrettyString( + commitOperations.getIOStatistics()); + }, + always(NO_HEAD_OR_LIST), // no probes for the dest path + with(FAKE_DIRECTORIES_DELETED, 0), // no fake dirs + with(OBJECT_DELETE_REQUEST, 0)); // no deletes + + LOG.info("Final Statistics {}", + IOStatisticsLogging.ioStatisticsToPrettyString(stream.getIOStatistics())); + } + + /** + * saving pending files MUST NOT trigger HEAD/LIST calls + * when created under a magic path; when opening + * with an S3AFileStatus the HEAD will be skipped too. + */ + @Test + public void testCostOfSavingLoadingPendingFile() throws Throwable { + describe("Verify costs of saving .pending file under a magic path"); + + S3AFileSystem fs = getFileSystem(); + Path partDir = methodSubPath("file.pending"); + Path destFile = new Path(partDir, "file.pending"); + Path magicDest = makeMagic(destFile); + // create a pending file with minimal values needed + // for validation to work + final SinglePendingCommit commit = new SinglePendingCommit(); + commit.touch(System.currentTimeMillis()); + commit.setUri(destFile.toUri().toString()); + commit.setBucket(fs.getBucket()); + commit.setLength(0); + commit.setDestinationKey(fs.pathToKey(destFile)); + commit.setUploadId("uploadId"); + commit.setEtags(new ArrayList<>()); + // fail fast if the commit data is incomplete + commit.validate(); + + // save the file: no checks will be made + verifyMetrics(() -> { + commit.save(fs, magicDest, + SinglePendingCommit.serializer()); + return commit.toString(); + }, + with(COMMITTER_MAGIC_FILES_CREATED, 0), + always(NO_HEAD_OR_LIST), + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0) + ); + + LOG.info("File written; Validating"); + testHelper.assertFileLacksMarkerHeader(magicDest); + FileStatus status = fs.getFileStatus(magicDest); + + LOG.info("Reading file {}", status); + // opening a file with a status passed in will skip the HEAD + verifyMetrics(() -> + PersistentCommitData.load(fs, status, SinglePendingCommit.serializer()), + always(NO_HEAD_OR_LIST), + with(ACTION_HTTP_GET_REQUEST, 1)); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java index 3f0e2e7a13..a76a65be8b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java @@ -23,8 +23,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.UUID; -import com.amazonaws.services.s3.model.PartETag; import org.assertj.core.api.Assertions; import org.junit.Test; import org.slf4j.Logger; @@ -33,12 +33,15 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.auth.ProgressCounter; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker; import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitter; import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory; @@ -52,11 +55,13 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.S3ATestUtils.*; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; -import static org.apache.hadoop.fs.s3a.commit.CommitOperations.extractMagicFileLength; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*; +import static org.apache.hadoop.fs.s3a.commit.CommitterTestHelper.assertIsMagicStream; import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*; import static org.apache.hadoop.fs.s3a.Constants.*; +import static org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext.EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS; import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.*; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Test the low-level binding of the S3A FS to the magic commit mechanism, @@ -69,6 +74,8 @@ public class ITestCommitOperations extends AbstractCommitITest { private static final byte[] DATASET = dataset(1000, 'a', 32); private static final String S3A_FACTORY_KEY = String.format( COMMITTER_FACTORY_SCHEME_PATTERN, "s3a"); + private static final String JOB_ID = UUID.randomUUID().toString(); + private ProgressCounter progress; @Override @@ -94,11 +101,13 @@ public void testCreateTrackerNormalPath() throws Throwable { MagicCommitIntegration integration = new MagicCommitIntegration(fs, true); String filename = "notdelayed.txt"; - Path destFile = methodPath(filename); + Path destFile = methodSubPath(filename); String origKey = fs.pathToKey(destFile); - PutTracker tracker = integration.createTracker(destFile, origKey); - assertFalse("wrong type: " + tracker + " for " + destFile, - tracker instanceof MagicCommitTracker); + PutTracker tracker = integration.createTracker(destFile, origKey, + EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS); + Assertions.assertThat(tracker) + .describedAs("Tracker for %s", destFile) + .isNotInstanceOf(MagicCommitTracker.class); } /** @@ -111,36 +120,45 @@ public void testCreateTrackerMagicPath() throws Throwable { MagicCommitIntegration integration = new MagicCommitIntegration(fs, true); String filename = "delayed.txt"; - Path destFile = methodPath(filename); + Path destFile = methodSubPath(filename); String origKey = fs.pathToKey(destFile); Path pendingPath = makeMagic(destFile); verifyIsMagicCommitPath(fs, pendingPath); String pendingPathKey = fs.pathToKey(pendingPath); - assertTrue("wrong path of " + pendingPathKey, - pendingPathKey.endsWith(filename)); + Assertions.assertThat(pendingPathKey) + .describedAs("pending path") + .endsWith(filename); final List elements = splitPathToElements(pendingPath); - assertEquals("splitPathToElements()", filename, lastElement(elements)); + Assertions.assertThat(lastElement(elements)) + .describedAs("splitPathToElements(%s)", pendingPath) + .isEqualTo(filename); List finalDestination = finalDestination(elements); - assertEquals("finalDestination()", - filename, - lastElement(finalDestination)); - final String destKey = elementsToKey(finalDestination); - assertEquals("destination key", origKey, destKey); + Assertions.assertThat(lastElement(finalDestination)) + .describedAs("finalDestination(%s)", pendingPath) + .isEqualTo(filename); + Assertions.assertThat(elementsToKey(finalDestination)) + .describedAs("destination key") + .isEqualTo(origKey); PutTracker tracker = integration.createTracker(pendingPath, - pendingPathKey); - assertTrue("wrong type: " + tracker + " for " + pendingPathKey, - tracker instanceof MagicCommitTracker); - assertEquals("tracker destination key", origKey, tracker.getDestKey()); + pendingPathKey, EMPTY_BLOCK_OUTPUT_STREAM_STATISTICS); + Assertions.assertThat(tracker) + .describedAs("Tracker for %s", pendingPathKey) + .isInstanceOf(MagicCommitTracker.class); + Assertions.assertThat(tracker.getDestKey()) + .describedAs("tracker destination key") + .isEqualTo(origKey); - Path pendingSuffixedPath = new Path(pendingPath, - "part-0000" + PENDING_SUFFIX); - assertFalse("still a delayed complete path " + pendingSuffixedPath, - fs.isMagicCommitPath(pendingSuffixedPath)); - Path pendingSet = new Path(pendingPath, - "part-0000" + PENDINGSET_SUFFIX); - assertFalse("still a delayed complete path " + pendingSet, - fs.isMagicCommitPath(pendingSet)); + assertNotDelayedWrite(new Path(pendingPath, + "part-0000" + PENDING_SUFFIX)); + assertNotDelayedWrite(new Path(pendingPath, + "part-0000" + PENDINGSET_SUFFIX)); + } + + private void assertNotDelayedWrite(Path pendingSuffixedPath) { + Assertions.assertThat(getFileSystem().isMagicCommitPath(pendingSuffixedPath)) + .describedAs("Expected %s to not be magic/delayed write", pendingSuffixedPath) + .isFalse(); } @Test @@ -148,7 +166,7 @@ public void testCreateAbortEmptyFile() throws Throwable { describe("create then abort an empty file; throttled"); S3AFileSystem fs = getFileSystem(); String filename = "empty-abort.txt"; - Path destFile = methodPath(filename); + Path destFile = methodSubPath(filename); Path pendingFilePath = makeMagic(destFile); touch(fs, pendingFilePath); @@ -160,13 +178,22 @@ public void testCreateAbortEmptyFile() throws Throwable { // abort,; rethrow on failure LOG.info("Abort call"); - actions.abortAllSinglePendingCommits(pendingDataPath.getParent(), true) - .maybeRethrow(); + Path parent = pendingDataPath.getParent(); + try (CommitContext commitContext = + actions.createCommitContextForTesting(parent, JOB_ID, 0)) { + actions.abortAllSinglePendingCommits(parent, commitContext, true) + .maybeRethrow(); + } assertPathDoesNotExist("pending file not deleted", pendingDataPath); assertPathDoesNotExist("dest file was created", destFile); } + /** + * Create a new commit operations instance for the test FS. + * @return commit operations. + * @throws IOException IO failure. + */ private CommitOperations newCommitOperations() throws IOException { return new CommitOperations(getFileSystem()); @@ -198,10 +225,16 @@ public void testCommitSmallFile() throws Throwable { @Test public void testAbortNonexistentDir() throws Throwable { describe("Attempt to abort a directory that does not exist"); - Path destFile = methodPath("testAbortNonexistentPath"); - newCommitOperations() - .abortAllSinglePendingCommits(destFile, true) - .maybeRethrow(); + Path destFile = methodSubPath("testAbortNonexistentPath"); + final CommitOperations operations = newCommitOperations(); + try (CommitContext commitContext + = operations.createCommitContextForTesting(destFile, JOB_ID, 0)) { + final CommitOperations.MaybeIOE outcome = operations + .abortAllSinglePendingCommits(destFile, commitContext, true); + outcome.maybeRethrow(); + Assertions.assertThat(outcome) + .isEqualTo(CommitOperations.MaybeIOE.NONE); + } } @Test @@ -244,7 +277,7 @@ public void testBaseRelativePath() throws Throwable { describe("Test creating file with a __base marker and verify that it ends" + " up in where expected"); S3AFileSystem fs = getFileSystem(); - Path destDir = methodPath("testBaseRelativePath"); + Path destDir = methodSubPath("testBaseRelativePath"); fs.delete(destDir, true); Path pendingBaseDir = new Path(destDir, MAGIC + "/child/" + BASE); String child = "subdir/child.txt"; @@ -270,14 +303,17 @@ public void testMarkerFileRename() fs.delete(destDir, true); Path magicDest = makeMagic(destFile); Path magicDir = magicDest.getParent(); - fs.mkdirs(magicDir); + fs.mkdirs(magicDest); // use the builder API to verify it works exactly the // same. - try (FSDataOutputStream stream = fs.createFile(magicDest) - .overwrite(true) - .recursive() - .build()) { + FSDataOutputStreamBuilder builder = fs.createFile(magicDest) + .overwrite(true); + builder.recursive(); + // this has a broken return type; not sure why + builder.must(FS_S3A_CREATE_PERFORMANCE, true); + + try (FSDataOutputStream stream = builder.build()) { assertIsMagicStream(stream); stream.write(DATASET); } @@ -286,9 +322,7 @@ public void testMarkerFileRename() fs.rename(magicDest, magic2); // the renamed file has no header - Assertions.assertThat(extractMagicFileLength(fs, magic2)) - .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + magic2) - .isEmpty(); + getTestHelper().assertFileLacksMarkerHeader(magic2); // abort the upload, which is driven by the .pending files // there must be 1 deleted file; during test debugging with aborted // runs there may be more. @@ -298,17 +332,6 @@ public void testMarkerFileRename() .isGreaterThanOrEqualTo(1); } - /** - * Assert that an output stream is magic. - * @param stream stream to probe. - */ - protected void assertIsMagicStream(final FSDataOutputStream stream) { - Assertions.assertThat(stream.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT)) - .describedAs("Stream capability %s in stream %s", - STREAM_CAPABILITY_MAGIC_OUTPUT, stream) - .isTrue(); - } - /** * Create a file through the magic commit mechanism. * @param filename file to create (with __magic path.) @@ -318,39 +341,27 @@ protected void assertIsMagicStream(final FSDataOutputStream stream) { private void createCommitAndVerify(String filename, byte[] data) throws Exception { S3AFileSystem fs = getFileSystem(); - Path destFile = methodPath(filename); + Path destFile = methodSubPath(filename); fs.delete(destFile.getParent(), true); Path magicDest = makeMagic(destFile); assertPathDoesNotExist("Magic file should not exist", magicDest); long dataSize = data != null ? data.length : 0; - try(FSDataOutputStream stream = fs.create(magicDest, true)) { + try (FSDataOutputStream stream = fs.create(magicDest, true)) { assertIsMagicStream(stream); if (dataSize > 0) { stream.write(data); } - stream.close(); } - FileStatus status = fs.getFileStatus(magicDest); - assertEquals("Magic marker file is not zero bytes: " + status, - 0, 0); - Assertions.assertThat(extractMagicFileLength(fs, - magicDest)) - .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + magicDest) - .isNotEmpty() - .hasValue(dataSize); + getTestHelper().assertIsMarkerFile(magicDest, dataSize); commit(filename, destFile); verifyFileContents(fs, destFile, data); // the destination file doesn't have the attribute - Assertions.assertThat(extractMagicFileLength(fs, - destFile)) - .describedAs("XAttribute " + XA_MAGIC_MARKER + " of " + destFile) - .isEmpty(); + getTestHelper().assertFileLacksMarkerHeader(destFile); } /** * Commit the file, with before and after checks on the dest and magic * values. - * Failures can be set; they'll be reset after the commit. * @param filename filename of file * @param destFile destination path of file * @throws Exception any failure of the operation @@ -371,20 +382,21 @@ private void commit(String filename, Path destFile) throws IOException { + final CommitOperations actions = newCommitOperations(); validateIntermediateAndFinalPaths(magicFile, destFile); SinglePendingCommit commit = SinglePendingCommit.load(getFileSystem(), - validatePendingCommitData(filename, magicFile)); + validatePendingCommitData(filename, magicFile), + null, + SinglePendingCommit.serializer()); - commitOrFail(destFile, commit, newCommitOperations()); - - verifyCommitExists(commit); + commitOrFail(destFile, commit, actions); } private void commitOrFail(final Path destFile, final SinglePendingCommit commit, final CommitOperations actions) throws IOException { - try (CommitOperations.CommitContext commitContext - = actions.initiateCommitOperation(destFile)) { + try (CommitContext commitContext + = actions.createCommitContextForTesting(destFile, JOB_ID, 0)) { commitContext.commitOrFail(commit); } } @@ -401,26 +413,6 @@ private void validateIntermediateAndFinalPaths(Path magicFilePath, assertPathDoesNotExist("dest file was created", destFile); } - /** - * Verify that the path at the end of a commit exists. - * This does not validate the size. - * @param commit commit to verify - * @throws FileNotFoundException dest doesn't exist - * @throws ValidationFailure commit arg is invalid - * @throws IOException invalid commit, IO failure - */ - private void verifyCommitExists(SinglePendingCommit commit) - throws FileNotFoundException, ValidationFailure, IOException { - commit.validate(); - // this will force an existence check - Path path = getFileSystem().keyToQualifiedPath(commit.getDestinationKey()); - FileStatus status = getFileSystem().getFileStatus(path); - LOG.debug("Destination entry: {}", status); - if (!status.isFile()) { - throw new PathCommitException(path, "Not a file: " + status); - } - } - /** * Validate that a pending commit data file exists, load it and validate * its contents. @@ -443,14 +435,19 @@ private Path validatePendingCommitData(String filename, SinglePendingCommit persisted = SinglePendingCommit.serializer() .load(fs, pendingDataPath); persisted.validate(); - assertTrue("created timestamp wrong in " + persisted, - persisted.getCreated() > 0); - assertTrue("saved timestamp wrong in " + persisted, - persisted.getSaved() > 0); + Assertions.assertThat(persisted.getCreated()) + .describedAs("Created timestamp in %s", persisted) + .isGreaterThan(0); + Assertions.assertThat(persisted.getSaved()) + .describedAs("saved timestamp in %s", persisted) + .isGreaterThan(0); List etags = persisted.getEtags(); - assertEquals("etag list " + persisted, 1, etags.size()); - List partList = CommitOperations.toPartEtags(etags); - assertEquals("part list " + persisted, 1, partList.size()); + Assertions.assertThat(etags) + .describedAs("Etag list") + .hasSize(1); + Assertions.assertThat(CommitOperations.toPartEtags(etags)) + .describedAs("Etags to parts") + .hasSize(1); return pendingDataPath; } @@ -460,24 +457,16 @@ private Path validatePendingCommitData(String filename, * @return new path * @throws IOException failure to create/parse the path. */ - private Path methodPath(String filename) throws IOException { + private Path methodSubPath(String filename) throws IOException { return new Path(methodPath(), filename); } - /** - * Get a unique path for a method. - * @return a path - * @throws IOException - */ - protected Path methodPath() throws IOException { - return path(getMethodName()); - } - @Test public void testUploadEmptyFile() throws Throwable { + describe("Upload a zero byte file to a magic path"); File tempFile = File.createTempFile("commit", ".txt"); CommitOperations actions = newCommitOperations(); - Path dest = methodPath("testUploadEmptyFile"); + Path dest = methodSubPath("testUploadEmptyFile"); S3AFileSystem fs = getFileSystem(); fs.delete(dest, false); @@ -492,11 +481,14 @@ public void testUploadEmptyFile() throws Throwable { commitOrFail(dest, pendingCommit, actions); - FileStatus status = verifyPathExists(fs, - "uploaded file commit", dest); progress.assertCount("Progress counter should be 1.", 1); - assertEquals("File length in " + status, 0, status.getLen()); + FileStatus status = verifyPathExists(fs, + "uploaded file commit", dest); + Assertions.assertThat(status.getLen()) + .describedAs("Committed File file %s: %s", dest, status) + .isEqualTo(0); + getTestHelper().assertFileLacksMarkerHeader(dest); } @Test @@ -505,7 +497,7 @@ public void testUploadSmallFile() throws Throwable { String text = "hello, world"; FileUtils.write(tempFile, text, "UTF-8"); CommitOperations actions = newCommitOperations(); - Path dest = methodPath("testUploadSmallFile"); + Path dest = methodSubPath("testUploadSmallFile"); S3AFileSystem fs = getFileSystem(); fs.delete(dest, true); @@ -523,51 +515,57 @@ public void testUploadSmallFile() throws Throwable { commitOrFail(dest, pendingCommit, actions); String s = readUTF8(fs, dest, -1); - assertEquals(text, s); + Assertions.assertThat(s) + .describedAs("contents of committed file %s", dest) + .isEqualTo(text); progress.assertCount("Progress counter should be 1.", 1); } - @Test(expected = FileNotFoundException.class) + @Test public void testUploadMissingFile() throws Throwable { File tempFile = File.createTempFile("commit", ".txt"); tempFile.delete(); CommitOperations actions = newCommitOperations(); - Path dest = methodPath("testUploadMissingile"); - - actions.uploadFileToPendingCommit(tempFile, dest, null, - DEFAULT_MULTIPART_SIZE, progress); - progress.assertCount("Progress counter should be 1.", - 1); + Path dest = methodSubPath("testUploadMissingFile"); + intercept(FileNotFoundException.class, () -> + actions.uploadFileToPendingCommit(tempFile, dest, null, + DEFAULT_MULTIPART_SIZE, progress)); + progress.assertCount("Progress counter should be 0.", + 0); } @Test public void testRevertCommit() throws Throwable { - Path destFile = methodPath("part-0000"); + describe("Revert a commit; the destination file will be deleted"); + Path destFile = methodSubPath("part-0000"); S3AFileSystem fs = getFileSystem(); touch(fs, destFile); - CommitOperations actions = newCommitOperations(); SinglePendingCommit commit = new SinglePendingCommit(); + CommitOperations actions = newCommitOperations(); commit.setDestinationKey(fs.pathToKey(destFile)); - - actions.revertCommit(commit); - - assertPathExists("parent of reverted commit", destFile.getParent()); + newCommitOperations().revertCommit(commit); + assertPathDoesNotExist("should have been reverted", destFile); } @Test public void testRevertMissingCommit() throws Throwable { - Path destFile = methodPath("part-0000"); + Path destFile = methodSubPath("part-0000"); S3AFileSystem fs = getFileSystem(); fs.delete(destFile, false); - CommitOperations actions = newCommitOperations(); SinglePendingCommit commit = new SinglePendingCommit(); commit.setDestinationKey(fs.pathToKey(destFile)); + newCommitOperations().revertCommit(commit); + assertPathDoesNotExist("should have been reverted", destFile); + } - actions.revertCommit(commit); - - assertPathExists("parent of reverted (nonexistent) commit", - destFile.getParent()); + @Test + public void testFailuresInAbortListing() throws Throwable { + Path path = path("testFailuresInAbort"); + getFileSystem().mkdirs(path); + LOG.info("Aborting"); + newCommitOperations().abortPendingUploadsUnderPath(path); + LOG.info("Abort completed"); } /** @@ -578,16 +576,16 @@ public void testRevertMissingCommit() throws Throwable { @Test public void testWriteNormalStream() throws Throwable { S3AFileSystem fs = getFileSystem(); - assumeMagicCommitEnabled(fs); Path destFile = path("normal"); try (FSDataOutputStream out = fs.create(destFile, true)) { out.writeChars("data"); assertFalse("stream has magic output: " + out, out.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT)); - out.close(); } FileStatus status = fs.getFileStatus(destFile); - assertTrue("Empty marker file: " + status, status.getLen() > 0); + Assertions.assertThat(status.getLen()) + .describedAs("Normal file %s: %s", destFile, status) + .isGreaterThan(0); } /** @@ -598,7 +596,7 @@ public void testBulkCommitFiles() throws Throwable { describe("verify bulk commit"); File localFile = File.createTempFile("commit", ".txt"); CommitOperations actions = newCommitOperations(); - Path destDir = methodPath("out"); + Path destDir = methodSubPath("out"); S3AFileSystem fs = getFileSystem(); fs.delete(destDir, false); @@ -612,7 +610,7 @@ public void testBulkCommitFiles() throws Throwable { destFile3); List commits = new ArrayList<>(3); - for (Path destination : destinations) { + for (Path destination: destinations) { SinglePendingCommit commit1 = actions.uploadFileToPendingCommit(localFile, destination, null, @@ -624,8 +622,8 @@ public void testBulkCommitFiles() throws Throwable { assertPathDoesNotExist("destination dir", destDir); assertPathDoesNotExist("subdirectory", subdir); LOG.info("Initiating commit operations"); - try (CommitOperations.CommitContext commitContext - = actions.initiateCommitOperation(destDir)) { + try (CommitContext commitContext + = actions.createCommitContextForTesting(destDir, JOB_ID, 0)) { LOG.info("Commit #1"); commitContext.commitOrFail(commits.get(0)); final String firstCommitContextString = commitContext.toString(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java index a8547d6728..2ad2568d5c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestS3ACommitterFactory.java @@ -157,7 +157,7 @@ public void testBindingsInFSConfig() throws Throwable { } /** - * Create an invalid committer via the FS binding, + * Create an invalid committer via the FS binding. */ public void testInvalidFileBinding() throws Throwable { taskConfRef.unset(FS_S3A_COMMITTER_NAME); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java deleted file mode 100644 index 7ff5c3d280..0000000000 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java +++ /dev/null @@ -1,569 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.s3a.commit; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hadoop.test.HadoopTestBase; - -import static org.apache.hadoop.test.LambdaTestUtils.intercept; - -/** - * Test Tasks class. - */ -@RunWith(Parameterized.class) -public class TestTasks extends HadoopTestBase { - private static final Logger LOG = LoggerFactory.getLogger(TestTasks.class); - public static final int ITEM_COUNT = 16; - private static final int FAILPOINT = 8; - - private final int numThreads; - /** - * Thread pool for task execution. - */ - private ExecutorService threadPool; - - /** - * Task submitter bonded to the thread pool, or - * null for the 0-thread case. - */ - Tasks.Submitter submitter; - private final CounterTask failingTask - = new CounterTask("failing committer", FAILPOINT, Item::commit); - - private final FailureCounter failures - = new FailureCounter("failures", 0, null); - private final CounterTask reverter - = new CounterTask("reverter", 0, Item::revert); - private final CounterTask aborter - = new CounterTask("aborter", 0, Item::abort); - - /** - * Test array for parameterized test runs: how many threads and - * to use. Threading makes some of the assertions brittle; there are - * more checks on single thread than parallel ops. - * @return a list of parameter tuples. - */ - @Parameterized.Parameters(name = "threads={0}") - public static Collection params() { - return Arrays.asList(new Object[][]{ - {0}, - {1}, - {3}, - {8}, - {16}, - }); - } - - - private List items; - - - /** - * Construct the parameterized test. - * @param numThreads number of threads - */ - public TestTasks(int numThreads) { - this.numThreads = numThreads; - } - - /** - * In a parallel test run there is more than one thread doing the execution. - * @return true if the threadpool size is >1 - */ - public boolean isParallel() { - return numThreads > 1; - } - - @Before - public void setup() { - items = IntStream.rangeClosed(1, ITEM_COUNT) - .mapToObj(i -> new Item(i, - String.format("With %d threads", numThreads))) - .collect(Collectors.toList()); - - if (numThreads > 0) { - threadPool = Executors.newFixedThreadPool(numThreads, - new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat(getMethodName() + "-pool-%d") - .build()); - submitter = new PoolSubmitter(); - } else { - submitter = null; - } - - } - - @After - public void teardown() { - if (threadPool != null) { - threadPool.shutdown(); - threadPool = null; - } - } - - private class PoolSubmitter implements Tasks.Submitter { - - @Override - public Future submit(final Runnable task) { - return threadPool.submit(task); - } - - } - - /** - * create the builder. - * @return pre-inited builder - */ - private Tasks.Builder builder() { - return Tasks.foreach(items).executeWith(submitter); - } - - private void assertRun(Tasks.Builder builder, - CounterTask task) throws IOException { - boolean b = builder.run(task); - assertTrue("Run of " + task + " failed", b); - } - - private void assertFailed(Tasks.Builder builder, - CounterTask task) throws IOException { - boolean b = builder.run(task); - assertFalse("Run of " + task + " unexpectedly succeeded", b); - } - - private String itemsToString() { - return "[" + items.stream().map(Item::toString) - .collect(Collectors.joining("\n")) +"]"; - } - - @Test - public void testSimpleInvocation() throws Throwable { - CounterTask t = new CounterTask("simple", 0, Item::commit); - assertRun(builder(), t); - t.assertInvoked("", ITEM_COUNT); - } - - @Test - public void testFailNoStoppingSuppressed() throws Throwable { - assertFailed(builder().suppressExceptions(), failingTask); - failingTask.assertInvoked("Continued through operations", ITEM_COUNT); - items.forEach(Item::assertCommittedOrFailed); - } - - @Test - public void testFailFastSuppressed() throws Throwable { - assertFailed(builder() - .suppressExceptions() - .stopOnFailure(), - failingTask); - if (isParallel()) { - failingTask.assertInvokedAtLeast("stop fast", FAILPOINT); - } else { - failingTask.assertInvoked("stop fast", FAILPOINT); - } - } - - @Test - public void testFailedCallAbortSuppressed() throws Throwable { - assertFailed(builder() - .stopOnFailure() - .suppressExceptions() - .abortWith(aborter), - failingTask); - failingTask.assertInvokedAtLeast("success", FAILPOINT); - if (!isParallel()) { - aborter.assertInvokedAtLeast("abort", 1); - // all uncommitted items were aborted - items.stream().filter(i -> !i.committed) - .map(Item::assertAborted); - items.stream().filter(i -> i.committed) - .forEach(i -> assertFalse(i.toString(), i.aborted)); - } - } - - @Test - public void testFailedCalledWhenNotStoppingSuppressed() throws Throwable { - assertFailed(builder() - .suppressExceptions() - .onFailure(failures), - failingTask); - failingTask.assertInvokedAtLeast("success", FAILPOINT); - // only one failure was triggered - failures.assertInvoked("failure event", 1); - } - - @Test - public void testFailFastCallRevertSuppressed() throws Throwable { - assertFailed(builder() - .stopOnFailure() - .revertWith(reverter) - .abortWith(aborter) - .suppressExceptions() - .onFailure(failures), - failingTask); - failingTask.assertInvokedAtLeast("success", FAILPOINT); - if (!isParallel()) { - aborter.assertInvokedAtLeast("abort", 1); - // all uncommitted items were aborted - items.stream().filter(i -> !i.committed) - .filter(i -> !i.failed) - .forEach(Item::assertAborted); - } - // all committed were reverted - items.stream().filter(i -> i.committed && !i.failed) - .forEach(Item::assertReverted); - // all reverted items are committed - items.stream().filter(i -> i.reverted) - .forEach(Item::assertCommitted); - - // only one failure was triggered - failures.assertInvoked("failure event", 1); - } - - @Test - public void testFailSlowCallRevertSuppressed() throws Throwable { - assertFailed(builder() - .suppressExceptions() - .revertWith(reverter) - .onFailure(failures), - failingTask); - failingTask.assertInvokedAtLeast("success", FAILPOINT); - // all committed were reverted - // identify which task failed from the set - int failing = failures.getItem().id; - items.stream() - .filter(i -> i.id != failing) - .filter(i -> i.committed) - .forEach(Item::assertReverted); - // all reverted items are committed - items.stream().filter(i -> i.reverted) - .forEach(Item::assertCommitted); - - // only one failure was triggered - failures.assertInvoked("failure event", 1); - } - - @Test - public void testFailFastExceptions() throws Throwable { - intercept(IOException.class, - () -> builder() - .stopOnFailure() - .run(failingTask)); - if (isParallel()) { - failingTask.assertInvokedAtLeast("stop fast", FAILPOINT); - } else { - failingTask.assertInvoked("stop fast", FAILPOINT); - } - } - - @Test - public void testFailSlowExceptions() throws Throwable { - intercept(IOException.class, - () -> builder() - .run(failingTask)); - failingTask.assertInvoked("continued through operations", ITEM_COUNT); - items.forEach(Item::assertCommittedOrFailed); - } - - @Test - public void testFailFastExceptionsWithAbortFailure() throws Throwable { - CounterTask failFirst = new CounterTask("task", 1, Item::commit); - CounterTask a = new CounterTask("aborter", 1, Item::abort); - intercept(IOException.class, - () -> builder() - .stopOnFailure() - .abortWith(a) - .run(failFirst)); - if (!isParallel()) { - // expect the other tasks to be aborted - a.assertInvokedAtLeast("abort", ITEM_COUNT - 1); - } - } - - @Test - public void testFailFastExceptionsWithAbortFailureStopped() throws Throwable { - CounterTask failFirst = new CounterTask("task", 1, Item::commit); - CounterTask a = new CounterTask("aborter", 1, Item::abort); - intercept(IOException.class, - () -> builder() - .stopOnFailure() - .stopAbortsOnFailure() - .abortWith(a) - .run(failFirst)); - if (!isParallel()) { - // expect the other tasks to be aborted - a.assertInvoked("abort", 1); - } - } - - /** - * Fail the last one committed, all the rest will be reverted. - * The actual ID of the last task has to be picke dup from the - * failure callback, as in the pool it may be one of any. - */ - @Test - public void testRevertAllSuppressed() throws Throwable { - CounterTask failLast = new CounterTask("task", ITEM_COUNT, Item::commit); - - assertFailed(builder() - .suppressExceptions() - .stopOnFailure() - .revertWith(reverter) - .abortWith(aborter) - .onFailure(failures), - failLast); - failLast.assertInvoked("success", ITEM_COUNT); - int abCount = aborter.getCount(); - int revCount = reverter.getCount(); - assertEquals(ITEM_COUNT, 1 + abCount + revCount); - // identify which task failed from the set - int failing = failures.getItem().id; - // all committed were reverted - items.stream() - .filter(i -> i.id != failing) - .filter(i -> i.committed) - .forEach(Item::assertReverted); - items.stream() - .filter(i -> i.id != failing) - .filter(i -> !i.committed) - .forEach(Item::assertAborted); - // all reverted items are committed - items.stream().filter(i -> i.reverted) - .forEach(Item::assertCommitted); - - // only one failure was triggered - failures.assertInvoked("failure event", 1); - } - - - /** - * The Item which tasks process. - */ - private final class Item { - private final int id; - private final String text; - - private volatile boolean committed, aborted, reverted, failed; - - private Item(int item, String text) { - this.id = item; - this.text = text; - } - - boolean commit() { - committed = true; - return true; - } - - boolean abort() { - aborted = true; - return true; - } - - boolean revert() { - reverted = true; - return true; - } - - boolean fail() { - failed = true; - return true; - } - - public Item assertCommitted() { - assertTrue(toString() + " was not committed in\n" - + itemsToString(), - committed); - return this; - } - - public Item assertCommittedOrFailed() { - assertTrue(toString() + " was not committed nor failed in\n" - + itemsToString(), - committed || failed); - return this; - } - - public Item assertAborted() { - assertTrue(toString() + " was not aborted in\n" - + itemsToString(), - aborted); - return this; - } - - public Item assertReverted() { - assertTrue(toString() + " was not reverted in\n" - + itemsToString(), - reverted); - return this; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("Item{"); - sb.append(String.format("[%02d]", id)); - sb.append(", committed=").append(committed); - sb.append(", aborted=").append(aborted); - sb.append(", reverted=").append(reverted); - sb.append(", failed=").append(failed); - sb.append(", text=").append(text); - sb.append('}'); - return sb.toString(); - } - } - - /** - * Class which can count invocations and, if limit > 0, will raise - * an exception on the specific invocation of {@link #note(Object)} - * whose count == limit. - */ - private class BaseCounter { - private final AtomicInteger counter = new AtomicInteger(0); - private final int limit; - private final String name; - private Item item; - private final Optional> action; - - /** - * Base counter, tracks items. - * @param name name for string/exception/logs. - * @param limit limit at which an exception is raised, 0 == never - * @param action optional action to invoke after the increment, - * before limit check - */ - BaseCounter(String name, - int limit, - Function action) { - this.name = name; - this.limit = limit; - this.action = Optional.ofNullable(action); - } - - /** - * Apply the action to an item; log at info afterwards with both the - * before and after string values of the item. - * @param i item to process. - * @throws IOException failure in the action - */ - void process(Item i) throws IOException { - this.item = i; - int count = counter.incrementAndGet(); - if (limit == count) { - i.fail(); - LOG.info("{}: Failed {}", this, i); - throw new IOException(String.format("%s: Limit %d reached for %s", - this, limit, i)); - } - String before = i.toString(); - action.map(a -> a.apply(i)); - LOG.info("{}: {} -> {}", this, before, i); - } - - int getCount() { - return counter.get(); - } - - Item getItem() { - return item; - } - - void assertInvoked(String text, int expected) { - assertEquals(toString() + ": " + text, expected, getCount()); - } - - void assertInvokedAtLeast(String text, int expected) { - int actual = getCount(); - assertTrue(toString() + ": " + text - + "-expected " + expected - + " invocations, but got " + actual - + " in " + itemsToString(), - expected <= actual); - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder( - "BaseCounter{"); - sb.append("name='").append(name).append('\''); - sb.append(", count=").append(counter.get()); - sb.append(", limit=").append(limit); - sb.append(", item=").append(item); - sb.append('}'); - return sb.toString(); - } - } - - private final class CounterTask - extends BaseCounter implements Tasks.Task { - - private CounterTask(String name, int limit, - Function action) { - super(name, limit, action); - } - - @Override - public void run(Item item) throws IOException { - process(item); - } - - } - - private final class FailureCounter - extends BaseCounter implements Tasks.FailureTask { - private Exception exception; - - private FailureCounter(String name, int limit, - Function action) { - super(name, limit, action); - } - - @Override - public void run(Item item, Exception ex) throws IOException { - process(item); - this.exception = ex; - } - - private Exception getException() { - return exception; - } - - } - -} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java index 5265163d83..32cab03770 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java @@ -25,8 +25,6 @@ import org.assertj.core.api.Assertions; import org.junit.Test; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; @@ -34,7 +32,6 @@ import org.apache.hadoop.fs.s3a.commit.AbstractITCommitProtocol; import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; import org.apache.hadoop.fs.s3a.commit.CommitUtils; import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection; import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl; @@ -45,7 +42,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.listAndFilter; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; -import static org.hamcrest.CoreMatchers.containsString; +import static org.apache.hadoop.util.functional.RemoteIterators.toList; /** * Test the magic committer's commit protocol. @@ -99,8 +96,9 @@ public MagicS3GuardCommitter createFailingCommitter( protected void validateTaskAttemptPathDuringWrite(Path p, final long expectedLength) throws IOException { String pathStr = p.toString(); - assertTrue("not magic " + pathStr, - pathStr.contains(MAGIC)); + Assertions.assertThat(pathStr) + .describedAs("Magic path") + .contains(MAGIC); assertPathDoesNotExist("task attempt visible", p); } @@ -116,9 +114,9 @@ protected void validateTaskAttemptPathAfterWrite(Path marker, // if you list the parent dir and find the marker, it // is really 0 bytes long String name = marker.getName(); - List filtered = listAndFilter(fs, + List filtered = toList(listAndFilter(fs, marker.getParent(), false, - (path) -> path.getName().equals(name)); + (path) -> path.getName().equals(name))); Assertions.assertThat(filtered) .hasSize(1); Assertions.assertThat(filtered.get(0)) @@ -126,14 +124,7 @@ protected void validateTaskAttemptPathAfterWrite(Path marker, "Listing should return 0 byte length"); // marker file is empty - FileStatus st = fs.getFileStatus(marker); - assertEquals("file length in " + st, 0, st.getLen()); - // xattr header - Assertions.assertThat(CommitOperations.extractMagicFileLength(fs, - marker)) - .describedAs("XAttribute " + XA_MAGIC_MARKER) - .isNotEmpty() - .hasValue(expectedLength); + getTestHelper().assertIsMarkerFile(marker, expectedLength); } /** @@ -151,8 +142,8 @@ protected void validateTaskAttemptWorkingDirectory( assertEquals("Wrong schema for working dir " + wd + " with committer " + committer, "s3a", wd.getScheme()); - assertThat(wd.getPath(), - containsString('/' + CommitConstants.MAGIC + '/')); + Assertions.assertThat(wd.getPath()) + .contains('/' + CommitConstants.MAGIC + '/'); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java index 3c15454e7e..aeea195aac 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java @@ -20,8 +20,8 @@ import java.io.IOException; import java.util.List; -import java.util.stream.Collectors; +import org.assertj.core.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,10 +33,11 @@ import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.CommitConstants; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; import org.apache.hadoop.fs.s3a.commit.CommitUtils; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles; import static org.apache.hadoop.fs.s3a.MultipartTestUtils.listMultipartUploads; @@ -54,6 +55,7 @@ public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles { private static final Logger LOG = LoggerFactory.getLogger( ITestS3AHugeMagicCommits.class); + private static final int COMMITTER_THREADS = 64; private Path magicDir; private Path jobDir; @@ -123,16 +125,16 @@ public void test_030_postCreationAssertions() throws Throwable { Path destDir = getHugefile().getParent(); assertPathExists("Magic dir", new Path(destDir, CommitConstants.MAGIC)); String destDirKey = fs.pathToKey(destDir); - List uploads = listMultipartUploads(fs, destDirKey); - assertEquals("Pending uploads: " - + uploads.stream() - .collect(Collectors.joining("\n")), 1, uploads.size()); + Assertions.assertThat(listMultipartUploads(fs, destDirKey)) + .describedAs("Pending uploads") + .hasSize(1); assertNotNull("jobDir", jobDir); - Pair>> - results = operations.loadSinglePendingCommits(jobDir, false); - try(CommitOperations.CommitContext commitContext - = operations.initiateCommitOperation(jobDir)) { + try(CommitContext commitContext + = operations.createCommitContextForTesting(jobDir, null, COMMITTER_THREADS)) { + Pair>> + results = operations.loadSinglePendingCommits(jobDir, false, commitContext + ); for (SinglePendingCommit singlePendingCommit : results.getKey().getCommits()) { commitContext.commitOrFail(singlePendingCommit); @@ -140,10 +142,9 @@ public void test_030_postCreationAssertions() throws Throwable { } timer.end("time to commit %s", pendingDataFile); // upload is no longer pending - uploads = listMultipartUploads(fs, destDirKey); - assertEquals("Pending uploads" - + uploads.stream().collect(Collectors.joining("\n")), - 0, operations.listPendingUploadsUnderPath(destDir).size()); + Assertions.assertThat(operations.listPendingUploadsUnderPath(destDir)) + .describedAs("Pending uploads undedr path") + .isEmpty(); // at this point, the huge file exists, so the normal assertions // on that file must be valid. Verify. super.test_030_postCreationAssertions(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java index d3da8185c8..beccf4f328 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java @@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.MockS3AFileSystem; +import org.apache.hadoop.fs.s3a.commit.files.SuccessData; import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientErrors; import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientResults; import org.apache.hadoop.fs.statistics.IOStatisticsSnapshot; @@ -76,11 +77,12 @@ public void commitJob(JobContext context) throws IOException { } @Override - protected void maybeCreateSuccessMarker(JobContext context, + protected SuccessData maybeCreateSuccessMarker(JobContext context, List filenames, final IOStatisticsSnapshot ioStatistics) throws IOException { //skipped + return null; } public ClientResults getResults() throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java index 6e13fd0227..6f29537624 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java @@ -105,10 +105,15 @@ public class StagingTestBase { /** The raw bucket URI Path before any canonicalization. */ public static final URI RAW_BUCKET_URI = RAW_BUCKET_PATH.toUri(); - public static Path outputPath = + + @SuppressWarnings("StaticNonFinalField") + private static Path outputPath = new Path("s3a://" + BUCKET + "/" + OUTPUT_PREFIX); - public static URI outputPathUri = outputPath.toUri(); - public static Path root; + + @SuppressWarnings("StaticNonFinalField") + private static URI outputPathUri = getOutputPath().toUri(); + @SuppressWarnings("StaticNonFinalField") + private static Path root; protected StagingTestBase() { } @@ -131,8 +136,8 @@ protected static S3AFileSystem createAndBindMockFSInstance(Configuration conf, URI uri = RAW_BUCKET_URI; wrapperFS.initialize(uri, conf); root = wrapperFS.makeQualified(new Path("/")); - outputPath = new Path(root, OUTPUT_PREFIX); - outputPathUri = outputPath.toUri(); + outputPath = new Path(getRoot(), OUTPUT_PREFIX); + outputPathUri = getOutputPath().toUri(); FileSystemTestHelper.addFileSystemForTesting(uri, conf, wrapperFS); return mockFs; } @@ -154,7 +159,7 @@ private static S3AFileSystem mockS3AFileSystemRobustly() { */ public static MockS3AFileSystem lookupWrapperFS(Configuration conf) throws IOException { - return (MockS3AFileSystem) FileSystem.get(outputPathUri, conf); + return (MockS3AFileSystem) FileSystem.get(getOutputPathUri(), conf); } public static void verifyCompletion(FileSystem mockS3) throws IOException { @@ -169,13 +174,13 @@ public static void verifyDeleted(FileSystem mockS3, Path path) public static void verifyDeleted(FileSystem mockS3, String child) throws IOException { - verifyDeleted(mockS3, new Path(outputPath, child)); + verifyDeleted(mockS3, new Path(getOutputPath(), child)); } public static void verifyCleanupTempFiles(FileSystem mockS3) throws IOException { verifyDeleted(mockS3, - new Path(outputPath, CommitConstants.TEMPORARY)); + new Path(getOutputPath(), CommitConstants.TEMPORARY)); } protected static void assertConflictResolution( @@ -189,7 +194,7 @@ protected static void assertConflictResolution( public static void pathsExist(FileSystem mockS3, String... children) throws IOException { for (String child : children) { - pathExists(mockS3, new Path(outputPath, child)); + pathExists(mockS3, new Path(getOutputPath(), child)); } } @@ -231,7 +236,7 @@ public static void mkdirsHasOutcome(FileSystem mockS3, public static void canDelete(FileSystem mockS3, String... children) throws IOException { for (String child : children) { - canDelete(mockS3, new Path(outputPath, child)); + canDelete(mockS3, new Path(getOutputPath(), child)); } } @@ -243,7 +248,7 @@ public static void canDelete(FileSystem mockS3, Path f) throws IOException { public static void verifyExistenceChecked(FileSystem mockS3, String child) throws IOException { - verifyExistenceChecked(mockS3, new Path(outputPath, child)); + verifyExistenceChecked(mockS3, new Path(getOutputPath(), child)); } public static void verifyExistenceChecked(FileSystem mockS3, Path path) @@ -262,6 +267,18 @@ public static void verifyMkdirsInvoked(FileSystem mockS3, Path path) verify(mockS3).mkdirs(path); } + protected static URI getOutputPathUri() { + return outputPathUri; + } + + static Path getRoot() { + return root; + } + + static Path getOutputPath() { + return outputPath; + } + /** * Provides setup/teardown of a MiniDFSCluster for tests that need one. */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java index cb7202b10d..4d24c07dac 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestDirectoryCommitterScale.java @@ -28,6 +28,7 @@ import java.util.stream.IntStream; import com.amazonaws.services.s3.model.PartETag; + import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; import org.assertj.core.api.Assertions; import org.junit.AfterClass; @@ -48,18 +49,21 @@ import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.util.DurationInfo; +import org.apache.hadoop.util.JsonSerialization; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.CONFLICT_MODE_APPEND; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.FS_S3A_COMMITTER_STAGING_CONFLICT_MODE; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.PENDINGSET_SUFFIX; import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.BUCKET; -import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.outputPath; -import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.outputPathUri; +import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.getOutputPath; +import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.getOutputPathUri; import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.pathIsDirectory; /** @@ -83,6 +87,7 @@ public class TestDirectoryCommitterScale public static final int TOTAL_COMMIT_COUNT = FILES_PER_TASK * TASKS; public static final int BLOCKS_PER_TASK = 1000; + private static final int COMMITTER_THREAD_COUNT = 100; private static File stagingDir; @@ -95,13 +100,13 @@ public class TestDirectoryCommitterScale @Override DirectoryCommitterForTesting newJobCommitter() throws Exception { - return new DirectoryCommitterForTesting(outputPath, + return new DirectoryCommitterForTesting(getOutputPath(), createTaskAttemptForJob()); } @BeforeClass public static void setupStaging() throws Exception { - stagingDir = File.createTempFile("staging", ""); + stagingDir = File.createTempFile("staging", null); stagingDir.delete(); stagingDir.mkdir(); stagingPath = new Path(stagingDir.toURI()); @@ -125,7 +130,7 @@ protected JobConf createJobConf() { JobConf conf = super.createJobConf(); conf.setInt( CommitConstants.FS_S3A_COMMITTER_THREADS, - 100); + COMMITTER_THREAD_COUNT); return conf; } @@ -149,7 +154,8 @@ public void test_010_createTaskFiles() throws Exception { */ private void createTasks() throws IOException { // create a stub multipart commit containing multiple files. - + JsonSerialization serializer = + SinglePendingCommit.serializer(); // step1: a list of tags. // this is the md5sum of hadoop 3.2.1.tar String tag = "9062dcf18ffaee254821303bbd11c72b"; @@ -164,12 +170,14 @@ private void createTasks() throws IOException { // these get overwritten base.setDestinationKey("/base"); base.setUploadId("uploadId"); - base.setUri(outputPathUri.toString()); + base.setUri(getOutputPathUri().toString()); + byte[] bytes = base.toBytes(serializer); SinglePendingCommit[] singles = new SinglePendingCommit[FILES_PER_TASK]; - byte[] bytes = base.toBytes(); + for (int i = 0; i < FILES_PER_TASK; i++) { - singles[i] = SinglePendingCommit.serializer().fromBytes(bytes); + + singles[i] = serializer.fromBytes(bytes); } // now create the files, using this as the template @@ -182,7 +190,7 @@ private void createTasks() throws IOException { String uploadId = String.format("%05d-task-%04d-file-%02d", uploadCount, task, i); // longer paths to take up more space. - Path p = new Path(outputPath, + Path p = new Path(getOutputPath(), "datasets/examples/testdirectoryscale/" + "year=2019/month=09/day=26/hour=20/second=53" + uploadId); @@ -199,7 +207,7 @@ private void createTasks() throws IOException { } Path path = new Path(stagingPath, String.format("task-%04d." + PENDINGSET_SUFFIX, task)); - pending.save(localFS, path, true); + pending.save(localFS, path, PendingSet.serializer()); } } @@ -211,12 +219,14 @@ public void test_020_loadFilesToAttempt() throws Exception { Configuration jobConf = getJobConf(); jobConf.set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND); - FileSystem mockS3 = getMockS3A(); - pathIsDirectory(mockS3, outputPath); - try (DurationInfo ignored = - new DurationInfo(LOG, "listing pending uploads")) { + S3AFileSystem mockS3 = getMockS3A(); + pathIsDirectory(mockS3, getOutputPath()); + final CommitOperations operations = new CommitOperations(getWrapperFS()); + try (CommitContext commitContext + = operations.createCommitContextForTesting(getOutputPath(), + null, COMMITTER_THREAD_COUNT)) { AbstractS3ACommitter.ActiveCommit activeCommit - = committer.listPendingUploadsToCommit(getJob()); + = committer.listPendingUploadsToCommit(commitContext); Assertions.assertThat(activeCommit.getSourceFiles()) .describedAs("Source files of %s", activeCommit) .hasSize(TASKS); @@ -232,7 +242,7 @@ public void test_030_commitFiles() throws Exception { jobConf.set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND); S3AFileSystem mockS3 = getMockS3A(); - pathIsDirectory(mockS3, outputPath); + pathIsDirectory(mockS3, getOutputPath()); try (DurationInfo ignored = new DurationInfo(LOG, "Committing Job")) { @@ -261,7 +271,7 @@ public void test_040_abortFiles() throws Exception { jobConf.set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND); FileSystem mockS3 = getMockS3A(); - pathIsDirectory(mockS3, outputPath); + pathIsDirectory(mockS3, getOutputPath()); committer.abortJob(getJob(), JobStatus.State.FAILED); } @@ -304,11 +314,11 @@ public Path getJobAttemptPath(JobContext context) { } @Override - protected void commitJobInternal(final JobContext context, + protected void commitJobInternal(final CommitContext commitContext, final ActiveCommit pending) throws IOException { activeCommit = pending; - super.commitJobInternal(context, pending); + super.commitJobInternal(commitContext, pending); } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java index 94c0b29a3c..11edf0d216 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java @@ -37,7 +37,6 @@ import org.apache.hadoop.util.Sets; import org.assertj.core.api.Assertions; -import org.hamcrest.core.StringStartsWith; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -58,6 +57,7 @@ import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; import org.apache.hadoop.fs.s3a.commit.PathCommitException; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; +import org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobContext; @@ -168,7 +168,7 @@ public void setupCommitter() throws Exception { this.tac = new TaskAttemptContextImpl( new Configuration(job.getConfiguration()), AID); - this.jobCommitter = new MockedStagingCommitter(outputPath, tac); + this.jobCommitter = new MockedStagingCommitter(getOutputPath(), tac); jobCommitter.setupJob(job); // get the task's configuration copy so modifications take effect @@ -183,7 +183,7 @@ public void setupCommitter() throws Exception { this.conf.set(BUFFER_DIR, String.format("%s/local-0/, %s/local-1 ", tmp, tmp)); - this.committer = new MockedStagingCommitter(outputPath, tac); + this.committer = new MockedStagingCommitter(getOutputPath(), tac); Paths.resetTempFolderCache(); } @@ -335,10 +335,11 @@ public void testAttemptPathConstructionWithSchema() throws Exception { config.set(BUFFER_DIR, "file:/tmp/mr-local-0,file:/tmp/mr-local-1"); - assertThat("Path should be the same with file scheme", + Assertions.assertThat( getLocalTaskAttemptTempDir(config, - jobUUID, tac.getTaskAttemptID()).toString(), - StringStartsWith.startsWith(commonPath)); + jobUUID, tac.getTaskAttemptID()).toString()) + .describedAs("Path should be the same with file scheme") + .startsWith(commonPath); } @Test @@ -379,7 +380,7 @@ public void testSingleTaskCommit() throws Exception { assertEquals("Should name the commits file with the task ID: " + results, "task_job_0001_r_000002", stats[0].getPath().getName()); - PendingSet pending = PendingSet.load(dfs, stats[0]); + PendingSet pending = PersistentCommitData.load(dfs, stats[0], PendingSet.serializer()); assertEquals("Should have one pending commit", 1, pending.size()); SinglePendingCommit commit = pending.getCommits().get(0); assertEquals("Should write to the correct bucket:" + results, @@ -419,7 +420,7 @@ public void testSingleTaskEmptyFileCommit() throws Exception { assertEquals("Should name the commits file with the task ID", "task_job_0001_r_000002", stats[0].getPath().getName()); - PendingSet pending = PendingSet.load(dfs, stats[0]); + PendingSet pending = PersistentCommitData.load(dfs, stats[0], PendingSet.serializer()); assertEquals("Should have one pending commit", 1, pending.size()); } @@ -442,7 +443,7 @@ public void testSingleTaskMultiFileCommit() throws Exception { "task_job_0001_r_000002", stats[0].getPath().getName()); List pending = - PendingSet.load(dfs, stats[0]).getCommits(); + PersistentCommitData.load(dfs, stats[0], PendingSet.serializer()).getCommits(); assertEquals("Should have correct number of pending commits", files.size(), pending.size()); @@ -717,7 +718,7 @@ private Set runTasks(JobContext jobContext, TaskAttemptContext attempt = new TaskAttemptContextImpl( new Configuration(jobContext.getConfiguration()), attemptID); MockedStagingCommitter taskCommitter = new MockedStagingCommitter( - outputPath, attempt); + getOutputPath(), attempt); commitTask(taskCommitter, attempt, numFiles); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java index 98075b827a..b92605ca25 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java @@ -29,8 +29,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.PathExistsException; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter; import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*; import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*; @@ -46,7 +49,7 @@ public class TestStagingDirectoryOutputCommitter @Override DirectoryStagingCommitter newJobCommitter() throws Exception { - return new DirectoryStagingCommitter(outputPath, + return new DirectoryStagingCommitter(getOutputPath(), createTaskAttemptForJob()); } @@ -63,7 +66,7 @@ public void testBadConflictMode() throws Throwable { public void testDefaultConflictResolution() throws Exception { getJob().getConfiguration().unset( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE); - pathIsDirectory(getMockS3A(), outputPath); + pathIsDirectory(getMockS3A(), getOutputPath()); verifyJobSetupAndCommit(); } @@ -75,7 +78,8 @@ public void testFailConflictResolution() throws Exception { } protected void verifyFailureConflictOutcome() throws Exception { - pathIsDirectory(getMockS3A(), outputPath); + final S3AFileSystem mockFS = getMockS3A(); + pathIsDirectory(mockFS, getOutputPath()); final DirectoryStagingCommitter committer = newJobCommitter(); // this should fail @@ -86,20 +90,23 @@ protected void verifyFailureConflictOutcome() throws Exception { // but there are no checks in job commit (HADOOP-15469) // this is done by calling the preCommit method directly, - committer.preCommitJob(getJob(), AbstractS3ACommitter.ActiveCommit.empty()); - reset(getMockS3A()); - pathDoesNotExist(getMockS3A(), outputPath); + final CommitContext commitContext = new CommitOperations(getWrapperFS()). + createCommitContext(getJob(), getOutputPath(), 0); + committer.preCommitJob(commitContext, AbstractS3ACommitter.ActiveCommit.empty()); + + reset(mockFS); + pathDoesNotExist(mockFS, getOutputPath()); committer.setupJob(getJob()); - verifyExistenceChecked(getMockS3A(), outputPath); - verifyMkdirsInvoked(getMockS3A(), outputPath); - verifyNoMoreInteractions(getMockS3A()); + verifyExistenceChecked(mockFS, getOutputPath()); + verifyMkdirsInvoked(mockFS, getOutputPath()); + verifyNoMoreInteractions(mockFS); - reset(getMockS3A()); - pathDoesNotExist(getMockS3A(), outputPath); + reset(mockFS); + pathDoesNotExist(mockFS, getOutputPath()); committer.commitJob(getJob()); - verifyCompletion(getMockS3A()); + verifyCompletion(mockFS); } @Test @@ -108,7 +115,7 @@ public void testAppendConflictResolution() throws Exception { getJob().getConfiguration().set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND); FileSystem mockS3 = getMockS3A(); - pathIsDirectory(mockS3, outputPath); + pathIsDirectory(mockS3, getOutputPath()); verifyJobSetupAndCommit(); } @@ -120,7 +127,7 @@ protected void verifyJobSetupAndCommit() FileSystem mockS3 = getMockS3A(); Mockito.reset(mockS3); - pathExists(mockS3, outputPath); + pathExists(mockS3, getOutputPath()); committer.commitJob(getJob()); verifyCompletion(mockS3); @@ -130,7 +137,7 @@ protected void verifyJobSetupAndCommit() public void testReplaceConflictResolution() throws Exception { FileSystem mockS3 = getMockS3A(); - pathIsDirectory(mockS3, outputPath); + pathIsDirectory(mockS3, getOutputPath()); getJob().getConfiguration().set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE); @@ -140,17 +147,17 @@ public void testReplaceConflictResolution() throws Exception { committer.setupJob(getJob()); Mockito.reset(mockS3); - pathExists(mockS3, outputPath); - canDelete(mockS3, outputPath); + pathExists(mockS3, getOutputPath()); + canDelete(mockS3, getOutputPath()); committer.commitJob(getJob()); - verifyDeleted(mockS3, outputPath); + verifyDeleted(mockS3, getOutputPath()); verifyCompletion(mockS3); } @Test public void testReplaceConflictFailsIfDestIsFile() throws Exception { - pathIsFile(getMockS3A(), outputPath); + pathIsFile(getMockS3A(), getOutputPath()); getJob().getConfiguration().set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE); @@ -166,7 +173,7 @@ public void testReplaceConflictFailsIfDestIsFile() throws Exception { @Test public void testAppendConflictFailsIfDestIsFile() throws Exception { - pathIsFile(getMockS3A(), outputPath); + pathIsFile(getMockS3A(), getOutputPath()); getJob().getConfiguration().set( FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java index 76a0de2253..64a9be0888 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java @@ -51,13 +51,13 @@ public class TestStagingPartitionedFileListing @Override PartitionedStagingCommitter newJobCommitter() throws IOException { - return new PartitionedStagingCommitter(outputPath, + return new PartitionedStagingCommitter(getOutputPath(), createTaskAttemptForJob()); } @Override PartitionedStagingCommitter newTaskCommitter() throws IOException { - return new PartitionedStagingCommitter(outputPath, getTAC()); + return new PartitionedStagingCommitter(getOutputPath(), getTAC()); } private FileSystem attemptFS; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java index 86b677c70a..28161979f0 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.s3a.commit.PathCommitException; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; -import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.fs.s3a.commit.impl.CommitContext; import org.apache.hadoop.mapreduce.TaskAttemptContext; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -67,7 +67,7 @@ private final class PartitionedStagingCommitterForTesting private PartitionedStagingCommitterForTesting(TaskAttemptContext context) throws IOException { - super(StagingTestBase.outputPath, context); + super(StagingTestBase.getOutputPath(), context); } /** @@ -75,14 +75,15 @@ private PartitionedStagingCommitterForTesting(TaskAttemptContext context) * This is quite complex as the mock pending uploads need to be saved * to a filesystem for the next stage of the commit process. * To simulate multiple commit, more than one .pendingset file is created, - * @param context job context + * @param commitContext job context * @return an active commit containing a list of paths to valid pending set * file. * @throws IOException IO failure */ + @SuppressWarnings("CollectionDeclaredAsConcreteClass") @Override protected ActiveCommit listPendingUploadsToCommit( - JobContext context) throws IOException { + CommitContext commitContext) throws IOException { LocalFileSystem localFS = FileSystem.getLocal(getConf()); ActiveCommit activeCommit = new ActiveCommit(localFS, @@ -109,17 +110,17 @@ protected ActiveCommit listPendingUploadsToCommit( File file = File.createTempFile("staging", ".pendingset"); file.deleteOnExit(); Path path = new Path(file.toURI()); - pendingSet.save(localFS, path, true); + pendingSet.save(localFS, path, PendingSet.serializer()); activeCommit.add(localFS.getFileStatus(path)); } return activeCommit; } @Override - protected void abortJobInternal(JobContext context, + protected void abortJobInternal(CommitContext commitContext, boolean suppressExceptions) throws IOException { this.aborted = true; - super.abortJobInternal(context, suppressExceptions); + super.abortJobInternal(commitContext, suppressExceptions); } } @@ -242,7 +243,7 @@ public void testReplaceWithDeleteFailure() throws Exception { pathsExist(mockS3, "dateint=20161116/hour=14"); when(mockS3 .delete( - new Path(outputPath, "dateint=20161116/hour=14"), + new Path(getOutputPath(), "dateint=20161116/hour=14"), true)) .thenThrow(new PathCommitException("fake", "Fake IOException for delete")); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java index fb25210249..4e82b94314 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java @@ -47,13 +47,13 @@ public class TestStagingPartitionedTaskCommit @Override PartitionedStagingCommitter newJobCommitter() throws IOException { - return new PartitionedStagingCommitter(outputPath, + return new PartitionedStagingCommitter(getOutputPath(), createTaskAttemptForJob()); } @Override PartitionedStagingCommitter newTaskCommitter() throws Exception { - return new PartitionedStagingCommitter(outputPath, getTAC()); + return new PartitionedStagingCommitter(getOutputPath(), getTAC()); } // The set of files used by this test @@ -104,7 +104,7 @@ public void testFail() throws Exception { // test failure when one partition already exists reset(mockS3); - Path existsPath = new Path(outputPath, relativeFiles.get(1)).getParent(); + Path existsPath = new Path(getOutputPath(), relativeFiles.get(1)).getParent(); pathExists(mockS3, existsPath); intercept(PathExistsException.class, "", @@ -133,7 +133,7 @@ public void testAppend() throws Exception { // test success when one partition already exists reset(mockS3); - pathExists(mockS3, new Path(outputPath, relativeFiles.get(2)).getParent()); + pathExists(mockS3, new Path(getOutputPath(), relativeFiles.get(2)).getParent()); committer.commitTask(getTAC()); verifyFilesCreated(committer); @@ -178,7 +178,7 @@ public void testReplace() throws Exception { // test success when one partition already exists reset(mockS3); - pathExists(mockS3, new Path(outputPath, relativeFiles.get(3)).getParent()); + pathExists(mockS3, new Path(getOutputPath(), relativeFiles.get(3)).getParent()); committer.commitTask(getTAC()); verifyFilesCreated(committer); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java index 3a820bcc11..bb3031b32c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java @@ -52,7 +52,7 @@ protected String suitename() { @Override protected Configuration createConfiguration() { Configuration conf = super.createConfiguration(); - conf.setInt(FS_S3A_COMMITTER_THREADS, 1); + conf.setInt(FS_S3A_COMMITTER_THREADS, 4); // disable unique filenames so that the protocol tests of FileOutputFormat // and this test generate consistent names. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java index 991969b0f0..d28ee5172b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/terasort/ITestTerasortOnS3A.java @@ -330,7 +330,7 @@ public void test_140_teracomplete() throws Throwable { stage.accept("teravalidate"); stage.accept("overall"); String text = results.toString(); - File resultsFile = File.createTempFile("results", ".csv"); + File resultsFile = new File(getReportDir(), committerName + ".csv"); FileUtils.write(resultsFile, text, StandardCharsets.UTF_8); LOG.info("Results are in {}\n{}", resultsFile, text); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java new file mode 100644 index 0000000000..41180667e1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/StubContextAccessor.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.File; +import java.io.IOException; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.MockS3AFileSystem; +import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.audit.AuditTestSupport; +import org.apache.hadoop.fs.store.audit.AuditSpan; + +/** + * A Stub context acccessor for test. + */ +public final class StubContextAccessor + implements ContextAccessors { + + private final String bucket; + + /** + * Construct. + * @param bucket bucket to use when qualifying keys.]= + */ + public StubContextAccessor(String bucket) { + this.bucket = bucket; + } + + @Override + public Path keyToPath(final String key) { + return new Path("s3a://" + bucket + "/" + key); + } + + @Override + public String pathToKey(final Path path) { + return null; + } + + @Override + public File createTempFile(final String prefix, final long size) + throws IOException { + throw new UnsupportedOperationException("unsppported"); + } + + @Override + public String getBucketLocation() throws IOException { + return null; + } + + @Override + public Path makeQualified(final Path path) { + return path; + } + + @Override + public AuditSpan getActiveAuditSpan() { + return AuditTestSupport.NOOP_SPAN; + } + + @Override + public RequestFactory getRequestFactory() { + return MockS3AFileSystem.REQUEST_FACTORY; + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java new file mode 100644 index 0000000000..65d7aa6192 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestCreateFileBuilder.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.test.HadoopTestBase; +import org.apache.hadoop.util.Progressable; + +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; + +/** + * Unit test of {@link CreateFileBuilder}. + */ +public class TestCreateFileBuilder extends HadoopTestBase { + + private static final BuilderCallbacks CALLBACKS = new BuilderCallbacks(); + + private CreateFileBuilder mkBuilder() throws IOException { + return new CreateFileBuilder( + FileSystem.getLocal(new Configuration()), + new Path("/"), + CALLBACKS); + } + + private BuilderOutputStream unwrap(FSDataOutputStream out) { + OutputStream s = out.getWrappedStream(); + Assertions.assertThat(s) + .isInstanceOf(BuilderOutputStream.class); + return (BuilderOutputStream) s; + } + + private BuilderOutputStream build(FSDataOutputStreamBuilder builder) + throws IOException { + return unwrap(builder.build()); + } + + @Test + public void testSimpleBuild() throws Throwable { + Assertions.assertThat(build(mkBuilder().create())) + .matches(p -> !p.isOverwrite()) + .matches(p -> !p.isPerformance()); + } + + @Test + public void testAppendForbidden() throws Throwable { + intercept(UnsupportedOperationException.class, () -> + build(mkBuilder().append())); + } + + @Test + public void testPerformanceSupport() throws Throwable { + CreateFileBuilder builder = mkBuilder().create(); + builder.must(FS_S3A_CREATE_PERFORMANCE, true); + Assertions.assertThat(build(builder)) + .matches(p -> p.isPerformance()); + } + + @Test + public void testHeaderOptions() throws Throwable { + final CreateFileBuilder builder = mkBuilder().create() + .must(FS_S3A_CREATE_HEADER + ".retention", "permanent") + .opt(FS_S3A_CREATE_HEADER + ".owner", "engineering"); + final Map headers = build(builder).getHeaders(); + Assertions.assertThat(headers) + .containsEntry("retention", "permanent") + .containsEntry("owner", "engineering"); + } + + @Test + public void testIncompleteHeader() throws Throwable { + final CreateFileBuilder builder = mkBuilder().create() + .must(FS_S3A_CREATE_HEADER, "permanent"); + intercept(IllegalArgumentException.class, () -> + build(builder)); + } + + private static final class BuilderCallbacks implements + CreateFileBuilder.CreateFileBuilderCallbacks { + + @Override + public FSDataOutputStream createFileFromBuilder(final Path path, + final Progressable progress, + final CreateFileBuilder.CreateFileOptions options) throws IOException { + return new FSDataOutputStream( + new BuilderOutputStream( + progress, + options), + null); + } + } + + /** + * Stream which will be wrapped and which returns the flags used + * creating the object. + */ + private static final class BuilderOutputStream extends OutputStream { + + private final Progressable progress; + + + private final CreateFileBuilder.CreateFileOptions options; + + private BuilderOutputStream(final Progressable progress, + final CreateFileBuilder.CreateFileOptions options) { + this.progress = progress; + this.options = options; + } + + private boolean isOverwrite() { + return options.getFlags().contains(CreateFlag.OVERWRITE); + } + + private Progressable getProgress() { + return progress; + } + + private boolean isPerformance() { + return options.isPerformance(); + } + + private CreateFileBuilder.CreateFileOptions getOptions() { + return options; + } + + private Map getHeaders() { + return options.getHeaders(); + } + + @Override + public void write(final int b) throws IOException { + + } + + @Override + public String toString() { + return "BuilderOutputStream{" + + "progress=" + progress + + ", options=" + options + + "} " + super.toString(); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java index 9bc3aef83a..5c243bb820 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java @@ -90,7 +90,7 @@ public void testRequestFactoryWithCannedACL() throws Throwable { ObjectMetadata md = factory.newObjectMetadata(128); Assertions.assertThat( factory.newPutObjectRequest(path, md, - new ByteArrayInputStream(new byte[0])) + null, new ByteArrayInputStream(new byte[0])) .getCannedAcl()) .describedAs("ACL of PUT") .isEqualTo(acl); @@ -98,7 +98,8 @@ public void testRequestFactoryWithCannedACL() throws Throwable { .getCannedAccessControlList()) .describedAs("ACL of COPY") .isEqualTo(acl); - Assertions.assertThat(factory.newMultipartUploadRequest(path) + Assertions.assertThat(factory.newMultipartUploadRequest(path, + null) .getCannedACL()) .describedAs("ACL of MPU") .isEqualTo(acl); @@ -172,12 +173,12 @@ private void createFactoryObjects(RequestFactory factory) { a(factory.newListObjectsV1Request(path, "/", 1)); a(factory.newListNextBatchOfObjectsRequest(new ObjectListing())); a(factory.newListObjectsV2Request(path, "/", 1)); - a(factory.newMultipartUploadRequest(path)); + a(factory.newMultipartUploadRequest(path, null)); File srcfile = new File("/tmp/a"); a(factory.newPutObjectRequest(path, - factory.newObjectMetadata(-1), srcfile)); + factory.newObjectMetadata(-1), null, srcfile)); ByteArrayInputStream stream = new ByteArrayInputStream(new byte[0]); - a(factory.newPutObjectRequest(path, md, stream)); + a(factory.newPutObjectRequest(path, md, null, stream)); a(factory.newSelectRequest(path)); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java index 3511020aa6..8bbf52b578 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/AbstractS3ACostTest.java @@ -132,16 +132,7 @@ public void setup() throws Exception { .isEqualTo(isKeepingMarkers() ? DirectoryPolicy.MarkerPolicy.Keep : DirectoryPolicy.MarkerPolicy.Delete); - // All counter statistics of the filesystem are added as metrics. - // Durations too, as they have counters of success and failure. - OperationCostValidator.Builder builder = OperationCostValidator.builder( - getFileSystem()); - EnumSet.allOf(Statistic.class).stream() - .filter(s -> - s.getType() == StatisticTypeEnum.TYPE_COUNTER - || s.getType() == StatisticTypeEnum.TYPE_DURATION) - .forEach(s -> builder.withMetric(s)); - costValidator = builder.build(); + setupCostValidator(); // determine bulk delete settings final Configuration fsConf = getFileSystem().getConf(); @@ -154,6 +145,19 @@ public void setup() throws Exception { setSpanSource(fs); } + protected void setupCostValidator() { + // All counter statistics of the filesystem are added as metrics. + // Durations too, as they have counters of success and failure. + OperationCostValidator.Builder builder = OperationCostValidator.builder( + getFileSystem()); + EnumSet.allOf(Statistic.class).stream() + .filter(s -> + s.getType() == StatisticTypeEnum.TYPE_COUNTER + || s.getType() == StatisticTypeEnum.TYPE_DURATION) + .forEach(s -> builder.withMetric(s)); + costValidator = builder.build(); + } + public boolean isDeleting() { return isDeleting; } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java new file mode 100644 index 0000000000..39530d97bf --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestCreateFileCost.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.performance; + +import java.io.IOException; + +import org.assertj.core.api.Assertions; +import org.junit.Test; + +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FSDataOutputStreamBuilder; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + + +import static java.util.Objects.requireNonNull; +import static org.apache.hadoop.fs.contract.ContractTestUtils.toChar; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_HEADER; +import static org.apache.hadoop.fs.s3a.Constants.FS_S3A_CREATE_PERFORMANCE; +import static org.apache.hadoop.fs.s3a.Constants.XA_HEADER_PREFIX; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_BULK_DELETE_REQUEST; +import static org.apache.hadoop.fs.s3a.Statistic.OBJECT_DELETE_REQUEST; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_NO_OVERWRITE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.CREATE_FILE_OVERWRITE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_DIR_PROBE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.FILE_STATUS_FILE_PROBE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_DIR_MARKER; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.GET_FILE_STATUS_ON_FILE; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.HEAD_OPERATION; +import static org.apache.hadoop.fs.s3a.performance.OperationCost.NO_HEAD_OR_LIST; + +/** + * Assert cost of createFile operations, especially + * with the FS_S3A_CREATE_PERFORMANCE option. + */ +@SuppressWarnings("resource") +public class ITestCreateFileCost extends AbstractS3ACostTest { + + /** + * Create with markers kept, always. + */ + public ITestCreateFileCost() { + super(false); + } + + @Test + public void testCreateNoOverwrite() throws Throwable { + describe("Test file creation without overwrite"); + Path testFile = methodPath(); + // when overwrite is false, the path is checked for existence. + create(testFile, false, + CREATE_FILE_NO_OVERWRITE); + } + + @Test + public void testCreateOverwrite() throws Throwable { + describe("Test file creation with overwrite"); + Path testFile = methodPath(); + // when overwrite is true: only the directory checks take place. + create(testFile, true, CREATE_FILE_OVERWRITE); + } + + @Test + public void testCreateNoOverwriteFileExists() throws Throwable { + describe("Test cost of create file failing with existing file"); + Path testFile = file(methodPath()); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptOperation(FileAlreadyExistsException.class, "", + FILE_STATUS_FILE_PROBE, + () -> file(testFile, false)); + } + + @Test + public void testCreateFileOverDir() throws Throwable { + describe("Test cost of create file failing with existing dir"); + Path testFile = dir(methodPath()); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptOperation(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_DIR_MARKER, + () -> file(testFile, false)); + } + + /** + * Use the builder API. + * on s3a this skips parent checks, always. + */ + @Test + public void testCreateBuilderSequence() throws Throwable { + describe("Test builder file creation cost"); + Path testFile = methodPath(); + dir(testFile.getParent()); + + // s3a fs skips the recursive checks to avoid race + // conditions with other processes/threads deleting + // files and so briefly the path not being present + // only make sure the dest path isn't a directory. + buildFile(testFile, true, false, + FILE_STATUS_DIR_PROBE); + + // now there is a file there, an attempt with overwrite == false will + // fail on the first HEAD. + interceptOperation(FileAlreadyExistsException.class, "", + GET_FILE_STATUS_ON_FILE, + () -> buildFile(testFile, false, true, + GET_FILE_STATUS_ON_FILE)); + } + + @Test + public void testCreateFilePerformanceFlag() throws Throwable { + describe("createFile with performance flag skips safety checks"); + S3AFileSystem fs = getFileSystem(); + + Path path = methodPath(); + FSDataOutputStreamBuilder builder = fs.createFile(path) + .overwrite(false) + .recursive(); + + // this has a broken return type; something to do with the return value of + // the createFile() call. only fixable via risky changes to the FileSystem class + builder.must(FS_S3A_CREATE_PERFORMANCE, true); + + verifyMetrics(() -> build(builder), + always(NO_HEAD_OR_LIST), + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0)); + } + + @Test + public void testCreateFileRecursive() throws Throwable { + describe("createFile without performance flag performs overwrite safety checks"); + S3AFileSystem fs = getFileSystem(); + + final Path path = methodPath(); + FSDataOutputStreamBuilder builder = fs.createFile(path) + .recursive() + .overwrite(false); + + // include a custom header to probe for after + final String custom = "custom"; + builder.must(FS_S3A_CREATE_HEADER + ".h1", custom); + + verifyMetrics(() -> build(builder), + always(CREATE_FILE_NO_OVERWRITE)); + + // the header is there and the probe should be a single HEAD call. + String header = verifyMetrics(() -> + toChar(requireNonNull( + fs.getXAttr(path, XA_HEADER_PREFIX + "h1"), + "no header")), + always(HEAD_OPERATION)); + Assertions.assertThat(header) + .isEqualTo(custom); + } + + @Test + public void testCreateFileNonRecursive() throws Throwable { + describe("nonrecursive createFile does not check parents"); + S3AFileSystem fs = getFileSystem(); + + verifyMetrics(() -> + build(fs.createFile(methodPath()).overwrite(true)), + always(CREATE_FILE_OVERWRITE)); + } + + + @Test + public void testCreateNonRecursive() throws Throwable { + describe("nonrecursive createFile does not check parents"); + S3AFileSystem fs = getFileSystem(); + + verifyMetrics(() -> { + fs.createNonRecursive(methodPath(), + true, 1000, (short)1, 0L, null) + .close(); + return ""; + }, + always(CREATE_FILE_OVERWRITE)); + } + + private FSDataOutputStream build(final FSDataOutputStreamBuilder builder) + throws IOException { + FSDataOutputStream out = builder.build(); + out.close(); + return out; + } + + /** + * Shows how the performance option allows the FS to become ill-formed. + */ + @Test + public void testPerformanceFlagPermitsInvalidStores() throws Throwable { + describe("createFile with performance flag over a directory"); + S3AFileSystem fs = getFileSystem(); + + Path path = methodPath(); + Path child = new Path(path, "child"); + ContractTestUtils.touch(fs, child); + try { + FSDataOutputStreamBuilder builder = fs.createFile(path) + .overwrite(false); + // this has a broken return type; a java typesystem quirk. + builder.must(FS_S3A_CREATE_PERFORMANCE, true); + + verifyMetrics(() -> build(builder), + always(NO_HEAD_OR_LIST), + with(OBJECT_BULK_DELETE_REQUEST, 0), + with(OBJECT_DELETE_REQUEST, 0)); + // the file is there + assertIsFile(path); + // the child is there + assertIsFile(child); + + // delete the path + fs.delete(path, true); + // the child is still there + assertIsFile(child); + // and the directory exists again + assertIsDirectory(path); + } finally { + // always delete the child, so if the test suite fails, the + // store is at least well-formed. + fs.delete(child, true); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java index 01cadc7c86..be4de7942f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/ITestS3ADeleteCost.java @@ -266,9 +266,10 @@ public void testDirMarkersFileCreation() throws Throwable { final int directories = directoriesInPath(srcDir); verifyMetrics(() -> { - file(new Path(srcDir, "source.txt")); + final Path srcPath = new Path(srcDir, "source.txt"); + file(srcPath); LOG.info("Metrics: {}\n{}", getMetricSummary(), getFileSystem()); - return "after touch(fs, srcFilePath) " + getMetricSummary(); + return "after touch(fs, " + srcPath + ")" + getMetricSummary(); }, with(DIRECTORIES_CREATED, 0), with(DIRECTORIES_DELETED, 0), @@ -276,10 +277,10 @@ public void testDirMarkersFileCreation() throws Throwable { withWhenKeeping(getDeleteMarkerStatistic(), 0), withWhenKeeping(FAKE_DIRECTORIES_DELETED, 0), // delete all possible fake dirs above the file - withWhenDeleting(getDeleteMarkerStatistic(), - isBulkDelete() ? 1: directories), withWhenDeleting(FAKE_DIRECTORIES_DELETED, - directories)); + directories), + withWhenDeleting(getDeleteMarkerStatistic(), + isBulkDelete() ? 1: directories)); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java index 03bc10f86c..08a19600d5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCost.java @@ -57,7 +57,8 @@ public final class OperationCost { public static final int DELETE_MARKER_REQUEST = DELETE_OBJECT_REQUEST; /** - * No IO takes place. + * No Head or List IO takes place; other operations + * may still take place. */ public static final OperationCost NO_IO = new OperationCost(0, 0); @@ -87,7 +88,7 @@ public final class OperationCost { /** * Cost of getFileStatus on root directory. */ - public static final OperationCost ROOT_FILE_STATUS_PROBE = NO_IO; + public static final OperationCost ROOT_FILE_STATUS_PROBE = NO_HEAD_OR_LIST; /** * Cost of {@link org.apache.hadoop.fs.s3a.impl.StatusProbeEnum#ALL}. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java index 72e51ee3b9..3a8a1f6ad7 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/performance/OperationCostValidator.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -34,6 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AInstrumentation; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.Statistic; +import org.apache.hadoop.fs.s3a.statistics.StatisticTypeEnum; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.metrics2.lib.MutableCounter; import org.apache.hadoop.metrics2.lib.MutableMetric; @@ -274,7 +276,7 @@ public Builder withMetric(Statistic statistic) { /** * Add a varargs list of metrics. - * @param stat statistics to monitor. + * @param stats statistics to monitor. * @return this. */ public Builder withMetrics(Statistic...stats) { @@ -282,6 +284,20 @@ public Builder withMetrics(Statistic...stats) { return this; } + /** + * Add all counters and duration types to the + * metrics which can be asserted over. + * @return this. + */ + public Builder withAllCounters() { + EnumSet.allOf(Statistic.class).stream() + .filter(s -> + s.getType() == StatisticTypeEnum.TYPE_COUNTER + || s.getType() == StatisticTypeEnum.TYPE_DURATION) + .forEach(metrics::add); + return this; + } + /** * Instantiate. * @return the validator. diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java index 946e59e9e3..91ea0c8e62 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3ADirectoryPerformance.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.WriteOperationHelper; import org.apache.hadoop.fs.s3a.api.RequestFactory; +import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.util.functional.RemoteIterators; @@ -257,9 +258,9 @@ public void testMultiPagesListingPerformanceAndCorrectness() ObjectMetadata om = fs.newObjectMetadata(0L); PutObjectRequest put = requestFactory .newPutObjectRequest(fs.pathToKey(file), om, - new FailingInputStream()); + null, new FailingInputStream()); futures.add(submit(executorService, () -> - writeOperationHelper.putObject(put))); + writeOperationHelper.putObject(put, PutObjectOptions.keepingDirs()))); } LOG.info("Waiting for PUTs to complete"); waitForCompletion(futures); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java index 231cfd884e..b83d12b4c1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/ITestS3AMultipartUploadSizeLimits.java @@ -35,7 +35,7 @@ import org.apache.hadoop.fs.s3a.S3AInstrumentation; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.auth.ProgressCounter; -import org.apache.hadoop.fs.s3a.commit.CommitOperations; +import org.apache.hadoop.fs.s3a.commit.impl.CommitOperations; import static org.apache.hadoop.fs.StreamCapabilities.ABORTABLE_STREAM; import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;