HADOOP-18377. hadoop-aws build to add a -prefetch profile to run all tests with prefetching (#4914)

Contributed by Viraj Jasani
This commit is contained in:
Viraj Jasani 2022-09-20 02:26:13 -07:00 committed by GitHub
parent e68006cd70
commit 5b1657278c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 42 additions and 0 deletions

View File

@ -53,6 +53,9 @@
<fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit>
<!-- marker retention policy -->
<fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention>
<!-- Is prefetch enabled? -->
<fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled>
</properties>
<profiles>
@ -127,6 +130,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
<!-- Prefetch -->
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
</configuration>
</plugin>
@ -166,6 +171,8 @@
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<test.default.timeout>${test.integration.timeout}</test.default.timeout>
<!-- Prefetch -->
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<!-- Some tests cannot run in parallel. Tests that cover -->
<!-- access to the root directory must run in isolation -->
@ -215,6 +222,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
<!-- Prefetch -->
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<!-- Do a sequential run for tests that cannot handle -->
<!-- parallel execution. -->
@ -271,6 +280,8 @@
<!-- Markers-->
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
<!-- Prefetch -->
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
</configuration>
@ -332,6 +343,19 @@
</properties>
</profile>
<!-- Turn on prefetching-->
<profile>
<id>prefetch</id>
<activation>
<property>
<name>prefetch</name>
</property>
</activation>
<properties>
<fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled>
</properties>
</profile>
</profiles>
<build>

View File

@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with
a meaningful error ("S3 Select not supported") rather than a generic Bad Request
exception.
### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
The tests are run with prefetch if the `prefetch` property is set in the
maven build. This can be combined with the scale tests as well.
```bash
mvn verify -Dprefetch
mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
```
### Testing Requester Pays
By default, the requester pays tests will look for a bucket that exists on Amazon S3

View File

@ -575,6 +575,10 @@ public static Configuration prepareTestConfiguration(final Configuration conf) {
DEFAULT_DIRECTORY_MARKER_POLICY);
conf.set(DIRECTORY_MARKER_POLICY, directoryRetention);
boolean prefetchEnabled =
getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled);
return conf;
}