HADOOP-18377. hadoop-aws build to add a -prefetch profile to run all tests with prefetching (#4914)
Contributed by Viraj Jasani
This commit is contained in:
parent
e68006cd70
commit
5b1657278c
@ -53,6 +53,9 @@
|
||||
<fs.s3a.directory.marker.audit>false</fs.s3a.directory.marker.audit>
|
||||
<!-- marker retention policy -->
|
||||
<fs.s3a.directory.marker.retention></fs.s3a.directory.marker.retention>
|
||||
|
||||
<!-- Is prefetch enabled? -->
|
||||
<fs.s3a.prefetch.enabled>unset</fs.s3a.prefetch.enabled>
|
||||
</properties>
|
||||
|
||||
<profiles>
|
||||
@ -127,6 +130,8 @@
|
||||
<!-- Markers-->
|
||||
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
|
||||
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
|
||||
<!-- Prefetch -->
|
||||
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
@ -166,6 +171,8 @@
|
||||
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
|
||||
|
||||
<test.default.timeout>${test.integration.timeout}</test.default.timeout>
|
||||
<!-- Prefetch -->
|
||||
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
|
||||
</systemPropertyVariables>
|
||||
<!-- Some tests cannot run in parallel. Tests that cover -->
|
||||
<!-- access to the root directory must run in isolation -->
|
||||
@ -215,6 +222,8 @@
|
||||
<!-- Markers-->
|
||||
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
|
||||
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
|
||||
<!-- Prefetch -->
|
||||
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
|
||||
</systemPropertyVariables>
|
||||
<!-- Do a sequential run for tests that cannot handle -->
|
||||
<!-- parallel execution. -->
|
||||
@ -271,6 +280,8 @@
|
||||
<!-- Markers-->
|
||||
<fs.s3a.directory.marker.retention>${fs.s3a.directory.marker.retention}</fs.s3a.directory.marker.retention>
|
||||
<fs.s3a.directory.marker.audit>${fs.s3a.directory.marker.audit}</fs.s3a.directory.marker.audit>
|
||||
<!-- Prefetch -->
|
||||
<fs.s3a.prefetch.enabled>${fs.s3a.prefetch.enabled}</fs.s3a.prefetch.enabled>
|
||||
</systemPropertyVariables>
|
||||
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
|
||||
</configuration>
|
||||
@ -332,6 +343,19 @@
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
<!-- Turn on prefetching-->
|
||||
<profile>
|
||||
<id>prefetch</id>
|
||||
<activation>
|
||||
<property>
|
||||
<name>prefetch</name>
|
||||
</property>
|
||||
</activation>
|
||||
<properties>
|
||||
<fs.s3a.prefetch.enabled>true</fs.s3a.prefetch.enabled>
|
||||
</properties>
|
||||
</profile>
|
||||
|
||||
</profiles>
|
||||
|
||||
<build>
|
||||
|
@ -617,6 +617,20 @@ your `core-site.xml` file, so that trying to use S3 select fails fast with
|
||||
a meaningful error ("S3 Select not supported") rather than a generic Bad Request
|
||||
exception.
|
||||
|
||||
### <a name="enabling-prefetch"></a> Enabling prefetch for all tests
|
||||
|
||||
The tests are run with prefetch if the `prefetch` property is set in the
|
||||
maven build. This can be combined with the scale tests as well.
|
||||
|
||||
```bash
|
||||
mvn verify -Dprefetch
|
||||
|
||||
mvn verify -Dparallel-tests -Dprefetch -DtestsThreadCount=8
|
||||
|
||||
mvn verify -Dparallel-tests -Dprefetch -Dscale -DtestsThreadCount=8
|
||||
```
|
||||
|
||||
|
||||
### Testing Requester Pays
|
||||
|
||||
By default, the requester pays tests will look for a bucket that exists on Amazon S3
|
||||
|
@ -575,6 +575,10 @@ public static Configuration prepareTestConfiguration(final Configuration conf) {
|
||||
DEFAULT_DIRECTORY_MARKER_POLICY);
|
||||
conf.set(DIRECTORY_MARKER_POLICY, directoryRetention);
|
||||
|
||||
boolean prefetchEnabled =
|
||||
getTestPropertyBool(conf, PREFETCH_ENABLED_KEY, PREFETCH_ENABLED_DEFAULT);
|
||||
conf.setBoolean(PREFETCH_ENABLED_KEY, prefetchEnabled);
|
||||
|
||||
return conf;
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user