510 lines
21 KiB
XML
510 lines
21 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License. See accompanying LICENSE file.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<modelVersion>4.0.0</modelVersion>
|
|
<parent>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-project</artifactId>
|
|
<version>3.3.0-SNAPSHOT</version>
|
|
<relativePath>../../hadoop-project</relativePath>
|
|
</parent>
|
|
<artifactId>hadoop-aws</artifactId>
|
|
<version>3.3.0-SNAPSHOT</version>
|
|
<name>Apache Hadoop Amazon Web Services support</name>
|
|
<description>
|
|
This module contains code to support integration with Amazon Web Services.
|
|
It also declares the dependencies needed to work with AWS services.
|
|
</description>
|
|
<packaging>jar</packaging>
|
|
|
|
<properties>
|
|
<file.encoding>UTF-8</file.encoding>
|
|
<downloadSources>true</downloadSources>
|
|
<hadoop.tmp.dir>${project.build.directory}/test</hadoop.tmp.dir>
|
|
|
|
<!-- are scale tests enabled ? -->
|
|
<fs.s3a.scale.test.enabled>unset</fs.s3a.scale.test.enabled>
|
|
<!-- Size in MB of huge files. -->
|
|
<fs.s3a.scale.test.huge.filesize>unset</fs.s3a.scale.test.huge.filesize>
|
|
<!-- Size in MB of the partion size in huge file uploads. -->
|
|
<fs.s3a.scale.test.huge.partitionsize>unset</fs.s3a.scale.test.huge.partitionsize>
|
|
<!-- Timeout in seconds for scale tests.-->
|
|
<fs.s3a.scale.test.timeout>3600</fs.s3a.scale.test.timeout>
|
|
<!-- are scale tests enabled ? -->
|
|
<fs.s3a.s3guard.test.enabled>false</fs.s3a.s3guard.test.enabled>
|
|
<fs.s3a.s3guard.test.authoritative>false</fs.s3a.s3guard.test.authoritative>
|
|
<fs.s3a.s3guard.test.implementation>local</fs.s3a.s3guard.test.implementation>
|
|
<!-- Set a longer timeout for integration test (in milliseconds) -->
|
|
<test.integration.timeout>200000</test.integration.timeout>
|
|
|
|
</properties>
|
|
|
|
<profiles>
|
|
<profile>
|
|
<id>tests-off</id>
|
|
<activation>
|
|
<file>
|
|
<missing>src/test/resources/auth-keys.xml</missing>
|
|
</file>
|
|
</activation>
|
|
<properties>
|
|
<skipITs>true</skipITs>
|
|
</properties>
|
|
</profile>
|
|
<profile>
|
|
<id>tests-on</id>
|
|
<activation>
|
|
<file>
|
|
<exists>src/test/resources/auth-keys.xml</exists>
|
|
</file>
|
|
</activation>
|
|
<properties>
|
|
<skipITs>false</skipITs>
|
|
</properties>
|
|
</profile>
|
|
<profile>
|
|
<id>parallel-tests</id>
|
|
<activation>
|
|
<property>
|
|
<name>parallel-tests</name>
|
|
</property>
|
|
</activation>
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-maven-plugins</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>parallel-tests-createdir</id>
|
|
<goals>
|
|
<goal>parallel-tests-createdir</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<configuration>
|
|
<forkCount>${testsThreadCount}</forkCount>
|
|
<reuseForks>false</reuseForks>
|
|
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
|
<systemPropertyVariables>
|
|
<testsThreadCount>${testsThreadCount}</testsThreadCount>
|
|
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
|
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
|
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
|
|
|
<!-- Due to a Maven quirk, setting this to just -->
|
|
<!-- surefire.forkNumber won't do the parameter -->
|
|
<!-- substitution. Putting a prefix in front of it like -->
|
|
<!-- "fork-" makes it work. -->
|
|
<!-- Important: Those leading 0s are needed to guarantee that -->
|
|
<!-- trailing three chars are always numeric and unique -->
|
|
<test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
|
|
<!-- Propagate scale parameters -->
|
|
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
|
|
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
|
|
<fs.s3a.scale.test.huge.huge.partitionsize>${fs.s3a.scale.test.huge.partitionsize}</fs.s3a.scale.test.huge.huge.partitionsize>
|
|
<fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
|
|
</systemPropertyVariables>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-failsafe-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>default-integration-test</id>
|
|
<goals>
|
|
<goal>integration-test</goal>
|
|
<goal>verify</goal>
|
|
</goals>
|
|
<configuration>
|
|
<forkCount>${testsThreadCount}</forkCount>
|
|
<reuseForks>false</reuseForks>
|
|
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
|
|
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
|
|
<trimStackTrace>false</trimStackTrace>
|
|
<systemPropertyVariables>
|
|
<!-- Tell tests that they are being executed in parallel -->
|
|
<test.parallel.execution>true</test.parallel.execution>
|
|
<test.build.data>${test.build.data}/${surefire.forkNumber}</test.build.data>
|
|
<test.build.dir>${test.build.dir}/${surefire.forkNumber}</test.build.dir>
|
|
<hadoop.tmp.dir>${hadoop.tmp.dir}/${surefire.forkNumber}</hadoop.tmp.dir>
|
|
|
|
<!-- Due to a Maven quirk, setting this to just -->
|
|
<!-- surefire.forkNumber won't do the parameter -->
|
|
<!-- substitution. Putting a prefix in front of it like -->
|
|
<!-- "fork-" makes it work. -->
|
|
<test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
|
|
<!-- Propagate scale parameters -->
|
|
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
|
|
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
|
|
<fs.s3a.scale.test.huge.huge.partitionsize>${fs.s3a.scale.test.huge.partitionsize}</fs.s3a.scale.test.huge.huge.partitionsize>
|
|
<fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
|
|
<!-- S3Guard -->
|
|
<fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
|
|
<fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
|
|
<fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
|
|
|
|
<test.default.timeout>${test.integration.timeout}</test.default.timeout>
|
|
</systemPropertyVariables>
|
|
<!-- Some tests cannot run in parallel. Tests that cover -->
|
|
<!-- access to the root directory must run in isolation -->
|
|
<!-- from anything else that could modify the bucket. -->
|
|
<!-- S3A tests that cover multi-part upload must run in -->
|
|
<!-- isolation, because the file system is configured to -->
|
|
<!-- purge existing multi-part upload data on -->
|
|
<!-- initialization. MiniYARNCluster has not yet been -->
|
|
<!-- changed to handle parallel test execution gracefully. -->
|
|
<!-- Exclude all of these tests from parallel execution, -->
|
|
<!-- and instead run them sequentially in a separate -->
|
|
<!-- Surefire execution step later. -->
|
|
<includes>
|
|
<include>**/ITest*.java</include>
|
|
</includes>
|
|
<excludes>
|
|
<exclude>**/ITestS3AContractRootDir.java</exclude>
|
|
<exclude>**/ITestS3AFileContextStatistics.java</exclude>
|
|
<exclude>**/ITestS3AEncryptionSSEC*.java</exclude>
|
|
<exclude>**/ITestS3AHuge*.java</exclude>
|
|
<!-- this sets out to overlaod DynamoDB, so must be run standalone -->
|
|
<exclude>**/ITestDynamoDBMetadataStoreScale.java</exclude>
|
|
</excludes>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>sequential-integration-tests</id>
|
|
<goals>
|
|
<goal>integration-test</goal>
|
|
<goal>verify</goal>
|
|
</goals>
|
|
<configuration>
|
|
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
|
|
<trimStackTrace>false</trimStackTrace>
|
|
<systemPropertyVariables>
|
|
<!-- Tell tests that they are being executed sequentially -->
|
|
<test.parallel.execution>false</test.parallel.execution>
|
|
<!-- Propagate scale parameters -->
|
|
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
|
|
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
|
|
<fs.s3a.scale.test.huge.huge.partitionsize>${fs.s3a.scale.test.huge.partitionsize}</fs.s3a.scale.test.huge.huge.partitionsize>
|
|
<fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
|
|
<!-- S3Guard -->
|
|
<fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
|
|
<fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
|
|
<fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
|
|
</systemPropertyVariables>
|
|
<!-- Do a sequential run for tests that cannot handle -->
|
|
<!-- parallel execution. -->
|
|
<includes>
|
|
<include>**/ITestS3AContractRootDir.java</include>
|
|
<include>**/ITestS3AFileContextStatistics.java</include>
|
|
<include>**/ITestS3AHuge*.java</include>
|
|
<include>**/ITestS3AEncryptionSSEC*.java</include>
|
|
<!-- this sets out to overlaod DynamoDB, so must be run standalone -->
|
|
<include>**/ITestDynamoDBMetadataStoreScale.java</include>
|
|
</includes>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</profile>
|
|
<profile>
|
|
<id>sequential-tests</id>
|
|
<activation>
|
|
<property>
|
|
<name>!parallel-tests</name>
|
|
</property>
|
|
</activation>
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-failsafe-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>integration-test</goal>
|
|
<goal>verify</goal>
|
|
</goals>
|
|
<configuration>
|
|
<systemPropertyVariables>
|
|
<!-- Propagate scale parameters -->
|
|
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
|
|
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
|
|
<fs.s3a.scale.test.timeout>${fs.s3a.scale.test.timeout}</fs.s3a.scale.test.timeout>
|
|
<!-- S3Guard -->
|
|
<fs.s3a.s3guard.test.enabled>${fs.s3a.s3guard.test.enabled}</fs.s3a.s3guard.test.enabled>
|
|
<fs.s3a.s3guard.test.implementation>${fs.s3a.s3guard.test.implementation}</fs.s3a.s3guard.test.implementation>
|
|
<fs.s3a.s3guard.test.authoritative>${fs.s3a.s3guard.test.authoritative}</fs.s3a.s3guard.test.authoritative>
|
|
</systemPropertyVariables>
|
|
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
</profile>
|
|
|
|
<!-- Turn on scale tests-->
|
|
<profile>
|
|
<id>scale</id>
|
|
<activation>
|
|
<property>
|
|
<name>scale</name>
|
|
</property>
|
|
</activation>
|
|
<properties >
|
|
<fs.s3a.scale.test.enabled>true</fs.s3a.scale.test.enabled>
|
|
</properties>
|
|
</profile>
|
|
|
|
<!-- Turn on S3Guard tests-->
|
|
<profile>
|
|
<id>s3guard</id>
|
|
<activation>
|
|
<property>
|
|
<name>s3guard</name>
|
|
</property>
|
|
</activation>
|
|
<properties >
|
|
<fs.s3a.s3guard.test.enabled>true</fs.s3a.s3guard.test.enabled>
|
|
</properties>
|
|
</profile>
|
|
|
|
<!-- Switch to DynamoDB for S3Guard. Has no effect unless S3Guard is enabled -->
|
|
<profile>
|
|
<id>dynamo</id>
|
|
<activation>
|
|
<property>
|
|
<name>dynamo</name>
|
|
</property>
|
|
</activation>
|
|
<properties >
|
|
<fs.s3a.s3guard.test.implementation>dynamo</fs.s3a.s3guard.test.implementation>
|
|
</properties>
|
|
</profile>
|
|
|
|
<!-- Switch S3Guard from Authoritative=false to true
|
|
Has no effect unless S3Guard is enabled -->
|
|
<profile>
|
|
<id>auth</id>
|
|
<activation>
|
|
<property>
|
|
<name>auth</name>
|
|
</property>
|
|
</activation>
|
|
<properties >
|
|
<fs.s3a.s3guard.test.authoritative>true</fs.s3a.s3guard.test.authoritative>
|
|
</properties>
|
|
</profile>
|
|
|
|
</profiles>
|
|
|
|
<build>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.codehaus.mojo</groupId>
|
|
<artifactId>findbugs-maven-plugin</artifactId>
|
|
<configuration>
|
|
<findbugsXmlOutput>true</findbugsXmlOutput>
|
|
<xmlOutput>true</xmlOutput>
|
|
<excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
|
|
</excludeFilterFile>
|
|
<effort>Max</effort>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-surefire-plugin</artifactId>
|
|
<configuration>
|
|
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
|
|
<systemPropertyVariables>
|
|
<test.default.timeout>${test.integration.timeout}</test.default.timeout>
|
|
</systemPropertyVariables>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>deplist1</id>
|
|
<phase>compile</phase>
|
|
<goals>
|
|
<goal>list</goal>
|
|
</goals>
|
|
<configuration>
|
|
<!-- build a shellprofile for hadoop-aws optional tools -->
|
|
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>copy</id>
|
|
<phase>test-compile</phase>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
<configuration>
|
|
<includeScope>test</includeScope>
|
|
<includeTypes>so,dll,dylib</includeTypes>
|
|
<outputDirectory>${project.build.directory}/native-libs</outputDirectory>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<phase>package</phase>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
<configuration>
|
|
<outputDirectory>${project.build.directory}/lib</outputDirectory>
|
|
</configuration>
|
|
</execution>
|
|
<execution>
|
|
<id>deplist2</id>
|
|
<phase>compile</phase>
|
|
<goals>
|
|
<goal>list</goal>
|
|
</goals>
|
|
<configuration>
|
|
<!-- referenced by the s3guard command -->
|
|
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<dependencies>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
<scope>test</scope>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.amazonaws</groupId>
|
|
<artifactId>aws-java-sdk-bundle</artifactId>
|
|
<scope>compile</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>junit</groupId>
|
|
<artifactId>junit</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-all</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-core</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-yarn-server-tests</artifactId>
|
|
<scope>test</scope>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-hs</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-examples</artifactId>
|
|
<scope>test</scope>
|
|
<type>jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
<scope>test</scope>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-distcp</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-distcp</artifactId>
|
|
<scope>test</scope>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<!-- artifacts needed to bring up a Mini MR Yarn cluster-->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-app</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-app</artifactId>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
|
|
<scope>test</scope>
|
|
<type>test-jar</type>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-minikdc</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.hamcrest</groupId>
|
|
<artifactId>hamcrest-library</artifactId>
|
|
</dependency>
|
|
<!-- Used to create SSL certs for a secure Keystore -->
|
|
<dependency>
|
|
<groupId>org.bouncycastle</groupId>
|
|
<artifactId>bcprov-jdk15on</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.bouncycastle</groupId>
|
|
<artifactId>bcpkix-jdk15on</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
</dependencies>
|
|
</project>
|