4.0.0 org.apache.hadoop hadoop-project 3.5.0-SNAPSHOT ../../hadoop-project hadoop-azure Apache Hadoop Azure support This module contains code to support integration with Azure. Currently this consists of a filesystem client to read data from and write data to Azure Storage. jar UTF-8 true ${project.build.directory}/test unset unset unset 7200 10 1000 100 com.github.spotbugs spotbugs-maven-plugin true ${basedir}/dev-support/findbugs-exclude.xml Max org.apache.maven.plugins maven-checkstyle-plugin src/config/checkstyle.xml src/config/checkstyle-suppressions.xml org.apache.maven.plugins maven-jar-plugin test-jar org.apache.maven.plugins maven-dependency-plugin deplist compile list ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt org.eclipse.m2e lifecycle-mapping 1.0.0 org.apache.maven.plugins maven-enforcer-plugin [1.0.0,) enforce org.apache.hadoop hadoop-common provided org.apache.httpcomponents httpclient compile com.microsoft.azure azure-storage compile org.apache.commons commons-lang3 com.fasterxml.jackson.core jackson-core org.slf4j slf4j-api com.google.guava guava org.apache.hadoop.thirdparty hadoop-shaded-guava org.eclipse.jetty jetty-util-ajax compile org.wildfly.openssl wildfly-openssl compile com.fasterxml.jackson.core jackson-core provided com.fasterxml.jackson.core jackson-databind provided junit junit test org.apache.hadoop hadoop-common test test-jar org.apache.hadoop hadoop-mapreduce-client-jobclient test org.apache.hadoop hadoop-mapreduce-client-core provided org.apache.hadoop hadoop-mapreduce-client-core test test-jar org.apache.hadoop hadoop-yarn-server-tests test test-jar org.apache.hadoop hadoop-mapreduce-client-hs test org.apache.hadoop hadoop-mapreduce-examples test jar org.apache.hadoop hadoop-mapreduce-client-app test org.apache.hadoop hadoop-mapreduce-client-app test-jar test org.apache.hadoop hadoop-mapreduce-client-jobclient test test-jar org.apache.hadoop hadoop-distcp test org.apache.hadoop hadoop-distcp test test-jar ch.qos.reload4j reload4j test javax.ws.rs jsr311-api test org.mockito mockito-core 4.11.0 test org.apache.hadoop hadoop-minikdc test org.bouncycastle bcprov-jdk18on test org.bouncycastle bcpkix-jdk18on test org.assertj assertj-core test org.hamcrest hamcrest-library test parallel-tests-wasb parallel-tests wasb org.apache.hadoop hadoop-maven-plugins parallel-tests-createdir parallel-tests-createdir org.apache.maven.plugins maven-surefire-plugin default-test test ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} **/azure/Test*.java **/azure/**/Test*.java **/azure/**/TestRollingWindowAverage*.java serialized-test-wasb test 1 false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} **/azure/**/TestRollingWindowAverage*.java org.apache.maven.plugins maven-failsafe-plugin default-integration-test-wasb integration-test verify ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} false true ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} **/azure/ITest*.java **/azure/**/ITest*.java **/azure/ITestNativeFileSystemStatistics.java sequential-integration-tests-wasb integration-test verify ${fs.azure.scale.test.timeout} false false ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} **/azure/ITestNativeFileSystemStatistics.java parallel-tests-abfs parallel-tests abfs org.apache.hadoop hadoop-maven-plugins parallel-tests-createdir parallel-tests-createdir org.apache.maven.plugins maven-surefire-plugin default-test test ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} **/azurebfs/Test*.java **/azurebfs/**/Test*.java org.apache.maven.plugins maven-failsafe-plugin integration-test-abfs-parallel-classesAndMethods integration-test verify ${testsThreadCount} true both ${testsThreadCount} ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} false true ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.timeout} ${http.maxConnections} **/azurebfs/ITest*.java **/azurebfs/**/ITest*.java **/azurebfs/contract/ITest*.java **/azurebfs/ITestAzureBlobFileSystemE2EScale.java **/azurebfs/ITestAbfsReadWriteAndSeek.java **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/services/ITestReadBufferManager.java **/azurebfs/commit/*.java integration-test-abfs-parallel-classes integration-test verify ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} false true ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.timeout} **/azurebfs/ITestAzureBlobFileSystemE2EScale.java **/azurebfs/ITestAbfsReadWriteAndSeek.java **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/commit/*.java integration-test-abfs-sequential-contract-classes integration-test verify ${fs.azure.scale.test.timeout} false false ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} **/azurebfs/contract/ITest*.java parallel-tests parallel-tests both org.apache.hadoop hadoop-maven-plugins parallel-tests-createdir parallel-tests-createdir org.apache.maven.plugins maven-surefire-plugin default-test test ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} **/Test*.java **/TestRollingWindowAverage*.java serialized-test test 1 false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} **/TestRollingWindowAverage*.java org.apache.maven.plugins maven-failsafe-plugin default-integration-test integration-test verify ${testsThreadCount} false ${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true ${fs.azure.scale.test.timeout} false true ${test.build.data}/${surefire.forkNumber} ${test.build.dir}/${surefire.forkNumber} ${hadoop.tmp.dir}/${surefire.forkNumber} fork-000${surefire.forkNumber} ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} **/ITest*.java **/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java **/ITestFileSystemOperationsWithThreads.java **/ITestOutOfBandAzureBlobOperationsLive.java **/ITestNativeAzureFileSystemAuthorizationWithOwner.java **/ITestNativeAzureFileSystemConcurrencyLive.java **/ITestNativeAzureFileSystemLive.java **/ITestNativeAzureFSPageBlobLive.java **/ITestAzureBlobFileSystemRandomRead.java **/ITestWasbRemoteCallHelper.java **/ITestBlockBlobInputStream.java **/ITestWasbAbfsCompatibility.java **/ITestNativeFileSystemStatistics.java sequential-integration-tests integration-test verify ${fs.azure.scale.test.timeout} false false ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.huge.partitionsize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} **/ITestWasbAbfsCompatibility.java **/ITestFileSystemOperationsExceptionHandlingMultiThreaded.java **/ITestFileSystemOperationsWithThreads.java **/ITestOutOfBandAzureBlobOperationsLive.java **/ITestNativeAzureFileSystemAuthorizationWithOwner.java **/ITestNativeAzureFileSystemConcurrencyLive.java **/ITestNativeAzureFileSystemLive.java **/ITestNativeAzureFSPageBlobLive.java **/ITestAzureBlobFileSystemRandomRead.java **/ITestWasbRemoteCallHelper.java **/ITestBlockBlobInputStream.java **/ITestNativeFileSystemStatistics.java sequential-tests !parallel-tests org.apache.maven.plugins maven-failsafe-plugin integration-test verify ${fs.azure.scale.test.enabled} ${fs.azure.scale.test.huge.filesize} ${fs.azure.scale.test.timeout} ${fs.azure.scale.test.list.performance.threads} ${fs.azure.scale.test.list.performance.files} ${http.maxConnections} ${fs.azure.scale.test.timeout} false scale scale true