From a996d889ec035e3b6134e94b50efc990525c3bbb Mon Sep 17 00:00:00 2001 From: Sneha Vijayarajan Date: Thu, 20 Oct 2022 22:37:04 +0530 Subject: [PATCH] HADOOP-17767. ABFS: Update test scripts (#3124) Contributed by Sneha Vijayarajan --- hadoop-tools/hadoop-azure/.gitignore | 3 +- .../dev-support/testrun-scripts/runtests.sh | 166 ++++++++-- .../testrun-scripts/testsupport.sh | 295 +++++++----------- .../src/site/markdown/testing_azure.md | 111 +++++-- .../azurebfs/utils/CleanupTestContainers.java | 74 +++++ .../accountName_settings.xml.template | 185 +++++++++++ .../resources/azure-auth-keys.xml.template | 156 +-------- .../src/test/resources/azure-test.xml | 45 ++- 8 files changed, 634 insertions(+), 401 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java create mode 100644 hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template diff --git a/hadoop-tools/hadoop-azure/.gitignore b/hadoop-tools/hadoop-azure/.gitignore index 0e17efaa1e..522210137e 100644 --- a/hadoop-tools/hadoop-azure/.gitignore +++ b/hadoop-tools/hadoop-azure/.gitignore @@ -1,5 +1,6 @@ .checkstyle bin/ -src/test/resources/combinationConfigFiles src/test/resources/abfs-combination-test-configs.xml dev-support/testlogs +src/test/resources/accountSettings/* +!src/test/resources/accountSettings/accountName_settings.xml.template diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh index 25d9593d57..400d0a2383 100755 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh @@ -2,7 +2,7 @@ # shellcheck disable=SC2034 # unused variables are global in nature and used in testsupport.sh - +test set -eo pipefail # Licensed to the Apache Software Foundation (ASF) under one or more @@ -22,36 +22,154 @@ set -eo pipefail # shellcheck disable=SC1091 . dev-support/testrun-scripts/testsupport.sh +init -begin +resourceDir=src/test/resources/ +logdir=dev-support/testlogs/ +azureTestXml=azure-auth-keys.xml +azureTestXmlPath=$resourceDir$azureTestXml +processCount=8 -### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES. -### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep. +## SECTION: TEST COMBINATION METHODS +runHNSOAuthTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("OAuth") + triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} -combination=HNS-OAuth -properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" -"fs.azure.account.auth.type") -values=("{account name}.dfs.core.windows.net" "true" "OAuth") -generateconfigs +runHNSSharedKeyTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("SharedKey") + triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} -combination=AppendBlob-HNS-OAuth -properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" -"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") -values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true") -generateconfigs +runNonHNSSharedKeyTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type") + VALUES=("SharedKey") + triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} -combination=HNS-SharedKey -properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type") -values=("{account name}.dfs.core.windows.net" "true" "SharedKey") -generateconfigs +runAppendBlobHNSOAuthTest() +{ + accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath) + PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled") + VALUES=("OAuth" "true") + triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers" +} -combination=NonHNS-SharedKey -properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type") -values=("{account name}.dfs.core.windows.net" "false" "SharedKey") -generateconfigs +runTest=false +cleanUpTestContainers=false +echo 'Ensure below are complete before running script:' +echo '1. Account specific settings file is present.' +echo ' Copy accountName_settings.xml.template to accountName_settings.xml' +echo ' where accountName in copied file name should be the test account name without domain' +echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings' +echo ' folder. New account settings file to be added to same folder.)' +echo ' Follow instructions in the template to populate settings correctly for the account' +echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName' +echo ' where accountNames should be the test account names without domain' +echo ' ' +echo ' ' +echo 'Choose action:' +echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]' +select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN +do + case $scriptMode in + SET_ACTIVE_TEST_CONFIG) + runTest=false + break + ;; + RUN_TEST) + runTest=true + read -r -p "Enter parallel test run process count [default - 8]: " processCount + processCount=${processCount:-8} + break + ;; + CLEAN_UP_OLD_TEST_CONTAINERS) + runTest=false + cleanUpTestContainers=true + break + ;; + SET_OR_CHANGE_TEST_ACCOUNT) + runTest=false + cleanUpTestContainers=false + accountSettingsFile="src/test/resources/azure-auth-keys.xml" + if [[ ! -f "$accountSettingsFile" ]]; + then + logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template" + cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile + fi + vi $accountSettingsFile + exit 0 + break + ;; + PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN) + runTest=false + cleanUpTestContainers=false + logFilePaths=/tmp/logPaths + find target/ -name "*output.txt" > $logFilePaths + logOutput "$(cat $logFilePaths)" + rm $logFilePaths + exit 0 + break + ;; + *) logOutput "ERROR: Invalid selection" + ;; + esac +done -### DO NOT EDIT THE LINES BELOW. +## SECTION: COMBINATION DEFINITIONS AND TRIGGER -runtests "$@" +echo ' ' +echo 'Set the active test combination to run the action:' +select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit +do + case $combo in + HNS-OAuth) + runHNSOAuthTest + break + ;; + HNS-SharedKey) + runHNSSharedKeyTest + break + ;; + nonHNS-SharedKey) + runNonHNSSharedKeyTest + break + ;; + AppendBlob-HNS-OAuth) + runAppendBlobHNSOAuthTest + break + ;; + AllCombinationsTestRun) + if [ $runTest == false ] + then + logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST." + break + fi + runHNSOAuthTest + runHNSSharedKeyTest + runNonHNSSharedKeyTest + runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always + break + ;; + Quit) + exit 0 + ;; + *) logOutput "ERROR: Invalid selection" + ;; + esac +done + +if [ $runTest == true ] +then + printAggregate +fi diff --git a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh index 1b118ae1e8..28f96edd27 100644 --- a/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh +++ b/hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh @@ -15,117 +15,88 @@ # See the License for the specific language governing permissions and # limitations under the License. -testresourcesdir=src/test/resources -combconfsdir=$testresourcesdir/combinationConfigFiles -combtestfile=$testresourcesdir/abfs-combination-test-configs.xml +resourceDir=src/test/resources/ +accountSettingsFolderName=accountSettings +combtestfile=$resourceDir +combtestfile+=abfs-combination-test-configs.xml +logdir=dev-support/testlogs/ -logdir=dev-support/testlogs testresultsregex="Results:(\n|.)*?Tests run:" -testresultsfilename= -starttime= -threadcount= -defaultthreadcount=8 +accountConfigFileSuffix="_settings.xml" +testOutputLogFolder=$logdir +testlogfilename=combinationTestLogFile -properties= -values= +fullRunStartTime=$(date +%s) +STARTTIME=$(date +%s) +ENDTIME=$(date +%s) -validate() { - if [ -z "$threadcount" ] ; then - threadcount=$defaultthreadcount - fi - numberegex='^[0-9]+$' - if ! [[ $threadcount =~ $numberegex ]] ; then - echo "Exiting. The script param (threadcount) should be a number" - exit -1 - fi - if [ -z "$combination" ]; then - echo "Exiting. combination cannot be empty" - exit -1 - fi - propertiessize=${#properties[@]} - valuessize=${#values[@]} - if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then - echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination" - exit -1 - fi - - for filename in "${combinations[@]}"; do - if [[ ! -f "$combconfsdir/$filename.xml" ]]; then - echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist." - exit -1 - fi - done -} - -checkdependencies() { - if ! [ "$(command -v pcregrep)" ]; then - echo "Exiting. pcregrep is required to run the script." - exit -1 - fi - if ! [ "$(command -v xmlstarlet)" ]; then - echo "Exiting. xmlstarlet is required to run the script." - exit -1 - fi -} - -cleancombinationconfigs() { - rm -rf $combconfsdir - mkdir -p $combconfsdir -} - -generateconfigs() { - combconffile="$combconfsdir/$combination.xml" - rm -rf "$combconffile" - cat > "$combconffile" << ENDOFFILE - - - -ENDOFFILE - - propertiessize=${#properties[@]} - valuessize=${#values[@]} - if [ "$propertiessize" -ne "$valuessize" ]; then - echo "Exiting. Number of properties and values differ for $combination" - exit -1 - fi - for ((i = 0; i < propertiessize; i++)); do - key=${properties[$i]} - val=${values[$i]} - changeconf "$key" "$val" - done - formatxml "$combconffile" -} - -formatxml() { - xmlstarlet fo -s 2 "$1" > "$1.tmp" - mv "$1.tmp" "$1" -} - -setactiveconf() { - if [[ ! -f "$combconfsdir/$combination.xml" ]]; then - echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist." - exit -1 +outputFormatOn="\033[0;95m" +outputFormatOff="\033[0m" + +triggerRun() +{ + echo ' ' + combination=$1 + accountName=$2 + runTest=$3 + processcount=$4 + cleanUpTestContainers=$5 + + if [ -z "$accountName" ]; then + logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account." + exit 1; fi + accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix rm -rf $combtestfile cat > $combtestfile << ENDOFFILE ENDOFFILE + propertiessize=${#PROPERTIES[@]} + valuessize=${#VALUES[@]} + if [ "$propertiessize" -ne "$valuessize" ]; then + logOutput "Exiting. Number of properties and values differ for $combination" + exit 1 + fi + for ((i = 0; i < propertiessize; i++)); do + key=${PROPERTIES[$i]} + val=${VALUES[$i]} + echo "Combination specific property setting: [ key=$key , value=$val ]" + changeconf "$key" "$val" + done + formatxml "$combtestfile" xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile - xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile + xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile formatxml $combtestfile -} + echo ' ' + echo "Activated [$combtestfile] - for account: $accountName for combination $combination" + testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt" + touch "$testlogfilename" -changeconf() { - xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile" - xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile" - if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile" + if [ "$runTest" == true ] then - echo "Exiting. Changing config property failed." - exit -1 + STARTTIME=$(date +%s) + echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]" + logOutput "Test run report can be seen in $testlogfilename" + mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true + ENDTIME=$(date +%s) + summary fi + + if [ "$cleanUpTestContainers" == true ] + then + mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true + if grep -q "There are test failures" "$testlogfilename"; + then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename" + pcregrep -M "$testresultsregex" "$testlogfilename" + exit 0 + fi + + logOutput "Delete test containers - complete. Test run logs in - $testlogfilename" + fi + } summary() { @@ -134,17 +105,42 @@ summary() { echo "$combination" echo "========================" pcregrep -M "$testresultsregex" "$testlogfilename" - } >> "$testresultsfilename" + } >> "$aggregatedTestResult" printf "\n----- Test results -----\n" pcregrep -M "$testresultsregex" "$testlogfilename" - secondstaken=$((ENDTIME - STARTTIME)) mins=$((secondstaken / 60)) secs=$((secondstaken % 60)) printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs" - echo "Find test logs for the combination ($combination) in: $testlogfilename" - echo "Find consolidated test results in: $testresultsfilename" - echo "----------" + echo "Find test result for the combination ($combination) in: $testlogfilename" + logOutput "Consolidated test result is saved in: $aggregatedTestResult" + echo "------------------------" +} + +checkdependencies() { + if ! [ "$(command -v pcregrep)" ]; then + logOutput "Exiting. pcregrep is required to run the script." + exit 1 + fi + if ! [ "$(command -v xmlstarlet)" ]; then + logOutput "Exiting. xmlstarlet is required to run the script." + exit 1 + fi +} + +formatxml() { + xmlstarlet fo -s 2 "$1" > "$1.tmp" + mv "$1.tmp" "$1" +} + +changeconf() { + xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile" + xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile" + if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile" + then + logOutput "Exiting. Changing config property failed." + exit 1 + fi } init() { @@ -153,89 +149,24 @@ init() { then echo "" echo "Exiting. Build failed." - exit -1 - fi - starttime=$(date +"%Y-%m-%d_%H-%M-%S") - mkdir -p "$logdir" - testresultsfilename="$logdir/$starttime/Test-Results.txt" - if [[ -z "$combinations" ]]; then - combinations=( $( ls $combconfsdir/*.xml )) - fi -} - -runtests() { - parseoptions "$@" - validate - if [ -z "$starttime" ]; then - init - fi - shopt -s nullglob - for combconffile in "${combinations[@]}"; do - STARTTIME=$(date +%s) - combination=$(basename "$combconffile" .xml) - mkdir -p "$logdir/$starttime" - testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt" - printf "\nRunning the combination: %s..." "$combination" - setactiveconf - mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true - ENDTIME=$(date +%s) - summary - done -} - -begin() { - cleancombinationconfigs -} - -parseoptions() { -runactivate=0 -runtests=0 - while getopts ":c:a:t:" option; do - case "${option}" in - a) - if [[ "$runactivate" -eq "1" ]]; then - echo "-a Option is not multivalued" - exit 1 - fi - runactivate=1 - combination=$(basename "$OPTARG" .xml) - ;; - c) - runtests=1 - combination=$(basename "$OPTARG" .xml) - combinations+=("$combination") - ;; - t) - threadcount=$OPTARG - ;; - *|?|h) - if [[ -z "$combinations" ]]; then - combinations=( $( ls $combconfsdir/*.xml )) - fi - combstr="" - for combconffile in "${combinations[@]}"; do - combname=$(basename "$combconffile" .xml) - combstr="${combname}, ${combstr}" - done - combstr=${combstr:0:-2} - - echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]" - echo "" - echo "Where:" - echo " -a COMBINATION_NAME Specify the combination name which needs to be activated." - echo " Configured combinations: ${combstr}" - echo " -c COMBINATION_NAME Specify the combination name for test runs" - echo " -t THREAD_COUNT Specify the thread count" - exit 1 - ;; - esac - done - if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then - echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together" exit 1 fi - if [[ "$runactivate" -eq "1" ]]; then - setactiveconf - exit 0 - fi + starttime=$(date +"%Y-%m-%d_%H-%M-%S") + testOutputLogFolder+=$starttime + mkdir -p "$testOutputLogFolder" + aggregatedTestResult="$testOutputLogFolder/Test-Results.txt" + } + + printAggregate() { + echo :::: AGGREGATED TEST RESULT :::: + cat "$aggregatedTestResult" + fullRunEndTime=$(date +%s) + fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime)) + mins=$((fullRunTimeInSecs / 60)) + secs=$((fullRunTimeInSecs % 60)) + printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs" + } + +logOutput() { + echo -e "$outputFormatOn" "$1" "$outputFormatOff" } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md index 933f86be3e..e256fbef54 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md @@ -602,26 +602,76 @@ various test combinations, it will: 2. Run tests for all combinations 3. Summarize results across all the test combination runs. -As a pre-requisite step, fill config values for test accounts and credentials -needed for authentication in `src/test/resources/azure-auth-keys.xml.template` -and rename as `src/test/resources/azure-auth-keys.xml`. +Below are the pre-requiste steps to follow: +1. Copy -**To add a new test combination:** Templates for mandatory test combinations -for PR validation are present in `dev-support/testrun-scripts/runtests.sh`. -If a new one needs to be added, add a combination set within -`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined -and -1. Provide a new combination name -2. Update properties and values array which need to be effective for the test -combination -3. Call generateconfigs + ./src/test/resources/azure-auth-keys.xml.template + TO + ./src/test/resources/azure-auth-keys.xml + Update account names that should be used in the test run for HNS and non-HNS + combinations in the 2 properties present in the xml (account name should be + without domain part), namely + + fs.azure.hnsTestAccountName + fs.azure.nonHnsTestAccountName + azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented. + +``` +XInclude is supported, so for extra security secrets may be +kept out of the source tree then referenced through an an XInclude element: + + +``` + +2. Create account config files (one config file per account) in folder: + + ./src/test/resources/accountSettings/ + Follow the instruction in the start of the template file + + accountName_settings.xml.template + within accountSettings folder while creating account config file. + New files created in folder accountSettings is listed in .gitignore to + prevent accidental cred leaks. **To run PR validation:** Running command -* `dev-support/testrun-scripts/runtests.sh` will generate configurations for -each of the combinations defined and run tests for all the combinations. -* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific -combinations can be provided with -c option. If combinations are provided -with -c option, tests for only those combinations will be run. +* `dev-support/testrun-scripts/runtests.sh` will prompt as below: +```bash +Choose action: +[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs] +1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT +2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN +3) CLEAN_UP_OLD_TEST_CONTAINERS +#? 2 +``` +Enter 1: for setting active combination for IDE test run/single mvn test class runs. + +Enter 2: for choosing the combination to choose for mvn full test suite. + +Enter 3: For clean-up of any abruptly ending test leaving auto generated test +container on the account. + +Enter 4: To create/modify the config file that decides the account to use for specific test combination. + +Enter 5: To print the log4j paths the last test runs. + +On next prompt, current list of combinations to choose are provided. +Sample for Run_TEST action: +```bash +Enter parallel test run process count [default - 8]: 4 +Set the active test combination to run the action: +1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun +2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit +#? 1 + +Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ] + +Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth +Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4] +Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt +```` + +Provide the option for the action chosen first. **Test logs:** Test runs will create a folder within dev-support/testlogs to save the test logs. Folder name will be the test start timestamp. The mvn verify @@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as Test-Results.log in the same folder. When run for PR validation, the consolidated test results needs to be pasted into the PR comment section. -**To generate config for use in IDE:** Running command with -a (activate) option -`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update -the effective config relevant for the specific test combination. Hence the same -config files used by the mvn test runs can be used for IDE without any manual -updates needed within config file. +**To add a new test combination:** Templates for mandatory test combinations +for PR validation are present in `dev-support/testrun-scripts/runtests.sh`. +If a new one needs to be added, add a combination to +`dev-support/testrun-scripts/runtests.sh`. +(Refer to current active combinations within +`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and +`SECTION: TEST COMBINATION METHODS` in the script). -**Other command line options:** -* -a Specify the combination name which needs to be -activated. This is to be used to generate config for use in IDE. -* -c Specify the combination name for test runs. If this -config is specified, tests for only the specified combinations will run. All -combinations of tests will be running if this config is not specified. -* -t ABFS mvn tests are run in parallel mode. Tests by default -are run with 8 thread count. It can be changed by providing -t +**Test Configuration Details:** -In order to test ABFS, please add the following configuration to your -`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include -compatibility tests which require WASB credentials, in addition to the ABFS -credentials. + Note that the ABFS tests include compatibility tests which require WASB + credentials, in addition to the ABFS credentials. ```xml diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java new file mode 100644 index 0000000000..b8272319ab --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.microsoft.azure.storage.CloudStorageAccount; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAccountAndKey; + +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; +import org.apache.hadoop.fs.azurebfs.AbfsConfiguration; + +/** + * This looks like a test, but it is really a command to invoke to + * clean up containers created in other test runs. + * + */ +public class CleanupTestContainers extends AbstractAbfsIntegrationTest { + private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class); + private static final String CONTAINER_PREFIX = "abfs-testcontainer-"; + + public CleanupTestContainers() throws Exception { + } + + @org.junit.Test + public void testDeleteContainers() throws Throwable { + int count = 0; + AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration(); + String accountName = abfsConfig.getAccountName().split("\\.")[0]; + LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName()); + + String accountKey = abfsConfig.getStorageAccountKey(); + if ((accountKey == null) || (accountKey.isEmpty())) { + LOG.debug("Clean up not possible. Account ket not present in config"); + } + final StorageCredentials credentials; + credentials = new StorageCredentialsAccountAndKey( + accountName, accountKey); + CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true); + CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); + Iterable containers + = blobClient.listContainers(CONTAINER_PREFIX); + for (CloudBlobContainer container : containers) { + LOG.info("Container {} URI {}", + container.getName(), + container.getUri()); + if (container.deleteIfExists()) { + count++; + LOG.info("Current deleted test containers count - #{}", count); + } + } + LOG.info("Summary: Deleted {} test containers", count); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template new file mode 100644 index 0000000000..062b2f4bf3 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template @@ -0,0 +1,185 @@ + + + + + + + + fs.azure.abfs.account.name + ACCOUNTNAME.dfs.core.windows.net + + + fs.contract.test.fs.abfs + abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net + + + fs.contract.test.fs.abfss + abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net + + + fs.contract.test.fs.wasb + wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net + + + fs.azure.wasb.account.name + ACCOUNTNAME.blob.core.windows.net + + + fs.azure.scale.test.enabled + true + + + fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net + IS_NAMESPACE_ENABLED + + + fs.azure.test.namespace.enabled + IS_NAMESPACE_ENABLED + + + fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net + IS_NAMESPACE_ENABLED + + + fs.azure.account.hns.enabled + IS_NAMESPACE_ENABLED + + + fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net + ACCOUNT_KEY + + + fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net + ACCOUNT_KEY + + + fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net + ACCOUNT_KEY + + + fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net + ACCOUNT_KEY + + + + + fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net + https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token + + + fs.azure.account.oauth2.client.endpoint + https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token + + + fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net + org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider + + + fs.azure.account.oauth.provider.type + org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider + + + fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net + SUPERUSER_CLIENT_ID + + + fs.azure.account.oauth2.client.id + SUPERUSER_CLIENT_ID + + + fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net + SUPERUSER_CLIENT_SECRET + + + fs.azure.account.oauth2.client.secret + SUPERUSER_CLIENT_SECRET + + + + + fs.azure.enable.check.access + true + + + fs.azure.account.test.oauth2.client.id + NO_RBAC_USER_CLIENT_ID + + + fs.azure.account.test.oauth2.client.secret + NO_RBAC_USER_CLIENT_SECRET + + + fs.azure.check.access.testuser.guid + NO_RBAC_USER_OID + + + + + fs.azure.account.oauth2.contributor.client.id + CONTRIBUTOR_RBAC_USER_CLIENT_ID + + + fs.azure.account.oauth2.contributor.client.secret + CONTRIBUTOR_RBAC_USER_CLIENT_SECRET + + + + + fs.azure.account.oauth2.reader.client.id + READER_RBAC_USER_CLIENT_ID + + + fs.azure.account.oauth2.reader.client.secret + READER_RBAC_USER_CLIENT_ID + + diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template index 12dbbfab47..636816551d 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template @@ -14,162 +14,16 @@ --> - - - - - - - fs.azure.account.auth.type - SharedKey - - - - - - fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net - {ACCOUNT_ACCESS_KEY} - Account access key - - - - fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net - - org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider - OAuth token provider implementation class - - - - fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net - - https://login.microsoftonline.com/{TENANTID}/oauth2/token - Token end point, this can be found through Azure portal - - - - - fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net - - {client id} - AAD client id. - - - - fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net - - {client secret} - AAD client secret - - - - - fs.contract.test.fs.abfs - abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net - - - fs.contract.test.fs.abfss - abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net - - - - - fs.azure.wasb.account.name - {WASB_ACCOUNT_NAME}.blob.core.windows.net - - - fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net - WASB account key - - - fs.contract.test.fs.wasb - wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net - - - - - - fs.azure.account.oauth2.contributor.client.id - {Client id of SP with RBAC Storage Blob Data Contributor} - - - fs.azure.account.oauth2.contributor.client.secret - {Client secret of SP with RBAC Storage Blob Data Contributor} - - - fs.azure.account.oauth2.reader.client.id - {Client id of SP with RBAC Storage Blob Data Reader} - - - fs.azure.account.oauth2.reader.client.secret - {Client secret of SP with RBAC Storage Blob Data Reader} - - - - + - fs.azure.account.test.oauth2.client.id - {client id} - The client id(app id) for the app created on step 1 - - - - fs.azure.account.test.oauth2.client.secret - {client secret} - -The client secret(application's secret) for the app created on step 1 - - - - fs.azure.check.access.testuser.guid - {guid} - The guid fetched on step 2 - - - fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core -.windows.net - https://login.microsoftonline.com/{TENANTID}/oauth2/token - -Token end point. This can be found through Azure portal. As part of CheckAccess -test cases. The access will be tested for an FS instance created with the -above mentioned client credentials. So this configuration is necessary to -create the test FS instance. - + fs.azure.hnsTestAccountName + - - fs.azure.test.appendblob.enabled - false - If made true, tests will be running under the assumption that - append blob is enabled and the root directory and contract test root - directory will be part of the append blob directories. Should be false for - non-HNS accounts. - + fs.azure.nonHnsTestAccountName + diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml index 24ffeb5d10..6730021974 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml @@ -28,11 +28,8 @@ false - - fs.azure.test.namespace.enabled - true - - + + fs.azure.abfs.latency.track false @@ -43,9 +40,6 @@ true - - - fs.azure.user.agent.prefix @@ -59,7 +53,40 @@ STORE THE CONFIGURATION PROPERTIES WITHIN IT. TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS, THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING - IT IN PATCHES OR COMMITS. --> + IT IN PATCHES OR COMMITS. + + TEST SCRIPT RUNS: + ================ + FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE + TRIGGERED OVER SCRIPT: + ./dev-support/testrun-scripts/runtests.sh + (FROM hadoop-azure ROOT PROJECT PATH) + + TO USE THE TEST SCRIPT, + 1. COPY + ./src/test/resources/azure-auth-keys.xml.template + TO + ./src/test/resources/azure-auth-keys.xml + UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN + FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES + PRESENT IN THE XML, NAMELY + fs.azure.hnsTestAccountName and + fs.azure.nonHnsTestAccountName + (ACCOUNT NAME SHOULD BE WITHOUT DOMAIN) + + azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY + ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED. + + 2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE + PER ACCOUNT) IN FOLDER: + ./src/test/resources/accountSettings/ + + FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE + accountName_settings.xml.template + WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE. + + NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore + TO PREVENT ACCIDENTAL CRED LEAKS. -->