HADOOP-17767. ABFS: Update test scripts (#3124)

Contributed by Sneha Vijayarajan

hadoop-tools/hadoop-azure/.gitignore

@@ -1,5 +1,6 @@
 .checkstyle
 bin/
-src/test/resources/combinationConfigFiles
 src/test/resources/abfs-combination-test-configs.xml
 dev-support/testlogs
+src/test/resources/accountSettings/*
+!src/test/resources/accountSettings/accountName_settings.xml.template

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh

@@ -2,7 +2,7 @@
 
 # shellcheck disable=SC2034
 # unused variables are global in nature and used in testsupport.sh
-
+test
 set -eo pipefail
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,36 +22,154 @@ set -eo pipefail
 
 # shellcheck disable=SC1091
 . dev-support/testrun-scripts/testsupport.sh
+init
 
-begin
+resourceDir=src/test/resources/
+logdir=dev-support/testlogs/
+azureTestXml=azure-auth-keys.xml
+azureTestXmlPath=$resourceDir$azureTestXml
+processCount=8
 
-### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
+## SECTION: TEST COMBINATION METHODS
-### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
 
+runHNSOAuthTest()
+{
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
+  PROPERTIES=("fs.azure.account.auth.type")
+  VALUES=("OAuth")
+  triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-OAuth
+runHNSSharedKeyTest()
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
+{
-"fs.azure.account.auth.type")
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
-values=("{account name}.dfs.core.windows.net" "true" "OAuth")
+  PROPERTIES=("fs.azure.account.auth.type")
-generateconfigs
+  VALUES=("SharedKey")
+  triggerRun "HNS-SharedKey" "$accountName"  "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=AppendBlob-HNS-OAuth
+runNonHNSSharedKeyTest()
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
+{
-"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
-values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
+  PROPERTIES=("fs.azure.account.auth.type")
-generateconfigs
+  VALUES=("SharedKey")
+  triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=HNS-SharedKey
+runAppendBlobHNSOAuthTest()
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
+{
-values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
+  accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
-generateconfigs
+  PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
+  VALUES=("OAuth" "true")
+  triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
+}
 
-combination=NonHNS-SharedKey
+runTest=false
-properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
+cleanUpTestContainers=false
-values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
+echo 'Ensure below are complete before running script:'
-generateconfigs
+echo '1. Account specific settings file is present.'
+echo '   Copy accountName_settings.xml.template to accountName_settings.xml'
+echo '   where accountName in copied file name should be the test account name without domain'
+echo '   (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
+echo '   folder. New account settings file to be added to same folder.)'
+echo '   Follow instructions in the template to populate settings correctly for the account'
+echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
+echo '   where accountNames should be the test account names without domain'
+echo ' '
+echo ' '
+echo 'Choose action:'
+echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
+select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+do
+  case $scriptMode in
+  SET_ACTIVE_TEST_CONFIG)
+    runTest=false
+    break
+    ;;
+  RUN_TEST)
+    runTest=true
+    read -r -p "Enter parallel test run process count [default - 8]: " processCount
+    processCount=${processCount:-8}
+    break
+    ;;
+  CLEAN_UP_OLD_TEST_CONTAINERS)
+    runTest=false
+    cleanUpTestContainers=true
+    break
+    ;;
+  SET_OR_CHANGE_TEST_ACCOUNT)
+    runTest=false
+    cleanUpTestContainers=false
+    accountSettingsFile="src/test/resources/azure-auth-keys.xml"
+    if [[ ! -f "$accountSettingsFile" ]];
+    then
+      logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
+      cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
+    fi
 
+    vi $accountSettingsFile
+    exit 0
+    break
+    ;;
+  PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
+    runTest=false
+    cleanUpTestContainers=false
+    logFilePaths=/tmp/logPaths
+    find target/ -name "*output.txt" > $logFilePaths
+    logOutput "$(cat $logFilePaths)"
+    rm $logFilePaths
+    exit 0
+    break
+    ;;
+  *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
 
-### DO NOT EDIT THE LINES BELOW.
+## SECTION: COMBINATION DEFINITIONS AND TRIGGER
 
-runtests "$@"
+echo ' '
+echo 'Set the active test combination to run the action:'
+select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
+do
+   case $combo in
+      HNS-OAuth)
+         runHNSOAuthTest
+         break
+         ;;
+      HNS-SharedKey)
+         runHNSSharedKeyTest
+         break
+         ;;
+      nonHNS-SharedKey)
+         runNonHNSSharedKeyTest
+         break
+         ;;
+       AppendBlob-HNS-OAuth)
+         runAppendBlobHNSOAuthTest
+         break
+         ;;
+      AllCombinationsTestRun)
+        if [ $runTest == false ]
+        then
+          logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
+          break
+        fi
+         runHNSOAuthTest
+         runHNSSharedKeyTest
+         runNonHNSSharedKeyTest
+         runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
+         break
+         ;;
+      Quit)
+         exit 0
+         ;;
+      *) logOutput "ERROR: Invalid selection"
+      ;;
+   esac
+done
+
+if [ $runTest == true ]
+then
+  printAggregate
+fi

hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh

@@ -15,117 +15,88 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-testresourcesdir=src/test/resources
+resourceDir=src/test/resources/
-combconfsdir=$testresourcesdir/combinationConfigFiles
+accountSettingsFolderName=accountSettings
-combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
+combtestfile=$resourceDir
+combtestfile+=abfs-combination-test-configs.xml
+logdir=dev-support/testlogs/
 
-logdir=dev-support/testlogs
 testresultsregex="Results:(\n|.)*?Tests run:"
-testresultsfilename=
+accountConfigFileSuffix="_settings.xml"
-starttime=
+testOutputLogFolder=$logdir
-threadcount=
+testlogfilename=combinationTestLogFile
-defaultthreadcount=8
 
-properties=
+fullRunStartTime=$(date +%s)
-values=
+STARTTIME=$(date +%s)
+ENDTIME=$(date +%s)
 
-validate() {
+outputFormatOn="\033[0;95m"
-  if [ -z "$threadcount" ] ; then
+outputFormatOff="\033[0m"
-    threadcount=$defaultthreadcount
+
-  fi
+triggerRun()
-  numberegex='^[0-9]+$'
+{
-  if ! [[ $threadcount =~ $numberegex ]] ; then
+  echo ' '
-    echo "Exiting. The script param (threadcount) should be a number"
+  combination=$1
-    exit -1
+  accountName=$2
-  fi
+  runTest=$3
-  if [ -z "$combination" ]; then
+  processcount=$4
-   echo "Exiting. combination cannot be empty"
+  cleanUpTestContainers=$5
-   exit -1
+
-  fi
+  if [ -z "$accountName" ]; then
-  propertiessize=${#properties[@]}
+    logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
-  valuessize=${#values[@]}
+    exit 1;
-  if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
-    echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
-    exit -1
-  fi
-
-  for filename in "${combinations[@]}"; do
-    if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
-      echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
-      exit -1
-    fi
-  done
-}
-
-checkdependencies() {
-  if ! [ "$(command -v pcregrep)" ]; then
-    echo "Exiting. pcregrep is required to run the script."
-    exit -1
-  fi
-  if ! [ "$(command -v xmlstarlet)" ]; then
-    echo "Exiting. xmlstarlet is required to run the script."
-    exit -1
-  fi
-}
-
-cleancombinationconfigs() {
-  rm -rf $combconfsdir
-  mkdir -p $combconfsdir
-}
-
-generateconfigs() {
-  combconffile="$combconfsdir/$combination.xml"
-  rm -rf "$combconffile"
-  cat > "$combconffile" << ENDOFFILE
-<configuration>
-
-</configuration>
-ENDOFFILE
-
-  propertiessize=${#properties[@]}
-  valuessize=${#values[@]}
-  if [ "$propertiessize" -ne "$valuessize" ]; then
-    echo "Exiting. Number of properties and values differ for $combination"
-    exit -1
-  fi
-  for ((i = 0; i < propertiessize; i++)); do
-    key=${properties[$i]}
-    val=${values[$i]}
-    changeconf "$key" "$val"
-  done
-  formatxml "$combconffile"
-}
-
-formatxml() {
-  xmlstarlet fo -s 2 "$1" > "$1.tmp"
-  mv "$1.tmp" "$1"
-}
-
-setactiveconf() {
-  if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
-    echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
-    exit -1
   fi
+  accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
   rm -rf $combtestfile
   cat > $combtestfile << ENDOFFILE
 <configuration>
 
 </configuration>
 ENDOFFILE
+  propertiessize=${#PROPERTIES[@]}
+  valuessize=${#VALUES[@]}
+  if [ "$propertiessize" -ne "$valuessize" ]; then
+    logOutput "Exiting. Number of properties and values differ for $combination"
+    exit 1
+  fi
+  for ((i = 0; i < propertiessize; i++)); do
+    key=${PROPERTIES[$i]}
+    val=${VALUES[$i]}
+    echo "Combination specific property setting: [ key=$key , value=$val ]"
+    changeconf "$key" "$val"
+  done
+  formatxml "$combtestfile"
   xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
-  xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
+  xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
   xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
   formatxml $combtestfile
-}
+  echo ' '
+  echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
+  testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
+  touch "$testlogfilename"
 
-changeconf() {
+  if [ "$runTest" == true ]
-  xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
-  xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
-  if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
   then
-    echo "Exiting. Changing config property failed."
+    STARTTIME=$(date +%s)
-    exit -1
+    echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
+    logOutput "Test run report can be seen in $testlogfilename"
+    mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
+    ENDTIME=$(date +%s)
+    summary
   fi
+
+  if [ "$cleanUpTestContainers" == true ]
+  then
+    mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
+    if grep -q "There are test failures" "$testlogfilename";
+    then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
+    pcregrep -M "$testresultsregex" "$testlogfilename"
+    exit 0
+    fi
+
+    logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
+  fi
+
 }
 
 summary() {
@@ -134,17 +105,42 @@ summary() {
     echo "$combination"
     echo "========================"
     pcregrep -M "$testresultsregex" "$testlogfilename"
-  } >> "$testresultsfilename"
+  } >> "$aggregatedTestResult"
   printf "\n----- Test results -----\n"
   pcregrep -M "$testresultsregex" "$testlogfilename"
-
   secondstaken=$((ENDTIME - STARTTIME))
   mins=$((secondstaken / 60))
   secs=$((secondstaken % 60))
   printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
-  echo "Find test logs for the combination ($combination) in: $testlogfilename"
+  echo "Find test result for the combination ($combination) in: $testlogfilename"
-  echo "Find consolidated test results in: $testresultsfilename"
+  logOutput "Consolidated test result is saved in: $aggregatedTestResult"
-  echo "----------"
+  echo "------------------------"
+}
+
+checkdependencies() {
+  if ! [ "$(command -v pcregrep)" ]; then
+    logOutput "Exiting. pcregrep is required to run the script."
+    exit 1
+  fi
+  if ! [ "$(command -v xmlstarlet)" ]; then
+    logOutput "Exiting. xmlstarlet is required to run the script."
+    exit 1
+  fi
+}
+
+formatxml() {
+  xmlstarlet fo -s 2 "$1" > "$1.tmp"
+  mv "$1.tmp" "$1"
+}
+
+changeconf() {
+  xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
+  xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
+  if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
+  then
+    logOutput "Exiting. Changing config property failed."
+    exit 1
+  fi
 }
 
 init() {
@@ -153,89 +149,24 @@ init() {
   then
     echo ""
     echo "Exiting. Build failed."
-    exit -1
-  fi
-  starttime=$(date +"%Y-%m-%d_%H-%M-%S")
-  mkdir -p "$logdir"
-  testresultsfilename="$logdir/$starttime/Test-Results.txt"
-  if [[ -z "$combinations" ]]; then
-    combinations=( $( ls $combconfsdir/*.xml ))
-  fi
-}
-
-runtests() {
-  parseoptions "$@"
-  validate
-  if [ -z "$starttime" ]; then
-    init
-  fi
-  shopt -s nullglob
-  for combconffile in "${combinations[@]}"; do
-    STARTTIME=$(date +%s)
-    combination=$(basename "$combconffile" .xml)
-    mkdir -p "$logdir/$starttime"
-    testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
-    printf "\nRunning the combination: %s..." "$combination"
-    setactiveconf
-    mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
-    ENDTIME=$(date +%s)
-    summary
-  done
-}
-
-begin() {
-  cleancombinationconfigs
-}
-
-parseoptions() {
-runactivate=0
-runtests=0
-  while getopts ":c:a:t:" option; do
-    case "${option}" in
-      a)
-        if [[ "$runactivate" -eq "1" ]]; then
-          echo "-a Option is not multivalued"
-          exit 1
-        fi
-        runactivate=1
-        combination=$(basename "$OPTARG" .xml)
-        ;;
-      c)
-        runtests=1
-        combination=$(basename "$OPTARG" .xml)
-        combinations+=("$combination")
-        ;;
-      t)
-        threadcount=$OPTARG
-        ;;
-      *|?|h)
-        if [[ -z "$combinations" ]]; then
-          combinations=( $( ls $combconfsdir/*.xml ))
-        fi
-      combstr=""
-        for combconffile in "${combinations[@]}"; do
-          combname=$(basename "$combconffile" .xml)
-          combstr="${combname}, ${combstr}"
-        done
-        combstr=${combstr:0:-2}
-
-        echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
-        echo ""
-        echo "Where:"
-        echo "  -a COMBINATION_NAME   Specify the combination name which needs to be activated."
-        echo "                        Configured combinations: ${combstr}"
-        echo "  -c COMBINATION_NAME   Specify the combination name for test runs"
-        echo "  -t THREAD_COUNT       Specify the thread count"
-        exit 1
-        ;;
-    esac
-  done
-  if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
-    echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
     exit 1
   fi
-  if [[ "$runactivate" -eq "1" ]]; then
+  starttime=$(date +"%Y-%m-%d_%H-%M-%S")
-        setactiveconf
+  testOutputLogFolder+=$starttime
-        exit 0
+  mkdir -p "$testOutputLogFolder"
-  fi
+  aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
+ }
+
+ printAggregate() {
+   echo  :::: AGGREGATED TEST RESULT ::::
+   cat "$aggregatedTestResult"
+  fullRunEndTime=$(date +%s)
+  fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
+  mins=$((fullRunTimeInSecs / 60))
+  secs=$((fullRunTimeInSecs % 60))
+  printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
+ }
+
+logOutput() {
+  echo -e "$outputFormatOn" "$1" "$outputFormatOff"
 }

hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md

@@ -602,26 +602,76 @@ various test combinations, it will:
 2. Run tests for all combinations
 3. Summarize results across all the test combination runs.
 
-As a pre-requisite step, fill config values for test accounts and credentials
+Below are the pre-requiste steps to follow:
-needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
+1. Copy
-and rename as `src/test/resources/azure-auth-keys.xml`.
 
-**To add a new test combination:** Templates for mandatory test combinations
+        ./src/test/resources/azure-auth-keys.xml.template
-for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
+        TO
-If a new one needs to be added, add a combination set within
+        ./src/test/resources/azure-auth-keys.xml
-`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
+  Update account names that should be used in the test run for HNS and non-HNS
-and
+  combinations in the 2 properties present in the xml (account name should be
-1. Provide a new combination name
+  without domain part), namely
-2. Update properties and values array which need to be effective for the test
+
-combination
+    fs.azure.hnsTestAccountName
-3. Call generateconfigs
+    fs.azure.nonHnsTestAccountName
+  azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
+
+```
+XInclude is supported, so for extra security secrets may be
+kept out of the source tree then referenced through an an XInclude element:
+
+      <include xmlns="http://www.w3.org/2001/XInclude"
+        href="/users/self/.secrets/auth-keys.xml" />
+```
+
+2. Create account config files (one config file per account) in folder:
+
+        ./src/test/resources/accountSettings/
+   Follow the instruction in the start of the template file
+
+        accountName_settings.xml.template
+   within accountSettings folder while creating account config file.
+   New files created in folder accountSettings is listed in .gitignore to
+   prevent accidental cred leaks.
 
 **To run PR validation:** Running command
-* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
+* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
-each of the combinations defined and run tests for all the combinations.
+```bash
-* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
+Choose action:
-combinations can be provided with -c option. If combinations are provided
+[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
-with -c option, tests for only those combinations will be run.
+1) SET_ACTIVE_TEST_CONFIG               4) SET_OR_CHANGE_TEST_ACCOUNT
+2) RUN_TEST                             5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
+3) CLEAN_UP_OLD_TEST_CONTAINERS
+#? 2
+```
+Enter 1: for setting active combination for IDE test run/single mvn test class runs.
+
+Enter 2: for choosing the combination to choose for mvn full test suite.
+
+Enter 3: For clean-up of any abruptly ending test leaving auto generated test
+container on the account.
+
+Enter 4: To create/modify the config file that decides the account to use for specific test combination.
+
+Enter 5: To print the log4j paths the last test runs.
+
+On next prompt, current list of combinations to choose are provided.
+Sample for Run_TEST action:
+```bash
+Enter parallel test run process count [default - 8]: 4
+Set the active test combination to run the action:
+1) HNS-OAuth               3) nonHNS-SharedKey        5) AllCombinationsTestRun
+2) HNS-SharedKey           4) AppendBlob-HNS-OAuth    6) Quit
+#? 1
+
+Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
+
+Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
+Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
+Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
+````
+
+Provide the option for the action chosen first.
 
 **Test logs:** Test runs will create a folder within dev-support/testlogs to
 save the test logs. Folder name will be the test start timestamp. The mvn verify
@@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
 Test-Results.log in the same folder. When run for PR validation, the
 consolidated test results needs to be pasted into the PR comment section.
 
-**To generate config for use in IDE:** Running command with -a (activate) option
+**To add a new test combination:** Templates for mandatory test combinations
-`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
+for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
-the effective config relevant for the specific test combination. Hence the same
+If a new one needs to be added, add a combination to
-config files used by the mvn test runs can be used for IDE without any manual
+`dev-support/testrun-scripts/runtests.sh`.
-updates needed within config file.
+(Refer to current active combinations within
+`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
+`SECTION: TEST COMBINATION METHODS` in the script).
 
-**Other command line options:**
+**Test Configuration Details:**
-* -a <COMBINATION_NAME> Specify the combination name which needs to be
-activated. This is to be used to generate config for use in IDE.
-* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
-config is specified, tests for only the specified combinations will run. All
-combinations of tests will be running if this config is not specified.
-* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
-are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
 
-In order to test ABFS, please add the following configuration to your
+ Note that the ABFS tests include compatibility tests which require WASB
-`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
+ credentials, in addition to the ABFS credentials.
-compatibility tests which require WASB credentials, in addition to the ABFS
-credentials.
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
+
+import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
+import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
+
+/**
+ * This looks like a test, but it is really a command to invoke to
+ * clean up containers created in other test runs.
+ *
+ */
+public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
+  private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
+  private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
+
+  public CleanupTestContainers() throws Exception {
+  }
+
+  @org.junit.Test
+  public void testDeleteContainers() throws Throwable {
+    int count = 0;
+    AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
+    String accountName = abfsConfig.getAccountName().split("\\.")[0];
+    LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
+
+    String accountKey = abfsConfig.getStorageAccountKey();
+    if ((accountKey == null) || (accountKey.isEmpty())) {
+      LOG.debug("Clean up not possible. Account ket not present in config");
+    }
+    final StorageCredentials credentials;
+    credentials = new StorageCredentialsAccountAndKey(
+        accountName, accountKey);
+    CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
+    CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
+    Iterable<CloudBlobContainer> containers
+        = blobClient.listContainers(CONTAINER_PREFIX);
+    for (CloudBlobContainer container : containers) {
+      LOG.info("Container {} URI {}",
+          container.getName(),
+          container.getUri());
+      if (container.deleteIfExists()) {
+        count++;
+        LOG.info("Current deleted test containers count - #{}", count);
+      }
+    }
+    LOG.info("Summary: Deleted {} test containers", count);
+  }
+}

hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template

@@ -0,0 +1,185 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  NOTE:
+  Each config file will hold configs for one storage account.
+  ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
+  Below template collects information needed for both Auth settings on the account.
+
+  OAuth test mode works with token provider type - ClientCredsTokenProvider
+  For specific details on test config or pre-requisite, refer to
+  https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
+
+  Pre-requisite before configuring:
+  1. An ABFS account
+  2. A container within account
+  3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
+  4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
+  5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
+  6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
+
+  Replace below strings with actual value:
+  ## ACCOUNT SETTINGS ##
+    1. ACCOUNTNAME -> to account name without domain
+    2. IS_NAMESPACE_ENABLED -> to true or false
+    3. CONTAINER_NAME -> name of an existing container
+    4. ACCOUNT_KEY -> Account access key
+  ## IDENTITY SETTINGS ##
+    5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
+    6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
+    7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
+
+    8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
+    9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
+    10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
+
+    11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
+    12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
+
+    13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
+    14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
+-->
+
+<configuration>
+
+  <!-- ACCOUNT SETTINGS -->
+  <property>
+    <name>fs.azure.abfs.account.name</name>
+    <value>ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.abfs</name>
+    <value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.abfss</name>
+    <value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.wasb.account.name</name>
+    <value>ACCOUNTNAME.blob.core.windows.net</value>
+  </property>
+  <property>
+    <name>fs.azure.scale.test.enabled</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.test.namespace.enabled</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.hns.enabled</name>
+    <value>IS_NAMESPACE_ENABLED</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+  <property>
+    <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
+    <value>ACCOUNT_KEY</value>
+  </property>
+
+  <!-- SUPERUSER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.endpoint</name>
+    <value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth.provider.type</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>SUPERUSER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.id</name>
+    <value>SUPERUSER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
+    <value>SUPERUSER_CLIENT_SECRET</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.client.secret</name>
+    <value>SUPERUSER_CLIENT_SECRET</value>
+  </property>
+
+  <!-- NO RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.enable.check.access</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>fs.azure.account.test.oauth2.client.id</name>
+    <value>NO_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.test.oauth2.client.secret</name>
+    <value>NO_RBAC_USER_CLIENT_SECRET</value>
+  </property>
+  <property>
+    <name>fs.azure.check.access.testuser.guid</name>
+    <value>NO_RBAC_USER_OID</value>
+  </property>
+
+  <!-- CONTRIBUTOR RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.id</name>
+    <value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.contributor.client.secret</name>
+    <value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
+  </property>
+
+  <!-- READER RBAC USER SETTINGS -->
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.id</name>
+    <value>READER_RBAC_USER_CLIENT_ID</value>
+  </property>
+  <property>
+    <name>fs.azure.account.oauth2.reader.client.secret</name>
+    <value>READER_RBAC_USER_CLIENT_ID</value>
+  </property>
+</configuration>

hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template

@@ -14,162 +14,16 @@
 -->
 
 <configuration>
-
+  <!-- Account names without domain -->
-  <!--====================== IMPORTANT!! ========================-->
-  <!--
-      1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
-      2. UPDATE runtests.sh
-         A.  "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
-             ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
-             SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
-         B.  PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
-             DIFFERENT VARIANTS OF CONFIGS.
-      3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
-      4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
-  -->
-
-  <!--=============== Auth type ===============-->
-  <property>
-    <name>fs.azure.account.auth.type</name>
-    <value>SharedKey</value>
-  </property>
-
-  <!--=============== Auth related accounts ===============-->
-  <!-- This set of configs needs to be provided for all the accounts with which
-   the tests needs to be ran. -->
-  <property>
-    <name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
-    <value>{ACCOUNT_ACCESS_KEY}</value>
-    <description>Account access key</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
-    <description>OAuth token provider implementation class</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>Token end point, this can be found through Azure portal
-    </description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>{client id}</value>
-    <description>AAD client id.</description>
-  </property>
-  <property>
-    <name>
-      fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
-    </name>
-    <value>{client secret}</value>
-    <description>AAD client secret</description>
-  </property>
-
-  <!--=============== Configs for hadoop contract tests for ABFS ===============-->
-  <property>
-    <name>fs.contract.test.fs.abfs</name>
-    <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.contract.test.fs.abfss</name>
-    <value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
-  </property>
-
-  <!--=============== WASB Configs ===============-->
-  <property>
-    <name>fs.azure.wasb.account.name</name>
-    <value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
-  </property>
-  <property>
-    <name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
-    <value>WASB account key</value>
-  </property>
-  <property>
-    <name>fs.contract.test.fs.wasb</name>
-    <value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
-    </value>
-  </property>
-
-  <!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.id</name>
-    <value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.secret</name>
-    <value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.id</name>
-    <value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.secret</name>
-    <value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
-  </property>
-
-<!--===========================   FOR CheckAccess =========================-->
-<!-- To run ABFS CheckAccess tests, you must register an app, with no role
- assignments, and set the configuration discussed below:
-
-    1) Register a new app with no RBAC
-    2) As part of the test configs you need to provide the guid for the above
-created app. Please follow the below steps to fetch the guid.
-      a) Get an access token with the above created app. Please refer the
- following documentation for the same. https://docs.microsoft
-.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
-      b) Decode the token fetched with the above step. You may use https
-://jwt.ms/ to decode the token
-      d) The oid field in the decoded string is the guid.
-    3) Set the following configurations:
--->
 
   <property>
-    <name>fs.azure.account.test.oauth2.client.id</name>
+    <name>fs.azure.hnsTestAccountName</name>
-    <value>{client id}</value>
+    <value></value>
-    <description>The client id(app id) for the app created on step 1
-    </description>
-  </property>
-  <property>
-    <name>fs.azure.account.test.oauth2.client.secret</name>
-    <value>{client secret}</value>
-    <description>
-The client secret(application's secret) for the app created on step 1
-    </description>
-  </property>
-  <property>
-    <name>fs.azure.check.access.testuser.guid</name>
-    <value>{guid}</value>
-    <description>The guid fetched on step 2</description>
-  </property>
-  <property>
-    <name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
-.windows.net</name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>
-Token end point. This can be found through Azure portal. As part of CheckAccess
-test cases. The access will be tested for an FS instance created with the
-above mentioned client credentials. So this configuration is necessary to
-create the test FS instance.
-    </description>
   </property>
 
-  <!--========== Append blob related configs ===========-->
   <property>
-    <name>fs.azure.test.appendblob.enabled</name>
+    <name>fs.azure.nonHnsTestAccountName</name>
-    <value>false</value>
+    <value></value>
-    <description>If made true, tests will be running under the assumption that
-      append blob is enabled and the root directory and contract test root
-      directory will be part of the append blob directories. Should be false for
-      non-HNS accounts.
-    </description>
   </property>
 
 </configuration>

hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml

@@ -28,11 +28,8 @@
     <value>false</value>
   </property>
 
-  <property>
+  <!--====================  ABFS CONFIGURATION ====================-->
-    <name>fs.azure.test.namespace.enabled</name>
+  <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
-    <value>true</value>
-  </property>
-
   <property>
     <name>fs.azure.abfs.latency.track</name>
     <value>false</value>
@@ -43,9 +40,6 @@
     <value>true</value>
   </property>
 
-  <!--====================  ABFS CONFIGURATION ====================-->
-  <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
-
   <!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
   <property>
     <name>fs.azure.user.agent.prefix</name>
@@ -59,7 +53,40 @@
       STORE THE CONFIGURATION PROPERTIES WITHIN IT.
       TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
       THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
-      IT IN PATCHES OR COMMITS. -->
+      IT IN PATCHES OR COMMITS.
+
+      TEST SCRIPT RUNS:
+      ================
+      FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
+      TRIGGERED OVER SCRIPT:
+      ./dev-support/testrun-scripts/runtests.sh
+      (FROM hadoop-azure ROOT PROJECT PATH)
+
+      TO USE THE TEST SCRIPT,
+      1. COPY
+         ./src/test/resources/azure-auth-keys.xml.template
+         TO
+         ./src/test/resources/azure-auth-keys.xml
+         UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
+         FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
+         PRESENT IN THE XML, NAMELY
+         fs.azure.hnsTestAccountName and
+         fs.azure.nonHnsTestAccountName
+         (ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
+
+         azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
+         ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
+
+      2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
+         PER ACCOUNT) IN FOLDER:
+         ./src/test/resources/accountSettings/
+
+         FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
+         accountName_settings.xml.template
+         WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
+
+         NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
+         TO PREVENT ACCIDENTAL CRED LEAKS. -->
   <!--=============================================================-->
 
   <include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">