big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17767. ABFS: Update test scripts (#3124)

Browse Source 
Contributed by Sneha Vijayarajan
This commit is contained in:
Sneha Vijayarajan 2022-10-20 22:37:04 +05:30 committed by GitHub
parent 9adf0ca089
commit a996d889ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 634 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-tools/hadoop-azure/.gitignore vendored
View File

@ -1,5 +1,6 @@
.checkstyle
bin/
src/test/resources/combinationConfigFiles
src/test/resources/abfs-combination-test-configs.xml
dev-support/testlogs
src/test/resources/accountSettings/*
!src/test/resources/accountSettings/accountName_settings.xml.template

166
hadoop-tools/hadoop-azure/dev-support/testrun-scripts/runtests.sh
View File

@ -2,7 +2,7 @@
# shellcheck disable=SC2034
# unused variables are global in nature and used in testsupport.sh
test
set -eo pipefail
# Licensed to the Apache Software Foundation (ASF) under one or more
@ -22,36 +22,154 @@ set -eo pipefail
# shellcheck disable=SC1091
. dev-support/testrun-scripts/testsupport.sh
init
begin
resourceDir=src/test/resources/
logdir=dev-support/testlogs/
azureTestXml=azure-auth-keys.xml
azureTestXmlPath=$resourceDir$azureTestXml
processCount=8
### ADD THE TEST COMBINATIONS BELOW. DO NOT EDIT THE ABOVE LINES.
### THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep.
## SECTION: TEST COMBINATION METHODS
runHNSOAuthTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("OAuth")
triggerRun "HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}
combination=HNS-OAuth
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
"fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "true" "OAuth")
generateconfigs
runHNSSharedKeyTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("SharedKey")
triggerRun "HNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}
combination=AppendBlob-HNS-OAuth
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled"
"fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
values=("{account name}.dfs.core.windows.net" "true" "OAuth" "true")
generateconfigs
runNonHNSSharedKeyTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.nonHnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type")
VALUES=("SharedKey")
triggerRun "NonHNS-SharedKey" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}
combination=HNS-SharedKey
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "true" "SharedKey")
generateconfigs
runAppendBlobHNSOAuthTest()
{
accountName=$(xmlstarlet sel -t -v '//property[name = "fs.azure.hnsTestAccountName"]/value' -n $azureTestXmlPath)
PROPERTIES=("fs.azure.account.auth.type" "fs.azure.test.appendblob.enabled")
VALUES=("OAuth" "true")
triggerRun "AppendBlob-HNS-OAuth" "$accountName" "$runTest" $processCount "$cleanUpTestContainers"
}
combination=NonHNS-SharedKey
properties=("fs.azure.abfs.account.name" "fs.azure.test.namespace.enabled" "fs.azure.account.auth.type")
values=("{account name}.dfs.core.windows.net" "false" "SharedKey")
generateconfigs
runTest=false
cleanUpTestContainers=false
echo 'Ensure below are complete before running script:'
echo '1. Account specific settings file is present.'
echo ' Copy accountName_settings.xml.template to accountName_settings.xml'
echo ' where accountName in copied file name should be the test account name without domain'
echo ' (accountName_settings.xml.template is present in src/test/resources/accountName_settings'
echo ' folder. New account settings file to be added to same folder.)'
echo ' Follow instructions in the template to populate settings correctly for the account'
echo '2. In azure-auth-keys.xml, update properties fs.azure.hnsTestAccountName and fs.azure.nonHnsTestAccountName'
echo ' where accountNames should be the test account names without domain'
echo ' '
echo ' '
echo 'Choose action:'
echo '[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]'
select scriptMode in SET_ACTIVE_TEST_CONFIG RUN_TEST CLEAN_UP_OLD_TEST_CONTAINERS SET_OR_CHANGE_TEST_ACCOUNT PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
do
case $scriptMode in
SET_ACTIVE_TEST_CONFIG)
runTest=false
break
;;
RUN_TEST)
runTest=true
read -r -p "Enter parallel test run process count [default - 8]: " processCount
processCount=${processCount:-8}
break
;;
CLEAN_UP_OLD_TEST_CONTAINERS)
runTest=false
cleanUpTestContainers=true
break
;;
SET_OR_CHANGE_TEST_ACCOUNT)
runTest=false
cleanUpTestContainers=false
accountSettingsFile="src/test/resources/azure-auth-keys.xml"
if [[ ! -f "$accountSettingsFile" ]];
then
logOutput "No settings present. Creating new settings file ($accountSettingsFile) from template"
cp src/test/resources/azure-auth-keys.xml.template $accountSettingsFile
fi
vi $accountSettingsFile
exit 0
break
;;
PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN)
runTest=false
cleanUpTestContainers=false
logFilePaths=/tmp/logPaths
find target/ -name "*output.txt" > $logFilePaths
logOutput "$(cat $logFilePaths)"
rm $logFilePaths
exit 0
break
;;
*) logOutput "ERROR: Invalid selection"
;;
esac
done
### DO NOT EDIT THE LINES BELOW.
## SECTION: COMBINATION DEFINITIONS AND TRIGGER
runtests "$@"
echo ' '
echo 'Set the active test combination to run the action:'
select combo in HNS-OAuth HNS-SharedKey nonHNS-SharedKey AppendBlob-HNS-OAuth AllCombinationsTestRun Quit
do
case $combo in
HNS-OAuth)
runHNSOAuthTest
break
;;
HNS-SharedKey)
runHNSSharedKeyTest
break
;;
nonHNS-SharedKey)
runNonHNSSharedKeyTest
break
;;
AppendBlob-HNS-OAuth)
runAppendBlobHNSOAuthTest
break
;;
AllCombinationsTestRun)
if [ $runTest == false ]
then
logOutput "ERROR: Invalid selection for SET_ACTIVE_TEST_CONFIG. This is applicable only for RUN_TEST."
break
fi
runHNSOAuthTest
runHNSSharedKeyTest
runNonHNSSharedKeyTest
runAppendBlobHNSOAuthTest ## Keep this as the last run scenario always
break
;;
Quit)
exit 0
;;
*) logOutput "ERROR: Invalid selection"
;;
esac
done
if [ $runTest == true ]
then
printAggregate
fi

295
hadoop-tools/hadoop-azure/dev-support/testrun-scripts/testsupport.sh
View File

@ -15,117 +15,88 @@
# See the License for the specific language governing permissions and
# limitations under the License.
testresourcesdir=src/test/resources
combconfsdir=$testresourcesdir/combinationConfigFiles
combtestfile=$testresourcesdir/abfs-combination-test-configs.xml
resourceDir=src/test/resources/
accountSettingsFolderName=accountSettings
combtestfile=$resourceDir
combtestfile+=abfs-combination-test-configs.xml
logdir=dev-support/testlogs/
logdir=dev-support/testlogs
testresultsregex="Results:(\n|.)*?Tests run:"
testresultsfilename=
starttime=
threadcount=
defaultthreadcount=8
accountConfigFileSuffix="_settings.xml"
testOutputLogFolder=$logdir
testlogfilename=combinationTestLogFile
properties=
values=
fullRunStartTime=$(date +%s)
STARTTIME=$(date +%s)
ENDTIME=$(date +%s)
validate() {
if [ -z "$threadcount" ] ; then
threadcount=$defaultthreadcount
fi
numberegex='^[0-9]+$'
if ! [[ $threadcount =~ $numberegex ]] ; then
echo "Exiting. The script param (threadcount) should be a number"
exit -1
fi
if [ -z "$combination" ]; then
echo "Exiting. combination cannot be empty"
exit -1
fi
propertiessize=${#properties[@]}
valuessize=${#values[@]}
if [ "$propertiessize" -lt 1 ] || [ "$valuessize" -lt 1 ] || [ "$propertiessize" -ne "$valuessize" ]; then
echo "Exiting. Both properties and values arrays has to be populated and of same size. Please check for combination $combination"
exit -1
fi
for filename in "${combinations[@]}"; do
if [[ ! -f "$combconfsdir/$filename.xml" ]]; then
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
exit -1
fi
done
}
checkdependencies() {
if ! [ "$(command -v pcregrep)" ]; then
echo "Exiting. pcregrep is required to run the script."
exit -1
fi
if ! [ "$(command -v xmlstarlet)" ]; then
echo "Exiting. xmlstarlet is required to run the script."
exit -1
fi
}
cleancombinationconfigs() {
rm -rf $combconfsdir
mkdir -p $combconfsdir
}
generateconfigs() {
combconffile="$combconfsdir/$combination.xml"
rm -rf "$combconffile"
cat > "$combconffile" << ENDOFFILE
<configuration>
</configuration>
ENDOFFILE
propertiessize=${#properties[@]}
valuessize=${#values[@]}
if [ "$propertiessize" -ne "$valuessize" ]; then
echo "Exiting. Number of properties and values differ for $combination"
exit -1
fi
for ((i = 0; i < propertiessize; i++)); do
key=${properties[$i]}
val=${values[$i]}
changeconf "$key" "$val"
done
formatxml "$combconffile"
}
formatxml() {
xmlstarlet fo -s 2 "$1" > "$1.tmp"
mv "$1.tmp" "$1"
}
setactiveconf() {
if [[ ! -f "$combconfsdir/$combination.xml" ]]; then
echo "Exiting. Combination config file ($combconfsdir/$combination.xml) does not exist."
exit -1
outputFormatOn="\033[0;95m"
outputFormatOff="\033[0m"
triggerRun()
{
echo ' '
combination=$1
accountName=$2
runTest=$3
processcount=$4
cleanUpTestContainers=$5
if [ -z "$accountName" ]; then
logOutput "ERROR: Test account not configured. Re-run the script and choose SET_OR_CHANGE_TEST_ACCOUNT to configure the test account."
exit 1;
fi
accountConfigFile=$accountSettingsFolderName/$accountName$accountConfigFileSuffix
rm -rf $combtestfile
cat > $combtestfile << ENDOFFILE
<configuration>
</configuration>
ENDOFFILE
propertiessize=${#PROPERTIES[@]}
valuessize=${#VALUES[@]}
if [ "$propertiessize" -ne "$valuessize" ]; then
logOutput "Exiting. Number of properties and values differ for $combination"
exit 1
fi
for ((i = 0; i < propertiessize; i++)); do
key=${PROPERTIES[$i]}
val=${VALUES[$i]}
echo "Combination specific property setting: [ key=$key , value=$val ]"
changeconf "$key" "$val"
done
formatxml "$combtestfile"
xmlstarlet ed -P -L -s /configuration -t elem -n include -v "" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "combinationConfigFiles/$combination.xml" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n href -v "$accountConfigFile" $combtestfile
xmlstarlet ed -P -L -i /configuration/include -t attr -n xmlns -v "http://www.w3.org/2001/XInclude" $combtestfile
formatxml $combtestfile
}
echo ' '
echo "Activated [$combtestfile] - for account: $accountName for combination $combination"
testlogfilename="$testOutputLogFolder/Test-Logs-$combination.txt"
touch "$testlogfilename"
changeconf() {
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combconffile"
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combconffile"
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combconffile"
if [ "$runTest" == true ]
then
echo "Exiting. Changing config property failed."
exit -1
STARTTIME=$(date +%s)
echo "Running test for combination $combination on account $accountName [ProcessCount=$processcount]"
logOutput "Test run report can be seen in $testlogfilename"
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount="$processcount" verify >> "$testlogfilename" || true
ENDTIME=$(date +%s)
summary
fi
if [ "$cleanUpTestContainers" == true ]
then
mvn test -Dtest=org.apache.hadoop.fs.azurebfs.utils.CleanupTestContainers >> "$testlogfilename" || true
if grep -q "There are test failures" "$testlogfilename";
then logOutput "ERROR: All test containers could not be deleted. Detailed error cause in $testlogfilename"
pcregrep -M "$testresultsregex" "$testlogfilename"
exit 0
fi
logOutput "Delete test containers - complete. Test run logs in - $testlogfilename"
fi
}
summary() {
@ -134,17 +105,42 @@ summary() {
echo "$combination"
echo "========================"
pcregrep -M "$testresultsregex" "$testlogfilename"
} >> "$testresultsfilename"
} >> "$aggregatedTestResult"
printf "\n----- Test results -----\n"
pcregrep -M "$testresultsregex" "$testlogfilename"
secondstaken=$((ENDTIME - STARTTIME))
mins=$((secondstaken / 60))
secs=$((secondstaken % 60))
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
echo "Find test logs for the combination ($combination) in: $testlogfilename"
echo "Find consolidated test results in: $testresultsfilename"
echo "----------"
echo "Find test result for the combination ($combination) in: $testlogfilename"
logOutput "Consolidated test result is saved in: $aggregatedTestResult"
echo "------------------------"
}
checkdependencies() {
if ! [ "$(command -v pcregrep)" ]; then
logOutput "Exiting. pcregrep is required to run the script."
exit 1
fi
if ! [ "$(command -v xmlstarlet)" ]; then
logOutput "Exiting. xmlstarlet is required to run the script."
exit 1
fi
}
formatxml() {
xmlstarlet fo -s 2 "$1" > "$1.tmp"
mv "$1.tmp" "$1"
}
changeconf() {
xmlstarlet ed -P -L -d "/configuration/property[name='$1']" "$combtestfile"
xmlstarlet ed -P -L -s /configuration -t elem -n propertyTMP -v "" -s /configuration/propertyTMP -t elem -n name -v "$1" -r /configuration/propertyTMP -v property "$combtestfile"
if ! xmlstarlet ed -P -L -s "/configuration/property[name='$1']" -t elem -n value -v "$2" "$combtestfile"
then
logOutput "Exiting. Changing config property failed."
exit 1
fi
}
init() {
@ -153,89 +149,24 @@ init() {
then
echo ""
echo "Exiting. Build failed."
exit -1
fi
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
mkdir -p "$logdir"
testresultsfilename="$logdir/$starttime/Test-Results.txt"
if [[ -z "$combinations" ]]; then
combinations=( $( ls $combconfsdir/*.xml ))
fi
}
runtests() {
parseoptions "$@"
validate
if [ -z "$starttime" ]; then
init
fi
shopt -s nullglob
for combconffile in "${combinations[@]}"; do
STARTTIME=$(date +%s)
combination=$(basename "$combconffile" .xml)
mkdir -p "$logdir/$starttime"
testlogfilename="$logdir/$starttime/Test-Logs-$combination.txt"
printf "\nRunning the combination: %s..." "$combination"
setactiveconf
mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=$threadcount verify >> "$testlogfilename" || true
ENDTIME=$(date +%s)
summary
done
}
begin() {
cleancombinationconfigs
}
parseoptions() {
runactivate=0
runtests=0
while getopts ":c:a:t:" option; do
case "${option}" in
a)
if [[ "$runactivate" -eq "1" ]]; then
echo "-a Option is not multivalued"
exit 1
fi
runactivate=1
combination=$(basename "$OPTARG" .xml)
;;
c)
runtests=1
combination=$(basename "$OPTARG" .xml)
combinations+=("$combination")
;;
t)
threadcount=$OPTARG
;;
*|?|h)
if [[ -z "$combinations" ]]; then
combinations=( $( ls $combconfsdir/*.xml ))
fi
combstr=""
for combconffile in "${combinations[@]}"; do
combname=$(basename "$combconffile" .xml)
combstr="${combname}, ${combstr}"
done
combstr=${combstr:0:-2}
echo "Usage: $0 [-n] [-a COMBINATION_NAME] [-c COMBINATION_NAME] [-t THREAD_COUNT]"
echo ""
echo "Where:"
echo " -a COMBINATION_NAME Specify the combination name which needs to be activated."
echo " Configured combinations: ${combstr}"
echo " -c COMBINATION_NAME Specify the combination name for test runs"
echo " -t THREAD_COUNT Specify the thread count"
exit 1
;;
esac
done
if [[ "$runactivate" -eq "1" && "$runtests" -eq "1" ]]; then
echo "Both activate (-a option) and test run combinations (-c option) cannot be specified together"
exit 1
fi
if [[ "$runactivate" -eq "1" ]]; then
setactiveconf
exit 0
fi
starttime=$(date +"%Y-%m-%d_%H-%M-%S")
testOutputLogFolder+=$starttime
mkdir -p "$testOutputLogFolder"
aggregatedTestResult="$testOutputLogFolder/Test-Results.txt"
}
printAggregate() {
echo :::: AGGREGATED TEST RESULT ::::
cat "$aggregatedTestResult"
fullRunEndTime=$(date +%s)
fullRunTimeInSecs=$((fullRunEndTime - fullRunStartTime))
mins=$((fullRunTimeInSecs / 60))
secs=$((fullRunTimeInSecs % 60))
printf "\nTime taken: %s mins %s secs.\n" "$mins" "$secs"
}
logOutput() {
echo -e "$outputFormatOn" "$1" "$outputFormatOff"
}

111
hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
View File

@ -602,26 +602,76 @@ various test combinations, it will:
2. Run tests for all combinations
3. Summarize results across all the test combination runs.
As a pre-requisite step, fill config values for test accounts and credentials
needed for authentication in `src/test/resources/azure-auth-keys.xml.template`
and rename as `src/test/resources/azure-auth-keys.xml`.
Below are the pre-requiste steps to follow:
1. Copy
**To add a new test combination:** Templates for mandatory test combinations
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
If a new one needs to be added, add a combination set within
`dev-support/testrun-scripts/runtests.sh` similar to the ones already defined
and
1. Provide a new combination name
2. Update properties and values array which need to be effective for the test
combination
3. Call generateconfigs
./src/test/resources/azure-auth-keys.xml.template
TO
./src/test/resources/azure-auth-keys.xml
Update account names that should be used in the test run for HNS and non-HNS
combinations in the 2 properties present in the xml (account name should be
without domain part), namely
fs.azure.hnsTestAccountName
fs.azure.nonHnsTestAccountName
azure-auth-keys.xml is listed in .gitignore, so any accidental account name leak is prevented.
```
XInclude is supported, so for extra security secrets may be
kept out of the source tree then referenced through an an XInclude element:
<include xmlns="http://www.w3.org/2001/XInclude"
href="/users/self/.secrets/auth-keys.xml" />
```
2. Create account config files (one config file per account) in folder:
./src/test/resources/accountSettings/
Follow the instruction in the start of the template file
accountName_settings.xml.template
within accountSettings folder while creating account config file.
New files created in folder accountSettings is listed in .gitignore to
prevent accidental cred leaks.
**To run PR validation:** Running command
* `dev-support/testrun-scripts/runtests.sh` will generate configurations for
each of the combinations defined and run tests for all the combinations.
* `dev-support/testrun-scripts/runtests.sh -c {combinationname}` Specific
combinations can be provided with -c option. If combinations are provided
with -c option, tests for only those combinations will be run.
* `dev-support/testrun-scripts/runtests.sh` will prompt as below:
```bash
Choose action:
[Note - SET_ACTIVE_TEST_CONFIG will help activate the config for IDE/single test class runs]
1) SET_ACTIVE_TEST_CONFIG 4) SET_OR_CHANGE_TEST_ACCOUNT
2) RUN_TEST 5) PRINT_LOG4J_LOG_PATHS_FROM_LAST_RUN
3) CLEAN_UP_OLD_TEST_CONTAINERS
#? 2
```
Enter 1: for setting active combination for IDE test run/single mvn test class runs.
Enter 2: for choosing the combination to choose for mvn full test suite.
Enter 3: For clean-up of any abruptly ending test leaving auto generated test
container on the account.
Enter 4: To create/modify the config file that decides the account to use for specific test combination.
Enter 5: To print the log4j paths the last test runs.
On next prompt, current list of combinations to choose are provided.
Sample for Run_TEST action:
```bash
Enter parallel test run process count [default - 8]: 4
Set the active test combination to run the action:
1) HNS-OAuth 3) nonHNS-SharedKey 5) AllCombinationsTestRun
2) HNS-SharedKey 4) AppendBlob-HNS-OAuth 6) Quit
#? 1
Combination specific property setting: [ key=fs.azure.account.auth.type , value=OAuth ]
Activated [src/test/resources/abfs-combination-test-configs.xml] - for account: snvijayacontracttest for combination HNS-OAuth
Running test for combination HNS-OAuth on account snvijayacontracttest [ProcessCount=4]
Test run report can be seen in dev-support/testlogs/2022-10-07_05-23-22/Test-Logs-HNS-OAuth.txt
````
Provide the option for the action chosen first.
**Test logs:** Test runs will create a folder within dev-support/testlogs to
save the test logs. Folder name will be the test start timestamp. The mvn verify
@ -632,25 +682,18 @@ consolidated results of all the combination runs will be saved into a file as
Test-Results.log in the same folder. When run for PR validation, the
consolidated test results needs to be pasted into the PR comment section.
**To generate config for use in IDE:** Running command with -a (activate) option
`dev-support/testrun-scripts/runtests.sh -a {combination name}` will update
the effective config relevant for the specific test combination. Hence the same
config files used by the mvn test runs can be used for IDE without any manual
updates needed within config file.
**To add a new test combination:** Templates for mandatory test combinations
for PR validation are present in `dev-support/testrun-scripts/runtests.sh`.
If a new one needs to be added, add a combination to
`dev-support/testrun-scripts/runtests.sh`.
(Refer to current active combinations within
`SECTION: COMBINATION DEFINITIONS AND TRIGGER` and
`SECTION: TEST COMBINATION METHODS` in the script).
**Other command line options:**
* -a <COMBINATION_NAME> Specify the combination name which needs to be
activated. This is to be used to generate config for use in IDE.
* -c <COMBINATION_NAME> Specify the combination name for test runs. If this
config is specified, tests for only the specified combinations will run. All
combinations of tests will be running if this config is not specified.
* -t <THREAD_COUNT> ABFS mvn tests are run in parallel mode. Tests by default
are run with 8 thread count. It can be changed by providing -t <THREAD_COUNT>
**Test Configuration Details:**
In order to test ABFS, please add the following configuration to your
`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
compatibility tests which require WASB credentials, in addition to the ABFS
credentials.
Note that the ABFS tests include compatibility tests which require WASB
credentials, in addition to the ABFS credentials.
```xml
<?xml version="1.0" encoding="UTF-8"?>

74
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanupTestContainers.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azurebfs.utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.blob.CloudBlobClient;
import com.microsoft.azure.storage.blob.CloudBlobContainer;
import com.microsoft.azure.storage.StorageCredentials;
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.AbfsConfiguration;
/**
* This looks like a test, but it is really a command to invoke to
* clean up containers created in other test runs.
*
*/
public class CleanupTestContainers extends AbstractAbfsIntegrationTest {
private static final Logger LOG = LoggerFactory.getLogger(CleanupTestContainers.class);
private static final String CONTAINER_PREFIX = "abfs-testcontainer-";
public CleanupTestContainers() throws Exception {
}
@org.junit.Test
public void testDeleteContainers() throws Throwable {
int count = 0;
AbfsConfiguration abfsConfig = getAbfsStore(getFileSystem()).getAbfsConfiguration();
String accountName = abfsConfig.getAccountName().split("\\.")[0];
LOG.debug("Deleting test containers in account - {}", abfsConfig.getAccountName());
String accountKey = abfsConfig.getStorageAccountKey();
if ((accountKey == null) || (accountKey.isEmpty())) {
LOG.debug("Clean up not possible. Account ket not present in config");
}
final StorageCredentials credentials;
credentials = new StorageCredentialsAccountAndKey(
accountName, accountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
Iterable<CloudBlobContainer> containers
= blobClient.listContainers(CONTAINER_PREFIX);
for (CloudBlobContainer container : containers) {
LOG.info("Container {} URI {}",
container.getName(),
container.getUri());
if (container.deleteIfExists()) {
count++;
LOG.info("Current deleted test containers count - #{}", count);
}
}
LOG.info("Summary: Deleted {} test containers", count);
}
}

185
hadoop-tools/hadoop-azure/src/test/resources/accountSettings/accountName_settings.xml.template Normal file
View File

@ -0,0 +1,185 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
NOTE:
Each config file will hold configs for one storage account.
ABFS Driver tests run with different Auth settings, OAuth and SharedKey, on same test account.
Below template collects information needed for both Auth settings on the account.
OAuth test mode works with token provider type - ClientCredsTokenProvider
For specific details on test config or pre-requisite, refer to
https://github.com/apache/hadoop/blob/trunk/hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md
Pre-requisite before configuring:
1. An ABFS account
2. A container within account
3. A Service principal with RBAC - Storage Blob Data Owner (superuser) role on account
4. A Service principal with no RBAC roles assigned on account. This is needed for checkaccess tests (noRBACUser)
5. A Service principal with Storage Blob Data Contributor RBAC roles assigned on account. This is needed for RBAC tests (contributorRBACUser)
6. A Service principal with Storage Blob Data Reader RBAC roles assigned on account. This is needed for RBAC tests (readerRBACUser)
Replace below strings with actual value:
## ACCOUNT SETTINGS ##
1. ACCOUNTNAME -> to account name without domain
2. IS_NAMESPACE_ENABLED -> to true or false
3. CONTAINER_NAME -> name of an existing container
4. ACCOUNT_KEY -> Account access key
## IDENTITY SETTINGS ##
5. SUPERUSER_TENANT_ID -> AAD tenant ID of Superuser Service principal
6. SUPERUSER_CLIENT_ID -> SuperUser Service principal's client ID
7. SUPERUSER_CLIENT_SECRET -> SuperUser Service principal's client secret
8. NO_RBAC_USER_CLIENT_ID -> noRBACUser Service principal's client ID
9. NO_RBAC_USER_CLIENT_SECRET -> noRBACUser Service principal's client secret
10. NO_RBAC_USER_OID -> noRBACUser Service principal's OID
11. CONTRIBUTOR_RBAC_USER_CLIENT_ID -> contributorRBACUser Service principal's client ID
12. CONTRIBUTOR_RBAC_USER_CLIENT_SECRET -> contributorRBACUser Service principal's client secret
13. READER_RBAC_USER_CLIENT_ID -> readerRBACUser Service principal's client ID
14. READER_RBAC_USER_CLIENT_SECRET -> readerRBACUser Service principal's client secret
-->
<configuration>
<!-- ACCOUNT SETTINGS -->
<property>
<name>fs.azure.abfs.account.name</name>
<value>ACCOUNTNAME.dfs.core.windows.net</value>
</property>
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
</property>
<property>
<name>fs.contract.test.fs.abfss</name>
<value>abfss://CONTAINER_NAME@ACCOUNTNAME.dfs.core.windows.net</value>
</property>
<property>
<name>fs.contract.test.fs.wasb</name>
<value>wasb://CONTAINER_NAME@ACCOUNTNAME.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.wasb.account.name</name>
<value>ACCOUNTNAME.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.scale.test.enabled</name>
<value>true</value>
</property>
<property>
<name>fs.azure.test.namespace.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
<value>IS_NAMESPACE_ENABLED</value>
</property>
<property>
<name>fs.azure.test.namespace.enabled</name>
<value>IS_NAMESPACE_ENABLED</value>
</property>
<property>
<name>fs.azure.account.hns.enabled.ACCOUNTNAME.dfs.core.windows.net</name>
<value>IS_NAMESPACE_ENABLED</value>
</property>
<property>
<name>fs.azure.account.hns.enabled</name>
<value>IS_NAMESPACE_ENABLED</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
<value>ACCOUNT_KEY</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
<value>ACCOUNT_KEY</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
<value>ACCOUNT_KEY</value>
</property>
<property>
<name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
<value>ACCOUNT_KEY</value>
</property>
<!-- SUPERUSER SETTINGS -->
<property>
<name>fs.azure.account.oauth2.client.endpoint.ACCOUNTNAME.dfs.core.windows.net</name>
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
</property>
<property>
<name>fs.azure.account.oauth2.client.endpoint</name>
<value>https://login.microsoftonline.com/SUPERUSER_TENANT_ID/oauth2/token</value>
</property>
<property>
<name>fs.azure.account.oauth.provider.type.ACCOUNTNAME.dfs.core.windows.net</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
</property>
<property>
<name>fs.azure.account.oauth.provider.type</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
</property>
<property>
<name>fs.azure.account.oauth2.client.id.ACCOUNTNAME.dfs.core.windows.net</name>
<value>SUPERUSER_CLIENT_ID</value>
</property>
<property>
<name>fs.azure.account.oauth2.client.id</name>
<value>SUPERUSER_CLIENT_ID</value>
</property>
<property>
<name>fs.azure.account.oauth2.client.secret.ACCOUNTNAME.dfs.core.windows.net</name>
<value>SUPERUSER_CLIENT_SECRET</value>
</property>
<property>
<name>fs.azure.account.oauth2.client.secret</name>
<value>SUPERUSER_CLIENT_SECRET</value>
</property>
<!-- NO RBAC USER SETTINGS -->
<property>
<name>fs.azure.enable.check.access</name>
<value>true</value>
</property>
<property>
<name>fs.azure.account.test.oauth2.client.id</name>
<value>NO_RBAC_USER_CLIENT_ID</value>
</property>
<property>
<name>fs.azure.account.test.oauth2.client.secret</name>
<value>NO_RBAC_USER_CLIENT_SECRET</value>
</property>
<property>
<name>fs.azure.check.access.testuser.guid</name>
<value>NO_RBAC_USER_OID</value>
</property>
<!-- CONTRIBUTOR RBAC USER SETTINGS -->
<property>
<name>fs.azure.account.oauth2.contributor.client.id</name>
<value>CONTRIBUTOR_RBAC_USER_CLIENT_ID</value>
</property>
<property>
<name>fs.azure.account.oauth2.contributor.client.secret</name>
<value>CONTRIBUTOR_RBAC_USER_CLIENT_SECRET</value>
</property>
<!-- READER RBAC USER SETTINGS -->
<property>
<name>fs.azure.account.oauth2.reader.client.id</name>
<value>READER_RBAC_USER_CLIENT_ID</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.secret</name>
<value>READER_RBAC_USER_CLIENT_ID</value>
</property>
</configuration>

156
hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml.template
View File

@ -14,162 +14,16 @@
-->
<configuration>
<!--====================== IMPORTANT!! ========================-->
<!--
1. COPY THE CONTENTS OF THIS FILE TO "azure-auth-keys.xml" AND THEN EDIT.
2. UPDATE runtests.sh
A. "fs.azure.abfs.account.name" AND "fs.azure.test.namespace.enabled"
ARE MANDATORY WITH EVERY SCENARIO. AUTHTYPE BY DEFAULT WILL BE
SHAREDKEY AS CONFIGURED IN THIS TEMPLATE FILE.
B. PLEASE ADD MORE SCENARIOS IF THE CODE CHANGE REQUIRES TESTING WITH
DIFFERENT VARIANTS OF CONFIGS.
3. THE SCRIPT REQUIRES THE FOLLOWING UTILITIES xmlstarlet AND pcregrep
4. NOW THE SCRIPT CAN BE EXECUTED WITH ./runtests.sh
-->
<!--=============== Auth type ===============-->
<property>
<name>fs.azure.account.auth.type</name>
<value>SharedKey</value>
</property>
<!--=============== Auth related accounts ===============-->
<!-- This set of configs needs to be provided for all the accounts with which
the tests needs to be ran. -->
<property>
<name>fs.azure.account.key.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net</name>
<value>{ACCOUNT_ACCESS_KEY}</value>
<description>Account access key</description>
</property>
<property>
<name>
fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider</value>
<description>OAuth token provider implementation class</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>Token end point, this can be found through Azure portal
</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>{client id}</value>
<description>AAD client id.</description>
</property>
<property>
<name>
fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}.dfs.core.windows.net
</name>
<value>{client secret}</value>
<description>AAD client secret</description>
</property>
<!--=============== Configs for hadoop contract tests for ABFS ===============-->
<property>
<name>fs.contract.test.fs.abfs</name>
<value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
</property>
<property>
<name>fs.contract.test.fs.abfss</name>
<value>abfss://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
</property>
<!--=============== WASB Configs ===============-->
<property>
<name>fs.azure.wasb.account.name</name>
<value>{WASB_ACCOUNT_NAME}.blob.core.windows.net</value>
</property>
<property>
<name>fs.azure.account.key.{WASB_ACCOUNT_NAME}.blob.core.windows.net</name>
<value>WASB account key</value>
</property>
<property>
<name>fs.contract.test.fs.wasb</name>
<value>wasb://{WASB_FILESYSTEM}@{WASB_ACCOUNT_NAME}.blob.core.windows.net
</value>
</property>
<!--============= Configs for ITestAzureBlobFileSystemOauth tests ===============-->
<property>
<name>fs.azure.account.oauth2.contributor.client.id</name>
<value>{Client id of SP with RBAC Storage Blob Data Contributor}</value>
</property>
<property>
<name>fs.azure.account.oauth2.contributor.client.secret</name>
<value>{Client secret of SP with RBAC Storage Blob Data Contributor}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.id</name>
<value>{Client id of SP with RBAC Storage Blob Data Reader}</value>
</property>
<property>
<name>fs.azure.account.oauth2.reader.client.secret</name>
<value>{Client secret of SP with RBAC Storage Blob Data Reader}</value>
</property>
<!--=========================== FOR CheckAccess =========================-->
<!-- To run ABFS CheckAccess tests, you must register an app, with no role
assignments, and set the configuration discussed below:
1) Register a new app with no RBAC
2) As part of the test configs you need to provide the guid for the above
created app. Please follow the below steps to fetch the guid.
a) Get an access token with the above created app. Please refer the
following documentation for the same. https://docs.microsoft
.com/en-us/azure/active-directory/develop/v2-oauth2-client-creds-grant-flow#get-a-token
b) Decode the token fetched with the above step. You may use https
://jwt.ms/ to decode the token
d) The oid field in the decoded string is the guid.
3) Set the following configurations:
-->
<!-- Account names without domain -->
<property>
<name>fs.azure.account.test.oauth2.client.id</name>
<value>{client id}</value>
<description>The client id(app id) for the app created on step 1
</description>
</property>
<property>
<name>fs.azure.account.test.oauth2.client.secret</name>
<value>{client secret}</value>
<description>
The client secret(application's secret) for the app created on step 1
</description>
</property>
<property>
<name>fs.azure.check.access.testuser.guid</name>
<value>{guid}</value>
<description>The guid fetched on step 2</description>
</property>
<property>
<name>fs.azure.account.oauth2.client.endpoint.{account name}.dfs.core
.windows.net</name>
<value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
<description>
Token end point. This can be found through Azure portal. As part of CheckAccess
test cases. The access will be tested for an FS instance created with the
above mentioned client credentials. So this configuration is necessary to
create the test FS instance.
</description>
<name>fs.azure.hnsTestAccountName</name>
<value></value>
</property>
<!--========== Append blob related configs ===========-->
<property>
<name>fs.azure.test.appendblob.enabled</name>
<value>false</value>
<description>If made true, tests will be running under the assumption that
append blob is enabled and the root directory and contract test root
directory will be part of the append blob directories. Should be false for
non-HNS accounts.
</description>
<name>fs.azure.nonHnsTestAccountName</name>
<value></value>
</property>
</configuration>

45
hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml
View File

@ -28,11 +28,8 @@
<value>false</value>
</property>
<property>
<name>fs.azure.test.namespace.enabled</name>
<value>true</value>
</property>
<!--==================== ABFS CONFIGURATION ====================-->
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
<property>
<name>fs.azure.abfs.latency.track</name>
<value>false</value>
@ -43,9 +40,6 @@
<value>true</value>
</property>
<!--==================== ABFS CONFIGURATION ====================-->
<!-- SEE relevant section in "site/markdown/testing_azure.md"-->
<!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
<property>
<name>fs.azure.user.agent.prefix</name>
@ -59,7 +53,40 @@
STORE THE CONFIGURATION PROPERTIES WITHIN IT.
TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
IT IN PATCHES OR COMMITS. -->
IT IN PATCHES OR COMMITS.
TEST SCRIPT RUNS:
================
FOR EASIER TEST RUNS, TEST RUNS FOR VARIOUS COMBINATIONS CAN BE
TRIGGERED OVER SCRIPT:
./dev-support/testrun-scripts/runtests.sh
(FROM hadoop-azure ROOT PROJECT PATH)
TO USE THE TEST SCRIPT,
1. COPY
./src/test/resources/azure-auth-keys.xml.template
TO
./src/test/resources/azure-auth-keys.xml
UPDATE ACCOUNT NAMES THAT SHOULD BE USED IN THE TEST RUN
FOR HNS AND NON-HNS COMBINATIONS IN THE 2 PROPERTIES
PRESENT IN THE XML, NAMELY
fs.azure.hnsTestAccountName and
fs.azure.nonHnsTestAccountName
(ACCOUNT NAME SHOULD BE WITHOUT DOMAIN)
azure-auth-keys.xml IS LISTED IN .gitignore, SO ANY
ACCIDENTAL ACCOUNT NAME LEAK IS PREVENTED.
2. CREATE ACCOUNT CONFIG FILES (ONE CONFIG FILE
PER ACCOUNT) IN FOLDER:
./src/test/resources/accountSettings/
FOLLOW INSTRUCTIONS IN THE START OF THE TEMPLATE FILE
accountName_settings.xml.template
WITHIN accountSettings FOLDER WHILE CREATING ACCOUNT CONFIG FILE.
NEW FILES CREATED IN FOLDER accountSettings IS LISTED IN .gitignore
TO PREVENT ACCIDENTAL CRED LEAKS. -->
<!--=============================================================-->
<include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">