HADOOP-13929. ADLS connector should not check in contract-test-options.xml. (John Zhuge via lei)

.gitignore

@@ -17,6 +17,10 @@ target
 build
 dependency-reduced-pom.xml
 
+# Filesystem contract test options and credentials
+auth-keys.xml
+azure-auth-keys.xml
+
 # External tool builders
 */.externalToolBuilders
 */maven-eclipse.xml
@@ -24,8 +28,6 @@ dependency-reduced-pom.xml
 hadoop-common-project/hadoop-kms/downloads/
 hadoop-hdfs-project/hadoop-hdfs/downloads
 hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
-hadoop-common-project/hadoop-common/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-openstack/src/test/resources/contract-test-options.xml
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/tla/yarnregistry.toolbox
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/dist
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/tmp
@@ -41,10 +43,4 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/testem.log
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/dist
 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
 yarnregistry.pdf
-hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
-hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
-hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
 patchprocess/
-hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
-hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java

@@ -72,7 +72,7 @@
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public class AdlFileSystem extends FileSystem {
-  static final String SCHEME = "adl";
+  public static final String SCHEME = "adl";
   static final int DEFAULT_PORT = 443;
   private URI uri;
   private String userName;

hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md

@@ -224,7 +224,9 @@ commands demonstrate access to a storage account named `youraccount`.
 ## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
 The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without additional configuration by running mvn test. This includes tests against mocked storage, which is an in-memory emulation of Azure Data Lake Storage.
 
-A selection of tests can run against the Azure Data Lake Storage. To run tests against Adl storage. Please configure contract-test-options.xml with Adl account information mentioned in the above sections. Also turn on contract test execution flag to trigger tests against Azure Data Lake Storage.
+A selection of tests can run against the Azure Data Lake Storage. To run these
+tests, please create `src/test/resources/auth-keys.xml` with Adl account
+information mentioned in the above sections and the following properties.
 
         <property>
             <name>dfs.adl.test.contract.enable</name>

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/AdlStorageConfiguration.java

@@ -21,6 +21,8 @@
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
+import org.apache.hadoop.util.ReflectionUtils;
 
 import java.io.IOException;
 import java.net.URI;
@@ -30,15 +32,19 @@
  * Configure Adl storage file system.
  */
 public final class AdlStorageConfiguration {
+  static final String CONTRACT_XML = "adls.xml";
+
   private static final String CONTRACT_ENABLE_KEY =
       "dfs.adl.test.contract.enable";
+  private static final boolean CONTRACT_ENABLE_DEFAULT = false;
 
-  private static final String TEST_CONFIGURATION_FILE_NAME =
+  private static final String FILE_SYSTEM_KEY =
-      "contract-test-options.xml";
+      String.format("test.fs.%s.name", AdlFileSystem.SCHEME);
-  private static final String TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME =
+
-      "adls.xml";
+  private static final String FILE_SYSTEM_IMPL_KEY =
-  private static final String KEY_FILE_SYSTEM_IMPL = "fs.contract.test.fs";
+      String.format("fs.%s.impl", AdlFileSystem.SCHEME);
-  private static final String KEY_FILE_SYSTEM = "test.fs.adl.name";
+  private static final Class<?> FILE_SYSTEM_IMPL_DEFAULT =
+      AdlFileSystem.class;
 
   private static boolean isContractTestEnabled = false;
   private static Configuration conf = null;
@@ -48,8 +54,7 @@ private AdlStorageConfiguration() {
 
   public synchronized static Configuration getConfiguration() {
     Configuration newConf = new Configuration();
-    newConf.addResource(TEST_CONFIGURATION_FILE_NAME);
+    newConf.addResource(CONTRACT_XML);
-    newConf.addResource(TEST_SUPPORTED_TEST_CONFIGURATION_FILE_NAME);
     return newConf;
   }
 
@@ -58,7 +63,8 @@ public synchronized static boolean isContractTestEnabled() {
       conf = getConfiguration();
     }
 
-    isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY, false);
+    isContractTestEnabled = conf.getBoolean(CONTRACT_ENABLE_KEY,
+        CONTRACT_ENABLE_DEFAULT);
     return isContractTestEnabled;
   }
 
@@ -72,23 +78,15 @@ public synchronized static FileSystem createStorageConnector()
       return null;
     }
 
-    String fileSystem = conf.get(KEY_FILE_SYSTEM);
+    String fileSystem = conf.get(FILE_SYSTEM_KEY);
     if (fileSystem == null || fileSystem.trim().length() == 0) {
       throw new IOException("Default file system not configured.");
     }
-    String fileSystemImpl = conf.get(KEY_FILE_SYSTEM_IMPL);
-    if (fileSystemImpl == null || fileSystemImpl.trim().length() == 0) {
-      throw new IOException(
-          "Configuration " + KEY_FILE_SYSTEM_IMPL + "does not exist.");
-    }
-    FileSystem fs = null;
-    try {
-      fs = (FileSystem) Class.forName(fileSystemImpl).newInstance();
-    } catch (Exception e) {
-      throw new IOException("Could not instantiate the filesystem.");
-    }
 
-    fs.initialize(new URI(conf.get(KEY_FILE_SYSTEM)), conf);
+    Class<?> clazz = conf.getClass(FILE_SYSTEM_IMPL_KEY,
+        FILE_SYSTEM_IMPL_DEFAULT);
+    FileSystem fs = (FileSystem) ReflectionUtils.newInstance(clazz, conf);
+    fs.initialize(new URI(fileSystem), conf);
     return fs;
   }
 }

hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml

@@ -12,6 +12,17 @@
   limitations under the License. See accompanying LICENSE file.
 -->
 <configuration>
+  <!--
+  To run live tests:
+  # Create a file auth-keys.xml  - DO NOT ADD TO REVISION CONTROL
+  # Add property test.fs.adl.name to point to an ADLS filesystem URL
+  # Add property dfs.adl.test.contract.enable with value set to true
+  # Add the credentials for the service you are testing against
+  -->
+  <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
+    <fallback/>
+  </include>
+
   <property>
     <name>fs.contract.test.root-tests-enabled</name>
     <value>true</value>

hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml

@@ -1,61 +0,0 @@
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
-  <property>
-    <name>dfs.adls.oauth2.refresh.url</name>
-    <value>
-    </value>
-  </property>
-
-  <property>
-    <name>dfs.adls.oauth2.credential</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>dfs.adls.oauth2.client.id</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>dfs.adls.oauth2.access.token.provider.type</name>
-    <value>ClientCredential</value>
-    <description>
-      Supported provider type:
-
-      "ClientCredential" : Client id and client credentials(Provided
-      through configuration file) flow.
-
-      "RefreshToken" : Client id and refresh token(Provided
-      through configuration file)flow.
-
-      "Custom" : Custom AAD token management.
-    </description>
-  </property>
-
-  <property>
-    <name>dfs.adl.test.contract.enable</name>
-    <value>false</value>
-  </property>
-
-  <property>
-    <name>test.fs.adl.name</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>fs.contract.test.fs</name>
-    <value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
-  </property>
-</configuration>