big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-14091. AbstractFileSystem implementaion for 'wasbs' scheme. Contributed Varada Hemeswari.

Browse Source
This commit is contained in:
Mingliang Liu 2017-02-23 13:48:44 -08:00
parent a4d4a23785
commit 82607fce39
2 changed files with 104 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/Wasbs.java Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.azure;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DelegateToFileSystem;
/**
* WASB implementation of AbstractFileSystem for wasbs scheme.
* This impl delegates to the old FileSystem
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class Wasbs extends DelegateToFileSystem {
Wasbs(final URI theUri, final Configuration conf) throws IOException,
URISyntaxException {
super(theUri, new NativeAzureFileSystem(), conf, "wasbs", false);
}
@Override
public int getUriDefaultPort() {
return -1;
}
}

57
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java
View File

@ -44,6 +44,7 @@
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions;
import org.junit.After;
import org.junit.Assert;
@ -471,6 +472,62 @@ public void testWasbAsDefaultFileSystemHasNoPort() throws Exception {
assertTrue(afs instanceof Wasb);
assertEquals(-1, afs.getUri().getPort());
} finally {
testAccount.cleanup();
FileSystem.closeAll();
}
}
/**
* Tests the cases when the scheme specified is 'wasbs'.
*/
@Test
public void testAbstractFileSystemImplementationForWasbsScheme() throws Exception {
try {
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI("wasbs", authority, null, null, null);
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs");
conf.addResource("azure-test.xml");
FileSystem fs = FileSystem.get(conf);
assertTrue(fs instanceof NativeAzureFileSystem);
assertEquals("wasbs", fs.getScheme());
AbstractFileSystem afs = FileContext.getFileContext(conf)
.getDefaultFileSystem();
assertTrue(afs instanceof Wasbs);
assertEquals(-1, afs.getUri().getPort());
assertEquals("wasbs", afs.getUri().getScheme());
} finally {
testAccount.cleanup();
FileSystem.closeAll();
}
}
@Test
public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception {
try {
testAccount = AzureBlobStorageTestAccount.createMock();
Configuration conf = testAccount.getFileSystem().getConf();
String authority = testAccount.getFileSystem().getUri().getAuthority();
URI defaultUri = new URI("wasbs", authority, null, null, null);
conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
FileSystem fs = FileSystem.get(conf);
assertTrue(fs instanceof NativeAzureFileSystem);
assertEquals("wasbs", fs.getScheme());
// should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified
try{
FileContext.getFileContext(conf).getDefaultFileSystem();
fail("Should've thrown.");
}catch(UnsupportedFileSystemException e){
}
} finally {
testAccount.cleanup();
FileSystem.closeAll();
}
}