big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1891. Ozone fs shell command should work with default port when port number is not specified

Browse Source 
Signed-off-by: Anu Engineer <aengineer@apache.org>
This commit is contained in:
Siyao Meng 2019-08-02 12:54:04 -07:00 committed by Anu Engineer
parent 69b74e9016
commit 68c818415a
2 changed files with 50 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@ -54,6 +55,7 @@
import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -85,9 +87,10 @@ public class BasicOzoneFileSystem extends FileSystem {
private static final Pattern URL_SCHEMA_PATTERN =
Pattern.compile("([^\\.]+)\\.([^\\.]+)\\.{0,1}(.*)");
private static final String URI_EXCEPTION_TEXT = "Ozone file system url " +
"should be either one of the two forms: " +
private static final String URI_EXCEPTION_TEXT = "Ozone file system URL " +
"should be one of the following formats: " +
"o3fs://bucket.volume/key OR " +
"o3fs://bucket.volume.om-host.example.com/key OR " +
"o3fs://bucket.volume.om-host.example.com:5678/key";
@Override
@ -113,11 +116,17 @@ public void initialize(URI name, Configuration conf) throws IOException {
String omPort = String.valueOf(-1);
if (!isEmpty(remaining)) {
String[] parts = remaining.split(":");
if (parts.length != 2) {
// Array length should be either 1(host) or 2(host:port)
if (parts.length > 2) {
throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
}
omHost = parts[0];
omPort = parts[1];
if (parts.length == 2) {
omPort = parts[1];
} else {
// If port number is not specified, read it from config
omPort = String.valueOf(OmUtils.getOmRpcPort(conf));
}
if (!isNumber(omPort)) {
throw new IllegalArgumentException(URI_EXCEPTION_TEXT);
}

37
hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithMocks.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@ -78,6 +79,42 @@ public void testFSUriWithHostPortOverrides() throws Exception {
OzoneClientFactory.getRpcClient("local.host", 5899, conf);
}
@Test
public void testFSUriWithHostPortUnspecified() throws Exception {
Configuration conf = new OzoneConfiguration();
final int omPort = OmUtils.getOmRpcPort(conf);
OzoneClient ozoneClient = mock(OzoneClient.class);
ObjectStore objectStore = mock(ObjectStore.class);
OzoneVolume volume = mock(OzoneVolume.class);
OzoneBucket bucket = mock(OzoneBucket.class);
when(ozoneClient.getObjectStore()).thenReturn(objectStore);
when(objectStore.getVolume(eq("volume1"))).thenReturn(volume);
when(volume.getBucket("bucket1")).thenReturn(bucket);
PowerMockito.mockStatic(OzoneClientFactory.class);
PowerMockito.when(OzoneClientFactory.getRpcClient(eq("local.host"),
eq(omPort), eq(conf))).thenReturn(ozoneClient);
UserGroupInformation ugi = mock(UserGroupInformation.class);
PowerMockito.mockStatic(UserGroupInformation.class);
PowerMockito.when(UserGroupInformation.getCurrentUser()).thenReturn(ugi);
when(ugi.getShortUserName()).thenReturn("user1");
URI uri = new URI("o3fs://bucket1.volume1.local.host");
FileSystem fileSystem = FileSystem.get(uri, conf);
OzoneFileSystem ozfs = (OzoneFileSystem) fileSystem;
assertEquals(ozfs.getUri().getHost(), "bucket1.volume1.local.host");
// The URI doesn't contain a port number, expect -1 from getPort()
assertEquals(ozfs.getUri().getPort(), -1);
PowerMockito.verifyStatic();
// Check the actual port number in use
OzoneClientFactory.getRpcClient("local.host", omPort, conf);
}
@Test
public void testFSUriHostVersionDefault() throws Exception {
Configuration conf = new OzoneConfiguration();