From f2d9360b9d3f1f34fb7410635e1e31ad0b1212cf Mon Sep 17 00:00:00 2001 From: Xiaoyu Yao Date: Wed, 6 Sep 2017 15:12:13 -0700 Subject: [PATCH] HDFS-12199. Ozone: OzoneFileSystem: OzoneFileystem initialization code. Contributed by Mukul Kumar Singh. --- hadoop-project/pom.xml | 6 + hadoop-tools/hadoop-ozone/pom.xml | 126 ++++++++++ .../org/apache/hadoop/fs/ozone/Constants.java | 37 +++ .../java/org/apache/hadoop/fs/ozone/OzFs.java | 43 ++++ .../hadoop/fs/ozone/OzoneFileSystem.java | 217 ++++++++++++++++++ .../apache/hadoop/fs/ozone/package-info.java | 30 +++ .../fs/ozone/TestOzoneFileInterfaces.java | 99 ++++++++ .../src/test/resources/log4j.properties | 23 ++ hadoop-tools/hadoop-tools-dist/pom.xml | 6 + 9 files changed, 587 insertions(+) create mode 100644 hadoop-tools/hadoop-ozone/pom.xml create mode 100644 hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java create mode 100644 hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java create mode 100644 hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java create mode 100644 hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java create mode 100644 hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java create mode 100644 hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 5f5abc4128..10fb3b381c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -556,6 +556,12 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-ozone + ${project.version} + + org.apache.hadoop hadoop-kms diff --git a/hadoop-tools/hadoop-ozone/pom.xml b/hadoop-tools/hadoop-ozone/pom.xml new file mode 100644 index 0000000000..b089a3aa9b --- /dev/null +++ b/hadoop-tools/hadoop-ozone/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.1.0-SNAPSHOT + ../../hadoop-project + + hadoop-ozone + Apache Hadoop Ozone FileSystem + jar + + + UTF-8 + true + + + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + + false + false + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + + + + + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-common + test + test-jar + + + org.apache.hadoop + hadoop-hdfs + compile + + + org.apache.hadoop + hadoop-hdfs + test + test-jar + + + org.apache.hadoop + hadoop-hdfs-client + compile + + + + org.mockito + mockito-all + test + + + junit + junit + test + + + org.apache.hadoop + hadoop-distcp + test + + + org.apache.hadoop + hadoop-distcp + test + test-jar + + + diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java new file mode 100644 index 0000000000..45c4172e56 --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/Constants.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +/** + * Constants for Ozone FileSystem implementation. + */ +public class Constants { + + public static final String OZONE_URI_SCHEME = "ozfs"; + + public static final String OZONE_DEFAULT_USER = "hdfs"; + + public static final String OZONE_HTTP_SCHEME = "http://"; + + public static final String OZONE_USER_DIR = "/user"; + + private Constants() { + + } +} diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java new file mode 100644 index 0000000000..46dd645f76 --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * ozone implementation of AbstractFileSystem. + * This impl delegates to the OzoneFileSystem + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class OzFs extends DelegateToFileSystem { + + public OzFs(URI theUri, Configuration conf) + throws IOException, URISyntaxException { + super(theUri, new OzoneFileSystem(), conf, + Constants.OZONE_URI_SCHEME, false); + } +} diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java new file mode 100644 index 0000000000..ff3340e997 --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.EnumSet; +import java.util.Objects; + +import org.apache.hadoop.ozone.web.client.OzoneRestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.ozone.web.client.OzoneBucket; +import org.apache.hadoop.ozone.web.client.OzoneVolume; +import org.apache.hadoop.ozone.web.exceptions.OzoneException; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Progressable; + +import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER; +import static org.apache.hadoop.fs.ozone.Constants.OZONE_URI_SCHEME; +import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR; +import static org.apache.hadoop.fs.ozone.Constants.OZONE_HTTP_SCHEME; + +/** + * The Ozone Filesystem implementation. + * + * This subclass is marked as private as code should not be creating it + * directly; use {@link FileSystem#get(Configuration)} and variants to create + * one. If cast to {@link OzoneFileSystem}, extra methods and features may be + * accessed. Consider those private and unstable. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class OzoneFileSystem extends FileSystem { + static final Logger LOG = LoggerFactory.getLogger(OzoneFileSystem.class); + + /** The Ozone client for connecting to Ozone server. */ + private OzoneRestClient ozone; + private OzoneBucket bucket; + private URI uri; + private String userName; + private Path workingDir; + + @Override + public void initialize(URI name, Configuration conf) throws IOException { + super.initialize(name, conf); + setConf(conf); + Objects.requireNonNull(name.getScheme(), "No scheme provided in " + name); + assert getScheme().equals(name.getScheme()); + + uri = name; + Path path = new Path(name.getPath()); + String hostStr = name.getAuthority(); + String volumeStr = null; + String bucketStr = null; + + LOG.info("Ozone URI for ozfs initialization is " + uri); + while (path != null && !path.isRoot()) { + bucketStr = volumeStr; + volumeStr = path.getName(); + path = path.getParent(); + } + + if (hostStr == null) { + throw new IllegalArgumentException("No host provided in " + name); + } else if (volumeStr == null) { + throw new IllegalArgumentException("No volume provided in " + name); + } else if (bucketStr == null) { + throw new IllegalArgumentException("No bucket provided in " + name); + } + + try { + this.ozone = new OzoneRestClient(OZONE_HTTP_SCHEME + hostStr); + try { + this.userName = + UserGroupInformation.getCurrentUser().getShortUserName(); + } catch (IOException e) { + this.userName = OZONE_DEFAULT_USER; + } + this.ozone.setUserAuth(userName); + + OzoneVolume volume = ozone.getVolume(volumeStr); + this.bucket = volume.getBucket(bucketStr); + this.workingDir = new Path(OZONE_USER_DIR, this.userName) + .makeQualified(this.uri, this.workingDir); + } catch (OzoneException oe) { + final String msg = "Ozone server exception when initializing file system"; + LOG.error(msg, oe); + throw new IOException(msg, oe); + } catch (URISyntaxException ue) { + final String msg = "Invalid Ozone endpoint " + name; + LOG.error(msg, ue); + throw new IOException(msg, ue); + } + } + + @Override + public void close() throws IOException { + try { + ozone.close(); + } finally { + super.close(); + } + } + + @Override + public URI getUri() { + return uri; + } + + @Override + public String getScheme() { + return OZONE_URI_SCHEME; + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + return null; + } + + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, + short replication, long blockSize, + Progressable progress) throws IOException { + return null; + } + + @Override + public FSDataOutputStream createNonRecursive(Path path, + FsPermission permission, + EnumSet flags, + int bufferSize, + short replication, + long blockSize, + Progressable progress) throws IOException { + return null; + } + + @Override + public FSDataOutputStream append(Path f, int bufferSize, + Progressable progress) throws IOException { + return null; + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + return false; + } + + @Override + public boolean delete(Path f, boolean recursive) throws IOException { + return false; + } + + @Override + public FileStatus[] listStatus(Path f) throws IOException { + return null; + } + + @Override + public void setWorkingDirectory(Path newDir) { + workingDir = newDir; + } + + @Override + public Path getWorkingDirectory() { + return workingDir; + } + + @Override + public boolean mkdirs(Path f, FsPermission permission) throws IOException { + return false; + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + return null; + } + + @Override + public String toString() { + return "OzoneFileSystem{URI=" + uri + ", " + + "workingDir=" + workingDir + ", " + + "userName=" + userName + ", " + + "statistics=" + statistics + + "}"; + } +} diff --git a/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java new file mode 100644 index 0000000000..93e82c3022 --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/main/java/org/apache/hadoop/fs/ozone/package-info.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Ozone Filesystem. + * + * Except for the exceptions, it should all be hidden as implementation details. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.ozone; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file diff --git a/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java new file mode 100644 index 0000000000..469753771a --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.ozone; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.web.exceptions.OzoneException; +import org.apache.hadoop.ozone.web.handlers.BucketArgs; +import org.apache.hadoop.ozone.web.handlers.UserArgs; +import org.apache.hadoop.ozone.web.handlers.VolumeArgs; +import org.apache.hadoop.ozone.web.interfaces.StorageHandler; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; +import org.junit.BeforeClass; +import org.junit.AfterClass; +import org.junit.Test; +import org.junit.Assert; + +import java.io.IOException; + +/** + * Test OzoneFileSystem Interfaces. + * + * This test will test the various interfaces i.e. + * create, read, write, getFileStatus + */ +public class TestOzoneFileInterfaces { + private static MiniOzoneCluster cluster = null; + private static FileSystem fs; + private static StorageHandler storageHandler; + + @BeforeClass + public static void init() throws IOException, OzoneException { + OzoneConfiguration conf = new OzoneConfiguration(); + cluster = new MiniOzoneCluster.Builder(conf) + .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); + storageHandler = + new ObjectStoreHandler(conf).getStorageHandler(); + + // create a volume and a bucket to be used by OzoneFileSystem + String userName = "user" + RandomStringUtils.randomNumeric(5); + String adminName = "admin" + RandomStringUtils.randomNumeric(5); + String volumeName = "volume" + RandomStringUtils.randomNumeric(5); + String bucketName = "bucket" + RandomStringUtils.randomNumeric(5); + UserArgs userArgs = new UserArgs(null, OzoneUtils.getRequestID(), + null, null, null, null); + VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs); + volumeArgs.setUserName(userName); + volumeArgs.setAdminName(adminName); + storageHandler.createVolume(volumeArgs); + BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs); + storageHandler.createBucket(bucketArgs); + + // Fetch the host and port for File System init + DataNode dataNode = cluster.getDataNodes().get(0); + int port = dataNode.getInfoPort(); + String host = dataNode.getDatanodeHostname(); + + // Set the fs.defaultFS and start the filesystem + String uri = String.format("%s://%s:%d/%s/%s", + Constants.OZONE_URI_SCHEME, host, port, volumeName, bucketName); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, uri); + fs = FileSystem.get(conf); + } + + @AfterClass + public static void teardown() throws IOException { + fs.close(); + storageHandler.close(); + cluster.shutdown(); + } + + @Test + public void testFileSystemInit() throws IOException { + Assert.assertTrue(fs instanceof OzoneFileSystem); + Assert.assertEquals(fs.getUri().getScheme(), Constants.OZONE_URI_SCHEME); + } +} diff --git a/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties new file mode 100644 index 0000000000..3bf1619d22 --- /dev/null +++ b/hadoop-tools/hadoop-ozone/src/test/resources/log4j.properties @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# log4j configuration used during build and unit tests + +log4j.rootLogger=INFO,stdout +log4j.threshold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n + +log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR + +# for debugging low level Ozone operations, uncomment this line +# log4j.logger.org.apache.hadoop.ozone=DEBUG diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index 42ce94c829..9d6bf69011 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -111,6 +111,12 @@ compile ${project.version} + + org.apache.hadoop + hadoop-ozone + compile + ${project.version} + org.apache.hadoop hadoop-sls