big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-651. Rename o3 to o3fs for Filesystem.

Browse Source
This commit is contained in:
Jitendra Pandey 2018-10-17 14:18:46 -07:00
parent 9abda83947
commit d93d515af5
9 changed files with 40 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
View File

@ -1788,19 +1788,6 @@
</property>
<!-- Ozone file system properties -->
<property>
<name>fs.o3.impl</name>
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
<description>The implementation class of the Ozone FileSystem.</description>
</property>
<property>
<name>fs.AbstractFileSystem.o3.impl</name>
<value>org.apache.hadoop.fs.ozone.OzFs</value>
<description>The implementation class of the OzFs AbstractFileSystem.</description>
</property>
<!-- ipc properties -->
<property>

2
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
View File

@ -103,7 +103,7 @@ public void initializeMemberVariables() {
xmlPrefixToSkipCompare.add("fs.s3a.");
// O3 properties are in a different subtree.
xmlPrefixToSkipCompare.add("fs.o3.");
xmlPrefixToSkipCompare.add("fs.o3fs.");
//ftp properties are in a different subtree.
// - org.apache.hadoop.fs.ftp.FTPFileSystem.

5
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -63,7 +63,10 @@ public final class OzoneConsts {
public static final String OZONE_USER = "user";
public static final String OZONE_REQUEST = "request";
public static final String OZONE_URI_SCHEME = "o3";
// Ozone File System scheme
public static final String OZONE_URI_SCHEME = "o3fs";
public static final String OZONE_RPC_SCHEME = "o3";
public static final String OZONE_HTTP_SCHEME = "http";
public static final String OZONE_URI_DELIMITER = "/";

2
hadoop-ozone/dist/src/main/compose/ozonefs/docker-config vendored
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm

52
hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot vendored
View File

@ -30,83 +30,83 @@ Create volume and bucket
Execute ozone sh bucket create http://ozoneManager/fstest2/bucket3
Check volume from ozonefs
${result} = Execute ozone fs -ls o3://bucket1.fstest/
${result} = Execute ozone fs -ls o3fs://bucket1.fstest/
Run ozoneFS tests
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep
Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep
Execute ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/testdir/deep/
Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} NOTICE.txt
Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir/deep/PUTFILE.txt
Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} PUTFILE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/
${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/
Should contain ${result} NOTICE.txt
Should contain ${result} PUTFILE.txt
Execute ozone fs -mv o3://bucket1.fstest/testdir/deep/NOTICE.txt o3://bucket1.fstest/testdir/deep/MOVED.TXT
Execute ozone fs -mv o3fs://bucket1.fstest/testdir/deep/NOTICE.txt o3fs://bucket1.fstest/testdir/deep/MOVED.TXT
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} MOVED.TXT
Should not contain ${result} NOTICE.txt
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir/deep/subdir1
Execute ozone fs -cp o3://bucket1.fstest/testdir/deep/MOVED.TXT o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep/subdir1
Execute ozone fs -cp o3fs://bucket1.fstest/testdir/deep/MOVED.TXT o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} subdir1/NOTICE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/testdir/deep/subdir1/
${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/subdir1/
Should contain ${result} NOTICE.txt
Execute ozone fs -cat o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
Execute ozone fs -cat o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
Should not contain ${result} Failed
Execute ozone fs -rm o3://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
Execute ozone fs -rm o3fs://bucket1.fstest/testdir/deep/subdir1/NOTICE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} NOTICE.txt
${result} = Execute ozone fs -rmdir o3://bucket1.fstest/testdir/deep/subdir1/
${result} = Execute ozone fs -rmdir o3fs://bucket1.fstest/testdir/deep/subdir1/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} subdir1
Execute ozone fs -touch o3://bucket1.fstest/testdir/TOUCHFILE.txt
Execute ozone fs -touch o3fs://bucket1.fstest/testdir/TOUCHFILE.txt
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} TOUCHFILE.txt
Execute ozone fs -rm -r o3://bucket1.fstest/testdir/
Execute ozone fs -rm -r o3fs://bucket1.fstest/testdir/
${result} = Execute ozone sh key list o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should not contain ${result} testdir
Execute rm -Rf localdir1
Execute mkdir localdir1
Execute cp NOTICE.txt localdir1/LOCAL.txt
Execute ozone fs -mkdir -p o3://bucket1.fstest/testdir1
Execute ozone fs -copyFromLocal localdir1 o3://bucket1.fstest/testdir1/
Execute ozone fs -put NOTICE.txt o3://bucket1.fstest/testdir1/NOTICE.txt
Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir1
Execute ozone fs -copyFromLocal localdir1 o3fs://bucket1.fstest/testdir1/
Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir1/NOTICE.txt
${result} = Execute ozone fs -ls -R o3://bucket1.fstest/testdir1/
${result} = Execute ozone fs -ls -R o3fs://bucket1.fstest/testdir1/
Should contain ${result} localdir1/LOCAL.txt
Should contain ${result} testdir1/NOTICE.txt
Execute ozone fs -mkdir -p o3://bucket2.fstest/testdir2
Execute ozone fs -mkdir -p o3://bucket3.fstest2/testdir3
Execute ozone fs -mkdir -p o3fs://bucket2.fstest/testdir2
Execute ozone fs -mkdir -p o3fs://bucket3.fstest2/testdir3
Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket2.fstest/testdir2/
Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket2.fstest/testdir2/
Execute ozone fs -cp o3://bucket1.fstest/testdir1/localdir1 o3://bucket3.fstest2/testdir3/
Execute ozone fs -cp o3fs://bucket1.fstest/testdir1/localdir1 o3fs://bucket3.fstest2/testdir3/
Execute ozone sh key put o3://ozoneManager/fstest/bucket1/KEY.txt NOTICE.txt
${result} = Execute ozone fs -ls o3://bucket1.fstest/KEY.txt
${result} = Execute ozone fs -ls o3fs://bucket1.fstest/KEY.txt
Should contain ${result} KEY.txt
${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3://bucket1.fstest/KEY.txt
${rc} ${result} = Run And Return Rc And Output ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/KEY.txt
Should Be Equal As Integers ${rc} 1
Should contain ${result} File exists
Execute rm -Rf GET.txt
Execute ozone fs -get o3://bucket1.fstest/KEY.txt GET.txt
Execute ozone fs -get o3fs://bucket1.fstest/KEY.txt GET.txt
Execute ls -l GET.txt
${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3://abcde.pqrs/
${rc} ${result} = Run And Return Rc And Output ozone fs -ls o3fs://abcde.pqrs/
Should Be Equal As Integers ${rc} 1
Should contain ${result} VOLUME_NOT_FOUND

6
hadoop-ozone/docs/content/OzoneFS.md
View File

@ -42,16 +42,16 @@ Please add the following entry to the core-site.xml.
{{< highlight xml >}}
<property>
<name>fs.o3.impl</name>
<name>fs.o3fs.impl</name>
<value>org.apache.hadoop.fs.ozone.OzoneFileSystem</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>o3://localhost:9864/volume/bucket</value>
<value>o3fs://localhost:9864/volume/bucket</value>
</property>
{{< /highlight >}}
This will make this bucket to be the default file system for HDFS dfs commands and register the o3 file system type..
This will make this bucket to be the default file system for HDFS dfs commands and register the o3fs file system type..
You also need to add the ozone-filesystem.jar file to the classpath:

6
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
View File

@ -35,7 +35,7 @@
import org.apache.hadoop.ozone.client.rest.OzoneException;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_HTTP_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RPC_SCHEME;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -83,7 +83,7 @@ protected URI verifyURI(String uri)
Configuration conf = new OzoneConfiguration();
String scheme = ozoneURI.getScheme();
if (ozoneURI.getScheme() == null || scheme.isEmpty()) {
scheme = OZONE_URI_SCHEME;
scheme = OZONE_RPC_SCHEME;
}
if (scheme.equals(OZONE_HTTP_SCHEME)) {
if (ozoneURI.getHost() != null) {
@ -96,7 +96,7 @@ protected URI verifyURI(String uri)
} else {
client = OzoneClientFactory.getRestClient(conf);
}
} else if (scheme.equals(OZONE_URI_SCHEME)) {
} else if (scheme.equals(OZONE_RPC_SCHEME)) {
if (ozoneURI.getHost() != null) {
if (ozoneURI.getPort() == -1) {
client = OzoneClientFactory.getRpcClient(ozoneURI.getHost());

2
hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
View File

@ -107,7 +107,7 @@ public void initialize(URI name, Configuration conf) throws IOException {
if (!matcher.matches()) {
throw new IllegalArgumentException("Ozone file system url should be "
+ "in the form o3://bucket.volume");
+ "in the form o3fs://bucket.volume");
}
String bucketStr = matcher.group(1);
String volumeStr = matcher.group(2);

2
hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileInterfaces.java
View File

@ -231,7 +231,7 @@ public void testPathToKey() throws Exception {
ozoneFs.pathToKey(new Path("key1/key2")));
assertEquals("key1/key2",
ozoneFs.pathToKey(new Path("o3://test1/key1/key2")));
ozoneFs.pathToKey(new Path("o3fs://test1/key1/key2")));
}
private String getCurrentUser() {