diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java index e56810cdca..e0c8150f44 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java @@ -82,7 +82,7 @@ public OzoneConfiguration createOzoneConfiguration() { if (configurationOverrides != null) { for (Entry entry : configurationOverrides.entrySet()) { ozoneConf - .set(entry.getKey(), configurationOverrides.get(entry.getValue())); + .set(entry.getKey(), entry.getValue()); } } return ozoneConf; diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java index b6005d1861..17d19389e0 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.client; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; @@ -56,6 +57,11 @@ public ObjectStore(Configuration conf, ClientProtocol proxy) { this.listCacheSize = HddsClientUtils.getListCacheSize(conf); } + @VisibleForTesting + protected ObjectStore() { + proxy = null; + } + /** * Creates the volume with default values. * @param volumeName Name of the volume to be created. @@ -96,7 +102,7 @@ public OzoneVolume getVolume(String volumeName) throws IOException { * @param volumePrefix Volume prefix to match * @return {@code Iterator} */ - public Iterator listVolumes(String volumePrefix) + public Iterator listVolumes(String volumePrefix) throws IOException { return listVolumes(volumePrefix, null); } @@ -111,7 +117,7 @@ public Iterator listVolumes(String volumePrefix) * @param prevVolume Volumes will be listed after this volume name * @return {@code Iterator} */ - public Iterator listVolumes(String volumePrefix, + public Iterator listVolumes(String volumePrefix, String prevVolume) throws IOException { return new VolumeIterator(null, volumePrefix, prevVolume); } @@ -127,7 +133,7 @@ public Iterator listVolumes(String volumePrefix, * @param prevVolume Volumes will be listed after this volume name * @return {@code Iterator} */ - public Iterator listVolumesByUser(String user, + public Iterator listVolumesByUser(String user, String volumePrefix, String prevVolume) throws IOException { if(Strings.isNullOrEmpty(user)) { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 97bd682975..751992ed6d 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.client; - +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; @@ -121,6 +121,23 @@ public OzoneBucket(Configuration conf, ClientProtocol proxy, OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)); } + @VisibleForTesting + OzoneBucket(String volumeName, String name, + ReplicationFactor defaultReplication, + ReplicationType defaultReplicationType, + List acls, StorageType storageType, Boolean versioning, + long creationTime) { + this.proxy = null; + this.volumeName = volumeName; + this.name = name; + this.defaultReplication = defaultReplication; + this.defaultReplicationType = defaultReplicationType; + this.acls = acls; + this.storageType = storageType; + this.versioning = versioning; + this.creationTime = creationTime; + } + /** * Returns Volume Name. * @@ -273,7 +290,7 @@ public OzoneKeyDetails getKey(String key) throws IOException { * @param keyPrefix Bucket prefix to match * @return {@code Iterator} */ - public Iterator listKeys(String keyPrefix) { + public Iterator listKeys(String keyPrefix) { return listKeys(keyPrefix, null); } @@ -287,7 +304,8 @@ public Iterator listKeys(String keyPrefix) { * @param prevKey Keys will be listed after this key name * @return {@code Iterator} */ - public Iterator listKeys(String keyPrefix, String prevKey) { + public Iterator listKeys(String keyPrefix, + String prevKey) { return new KeyIterator(keyPrefix, prevKey); } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java index f191507464..0d65d73fc3 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java @@ -24,6 +24,8 @@ import java.io.Closeable; import java.io.IOException; +import com.google.common.annotations.VisibleForTesting; + /** * OzoneClient connects to Ozone Cluster and * perform basic operations. @@ -84,6 +86,11 @@ public OzoneClient(Configuration conf, ClientProtocol proxy) { this.objectStore = new ObjectStore(conf, this.proxy); } + @VisibleForTesting + protected OzoneClient(ObjectStore objectStore) { + this.objectStore = objectStore; + this.proxy = null; + } /** * Returns the object store associated with the Ozone Cluster. * @return ObjectStore diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java index 77f882acff..e451b1ac24 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java @@ -18,17 +18,19 @@ package org.apache.hadoop.ozone.client; -import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; /** * A class that encapsulates OzoneVolume. @@ -94,6 +96,19 @@ public OzoneVolume(Configuration conf, ClientProtocol proxy, String name, this.listCacheSize = HddsClientUtils.getListCacheSize(conf); } + @VisibleForTesting + protected OzoneVolume(String name, String admin, String owner, + long quotaInBytes, + long creationTime, List acls) { + this.proxy = null; + this.name = name; + this.admin = admin; + this.owner = owner; + this.quotaInBytes = quotaInBytes; + this.creationTime = creationTime; + this.acls = acls; + } + /** * Returns Volume name. * @@ -208,12 +223,13 @@ public OzoneBucket getBucket(String bucketName) throws IOException { * @param bucketPrefix Bucket prefix to match * @return {@code Iterator} */ - public Iterator listBuckets(String bucketPrefix) { + public Iterator listBuckets(String bucketPrefix) { return listBuckets(bucketPrefix, null); } /** - * Returns Iterator to iterate over all buckets after prevBucket in the volume. + * Returns Iterator to iterate over all buckets after prevBucket in the + * volume. * If prevBucket is null it iterates from the first bucket in the volume. * The result can be restricted using bucket prefix, will return all * buckets if bucket prefix is null. @@ -222,7 +238,7 @@ public Iterator listBuckets(String bucketPrefix) { * @param prevBucket Buckets are listed after this bucket * @return {@code Iterator} */ - public Iterator listBuckets(String bucketPrefix, + public Iterator listBuckets(String bucketPrefix, String prevBucket) { return new BucketIterator(bucketPrefix, prevBucket); } diff --git a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml index 10dc61e40b..42d7d1d1a6 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3/docker-compose.yaml @@ -51,6 +51,8 @@ services: image: apache/hadoop-runner volumes: - ../..:/opt/hadoop + ports: + - 9878:9878 env_file: - ./docker-config command: ["ozone","s3g"] diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java index cc045d09d5..d6441908bb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java @@ -701,26 +701,29 @@ public void testListVolume() throws IOException, OzoneException { store.createVolume( volBaseNameB + i + "-" + RandomStringUtils.randomNumeric(5)); } - Iterator volIterator = store.listVolumes(volBase); + Iterator volIterator = store.listVolumes(volBase); int totalVolumeCount = 0; while(volIterator.hasNext()) { volIterator.next(); totalVolumeCount++; } Assert.assertEquals(20, totalVolumeCount); - Iterator volAIterator = store.listVolumes(volBaseNameA); + Iterator volAIterator = store.listVolumes( + volBaseNameA); for(int i = 0; i < 10; i++) { Assert.assertTrue(volAIterator.next().getName() .startsWith(volBaseNameA + i + "-")); } Assert.assertFalse(volAIterator.hasNext()); - Iterator volBIterator = store.listVolumes(volBaseNameB); + Iterator volBIterator = store.listVolumes( + volBaseNameB); for(int i = 0; i < 10; i++) { Assert.assertTrue(volBIterator.next().getName() .startsWith(volBaseNameB + i + "-")); } Assert.assertFalse(volBIterator.hasNext()); - Iterator iter = store.listVolumes(volBaseNameA + "1-"); + Iterator iter = store.listVolumes(volBaseNameA + + "1-"); Assert.assertTrue(iter.next().getName().startsWith(volBaseNameA + "1-")); Assert.assertFalse(iter.hasNext()); } @@ -751,7 +754,7 @@ public void testListBucket() volB.createBucket( bucketBaseNameB + i + "-" + RandomStringUtils.randomNumeric(5)); } - Iterator volABucketIter = + Iterator volABucketIter = volA.listBuckets("bucket-"); int volABucketCount = 0; while(volABucketIter.hasNext()) { @@ -759,7 +762,7 @@ public void testListBucket() volABucketCount++; } Assert.assertEquals(20, volABucketCount); - Iterator volBBucketIter = + Iterator volBBucketIter = volA.listBuckets("bucket-"); int volBBucketCount = 0; while(volBBucketIter.hasNext()) { @@ -768,7 +771,7 @@ public void testListBucket() } Assert.assertEquals(20, volBBucketCount); - Iterator volABucketAIter = + Iterator volABucketAIter = volA.listBuckets("bucket-a-"); int volABucketACount = 0; while(volABucketAIter.hasNext()) { @@ -776,7 +779,7 @@ public void testListBucket() volABucketACount++; } Assert.assertEquals(10, volABucketACount); - Iterator volBBucketBIter = + Iterator volBBucketBIter = volA.listBuckets("bucket-b-"); int volBBucketBCount = 0; while(volBBucketBIter.hasNext()) { @@ -784,13 +787,15 @@ public void testListBucket() volBBucketBCount++; } Assert.assertEquals(10, volBBucketBCount); - Iterator volABucketBIter = volA.listBuckets("bucket-b-"); + Iterator volABucketBIter = volA.listBuckets( + "bucket-b-"); for(int i = 0; i < 10; i++) { Assert.assertTrue(volABucketBIter.next().getName() .startsWith(bucketBaseNameB + i + "-")); } Assert.assertFalse(volABucketBIter.hasNext()); - Iterator volBBucketAIter = volB.listBuckets("bucket-a-"); + Iterator volBBucketAIter = volB.listBuckets( + "bucket-a-"); for(int i = 0; i < 10; i++) { Assert.assertTrue(volBBucketAIter.next().getName() .startsWith(bucketBaseNameA + i + "-")); @@ -805,7 +810,7 @@ public void testListBucketsOnEmptyVolume() String volume = "vol-" + RandomStringUtils.randomNumeric(5); store.createVolume(volume); OzoneVolume vol = store.getVolume(volume); - Iterator buckets = vol.listBuckets(""); + Iterator buckets = vol.listBuckets(""); while(buckets.hasNext()) { Assert.fail(); } @@ -889,7 +894,7 @@ public void testListKey() four.write(value); four.close(); } - Iterator volABucketAIter = + Iterator volABucketAIter = volAbucketA.listKeys("key-"); int volABucketAKeyCount = 0; while(volABucketAIter.hasNext()) { @@ -897,7 +902,7 @@ public void testListKey() volABucketAKeyCount++; } Assert.assertEquals(20, volABucketAKeyCount); - Iterator volABucketBIter = + Iterator volABucketBIter = volAbucketB.listKeys("key-"); int volABucketBKeyCount = 0; while(volABucketBIter.hasNext()) { @@ -905,7 +910,7 @@ public void testListKey() volABucketBKeyCount++; } Assert.assertEquals(20, volABucketBKeyCount); - Iterator volBBucketAIter = + Iterator volBBucketAIter = volBbucketA.listKeys("key-"); int volBBucketAKeyCount = 0; while(volBBucketAIter.hasNext()) { @@ -913,7 +918,7 @@ public void testListKey() volBBucketAKeyCount++; } Assert.assertEquals(20, volBBucketAKeyCount); - Iterator volBBucketBIter = + Iterator volBBucketBIter = volBbucketB.listKeys("key-"); int volBBucketBKeyCount = 0; while(volBBucketBIter.hasNext()) { @@ -921,7 +926,7 @@ public void testListKey() volBBucketBKeyCount++; } Assert.assertEquals(20, volBBucketBKeyCount); - Iterator volABucketAKeyAIter = + Iterator volABucketAKeyAIter = volAbucketA.listKeys("key-a-"); int volABucketAKeyACount = 0; while(volABucketAKeyAIter.hasNext()) { @@ -929,7 +934,7 @@ public void testListKey() volABucketAKeyACount++; } Assert.assertEquals(10, volABucketAKeyACount); - Iterator volABucketAKeyBIter = + Iterator volABucketAKeyBIter = volAbucketA.listKeys("key-b-"); for(int i = 0; i < 10; i++) { Assert.assertTrue(volABucketAKeyBIter.next().getName() @@ -947,7 +952,7 @@ public void testListKeyOnEmptyBucket() OzoneVolume vol = store.getVolume(volume); vol.createBucket(bucket); OzoneBucket buc = vol.getBucket(bucket); - Iterator keys = buc.listKeys(""); + Iterator keys = buc.listKeys(""); while(keys.hasNext()) { Assert.fail(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java index e866d206d0..fe96198636 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java @@ -305,7 +305,7 @@ static void runTestListBucket(ClientProtocol client) .build(); vol.createBucket(bucketName, bucketArgs); } - Iterator bucketIterator = vol.listBuckets(null); + Iterator bucketIterator = vol.listBuckets(null); int count = 0; while (bucketIterator.hasNext()) { @@ -324,7 +324,7 @@ static void runTestListBucket(ClientProtocol client) client.close(); } - private static int getSize(Iterator bucketIterator) { + private static int getSize(Iterator bucketIterator) { int count = 0; while (bucketIterator.hasNext()) { count++; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java index 1d97bf5d34..d8c824bdd6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java @@ -88,7 +88,8 @@ public Void call() throws Exception { OzoneVolume vol = client.getObjectStore().getVolume(volumeName); - Iterator bucketIterator = vol.listBuckets(prefix, startBucket); + Iterator bucketIterator = + vol.listBuckets(prefix, startBucket); List bucketList = new ArrayList<>(); while (maxBuckets > 0 && bucketIterator.hasNext()) { BucketInfo bucketInfo = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java index 0a710ac945..3c019a6551 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java @@ -91,7 +91,8 @@ public Void call() throws Exception { OzoneVolume vol = client.getObjectStore().getVolume(volumeName); OzoneBucket bucket = vol.getBucket(bucketName); - Iterator keyIterator = bucket.listKeys(prefix, startKey); + Iterator keyIterator = bucket.listKeys(prefix, + startKey); List keyInfos = new ArrayList<>(); while (maxKeys > 0 && keyIterator.hasNext()) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java index a54393c3da..948a45c5b5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java @@ -89,7 +89,7 @@ public Void call() throws Exception { "the length should be a positive number"); } - Iterator volumeIterator; + Iterator volumeIterator; if(userName != null) { volumeIterator = client.getObjectStore() .listVolumesByUser(userName, prefix, startVolume); diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java index b876dc28b1..8fbd688200 100644 --- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java +++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java @@ -638,7 +638,7 @@ private abstract class OzoneListingIterator { private final Path path; private final FileStatus status; private String pathKey; - private Iterator keyIterator; + private Iterator keyIterator; OzoneListingIterator(Path path) throws IOException { diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index 075c674432..bb9652376a 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -183,6 +183,17 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/CommonHeadersContainerResponseFilter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/CommonHeadersContainerResponseFilter.java new file mode 100644 index 0000000000..d2f2d659d3 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/CommonHeadersContainerResponseFilter.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerResponseContext; +import javax.ws.rs.container.ContainerResponseFilter; +import javax.ws.rs.ext.Provider; +import java.io.IOException; + +/** + * This class adds common header responses for all the requests. + */ +@Provider +public class CommonHeadersContainerResponseFilter implements + ContainerResponseFilter { + + @Override + public void filter(ContainerRequestContext containerRequestContext, + ContainerResponseContext containerResponseContext) throws IOException { + containerResponseContext.getHeaders().add("Server", "Ozone"); + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java new file mode 100644 index 0000000000..bfbeeec227 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/EndpointBase.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import javax.inject.Inject; +import javax.ws.rs.NotFoundException; +import java.io.IOException; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneVolume; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Basic helpers for all the REST endpoints. + */ +public class EndpointBase { + + @Inject + private OzoneClient client; + + protected OzoneBucket getBucket(String volumeName, String bucketName) + throws IOException { + return getVolume(volumeName).getBucket(bucketName); + } + + protected OzoneBucket getBucket(OzoneVolume volume, String bucketName) + throws IOException { + OzoneBucket bucket = null; + try { + bucket = volume.getBucket(bucketName); + } catch (Exception ex) { + if (ex.getMessage().contains("NOT_FOUND")) { + throw new NotFoundException("Bucket" + bucketName + " is not found"); + } else { + throw ex; + } + } + return bucket; + } + + protected OzoneVolume getVolume(String volumeName) throws IOException { + OzoneVolume volume = null; + try { + volume = client.getObjectStore().getVolume(volumeName); + } catch (Exception ex) { + if (ex.getMessage().contains("NOT_FOUND")) { + throw new NotFoundException("Volume " + volumeName + " is not found"); + } else { + throw ex; + } + } + return volume; + } + + @VisibleForTesting + public void setClient(OzoneClient ozoneClient) { + this.client = ozoneClient; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java index 4e3e48e1f7..061a2d7d0d 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java @@ -17,8 +17,11 @@ */ package org.apache.hadoop.ozone.s3; +import java.io.IOException; + import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,21 +38,29 @@ public class Gateway extends GenericCli { private static final Logger LOG = LoggerFactory.getLogger(Gateway.class); + private S3GatewayHttpServer httpServer; + public static void main(String[] args) throws Exception { new Gateway().run(args); } @Override public Void call() throws Exception { + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + OzoneConfigurationHolder.setConfiguration(ozoneConfiguration); + httpServer = new S3GatewayHttpServer(ozoneConfiguration, "s3gateway"); start(); return null; } - public void start() { + public void start() throws IOException { LOG.info("Starting Ozone S3 gateway"); + httpServer.start(); } - public void stop() { - LOG.info("Stoping Ozone S3 gateway"); + public void stop() throws Exception { + LOG.info("Stopping Ozone S3 gateway"); + httpServer.stop(); } + } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/GatewayApplication.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/GatewayApplication.java new file mode 100644 index 0000000000..c5a291b445 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/GatewayApplication.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import org.glassfish.jersey.server.ResourceConfig; + +/** + * JaxRS resource definition. + */ +public class GatewayApplication extends ResourceConfig { + public GatewayApplication() { + packages("org.apache.hadoop.ozone.s3"); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java new file mode 100644 index 0000000000..e43815717d --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Produces; +import javax.inject.Inject; +import java.io.IOException; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; + +/** + * This class creates the OzoneClient for the Rest endpoints. + */ +@ApplicationScoped +public class OzoneClientProducer { + + private OzoneConfiguration ozoneConfiguration; + + @Inject + public OzoneClientProducer( + OzoneConfiguration ozoneConfiguration) { + this.ozoneConfiguration = ozoneConfiguration; + } + + @Produces + public OzoneClient createClient() throws IOException { + return OzoneClientFactory.getClient(ozoneConfiguration); + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java new file mode 100644 index 0000000000..4aeab1f3c4 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import javax.enterprise.inject.Produces; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; + +/** + * Ozone Configuration factory. + *

+ * As the OzoneConfiguration is created by the CLI application here we inject + * it via a singleton instance to the Jax-RS/CDI instances. + */ +public class OzoneConfigurationHolder { + + private static OzoneConfiguration configuration; + + @Produces + public OzoneConfiguration configuration() { + return configuration; + } + + public static void setConfiguration( + OzoneConfiguration conf) { + OzoneConfigurationHolder.configuration = conf; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java new file mode 100644 index 0000000000..c340a50917 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * This class contains constants for configuration keys used in S3G. + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public final class S3GatewayConfigKeys { + + public static final String OZONE_S3G_HTTP_ENABLED_KEY = + "ozone.s3g.http.enabled"; + public static final String OZONE_S3G_HTTP_BIND_HOST_KEY = + "ozone.s3g.http-bind-host"; + public static final String OZONE_S3G_HTTPS_BIND_HOST_KEY = + "ozone.s3g.https-bind-host"; + public static final String OZONE_S3G_HTTP_ADDRESS_KEY = + "ozone.s3g.http-address"; + public static final String OZONE_S3G_HTTPS_ADDRESS_KEY = + "ozone.s3g.https-address"; + public static final String OZONE_S3G_KEYTAB_FILE = + "ozone.s3g.keytab.file"; + public static final String OZONE_S3G_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; + public static final int OZONE_S3G_HTTP_BIND_PORT_DEFAULT = 9878; + public static final int OZONE_S3G_HTTPS_BIND_PORT_DEFAULT = 9879; + public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL = + "ozone.s3g.authentication.kerberos.principal"; + + /** + * Never constructed. + */ + private S3GatewayConfigKeys() { + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java new file mode 100644 index 0000000000..f20b928b1f --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayHttpServer.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.server.BaseHttpServer; + +/** + * S3 Gateway specific configuration keys. + */ +public class S3GatewayHttpServer extends BaseHttpServer { + + public S3GatewayHttpServer(Configuration conf, + String name) throws IOException { + super(conf, name); + } + + @Override + protected String getHttpAddressKey() { + return S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; + } + + @Override + protected String getHttpBindHostKey() { + return S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_HOST_KEY; + } + + @Override + protected String getHttpsAddressKey() { + return S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; + } + + @Override + protected String getHttpsBindHostKey() { + return S3GatewayConfigKeys.OZONE_S3G_HTTPS_BIND_HOST_KEY; + } + + @Override + protected String getBindHostDefault() { + return S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_HOST_DEFAULT; + } + + @Override + protected int getHttpBindPortDefault() { + return S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_PORT_DEFAULT; + } + + @Override + protected int getHttpsBindPortDefault() { + return S3GatewayConfigKeys.OZONE_S3G_HTTPS_BIND_PORT_DEFAULT; + } + + @Override + protected String getKeytabFile() { + return S3GatewayConfigKeys.OZONE_S3G_KEYTAB_FILE; + } + + @Override + protected String getSpnegoPrincipal() { + return S3GatewayConfigKeys.OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL; + } + + @Override + protected String getEnabledKey() { + return S3GatewayConfigKeys.OZONE_S3G_HTTP_ENABLED_KEY; + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/DeleteBucket.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/DeleteBucket.java new file mode 100644 index 0000000000..bbeb7d52df --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/DeleteBucket.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.bucket; + +import javax.ws.rs.DELETE; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +import org.apache.hadoop.ozone.s3.EndpointBase; + +/** + * Delete a bucket. + */ +@Path("/{volume}/{bucket}") +public class DeleteBucket extends EndpointBase { + + @DELETE + @Produces(MediaType.APPLICATION_XML) + public void put( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName) throws IOException { + + getVolume(volumeName).deleteBucket(bucketName); + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/PutBucket.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/PutBucket.java new file mode 100644 index 0000000000..a46b771c52 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/PutBucket.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.bucket; + +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.IOException; + +import org.apache.hadoop.ozone.s3.EndpointBase; + +/** + * Create new bucket. + */ +@Path("/{volume}/{bucket}") +public class PutBucket extends EndpointBase { + + @PUT + @Produces(MediaType.APPLICATION_XML) + public void put( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName) throws IOException { + + getVolume(volumeName).createBucket(bucketName); + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/package-info.java new file mode 100644 index 0000000000..c099c69fe7 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/bucket/package-info.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Rest endpoint implementation for the bucket specific methods. + */ +@javax.xml.bind.annotation.XmlSchema( + namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", elementFormDefault = + javax.xml.bind.annotation.XmlNsForm.QUALIFIED, + xmlns = { + @javax.xml.bind.annotation.XmlNs(namespaceURI = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", prefix = "")}) + +package org.apache.hadoop.ozone.s3.bucket; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java new file mode 100644 index 0000000000..83e60476c7 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/CommonPrefix.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.commontypes; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; + +/** + * Directory name ("key prefix") in case of listing. + */ +@XmlAccessorType(XmlAccessType.FIELD) +public class CommonPrefix { + + @XmlElement(name = "Prefix") + private String prefix; + + public CommonPrefix(String prefix) { + this.prefix = prefix; + } + + public CommonPrefix() { + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/IsoDateAdapter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/IsoDateAdapter.java new file mode 100644 index 0000000000..281e00b4e3 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/IsoDateAdapter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.commontypes; + +import javax.xml.bind.annotation.adapters.XmlAdapter; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +/** + * A converter to convert Instant to standard date string. + */ +public class IsoDateAdapter extends XmlAdapter { + + private DateTimeFormatter iso8861Formatter; + + public IsoDateAdapter() { + iso8861Formatter = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mmX") + .withZone(ZoneOffset.UTC); + } + + @Override + public Instant unmarshal(String v) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public String marshal(Instant v) throws Exception { + return iso8861Formatter.format(v); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java new file mode 100644 index 0000000000..34cea28194 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/KeyMetadata.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.commontypes; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.time.Instant; + +/** + * Metadata object represents one key in the object store. + */ +@XmlAccessorType(XmlAccessType.FIELD) +public class KeyMetadata { + + @XmlElement(name = "Key") + private String key; // or the Object Name + + @XmlJavaTypeAdapter(IsoDateAdapter.class) + @XmlElement(name = "LastModified") + private Instant lastModified; + + @XmlElement(name = "ETag") + private String eTag; + + @XmlElement(name = "Size") + private long size; + + @XmlElement(name = "StorageClass") + private String storageClass; + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public Instant getLastModified() { + return lastModified; + } + + public void setLastModified(Instant lastModified) { + this.lastModified = lastModified; + } + + public String getETag() { + return eTag; + } + + public void setETag(String tag) { + this.eTag = tag; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + public String getStorageClass() { + return storageClass; + } + + public void setStorageClass(String storageClass) { + this.storageClass = storageClass; + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/package-info.java new file mode 100644 index 0000000000..dd916e8839 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Common classes required for S3 rest API's. + */ +@javax.xml.bind.annotation.XmlSchema( + namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", elementFormDefault = + javax.xml.bind.annotation.XmlNsForm.QUALIFIED, + xmlns = { + @javax.xml.bind.annotation.XmlNs(namespaceURI = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", prefix = "")}) +package org.apache.hadoop.ozone.s3.commontypes; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/DeleteObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/DeleteObject.java new file mode 100644 index 0000000000..d5ef70e324 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/DeleteObject.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.object; + +import javax.ws.rs.DELETE; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.s3.EndpointBase; + +/** + * Delete Object rest endpoint. + */ +@Path("/{volume}/{bucket}/{path:.+}") +public class DeleteObject extends EndpointBase { + + @DELETE + @Produces(MediaType.APPLICATION_XML) + public Response delete( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName, + @PathParam("path") String keyPath) throws IOException { + + OzoneBucket bucket = getBucket(volumeName, bucketName); + bucket.deleteKey(keyPath); + return Response. + ok() + .build(); + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java new file mode 100644 index 0000000000..e2841003a4 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/HeadObject.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.object; + +import javax.ws.rs.HEAD; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.s3.EndpointBase; + +/** + * Get object info rest endpoint. + */ +@Path("/{volume}/{bucket}/{path:.+}") +public class HeadObject extends EndpointBase { + + @HEAD + @Produces(MediaType.APPLICATION_XML) + public Response head( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName, + @PathParam("path") String keyPath, + @HeaderParam("Content-Length") long length, + InputStream body) throws IOException { + + OzoneBucket bucket = getBucket(volumeName, bucketName); + OzoneKeyDetails key = bucket.getKey(keyPath); + + return Response. + ok() + .header("Content-Length", key.getDataSize()) + .build(); + + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java new file mode 100644 index 0000000000..73f1343791 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObject.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.object; + +import javax.ws.rs.DefaultValue; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.time.Instant; +import java.util.Iterator; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.s3.EndpointBase; +import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; + +import org.apache.commons.lang3.StringUtils; + +/** + * List Object Rest endpoint. + */ +@Path("/{volume}/{bucket}") +public class ListObject extends EndpointBase { + + + @GET + @Produces(MediaType.APPLICATION_XML) + public ListObjectResponse get( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName, + @QueryParam("delimiter") String delimiter, + @QueryParam("encoding-type") String encodingType, + @QueryParam("marker") String marker, + @DefaultValue("1000") @QueryParam("max-keys") int maxKeys, + @QueryParam("prefix") String prefix, + @Context HttpHeaders hh) throws IOException { + + if (delimiter == null) { + delimiter = "/"; + } + if (prefix == null) { + prefix = ""; + } + + OzoneVolume volume = getVolume(volumeName); + OzoneBucket bucket = getBucket(volume, bucketName); + + Iterator ozoneKeyIterator = bucket.listKeys(prefix); + + ListObjectResponse response = new ListObjectResponse(); + response.setDelimiter(delimiter); + response.setName(bucketName); + response.setPrefix(prefix); + response.setMarker(""); + response.setMaxKeys(1000); + response.setEncodingType("url"); + response.setTruncated(false); + + String prevDir = null; + while (ozoneKeyIterator.hasNext()) { + OzoneKey next = ozoneKeyIterator.next(); + String relativeKeyName = next.getName().substring(prefix.length()); + + int depth = + StringUtils.countMatches(relativeKeyName, delimiter); + + if (prefix.length() > 0 && !prefix.endsWith(delimiter) + && relativeKeyName.length() > 0) { + response.addPrefix(prefix + "/"); + break; + } + if (depth > 0) { + String dirName = relativeKeyName + .substring(0, relativeKeyName.indexOf(delimiter)); + if (!dirName.equals(prevDir)) { + response.addPrefix( + prefix + dirName + delimiter); + prevDir = dirName; + } + } else if (relativeKeyName.endsWith(delimiter)) { + response.addPrefix(relativeKeyName); + } else if (relativeKeyName.length() > 0) { + KeyMetadata keyMetadata = new KeyMetadata(); + keyMetadata.setKey(next.getName()); + keyMetadata.setSize(next.getDataSize()); + keyMetadata.setETag("" + next.getModificationTime()); + keyMetadata.setStorageClass("STANDARD"); + keyMetadata + .setLastModified(Instant.ofEpochMilli(next.getModificationTime())); + response.addKey(keyMetadata); + } + } + return response; + } + +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObjectResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObjectResponse.java new file mode 100644 index 0000000000..a32fb93d06 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/ListObjectResponse.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.object; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.ozone.s3.commontypes.CommonPrefix; +import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; + +/** + * Response from the ListObject RPC Call. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "ListBucketResult", namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/") +public class ListObjectResponse { + + @XmlElement(name = "Name") + private String name; + + @XmlElement(name = "Prefix") + private String prefix; + + @XmlElement(name = "Marker") + private String marker; + + @XmlElement(name = "MaxKeys") + private int maxKeys; + + @XmlElement(name = "Delimiter") + private String delimiter = "/"; + + @XmlElement(name = "EncodingType") + private String encodingType = "url"; + + @XmlElement(name = "IsTruncated") + private boolean isTruncated; + + @XmlElement(name = "Contents") + private List contents = new ArrayList<>(); + + @XmlElement(name = "CommonPrefixes") + private List commonPrefixes = new ArrayList<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public String getMarker() { + return marker; + } + + public void setMarker(String marker) { + this.marker = marker; + } + + public int getMaxKeys() { + return maxKeys; + } + + public void setMaxKeys(int maxKeys) { + this.maxKeys = maxKeys; + } + + public String getDelimiter() { + return delimiter; + } + + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; + } + + public String getEncodingType() { + return encodingType; + } + + public void setEncodingType(String encodingType) { + this.encodingType = encodingType; + } + + public boolean isTruncated() { + return isTruncated; + } + + public void setTruncated(boolean truncated) { + isTruncated = truncated; + } + + public List getContents() { + return contents; + } + + public void setContents( + List contents) { + this.contents = contents; + } + + public List getCommonPrefixes() { + return commonPrefixes; + } + + public void setCommonPrefixes( + List commonPrefixes) { + this.commonPrefixes = commonPrefixes; + } + + public void addKey(KeyMetadata keyMetadata) { + contents.add(keyMetadata); + } + + public void addPrefix(String relativeKeyName) { + commonPrefixes.add(new CommonPrefix(relativeKeyName)); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java new file mode 100644 index 0000000000..53e63d8561 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/PutObject.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.s3.object; + +import javax.ws.rs.HeaderParam; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.ozone.s3.EndpointBase; + +/** + * File upload. + */ +@Path("/{volume}/{bucket}/{path:.+}") +public class PutObject extends EndpointBase { + + @PUT + @Produces(MediaType.APPLICATION_XML) + public void put( + @PathParam("volume") String volumeName, + @PathParam("bucket") String bucketName, + @PathParam("path") String keyPath, + @HeaderParam("Content-Length") long length, + InputStream body) throws IOException { + + OzoneBucket bucket = getBucket(volumeName, bucketName); + + OzoneOutputStream output = bucket + .createKey(keyPath, length, ReplicationType.STAND_ALONE, + ReplicationFactor.ONE); + + IOUtils.copy(body, output); + output.close(); + } +} diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/package-info.java new file mode 100644 index 0000000000..e255991d58 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/object/package-info.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Rest endpoint implementation for the Object specific methods. + */ +@javax.xml.bind.annotation.XmlSchema( + namespace = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", elementFormDefault = + javax.xml.bind.annotation.XmlNsForm.QUALIFIED, + xmlns = { + @javax.xml.bind.annotation.XmlNs(namespaceURI = "http://s3.amazonaws" + + ".com/doc/2006-03-01/", prefix = "")}) +package org.apache.hadoop.ozone.s3.object; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/package-info.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/package-info.java new file mode 100644 index 0000000000..9d41551c49 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains the top level generic classes of s3 gateway. + */ +package org.apache.hadoop.ozone.s3; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/resources/META-INF/beans.xml b/hadoop-ozone/s3gateway/src/main/resources/META-INF/beans.xml new file mode 100644 index 0000000000..cf00d2936a --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/resources/META-INF/beans.xml @@ -0,0 +1,20 @@ + + + + \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/beans.xml b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/beans.xml new file mode 100644 index 0000000000..cf00d2936a --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/beans.xml @@ -0,0 +1,20 @@ + + + + \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml new file mode 100644 index 0000000000..a3552f0700 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/resources/webapps/s3gateway/WEB-INF/web.xml @@ -0,0 +1,36 @@ + + + + jaxrs + org.glassfish.jersey.servlet.ServletContainer + + javax.ws.rs.Application + org.apache.hadoop.ozone.s3.GatewayApplication + + 1 + + + jaxrs + /* + + + + org.jboss.weld.environment.servlet.Listener + + + + \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java new file mode 100644 index 0000000000..0777856c85 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * ObjectStore implementation with in-memory state. + */ +public class ObjectStoreStub extends ObjectStore { + + public ObjectStoreStub() { + super(); + } + + private Map volumes = new HashMap<>(); + + @Override + public void createVolume(String volumeName) throws IOException { + createVolume(volumeName, + VolumeArgs.newBuilder() + .setAdmin("root") + .setOwner("root") + .setQuota("" + Integer.MAX_VALUE) + .setAcls(new ArrayList<>()).build()); + } + + @Override + public void createVolume(String volumeName, VolumeArgs volumeArgs) + throws IOException { + OzoneVolumeStub volume = + new OzoneVolumeStub(volumeName, + volumeArgs.getAdmin(), + volumeArgs.getOwner(), + Long.parseLong(volumeArgs.getQuota()), + System.currentTimeMillis(), + volumeArgs.getAcls()); + volumes.put(volumeName, volume); + } + + @Override + public OzoneVolume getVolume(String volumeName) throws IOException { + if (volumes.containsKey(volumeName)) { + return volumes.get(volumeName); + } else { + throw new IOException("VOLUME_NOT_FOUND"); + } + } + + @Override + public Iterator listVolumes(String volumePrefix) + throws IOException { + return volumes.values() + .stream() + .filter(volume -> volume.getName().startsWith(volumePrefix)) + .collect(Collectors.toList()) + .iterator(); + + } + + @Override + public Iterator listVolumes(String volumePrefix, + String prevVolume) throws IOException { + return volumes.values() + .stream() + .filter(volume -> volume.getName().compareTo(prevVolume) > 0) + .filter(volume -> volume.getName().startsWith(volumePrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public Iterator listVolumesByUser(String user, + String volumePrefix, String prevVolume) throws IOException { + return volumes.values() + .stream() + .filter(volume -> volume.getOwner().equals(user)) + .filter(volume -> volume.getName().compareTo(prevVolume) < 0) + .filter(volume -> volume.getName().startsWith(volumePrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public void deleteVolume(String volumeName) throws IOException { + volumes.remove(volumeName); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java new file mode 100644 index 0000000000..18dff71aa6 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.client; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; + +/** + * In-memory ozone bucket for testing. + */ +public class OzoneBucketStub extends OzoneBucket { + + private Map keyDetails = new HashMap<>(); + + private Map keyContents = new HashMap<>(); + + /** + * Constructs OzoneBucket instance. + * + * @param volumeName Name of the volume the bucket belongs to. + * @param bucketName Name of the bucket. + * @param acls ACLs associated with the bucket. + * @param storageType StorageType of the bucket. + * @param versioning versioning status of the bucket. + * @param creationTime creation time of the bucket. + */ + public OzoneBucketStub( + String volumeName, + String bucketName, + List acls, + StorageType storageType, Boolean versioning, + long creationTime) { + super(volumeName, + bucketName, + ReplicationFactor.ONE, + ReplicationType.STAND_ALONE, + acls, + storageType, + versioning, + creationTime); + } + + @Override + public OzoneOutputStream createKey(String key, long size) throws IOException { + return createKey(key, size, ReplicationType.STAND_ALONE, + ReplicationFactor.ONE); + } + + @Override + public OzoneOutputStream createKey(String key, long size, + ReplicationType type, ReplicationFactor factor) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = + new ByteArrayOutputStream((int) size) { + @Override + public void close() throws IOException { + keyContents.put(key, toByteArray()); + keyDetails.put(key, new OzoneKeyDetails( + getVolumeName(), + getName(), + key, + size, + System.currentTimeMillis(), + System.currentTimeMillis(), + new ArrayList<>() + )); + super.close(); + } + }; + return new OzoneOutputStream(byteArrayOutputStream); + } + + @Override + public OzoneInputStream readKey(String key) throws IOException { + return new OzoneInputStream(new ByteArrayInputStream(keyContents.get(key))); + } + + @Override + public OzoneKeyDetails getKey(String key) throws IOException { + return keyDetails.get(key); + } + + @Override + public Iterator listKeys(String keyPrefix) { + return keyDetails.values() + .stream() + .filter(key -> key.getName().startsWith(keyPrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public Iterator listKeys(String keyPrefix, + String prevKey) { + return keyDetails.values() + .stream() + .filter(key -> key.getName().compareTo(prevKey) > 0) + .filter(key -> key.getName().startsWith(keyPrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public void deleteKey(String key) throws IOException { + keyDetails.remove(key); + } + + @Override + public void renameKey(String fromKeyName, String toKeyName) + throws IOException { + throw new UnsupportedOperationException(); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java new file mode 100644 index 0000000000..3c7a253465 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneClientStub.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.client; + +import java.io.IOException; + +/** + * In-memory OzoneClient for testing. + */ +public class OzoneClientStub extends OzoneClient { + + public OzoneClientStub() { + super(new ObjectStoreStub()); + } + + @Override + public void close() throws IOException { + //NOOP. + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java new file mode 100644 index 0000000000..eb866e020c --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneVolumeStub.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.client; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.ozone.OzoneAcl; + +/** + * Ozone volume with in-memory state for testing. + */ +public class OzoneVolumeStub extends OzoneVolume { + + private Map buckets = new HashMap<>(); + + public OzoneVolumeStub(String name, String admin, String owner, + long quotaInBytes, + long creationTime, List acls) { + super(name, admin, owner, quotaInBytes, creationTime, acls); + } + + @Override + public void createBucket(String bucketName) throws IOException { + createBucket(bucketName, new BucketArgs.Builder() + .setStorageType(StorageType.DEFAULT) + .setVersioning(false) + .build()); + } + + @Override + public void createBucket(String bucketName, BucketArgs bucketArgs) + throws IOException { + buckets.put(bucketName, new OzoneBucketStub( + getName(), + bucketName, + bucketArgs.getAcls(), + bucketArgs.getStorageType(), + bucketArgs.getVersioning(), + System.currentTimeMillis())); + + } + + @Override + public OzoneBucket getBucket(String bucketName) throws IOException { + return buckets.get(bucketName); + } + + @Override + public Iterator listBuckets(String bucketPrefix) { + return buckets.values() + .stream() + .filter(bucket -> bucket.getName().startsWith(bucketPrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public Iterator listBuckets(String bucketPrefix, + String prevBucket) { + return buckets.values() + .stream() + .filter(bucket -> bucket.getName().compareTo(prevBucket) > 0) + .filter(bucket -> bucket.getName().startsWith(bucketPrefix)) + .collect(Collectors.toList()) + .iterator(); + } + + @Override + public void deleteBucket(String bucketName) throws IOException { + buckets.remove(bucketName); + } +} diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/package-info.java new file mode 100644 index 0000000000..10e42745b3 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * In-memory OzoneClient implementation to test REST endpoints. + */ +package org.apache.hadoop.ozone.client; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestBucketResponse.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestBucketResponse.java new file mode 100644 index 0000000000..efc69bea8d --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestBucketResponse.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.s3.bucket; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; + +import org.apache.hadoop.ozone.s3.object.ListObjectResponse; + +import org.junit.Test; + +/** + * Testing JAXB serialization. + */ +public class TestBucketResponse { + + @Test + public void serialize() throws JAXBException { + JAXBContext context = JAXBContext.newInstance(ListObjectResponse.class); + context.createMarshaller().marshal(new ListObjectResponse(), System.out); + } + +} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java new file mode 100644 index 0000000000..f5a6f7e2df --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/TestGetBucket.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.s3.bucket; + +import java.io.IOException; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; +import org.apache.hadoop.ozone.s3.object.ListObject; +import org.apache.hadoop.ozone.s3.object.ListObjectResponse; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Testing basic object list browsing. + */ +public class TestGetBucket { + + @Test + public void listRoot() throws IOException { + + ListObject getBucket = new ListObject(); + + OzoneClient client = createClientWithKeys("file1", "dir1/file2"); + + getBucket.setClient(client); + + ListObjectResponse getBucketResponse = + getBucket.get("vol1", "b1", "/", null, null, 100, "", null); + + Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); + Assert.assertEquals("dir1/", + getBucketResponse.getCommonPrefixes().get(0).getPrefix()); + + Assert.assertEquals(1, getBucketResponse.getContents().size()); + Assert.assertEquals("file1", + getBucketResponse.getContents().get(0).getKey()); + + } + + @Test + public void listDir() throws IOException { + + ListObject getBucket = new ListObject(); + + OzoneClient client = createClientWithKeys("dir1/file2", "dir1/dir2/file2"); + + getBucket.setClient(client); + + ListObjectResponse getBucketResponse = + getBucket.get("vol1", "b1", "/", null, null, 100, "dir1", null); + + Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); + Assert.assertEquals("dir1/", + getBucketResponse.getCommonPrefixes().get(0).getPrefix()); + + Assert.assertEquals(0, getBucketResponse.getContents().size()); + + } + + @Test + public void listSubDir() throws IOException { + + ListObject getBucket = new ListObject(); + OzoneClient ozoneClient = + createClientWithKeys("dir1/file2", "dir1/dir2/file2"); + + getBucket.setClient(ozoneClient); + + ListObjectResponse getBucketResponse = + getBucket.get("vol1", "b1", "/", null, null, 100, "dir1/", null); + + Assert.assertEquals(1, getBucketResponse.getCommonPrefixes().size()); + Assert.assertEquals("dir1/dir2/", + getBucketResponse.getCommonPrefixes().get(0).getPrefix()); + + Assert.assertEquals(1, getBucketResponse.getContents().size()); + Assert.assertEquals("dir1/file2", + getBucketResponse.getContents().get(0).getKey()); + + } + + private OzoneClient createClientWithKeys(String... keys) throws IOException { + OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createVolume("vol1"); + client.getObjectStore().getVolume("vol1").createBucket("b1"); + OzoneBucket bucket = + client.getObjectStore().getVolume("vol1").getBucket("b1"); + for (String key : keys) { + bucket.createKey(key, 0).close(); + } + return client; + } +} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/package-info.java new file mode 100644 index 0000000000..de09daef55 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/bucket/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Unit tests for the bucket related rest endpoints. + */ +package org.apache.hadoop.ozone.s3.bucket; \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestDeleteObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestDeleteObject.java new file mode 100644 index 0000000000..6c06a76742 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/TestDeleteObject.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.hadoop.ozone.s3.object; + +import java.io.IOException; + +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientStub; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test delete object. + */ +public class TestDeleteObject { + + @Test + public void delete() throws IOException { + //GIVEN + OzoneClient client = new OzoneClientStub(); + client.getObjectStore().createVolume("vol1"); + client.getObjectStore().getVolume("vol1").createBucket("b1"); + OzoneBucket bucket = + client.getObjectStore().getVolume("vol1").getBucket("b1"); + bucket.createKey("key1", 0).close(); + + DeleteObject rest = new DeleteObject(); + rest.setClient(client); + + //WHEN + rest.delete("vol1", "b1", "key1"); + + //THEN + Assert.assertFalse("Bucket Should not contain any key after delete", + bucket.listKeys("").hasNext()); + } +} \ No newline at end of file diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/package-info.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/package-info.java new file mode 100644 index 0000000000..897e2095a6 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/object/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Unit tests for the object related rest endpoints. + */ +package org.apache.hadoop.ozone.s3.object; \ No newline at end of file