diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2cb86f6573..e7b0e5d308 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -720,6 +720,8 @@ Release 2.8.0 - UNRELEASED HADOOP-12657. Add a option to skip newline on empty files with getMerge -nl. (Kanaka Kumar Avvaru via aajisaka) + HADOOP-11262. Enable YARN to use S3A. (Pieter Reuse via lei) + IMPROVEMENTS HADOOP-12458. Retries is typoed to spell Retires in parts of diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index a766f79a6d..bf60a25c51 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -936,6 +936,12 @@ for ldap providers in the same way as above does. The implementation class of the S3A Filesystem + + fs.AbstractFileSystem.s3a.impl + org.apache.hadoop.fs.s3a.S3A + The implementation class of the S3A AbstractFileSystem. + + io.seqfile.compress.blocksize 1000000 diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 12ec3751c3..0c736a6b0e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -1166,7 +1166,7 @@ public void testUnsupportedSymlink() throws IOException { fc.createSymlink(file, link, false); Assert.fail("Created a symlink on a file system that "+ "does not support symlinks."); - } catch (IOException e) { + } catch (UnsupportedOperationException e) { // Expected } createFile(file); diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 9ef99fb160..eb87d77b7c 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -128,6 +128,22 @@ junit test - + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + test + + + org.apache.hadoop + hadoop-yarn-server-tests + test + test-jar + + + org.apache.hadoop + hadoop-mapreduce-examples + test + jar + diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 99e85cfef5..fc99cb9607 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -113,4 +113,6 @@ public class Constants { public static final String S3N_FOLDER_SUFFIX = "_$folder$"; public static final String FS_S3A_BLOCK_SIZE = "fs.s3a.block.size"; public static final String FS_S3A = "s3a"; + + public static final int S3A_DEFAULT_PORT = -1; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java new file mode 100644 index 0000000000..d856d802d5 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3A.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.DelegateToFileSystem; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * S3A implementation of AbstractFileSystem. + * This impl delegates to the S3AFileSystem + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public class S3A extends DelegateToFileSystem{ + + public S3A(URI theUri, Configuration conf) + throws IOException, URISyntaxException { + super(theUri, new S3AFileSystem(), conf, "s3a", false); + } + + @Override + public int getUriDefaultPort() { + return Constants.S3A_DEFAULT_PORT; + } +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java index 85e0ef77d5..47caea8390 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java @@ -15,7 +15,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.fs.s3a; import org.apache.hadoop.fs.FileStatus; @@ -40,7 +39,12 @@ public S3AFileStatus(long length, long modification_time, Path path, public boolean isEmptyDirectory() { return isEmptyDirectory; } - + + @Override + public String getOwner() { + return System.getProperty("user.name"); + } + /** Compare if this object is equal to another object * @param o the object to be compared. * @return true if two file status has the same path name; false if not. @@ -60,4 +64,23 @@ public boolean equals(Object o) { public int hashCode() { return super.hashCode(); } + + /** Get the modification time of the file/directory. + * + * s3a uses objects as "fake" directories, which are not updated to + * reflect the accurate modification time. We choose to report the + * current time because some parts of the ecosystem (e.g. the + * HistoryServer) use modification time to ignore "old" directories. + * + * @return for files the modification time in milliseconds since January 1, + * 1970 UTC or for directories the current time. + */ + @Override + public long getModificationTime(){ + if(isDirectory()){ + return System.currentTimeMillis(); + } else { + return super.getModificationTime(); + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index ccb0cd26ac..71771bd9b9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -265,15 +265,24 @@ public void initialize(URI name, Configuration conf) throws IOException { * * @return "s3a" */ + @Override public String getScheme() { return "s3a"; } - /** Returns a URI whose scheme and authority identify this FileSystem.*/ + /** + * Returns a URI whose scheme and authority identify this FileSystem. + */ + @Override public URI getUri() { return uri; } + @Override + public int getDefaultPort() { + return Constants.S3A_DEFAULT_PORT; + } + /** * Returns the S3 client used by this filesystem. * @return AmazonS3Client @@ -339,8 +348,10 @@ public FSDataInputStream open(Path f, int bufferSize) * @throws IOException * @see #setPermission(Path, FsPermission) */ - public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, - int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { + @Override + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { String key = pathToKey(f); if (!overwrite && exists(f)) { diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 0260e26d4b..9fb0f362e4 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -47,10 +47,14 @@ recursive file-by-file operations. They take time at least proportional to the number of files, during which time partial updates may be visible. If the operations are interrupted, the filesystem is left in an intermediate state. +## Warning #2: Because Object stores don't track modification times of directories, +features of Hadoop relying on this can have unexpected behaviour. E.g. the +AggregatedLogDeletionService of YARN will not remove the appropriate logfiles. + For further discussion on these topics, please consult [The Hadoop FileSystem API Definition](../../../hadoop-project-dist/hadoop-common/filesystem/index.html). -## Warning #2: your AWS credentials are valuable +## Warning #3: your AWS credentials are valuable Your AWS credentials not only pay for services, they offer read and write access to the data. Anyone with the credentials can not only read your datasets @@ -301,6 +305,12 @@ If you do any of these: change your credentials immediately! The implementation class of the S3A Filesystem + + fs.AbstractFileSystem.s3a.impl + org.apache.hadoop.fs.s3a.S3A + The implementation class of the S3A AbstractFileSystem. + + ### S3AFastOutputStream **Warning: NEW in hadoop 2.7. UNSTABLE, EXPERIMENTAL: use at own risk** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index d9726db702..2308dd48cb 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; import org.junit.internal.AssumptionViolatedException; import java.io.IOException; @@ -51,4 +52,24 @@ public static S3AFileSystem createTestFileSystem(Configuration conf) throws fs1.initialize(testURI, conf); return fs1; } + + public static FileContext createTestFileContext(Configuration conf) throws + IOException { + String fsname = conf.getTrimmed(TestS3AFileSystemContract.TEST_FS_S3A_NAME, ""); + + boolean liveTest = !StringUtils.isEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(Constants.FS_S3A); + } + if (!liveTest) { + // This doesn't work with our JUnit 3 style test cases, so instead we'll + // make this whole class not run by default + throw new AssumptionViolatedException( + "No test filesystem in " + TestS3AFileSystemContract.TEST_FS_S3A_NAME); + } + FileContext fc = FileContext.getFileContext(testURI,conf); + return fc; + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java new file mode 100644 index 0000000000..34d78a5d26 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContext.java @@ -0,0 +1,23 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import org.apache.hadoop.fs.TestFileContext; + +/** + * Implementation of TestFileContext for S3a + */ +public class TestS3AFileContext extends TestFileContext{ + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java new file mode 100644 index 0000000000..b0c4d8424f --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextCreateMkdir.java @@ -0,0 +1,35 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextCreateMkdirBaseTest; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; + +/** + * Extends FileContextCreateMkdirBaseTest for a S3a FileContext + */ +public class TestS3AFileContextCreateMkdir + extends FileContextCreateMkdirBaseTest { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java new file mode 100644 index 0000000000..4d200d14e2 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextMainOperations.java @@ -0,0 +1,60 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextMainOperationsBaseTest; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +/** + * S3A implementation of FileContextMainOperationsBaseTest + */ +public class TestS3AFileContextMainOperations + extends FileContextMainOperationsBaseTest { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + + @Override + protected boolean listCorruptedBlocksSupported() { + return false; + } + + @Test + @Ignore + public void testCreateFlagAppendExistingFile() throws IOException { + //append not supported, so test removed + } + + @Test + @Ignore + public void testCreateFlagCreateAppendExistingFile() throws IOException { + //append not supported, so test removed + } + + @Test + @Ignore + public void testSetVerifyChecksum() throws IOException { + //checksums ignored, so test removed + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java new file mode 100644 index 0000000000..a9f4848367 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextStatistics.java @@ -0,0 +1,60 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.net.URI; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FCStatisticsBaseTest; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; + +/** + * S3a implementation of FCStatisticsBaseTest + */ +public class TestS3AFileContextStatistics extends FCStatisticsBaseTest { + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "test"), FileContext.DEFAULT_PERM, true); + } + + @After + public void tearDown() throws Exception { + fc.delete(fileContextTestHelper.getTestRootPath(fc, "test"), true); + } + + @Override + protected void verifyReadBytes(FileSystem.Statistics stats) { + // one blockSize for read, one for pread + Assert.assertEquals(2 * blockSize, stats.getBytesRead()); + } + + @Override + protected void verifyWrittenBytes(FileSystem.Statistics stats) { + //No extra bytes are written + Assert.assertEquals(blockSize, stats.getBytesWritten()); + } + + @Override + protected URI getFsUri() { + return fc.getHomeDirectory().toUri(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java new file mode 100644 index 0000000000..3da7b19a84 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextURI.java @@ -0,0 +1,43 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextURIBase; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +/** + * S3a implementation of FileContextURIBase + */ +public class TestS3AFileContextURI extends FileContextURIBase { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc1 = S3ATestUtils.createTestFileContext(conf); + fc2 = S3ATestUtils.createTestFileContext(conf); //different object, same FS + super.setUp(); + } + + @Test + @Ignore + public void testFileStatus() throws IOException { + //test disabled (the statistics tested with this method are not relevant for an S3FS) + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java new file mode 100644 index 0000000000..666f4c292a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/fileContext/TestS3AFileContextUtil.java @@ -0,0 +1,34 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.fileContext; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContextUtilBase; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.junit.Before; + +/** + * S3A implementation of FileContextUtilBase + */ +public class TestS3AFileContextUtil extends FileContextUtilBase { + + @Before + public void setUp() throws IOException, Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + super.setUp(); + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java new file mode 100644 index 0000000000..c6a5a39b8a --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3A.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.s3a.yarn; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Path; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.EnumSet; +import org.apache.hadoop.fs.s3a.S3ATestUtils; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestS3A { + private FileContext fc; + + @Rule + public final Timeout testTimeout = new Timeout(90000); + + @Before + public void setUp() throws Exception { + Configuration conf = new Configuration(); + fc = S3ATestUtils.createTestFileContext(conf); + } + + @After + public void tearDown() throws Exception { + fc.delete(getTestPath(),true); + } + + protected Path getTestPath() { + return new Path("/tests3afc"); + } + + @Test + public void testS3AStatus() throws Exception { + FsStatus fsStatus = fc.getFsStatus(null); + assertNotNull(fsStatus); + assertTrue("Used capacity should be positive: " + fsStatus.getUsed(), + fsStatus.getUsed() >= 0); + assertTrue("Remaining capacity should be positive: " + fsStatus + .getRemaining(), + fsStatus.getRemaining() >= 0); + assertTrue("Capacity should be positive: " + fsStatus.getCapacity(), + fsStatus.getCapacity() >= 0); + } + + @Test + public void testS3ACreateFileInSubDir() throws Exception { + Path dirPath = getTestPath(); + fc.mkdir(dirPath,FileContext.DIR_DEFAULT_PERM,true); + Path filePath = new Path(dirPath, "file"); + try (FSDataOutputStream file = fc.create(filePath, EnumSet.of(CreateFlag + .CREATE))) { + file.write(666); + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java new file mode 100644 index 0000000000..990d79f205 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/TestS3AMiniYarnCluster.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.fs.s3a.yarn; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.examples.WordCount; +import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.MiniYARNCluster; + +import org.junit.After; +import static org.junit.Assert.assertTrue; +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +public class TestS3AMiniYarnCluster { + + private final Configuration conf = new YarnConfiguration(); + private S3AFileSystem fs; + private MiniYARNCluster yarnCluster; + private final String rootPath = "/tests/MiniClusterWordCount/"; + + @Before + public void beforeTest() throws IOException { + fs = S3ATestUtils.createTestFileSystem(conf); + fs.mkdirs(new Path(rootPath + "input/")); + + yarnCluster = new MiniYARNCluster("MiniClusterWordCount", // testName + 1, // number of node managers + 1, // number of local log dirs per node manager + 1); // number of hdfs dirs per node manager + yarnCluster.init(conf); + yarnCluster.start(); + } + + @After + public void afterTest() throws IOException { + fs.delete(new Path(rootPath), true); + yarnCluster.stop(); + } + + @Test + public void testWithMiniCluster() throws Exception { + Path input = new Path(rootPath + "input/in.txt"); + input = input.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + Path output = new Path(rootPath + "output/"); + output = output.makeQualified(fs.getUri(), fs.getWorkingDirectory()); + + writeStringToFile(input, "first line\nsecond line\nthird line"); + + Job job = Job.getInstance(conf, "word count"); + job.setJarByClass(WordCount.class); + job.setMapperClass(WordCount.TokenizerMapper.class); + job.setCombinerClass(WordCount.IntSumReducer.class); + job.setReducerClass(WordCount.IntSumReducer.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(IntWritable.class); + FileInputFormat.addInputPath(job, input); + FileOutputFormat.setOutputPath(job, output); + + int exitCode = (job.waitForCompletion(true) ? 0 : 1); + assertEquals("Returned error code.", 0, exitCode); + + assertTrue(fs.exists(new Path(output, "_SUCCESS"))); + String outputAsStr = readStringFromFile(new Path(output, "part-r-00000")); + Map resAsMap = getResultAsMap(outputAsStr); + + assertEquals(4, resAsMap.size()); + assertEquals(1, (int) resAsMap.get("first")); + assertEquals(1, (int) resAsMap.get("second")); + assertEquals(1, (int) resAsMap.get("third")); + assertEquals(3, (int) resAsMap.get("line")); + } + + /** + * helper method + */ + private Map getResultAsMap(String outputAsStr) throws IOException { + Map result = new HashMap<>(); + for (String line : outputAsStr.split("\n")) { + String[] tokens = line.split("\t"); + result.put(tokens[0], Integer.parseInt(tokens[1])); + } + return result; + } + + /** + * helper method + */ + private void writeStringToFile(Path path, String string) throws IOException { + FileContext fc = S3ATestUtils.createTestFileContext(conf); + try (FSDataOutputStream file = fc.create(path, + EnumSet.of(CreateFlag.CREATE))) { + file.write(string.getBytes()); + } + } + + /** + * helper method + */ + private String readStringFromFile(Path path) { + try (FSDataInputStream in = fs.open(path)) { + long bytesLen = fs.getFileStatus(path).getLen(); + byte[] buffer = new byte[(int) bytesLen]; + IOUtils.readFully(in, buffer, 0, buffer.length); + return new String(buffer); + } catch (IOException e) { + throw new RuntimeException("Failed to read from [" + path + "]", e); + } + } + +} diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml index 3397769d3a..7d2046b588 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml +++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml @@ -23,29 +23,28 @@ - - hadoop.tmp.dir - target/build/test - A base for other temporary directories. - true - + + hadoop.tmp.dir + target/build/test + A base for other temporary directories. + true + - - - hadoop.security.authentication - simple - - - - + + + hadoop.security.authentication + simple + + + + +