From 86ae218893d018638e937c2528c8e84336254da7 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 25 Jul 2016 15:48:32 +0100 Subject: [PATCH] HADOOP-13188 S3A file-create should throw error rather than overwrite directories. Contributed by Steve Loughran --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 19 +++++++++++++++++-- .../contract/s3a/TestS3AContractCreate.java | 5 ----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 075f36c9ac..902f73ab43 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -588,9 +588,24 @@ public FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { String key = pathToKey(f); + S3AFileStatus status = null; + try { + // get the status or throw an FNFE + status = getFileStatus(f); + + // if the thread reaches here, there is something at the path + if (status.isDirectory()) { + // path references a directory: automatic error + throw new FileAlreadyExistsException(f + " is a directory"); + } + if (!overwrite) { + // path references a file and overwrite is disabled + throw new FileAlreadyExistsException(f + " already exists"); + } + LOG.debug("Overwriting file {}", f); + } catch (FileNotFoundException e) { + // this means the file is not found - if (!overwrite && exists(f)) { - throw new FileAlreadyExistsException(f + " already exists"); } instrumentation.fileCreated(); if (getConf().getBoolean(FAST_UPLOAD, DEFAULT_FAST_UPLOAD)) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java index 1d95ddfdb4..035252bf5f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/TestS3AContractCreate.java @@ -30,9 +30,4 @@ protected AbstractFSContract createContract(Configuration conf) { return new S3AContract(conf); } - @Override - public void testOverwriteEmptyDirectory() throws Throwable { - ContractTestUtils.skip( - "blobstores can't distinguish empty directories from files"); - } }