big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-15667. FileSystemMultipartUploader should verify that UploadHandle has non-0 length.

Browse Source 
Contributed by Ewan Higgs
This commit is contained in:
Steve Loughran 2018-08-30 14:33:16 +01:00
parent 781437c219
commit 2e6c1109dc
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
4 changed files with 61 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
View File

@ -68,6 +68,7 @@ public PartHandle putPart(Path filePath, InputStream inputStream,
throws IOException {
byte[] uploadIdByteArray = uploadId.toByteArray();
checkUploadId(uploadIdByteArray);
Path collectorPath = new Path(new String(uploadIdByteArray, 0,
uploadIdByteArray.length, Charsets.UTF_8));
Path partPath =
@ -101,6 +102,8 @@ public PathHandle complete(Path filePath,
List<Pair<Integer, PartHandle>> handles, UploadHandle multipartUploadId)
throws IOException {
checkUploadId(multipartUploadId.toByteArray());
if (handles.isEmpty()) {
throw new IOException("Empty upload");
}
@ -133,8 +136,7 @@ public PathHandle complete(Path filePath,
@Override
public void abort(Path filePath, UploadHandle uploadId) throws IOException {
byte[] uploadIdByteArray = uploadId.toByteArray();
Preconditions.checkArgument(uploadIdByteArray.length != 0,
"UploadId is empty");
checkUploadId(uploadIdByteArray);
Path collectorPath = new Path(new String(uploadIdByteArray, 0,
uploadIdByteArray.length, Charsets.UTF_8));

11
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MultipartUploader.java
View File

@ -21,6 +21,7 @@
import java.io.InputStream;
import java.util.List;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -91,4 +92,14 @@ public abstract PathHandle complete(Path filePath,
public abstract void abort(Path filePath, UploadHandle multipartUploadId)
throws IOException;
/**
* Utility method to validate uploadIDs
* @param uploadId
* @throws IllegalArgumentException
*/
protected void checkUploadId(byte[] uploadId)
throws IllegalArgumentException {
Preconditions.checkArgument(uploadId.length > 0,
"Empty UploadId is not valid");
}
}

43
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
View File

@ -297,4 +297,47 @@ public void testCompleteEmptyUpload() throws Exception {
() -> mpu.complete(dest, new ArrayList<>(), handle));
mpu.abort(dest, handle);
}
/**
* When we pass empty uploadID, putPart throws IllegalArgumentException.
* @throws Exception
*/
@Test
public void testPutPartEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when putPart uploadID is empty");
FileSystem fs = getFileSystem();
Path dest = path("testCompleteEmptyUpload");
MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
mpu.initialize(dest);
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
byte[] payload = generatePayload(1);
InputStream is = new ByteArrayInputStream(payload);
intercept(IllegalArgumentException.class,
() -> mpu.putPart(dest, is, 1, emptyHandle, payload.length));
}
/**
* When we pass empty uploadID, complete throws IllegalArgumentException.
* @throws Exception
*/
@Test
public void testCompleteEmptyUploadID() throws Exception {
describe("Expect IllegalArgumentException when complete uploadID is empty");
FileSystem fs = getFileSystem();
Path dest = path("testCompleteEmptyUpload");
MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
UploadHandle realHandle = mpu.initialize(dest);
UploadHandle emptyHandle =
BBUploadHandle.from(ByteBuffer.wrap(new byte[0]));
List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
byte[] payload = generatePayload(1);
InputStream is = new ByteArrayInputStream(payload);
PartHandle partHandle = mpu.putPart(dest, is, 1, realHandle,
payload.length);
partHandles.add(Pair.of(1, partHandle));
intercept(IllegalArgumentException.class,
() -> mpu.complete(dest, partHandles, emptyHandle));
}
}

10
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AMultipartUploader.java
View File

@ -84,9 +84,10 @@ public UploadHandle initialize(Path filePath) throws IOException {
public PartHandle putPart(Path filePath, InputStream inputStream,
int partNumber, UploadHandle uploadId, long lengthInBytes)
throws IOException {
final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper();
String key = s3a.pathToKey(filePath);
byte[] uploadIdBytes = uploadId.toByteArray();
checkUploadId(uploadIdBytes);
String key = s3a.pathToKey(filePath);
final WriteOperationHelper writeHelper = s3a.getWriteOperationHelper();
String uploadIdString = new String(uploadIdBytes, 0, uploadIdBytes.length,
Charsets.UTF_8);
UploadPartRequest request = writeHelper.newUploadPartRequest(key,
@ -155,11 +156,6 @@ protected MultipartUploader createMultipartUploader(FileSystem fs,
}
}
private void checkUploadId(byte[] uploadId) throws IllegalArgumentException {
Preconditions.checkArgument(uploadId.length > 0,
"Empty UploadId is not valid");
}
/**
* Build the payload for marshalling.
* @param eTag upload etag