HADOOP-18438: AliyunOSSFileSystemStore deleteObjects interface should return the objects that failed to delete (#4857)
Merged to trunk, thank @chenshuang778 for your contribution
This commit is contained in:
parent
52c72fafe4
commit
f6605f1b3a
@ -72,6 +72,7 @@
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
|
||||
|
||||
@ -203,31 +204,29 @@ public void deleteObjects(List<String> keysToDelete) throws IOException {
|
||||
|
||||
int retry = 10;
|
||||
int tries = 0;
|
||||
List<String> deleteFailed = keysToDelete;
|
||||
while(CollectionUtils.isNotEmpty(deleteFailed)) {
|
||||
while (CollectionUtils.isNotEmpty(keysToDelete)) {
|
||||
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucketName);
|
||||
deleteRequest.setKeys(deleteFailed);
|
||||
deleteRequest.setKeys(keysToDelete);
|
||||
// There are two modes to do batch delete:
|
||||
// 1. detail mode: DeleteObjectsResult.getDeletedObjects returns objects
|
||||
// which were deleted successfully.
|
||||
// 2. simple mode: DeleteObjectsResult.getDeletedObjects returns objects
|
||||
// which were deleted unsuccessfully.
|
||||
// Here, we choose the simple mode to do batch delete.
|
||||
deleteRequest.setQuiet(true);
|
||||
// 1. verbose mode: A list of all deleted objects is returned.
|
||||
// 2. quiet mode: No message body is returned.
|
||||
// Here, we choose the verbose mode to do batch delete.
|
||||
deleteRequest.setQuiet(false);
|
||||
DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest);
|
||||
statistics.incrementWriteOps(1);
|
||||
deleteFailed = result.getDeletedObjects();
|
||||
final List<String> deletedObjects = result.getDeletedObjects();
|
||||
keysToDelete = keysToDelete.stream().filter(item -> !deletedObjects.contains(item))
|
||||
.collect(Collectors.toList());
|
||||
tries++;
|
||||
if (tries == retry) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (tries == retry && CollectionUtils.isNotEmpty(deleteFailed)) {
|
||||
if (tries == retry && CollectionUtils.isNotEmpty(keysToDelete)) {
|
||||
// Most of time, it is impossible to try 10 times, expect the
|
||||
// Aliyun OSS service problems.
|
||||
throw new IOException("Failed to delete Aliyun OSS objects for " +
|
||||
tries + " times.");
|
||||
throw new IOException("Failed to delete Aliyun OSS objects for " + tries + " times.");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -18,9 +18,12 @@
|
||||
|
||||
package org.apache.hadoop.fs.aliyun.oss;
|
||||
|
||||
import com.aliyun.oss.model.OSSObjectSummary;
|
||||
import com.aliyun.oss.model.ObjectMetadata;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.contract.ContractTestUtils;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
@ -36,7 +39,10 @@
|
||||
import java.security.DigestOutputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.MAX_PAGING_KEYS_DEFAULT;
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
@ -128,4 +134,29 @@ public void testLargeUpload()
|
||||
writeRenameReadCompare(new Path("/test/xlarge"),
|
||||
Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeleteObjects() throws IOException, NoSuchAlgorithmException {
|
||||
// generate test files
|
||||
final int files = 10;
|
||||
final long size = 5 * 1024 * 1024;
|
||||
final String prefix = "dir";
|
||||
for (int i = 0; i < files; i++) {
|
||||
Path path = new Path(String.format("/%s/testFile-%d.txt", prefix, i));
|
||||
ContractTestUtils.generateTestFile(this.fs, path, size, 256, 255);
|
||||
}
|
||||
OSSListRequest listRequest =
|
||||
store.createListObjectsRequest(prefix, MAX_PAGING_KEYS_DEFAULT, null, null, true);
|
||||
List<String> keysToDelete = new ArrayList<>();
|
||||
OSSListResult objects = store.listObjects(listRequest);
|
||||
assertEquals(files, objects.getObjectSummaries().size());
|
||||
|
||||
// test delete files
|
||||
for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
|
||||
keysToDelete.add(objectSummary.getKey());
|
||||
}
|
||||
store.deleteObjects(keysToDelete);
|
||||
objects = store.listObjects(listRequest);
|
||||
assertEquals(0, objects.getObjectSummaries().size());
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user