big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18438: AliyunOSSFileSystemStore deleteObjects interface should return the objects that failed to delete (#4857)

Browse Source 
Merged to trunk, thank @chenshuang778  for your contribution
This commit is contained in:
陈爽-Jack Chen 2022-12-20 13:57:49 +08:00 committed by GitHub
parent 52c72fafe4
commit f6605f1b3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 43 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
View File

@ -72,6 +72,7 @@
import java.util.List; import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.stream.Collectors;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*; import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
@ -203,31 +204,29 @@ public void deleteObjects(List<String> keysToDelete) throws IOException {
int retry = 10; int retry = 10;
int tries = 0; int tries = 0;
List<String> deleteFailed = keysToDelete; while (CollectionUtils.isNotEmpty(keysToDelete)) {
while(CollectionUtils.isNotEmpty(deleteFailed)) {
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucketName); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucketName);
deleteRequest.setKeys(deleteFailed); deleteRequest.setKeys(keysToDelete);
// There are two modes to do batch delete: // There are two modes to do batch delete:
// 1. detail mode: DeleteObjectsResult.getDeletedObjects returns objects // 1. verbose mode: A list of all deleted objects is returned.
// which were deleted successfully. // 2. quiet mode: No message body is returned.
// 2. simple mode: DeleteObjectsResult.getDeletedObjects returns objects // Here, we choose the verbose mode to do batch delete.
// which were deleted unsuccessfully. deleteRequest.setQuiet(false);
// Here, we choose the simple mode to do batch delete.
deleteRequest.setQuiet(true);
DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest); DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest);
statistics.incrementWriteOps(1); statistics.incrementWriteOps(1);
deleteFailed = result.getDeletedObjects(); final List<String> deletedObjects = result.getDeletedObjects();
keysToDelete = keysToDelete.stream().filter(item -> !deletedObjects.contains(item))
.collect(Collectors.toList());
tries++; tries++;
if (tries == retry) { if (tries == retry) {
break; break;
} }
} }
if (tries == retry && CollectionUtils.isNotEmpty(deleteFailed)) { if (tries == retry && CollectionUtils.isNotEmpty(keysToDelete)) {
// Most of time, it is impossible to try 10 times, expect the // Most of time, it is impossible to try 10 times, expect the
// Aliyun OSS service problems. // Aliyun OSS service problems.
throw new IOException("Failed to delete Aliyun OSS objects for " + throw new IOException("Failed to delete Aliyun OSS objects for " + tries + " times.");
tries + " times.");
} }
} }

31
hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
View File

@ -18,9 +18,12 @@
package org.apache.hadoop.fs.aliyun.oss; package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.ObjectMetadata;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -36,7 +39,10 @@
import java.security.DigestOutputStream; import java.security.DigestOutputStream;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import static org.apache.hadoop.fs.aliyun.oss.Constants.MAX_PAGING_KEYS_DEFAULT;
import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -128,4 +134,29 @@ public void testLargeUpload()
writeRenameReadCompare(new Path("/test/xlarge"), writeRenameReadCompare(new Path("/test/xlarge"),
Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1); Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
} }
@Test
public void testDeleteObjects() throws IOException, NoSuchAlgorithmException {
// generate test files
final int files = 10;
final long size = 5 * 1024 * 1024;
final String prefix = "dir";
for (int i = 0; i < files; i++) {
Path path = new Path(String.format("/%s/testFile-%d.txt", prefix, i));
ContractTestUtils.generateTestFile(this.fs, path, size, 256, 255);
}
OSSListRequest listRequest =
store.createListObjectsRequest(prefix, MAX_PAGING_KEYS_DEFAULT, null, null, true);
List<String> keysToDelete = new ArrayList<>();
OSSListResult objects = store.listObjects(listRequest);
assertEquals(files, objects.getObjectSummaries().size());
// test delete files
for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
keysToDelete.add(objectSummary.getKey());
}
store.deleteObjects(keysToDelete);
objects = store.listObjects(listRequest);
assertEquals(0, objects.getObjectSummaries().size());
}
} }