big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18408. ABFS: ITestAbfsManifestCommitProtocol fails on nonHNS configuration (#4758)

Browse Source 
ITestAbfsManifestCommitProtocol  to set requireRenameResilience to false for nonHNS configuration  (#4758)

Contributed by Sree Bhattacharyya
This commit is contained in:
sreeb-msft 2022-09-02 17:03:12 +05:30 committed by GitHub
parent 37e213c3fc
commit c48ed3e96c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java
View File

@ -115,7 +115,7 @@ public void setup() throws Exception {
* Does this test suite require rename resilience in the store/FS?
* @return true if the store operations are resilient.
*/
protected boolean requireRenameResilience() {
protected boolean requireRenameResilience() throws IOException {
return false;
}

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -149,6 +149,17 @@ protected boolean getIsNamespaceEnabled(AzureBlobFileSystem fs)
return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false));
}
public static TracingContext getSampleTracingContext(AzureBlobFileSystem fs,
boolean needsPrimaryReqId) {
String correlationId, fsId;
TracingHeaderFormat format;
correlationId = "test-corr-id";
fsId = "test-filesystem-id";
format = TracingHeaderFormat.ALL_ID_FORMAT;
return new TracingContext(correlationId, fsId,
FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
}
public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
boolean needsPrimaryReqId) {
String correlationId, fsId;
@ -167,7 +178,6 @@ public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
}
@Before
public void setup() throws Exception {
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.

13
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
View File

@ -19,11 +19,13 @@
package org.apache.hadoop.fs.azurebfs.commit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure;
/**
* Rename failure logic on ABFS.
* This will go through the resilient rename operation.
@ -41,6 +43,11 @@ public ITestAbfsRenameStageFailure() throws Exception {
binding = new ABFSContractTestBinding();
}
protected boolean isNamespaceEnabled() throws AzureBlobFileSystemException {
AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
return fs.getAbfsStore().getIsNamespaceEnabled(AbstractAbfsIntegrationTest.getSampleTracingContext(fs, false));
}
@Override
public void setup() throws Exception {
binding.setup();
@ -58,8 +65,8 @@ protected AbstractFSContract createContract(final Configuration conf) {
}
@Override
protected boolean requireRenameResilience() {
return true;
protected boolean requireRenameResilience() throws AzureBlobFileSystemException {
return isNamespaceEnabled();
}
@Override