big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18408. ABFS: ITestAbfsManifestCommitProtocol fails on nonHNS configuration (#4758)

Browse Source 
ITestAbfsManifestCommitProtocol  to set requireRenameResilience to false for nonHNS configuration  (#4758)

Contributed by Sree Bhattacharyya
This commit is contained in:
sreeb-msft 2022-09-02 17:03:12 +05:30 committed by GitHub
parent 37e213c3fc
commit c48ed3e96c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/committer/manifest/TestRenameStageFailure.java
View File

@ -115,7 +115,7 @@ public void setup() throws Exception {
* Does this test suite require rename resilience in the store/FS? * Does this test suite require rename resilience in the store/FS?
* @return true if the store operations are resilient. * @return true if the store operations are resilient.
*/ */
protected boolean requireRenameResilience() { protected boolean requireRenameResilience() throws IOException {
return false; return false;
} }

12
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -149,6 +149,17 @@ protected boolean getIsNamespaceEnabled(AzureBlobFileSystem fs)
return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false)); return fs.getIsNamespaceEnabled(getTestTracingContext(fs, false));
} }
public static TracingContext getSampleTracingContext(AzureBlobFileSystem fs,
boolean needsPrimaryReqId) {
String correlationId, fsId;
TracingHeaderFormat format;
correlationId = "test-corr-id";
fsId = "test-filesystem-id";
format = TracingHeaderFormat.ALL_ID_FORMAT;
return new TracingContext(correlationId, fsId,
FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
}
public TracingContext getTestTracingContext(AzureBlobFileSystem fs, public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
boolean needsPrimaryReqId) { boolean needsPrimaryReqId) {
String correlationId, fsId; String correlationId, fsId;
@ -167,7 +178,6 @@ public TracingContext getTestTracingContext(AzureBlobFileSystem fs,
FSOperationType.TEST_OP, needsPrimaryReqId, format, null); FSOperationType.TEST_OP, needsPrimaryReqId, format, null);
} }
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {
//Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.

13
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/commit/ITestAbfsRenameStageFailure.java
View File

@ -19,11 +19,13 @@
package org.apache.hadoop.fs.azurebfs.commit; package org.apache.hadoop.fs.azurebfs.commit;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem;
import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding; import org.apache.hadoop.fs.azurebfs.contract.ABFSContractTestBinding;
import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract; import org.apache.hadoop.fs.azurebfs.contract.AbfsFileSystemContract;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.contract.AbstractFSContract; import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure; import org.apache.hadoop.mapreduce.lib.output.committer.manifest.TestRenameStageFailure;
/** /**
* Rename failure logic on ABFS. * Rename failure logic on ABFS.
* This will go through the resilient rename operation. * This will go through the resilient rename operation.
@ -41,6 +43,11 @@ public ITestAbfsRenameStageFailure() throws Exception {
binding = new ABFSContractTestBinding(); binding = new ABFSContractTestBinding();
} }
protected boolean isNamespaceEnabled() throws AzureBlobFileSystemException {
AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
return fs.getAbfsStore().getIsNamespaceEnabled(AbstractAbfsIntegrationTest.getSampleTracingContext(fs, false));
}
@Override @Override
public void setup() throws Exception { public void setup() throws Exception {
binding.setup(); binding.setup();
@ -58,8 +65,8 @@ protected AbstractFSContract createContract(final Configuration conf) {
} }
@Override @Override
protected boolean requireRenameResilience() { protected boolean requireRenameResilience() throws AzureBlobFileSystemException {
return true; return isNamespaceEnabled();
} }
@Override @Override