big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-18874: [ABFS] Add Server request ID in Exception Messages thrown to the caller. (#6004)

Browse Source 
Contributed by Anuj Modi
This commit is contained in:
Anuj Modi 2023-11-06 12:56:55 -08:00 committed by GitHub
parent d634deea4e
commit 597ceaae3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 95 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/exceptions/AbfsRestOperationException.java
View File

@ -84,20 +84,22 @@ private static String formatMessage(final AbfsHttpOperation abfsHttpOperation) {
// HEAD request response doesn't have StorageErrorCode, StorageErrorMessage. // HEAD request response doesn't have StorageErrorCode, StorageErrorMessage.
if (abfsHttpOperation.getMethod().equals("HEAD")) { if (abfsHttpOperation.getMethod().equals("HEAD")) {
return String.format( return String.format(
"Operation failed: \"%1$s\", %2$s, HEAD, %3$s", "Operation failed: \"%1$s\", %2$s, HEAD, %3$ss, rId: %4$s",
abfsHttpOperation.getStatusDescription(), abfsHttpOperation.getStatusDescription(),
abfsHttpOperation.getStatusCode(), abfsHttpOperation.getStatusCode(),
abfsHttpOperation.getMaskedUrl()); abfsHttpOperation.getMaskedUrl(),
abfsHttpOperation.getRequestId());
} }
return String.format( return String.format(
"Operation failed: \"%1$s\", %2$s, %3$s, %4$s, %5$s, \"%6$s\"", "Operation failed: \"%1$s\", %2$s, %3$s, %4$s, rId: %5$s, %6$s, \"%7$s\"",
abfsHttpOperation.getStatusDescription(), abfsHttpOperation.getStatusDescription(),
abfsHttpOperation.getStatusCode(), abfsHttpOperation.getStatusCode(),
abfsHttpOperation.getMethod(), abfsHttpOperation.getMethod(),
abfsHttpOperation.getMaskedUrl(), abfsHttpOperation.getMaskedUrl(),
abfsHttpOperation.getStorageErrorCode(), abfsHttpOperation.getRequestId(),
// Remove break line to ensure the request id and timestamp can be shown in console. abfsHttpOperation.getStorageErrorCode(),
abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " ")); // Remove break line to ensure the request id and timestamp can be shown in console.
abfsHttpOperation.getStorageErrorMessage().replaceAll("\\n", " "));
} }
} }

117
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
View File

@ -21,14 +21,12 @@
import java.io.IOException; import java.io.IOException;
import org.assertj.core.api.Assertions; import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
import org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider; import org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -43,7 +41,8 @@
* Verify the AbfsRestOperationException error message format. * Verify the AbfsRestOperationException error message format.
* */ * */
public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest{ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest{
private static final String RETRY_TEST_TOKEN_PROVIDER = "org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider"; private static final String RETRY_TEST_TOKEN_PROVIDER =
"org.apache.hadoop.fs.azurebfs.oauth2.RetryTestTokenProvider";
public ITestAbfsRestOperationException() throws Exception { public ITestAbfsRestOperationException() throws Exception {
super(); super();
@ -55,17 +54,35 @@ public void testAbfsRestOperationExceptionFormat() throws IOException {
Path nonExistedFilePath1 = new Path("nonExistedPath1"); Path nonExistedFilePath1 = new Path("nonExistedPath1");
Path nonExistedFilePath2 = new Path("nonExistedPath2"); Path nonExistedFilePath2 = new Path("nonExistedPath2");
try { try {
FileStatus fileStatus = fs.getFileStatus(nonExistedFilePath1); fs.getFileStatus(nonExistedFilePath1);
} catch (Exception ex) { } catch (Exception ex) {
String errorMessage = ex.getLocalizedMessage(); String errorMessage = ex.getLocalizedMessage();
String[] errorFields = errorMessage.split(","); String[] errorFields = errorMessage.split(",");
Assert.assertEquals(4, errorFields.length); // Expected Fields are: Message, StatusCode, Method, URL, ActivityId(rId)
Assertions.assertThat(errorFields)
.describedAs("Number of Fields in exception message are not as expected")
.hasSize(5);
// Check status message, status code, HTTP Request Type and URL. // Check status message, status code, HTTP Request Type and URL.
Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim()); Assertions.assertThat(errorFields[0].trim())
Assert.assertEquals("404", errorFields[1].trim()); .describedAs("Error Message Field in exception message is wrong")
Assert.assertEquals("HEAD", errorFields[2].trim()); .isEqualTo("Operation failed: \"The specified path does not exist.\"");
Assert.assertTrue(errorFields[3].trim().startsWith("http")); Assertions.assertThat(errorFields[1].trim())
.describedAs("Status Code Field in exception message "
+ "should be \"404\"")
.isEqualTo("404");
Assertions.assertThat(errorFields[2].trim())
.describedAs("Http Rest Method Field in exception message "
+ "should be \"HEAD\"")
.isEqualTo("HEAD");
Assertions.assertThat(errorFields[3].trim())
.describedAs("Url Field in exception message"
+ " should start with \"http\"")
.startsWith("http");
Assertions.assertThat(errorFields[4].trim())
.describedAs("ActivityId Field in exception message "
+ "should start with \"rId:\"")
.startsWith("rId:");
} }
try { try {
@ -74,18 +91,43 @@ public void testAbfsRestOperationExceptionFormat() throws IOException {
// verify its format // verify its format
String errorMessage = ex.getLocalizedMessage(); String errorMessage = ex.getLocalizedMessage();
String[] errorFields = errorMessage.split(","); String[] errorFields = errorMessage.split(",");
// Expected Fields are: Message, StatusCode, Method, URL, ActivityId(rId), StorageErrorCode, StorageErrorMessage.
Assertions.assertThat(errorFields) Assertions.assertThat(errorFields)
.describedAs("fields in exception of %s", ex) .describedAs("Number of Fields in exception message are not as expected")
.hasSize(6); .hasSize(7);
// Check status message, status code, HTTP Request Type and URL. // Check status message, status code, HTTP Request Type and URL.
Assert.assertEquals("Operation failed: \"The specified path does not exist.\"", errorFields[0].trim()); Assertions.assertThat(errorFields[0].trim())
Assert.assertEquals("404", errorFields[1].trim()); .describedAs("Error Message Field in exception message is wrong")
Assert.assertEquals("GET", errorFields[2].trim()); .isEqualTo("Operation failed: \"The specified path does not exist.\"");
Assert.assertTrue(errorFields[3].trim().startsWith("http")); Assertions.assertThat(errorFields[1].trim())
.describedAs("Status Code Field in exception message"
+ " should be \"404\"")
.isEqualTo("404");
Assertions.assertThat(errorFields[2].trim())
.describedAs("Http Rest Method Field in exception message"
+ " should be \"GET\"")
.isEqualTo("GET");
Assertions.assertThat(errorFields[3].trim())
.describedAs("Url Field in exception message"
+ " should start with \"http\"")
.startsWith("http");
Assertions.assertThat(errorFields[4].trim())
.describedAs("ActivityId Field in exception message"
+ " should start with \"rId:\"")
.startsWith("rId:");
// Check storage error code and storage error message. // Check storage error code and storage error message.
Assert.assertEquals("PathNotFound", errorFields[4].trim()); Assertions.assertThat(errorFields[5].trim())
Assert.assertTrue(errorFields[5].contains("RequestId") .describedAs("StorageErrorCode Field in exception message"
&& errorFields[5].contains("Time")); + " should be \"PathNotFound\"")
.isEqualTo("PathNotFound");
Assertions.assertThat(errorFields[6].trim())
.describedAs("StorageErrorMessage Field in exception message"
+ " should contain \"RequestId\"")
.contains("RequestId");
Assertions.assertThat(errorFields[6].trim())
.describedAs("StorageErrorMessage Field in exception message"
+ " should contain \"Time\"")
.contains("Time");
} }
} }
@ -101,7 +143,7 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
Configuration config = new Configuration(this.getRawConfiguration()); Configuration config = new Configuration(this.getRawConfiguration());
String accountName = config.get("fs.azure.abfs.account.name"); String accountName = config.get("fs.azure.abfs.account.name");
// Setup to configure custom token provider // Setup to configure custom token provider.
config.set("fs.azure.account.auth.type." + accountName, "Custom"); config.set("fs.azure.account.auth.type." + accountName, "Custom");
config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs" config.set("fs.azure.account.oauth.provider.type." + accountName, "org.apache.hadoop.fs"
+ ".azurebfs.oauth2.RetryTestTokenProvider"); + ".azurebfs.oauth2.RetryTestTokenProvider");
@ -109,24 +151,25 @@ public void testWithDifferentCustomTokenFetchRetry(int numOfRetries) throws Exce
// Stop filesystem creation as it will lead to calls to store. // Stop filesystem creation as it will lead to calls to store.
config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false"); config.set("fs.azure.createRemoteFileSystemDuringInitialization", "false");
final AzureBlobFileSystem fs1 = try (final AzureBlobFileSystem fs1 =
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
config); config)) {
RetryTestTokenProvider retryTestTokenProvider RetryTestTokenProvider retryTestTokenProvider
= RetryTestTokenProvider.getCurrentRetryTestProviderInstance( = RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
getAccessTokenProvider(fs1)); getAccessTokenProvider(fs1));
retryTestTokenProvider.resetStatusToFirstTokenFetch(); retryTestTokenProvider.resetStatusToFirstTokenFetch();
intercept(Exception.class, intercept(Exception.class,
()-> { () -> {
fs1.getFileStatus(new Path("/")); fs1.getFileStatus(new Path("/"));
}); });
// Number of retries done should be as configured // Number of retries done should be as configured
Assert.assertEquals( Assertions.assertThat(retryTestTokenProvider.getRetryCount())
"Number of token fetch retries done does not match with fs.azure" .describedAs("Number of token fetch retries done does not "
+ ".custom.token.fetch.retry.count configured", numOfRetries, + "match with fs.azure.custom.token.fetch.retry.count configured")
retryTestTokenProvider.getRetryCount()); .isEqualTo(numOfRetries);
}
} }
@Test @Test
@ -145,8 +188,10 @@ public void testAuthFailException() throws Exception {
final AzureBlobFileSystem fs = getFileSystem(config); final AzureBlobFileSystem fs = getFileSystem(config);
try { try {
fs.getFileStatus(new Path("/")); intercept(Exception.class,
fail("Should fail at auth token fetch call"); () -> {
fs.getFileStatus(new Path("/"));
});
} catch (AbfsRestOperationException e) { } catch (AbfsRestOperationException e) {
String errorDesc = "Should throw RestOp exception on AAD failure"; String errorDesc = "Should throw RestOp exception on AAD failure";
Assertions.assertThat(e.getStatusCode()) Assertions.assertThat(e.getStatusCode())