big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17325. WASB Test Failures

Browse Source 
Contributed by Ayush Saxena and Steve Loughran

Change-Id: I4bb76815bc1d11d1804dc67bafde68b6a995b974
This commit is contained in:
Steve Loughran 2020-11-23 17:22:01 +00:00
parent f13c7b1b02
commit 07b7d07388
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
4 changed files with 26 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java
View File

@ -655,9 +655,7 @@ public void testCanonicalServiceName() throws Exception {
// because the mock container does not exist, this call is expected to fail.
intercept(IllegalArgumentException.class,
"java.net.UnknownHostException",
() -> {
fs0.getCanonicalServiceName();
});
() -> fs0.getCanonicalServiceName());
conf.setBoolean(RETURN_URI_AS_CANONICAL_SERVICE_NAME_PROPERTY_NAME, true);
FileSystem fs1 = FileSystem.newInstance(defaultUri, conf);

7
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/InMemoryBlockBlobStore.java
View File

@ -25,6 +25,8 @@
import java.util.HashMap;
import java.util.Map;
import static java.util.Objects.requireNonNull;
/**
* A simple memory key-value store to help mock the Windows Azure Storage
* implementation for unit testing.
@ -163,7 +165,10 @@ public synchronized boolean exists(String key) {
@SuppressWarnings("unchecked")
public synchronized HashMap<String, String> getMetadata(String key) {
return (HashMap<String, String>) blobs.get(key).metadata.clone();
Entry entry = requireNonNull(blobs.get(key), "entry for " + key);
return (HashMap<String, String>) requireNonNull(entry.metadata,
"metadata for " + key)
.clone();
}
public synchronized HashMap<String, String> getContainerMetadata() {

20
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/MockStorageInterface.java
View File

@ -37,6 +37,7 @@
import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.net.URLCodec;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.fs.Path;
import org.apache.http.client.utils.URIBuilder;
import com.microsoft.azure.storage.AccessCondition;
@ -137,9 +138,20 @@ private static String convertUriToDecodedString(URI uri) {
private static URI convertKeyToEncodedUri(String key) {
try {
return new URIBuilder().setPath(key).build();
Path p = new Path(key);
URI unEncodedURI = p.toUri();
return new URIBuilder().setPath(unEncodedURI.getPath())
.setScheme(unEncodedURI.getScheme()).build();
} catch (URISyntaxException e) {
throw new AssertionError("Failed to encode key: " + key);
int i = e.getIndex();
String details;
if (i >= 0) {
details = " -- \"" + e.getInput().charAt(i) + "\"";
} else {
details = "";
}
throw new AssertionError("Failed to encode key: " + key
+ ": " + e + details);
}
}
@ -148,8 +160,8 @@ public CloudBlobContainerWrapper getContainerReference(String name)
throws URISyntaxException, StorageException {
String fullUri;
URIBuilder builder = new URIBuilder(baseUriString);
fullUri = builder.setPath(builder.getPath() + "/" + name).toString();
String path = builder.getPath() == null ? "" : builder.getPath() + "/";
fullUri = builder.setPath(path + name).toString();
MockCloudBlobContainerWrapper container = new MockCloudBlobContainerWrapper(
fullUri, name);
// Check if we have a pre-existing container with that name, and prime

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestBlobMetadata.java
View File

@ -202,8 +202,10 @@ public void testPermissionMetadata() throws Exception {
Path selfishFile = new Path("/noOneElse");
fs.create(selfishFile, justMe, true, 4096, fs.getDefaultReplication(),
fs.getDefaultBlockSize(), null).close();
String mockUri = AzureBlobStorageTestAccount.toMockUri(selfishFile);
assertNotNull("converted URI", mockUri);
HashMap<String, String> metadata = backingStore
.getMetadata(AzureBlobStorageTestAccount.toMockUri(selfishFile));
.getMetadata(mockUri);
assertNotNull(metadata);
String storedPermission = metadata.get("hdi_permission");
assertEquals(getExpectedPermissionString("rw-------"), storedPermission);