big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17643 WASB : Make metadata checks case insensitive (#2972)

Browse Source
This commit is contained in:
Anoop Sam John 2021-06-12 15:25:03 +05:30 committed by GitHub
parent 352949d070
commit 2cf952baf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 52 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
View File

@ -40,6 +40,7 @@
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@ -180,6 +181,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
*/
public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode";
/**
* Config to control case sensitive metadata key checks/retrieval. If this
* is false, blob metadata keys will be treated case insensitive.
*/
private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive";
private static final String PERMISSION_METADATA_KEY = "hdi_permission";
private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission";
private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder";
@ -353,6 +359,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private String delegationToken;
private boolean metadataKeyCaseSensitive;
/** The error message template when container is not accessible. */
public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+ "account %s in the configuration, and its container %s is not "
@ -574,6 +582,12 @@ public void initialize(URI uri, Configuration conf, AzureFileSystemInstrumentati
LOG.warn("Unable to initialize HBase root as an atomic rename directory.");
}
LOG.debug("Atomic rename directories: {} ", setToString(atomicRenameDirs));
metadataKeyCaseSensitive = conf
.getBoolean(KEY_BLOB_METADATA_KEY_CASE_SENSITIVE, true);
if (!metadataKeyCaseSensitive) {
LOG.info("{} configured as false. Blob metadata will be treated case insensitive.",
KEY_BLOB_METADATA_KEY_CASE_SENSITIVE);
}
}
/**
@ -1618,15 +1632,24 @@ private static void storeMetadataAttribute(CloudBlobWrapper blob,
blob.setMetadata(metadata);
}
private static String getMetadataAttribute(CloudBlobWrapper blob,
private String getMetadataAttribute(HashMap<String, String> metadata,
String... keyAlternatives) {
HashMap<String, String> metadata = blob.getMetadata();
if (null == metadata) {
return null;
}
for (String key : keyAlternatives) {
if (metadata.containsKey(key)) {
return metadata.get(key);
if (metadataKeyCaseSensitive) {
if (metadata.containsKey(key)) {
return metadata.get(key);
}
} else {
// See HADOOP-17643 for details on why this case insensitive metadata
// checks been added
for (Entry<String, String> entry : metadata.entrySet()) {
if (key.equalsIgnoreCase(entry.getKey())) {
return entry.getValue();
}
}
}
}
return null;
@ -1650,7 +1673,7 @@ private static void storePermissionStatus(CloudBlobWrapper blob,
}
private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) {
String permissionMetadataValue = getMetadataAttribute(blob,
String permissionMetadataValue = getMetadataAttribute(blob.getMetadata(),
PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY);
if (permissionMetadataValue != null) {
return PermissionStatusJsonSerializer.fromJSONString(
@ -1698,19 +1721,32 @@ private static void storeLinkAttribute(CloudBlobWrapper blob,
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
}
private static String getLinkAttributeValue(CloudBlobWrapper blob)
private String getLinkAttributeValue(CloudBlobWrapper blob)
throws UnsupportedEncodingException {
String encodedLinkTarget = getMetadataAttribute(blob,
String encodedLinkTarget = getMetadataAttribute(blob.getMetadata(),
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
return decodeMetadataAttribute(encodedLinkTarget);
}
private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
private boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
HashMap<String, String> metadata = blob.getMetadata();
return null != metadata
&& (metadata.containsKey(IS_FOLDER_METADATA_KEY) || metadata
.containsKey(OLD_IS_FOLDER_METADATA_KEY));
if (null != metadata) {
if (metadataKeyCaseSensitive) {
return metadata.containsKey(IS_FOLDER_METADATA_KEY)
|| metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY);
} else {
// See HADOOP-17643 for details on why this case insensitive metadata
// checks been added
for (String key : metadata.keySet()) {
if (key.equalsIgnoreCase(IS_FOLDER_METADATA_KEY)
|| key.equalsIgnoreCase(OLD_IS_FOLDER_METADATA_KEY)) {
return true;
}
}
}
}
return false;
}
private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
@ -1725,18 +1761,9 @@ private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
container.setMetadata(metadata);
}
private static String retrieveVersionAttribute(
CloudBlobContainerWrapper container) {
HashMap<String, String> metadata = container.getMetadata();
if (metadata == null) {
return null;
} else if (metadata.containsKey(VERSION_METADATA_KEY)) {
return metadata.get(VERSION_METADATA_KEY);
} else if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) {
return metadata.get(OLD_VERSION_METADATA_KEY);
} else {
return null;
}
private String retrieveVersionAttribute(CloudBlobContainerWrapper container) {
return getMetadataAttribute(container.getMetadata(), VERSION_METADATA_KEY,
OLD_VERSION_METADATA_KEY);
}
@Override
@ -2231,7 +2258,8 @@ public byte[] retrieveAttribute(String key, String attribute) throws IOException
CloudBlobWrapper blob = getBlobReference(key);
blob.downloadAttributes(getInstrumentedContext());
String value = getMetadataAttribute(blob, ensureValidAttributeName(attribute));
String value = getMetadataAttribute(blob.getMetadata(),
ensureValidAttributeName(attribute));
value = decodeMetadataAttribute(value);
return value == null ? null : value.getBytes(METADATA_ENCODING);
} catch (Exception e) {