big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-17076: ABFS: Delegation SAS Generator Updates

Browse Source 
Contributed by Thomas Marquardt.

DETAILS:
1) The authentication version in the service has been updated from Dec19 to Feb20, so need to update the client.
2) Add support and test cases for getXattr and setXAttr.
3) Update DelegationSASGenerator and related to use Duration instead of int for time periods.
4) Cleanup DelegationSASGenerator switch/case statement that maps operations to permissions.
5) Cleanup SASGenerator classes to use String.equals instead of ==.

TESTS:
Added tests for getXAttr and setXAttr.

All tests are passing against my account in eastus2euap:

 $mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=8 clean verify
 Tests run: 76, Failures: 0, Errors: 0, Skipped: 0
 Tests run: 441, Failures: 0, Errors: 0, Skipped: 33
 Tests run: 206, Failures: 0, Errors: 0, Skipped: 24
This commit is contained in:
Thomas Marquardt 2020-06-17 23:12:22 +00:00
parent d639c11986
commit 63d236c019
No known key found for this signature in database
GPG Key ID: AEB30C9E78868287
6 changed files with 42 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java
View File

@ -33,6 +33,7 @@
public interface SASTokenProvider { public interface SASTokenProvider {
String CHECK_ACCESS_OPERATION = "check-access"; String CHECK_ACCESS_OPERATION = "check-access";
String CREATE_DIRECTORY_OPERATION = "create-directory";
String CREATE_FILE_OPERATION = "create-file"; String CREATE_FILE_OPERATION = "create-file";
String DELETE_OPERATION = "delete"; String DELETE_OPERATION = "delete";
String DELETE_RECURSIVE_OPERATION = "delete-recursive"; String DELETE_RECURSIVE_OPERATION = "delete-recursive";
@ -40,7 +41,6 @@ public interface SASTokenProvider {
String GET_STATUS_OPERATION = "get-status"; String GET_STATUS_OPERATION = "get-status";
String GET_PROPERTIES_OPERATION = "get-properties"; String GET_PROPERTIES_OPERATION = "get-properties";
String LIST_OPERATION = "list"; String LIST_OPERATION = "list";
String CREATE_DIRECTORY_OPERATION = "create-directory";
String READ_OPERATION = "read"; String READ_OPERATION = "read";
String RENAME_SOURCE_OPERATION = "rename-source"; String RENAME_SOURCE_OPERATION = "rename-source";
String RENAME_DESTINATION_OPERATION = "rename-destination"; String RENAME_DESTINATION_OPERATION = "rename-destination";

16
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
View File

@ -25,6 +25,7 @@
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.junit.Assert;
import org.junit.Assume; import org.junit.Assume;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -365,4 +366,19 @@ public void testRootPath() throws Exception {
} }
assertEquals(0, count); assertEquals(0, count);
} }
@Test
// Test filesystem operations getXAttr and setXAttr
public void testProperties() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
Path reqPath = new Path(UUID.randomUUID().toString());
fs.create(reqPath).close();
final String propertyName = "user.mime_type";
final byte[] propertyValue = "text/plain".getBytes("utf-8");
fs.setXAttr(reqPath, propertyName, propertyValue);
assertArrayEquals(propertyValue, fs.getXAttr(reqPath, propertyName));
}
} }

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java
View File

@ -55,8 +55,8 @@ public void initialize(Configuration configuration, String accountName) throws I
String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET); String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET);
String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID); String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID); String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minusSeconds(SASGenerator.FIVE_MINUTES)); String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minus(SASGenerator.FIVE_MINUTES));
String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plusSeconds(SASGenerator.ONE_DAY)); String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plus(SASGenerator.ONE_DAY));
String skv = SASGenerator.AuthenticationVersion.Dec19.toString(); String skv = SASGenerator.AuthenticationVersion.Dec19.toString();
byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv); byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv);

32
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java
View File

@ -29,12 +29,12 @@
* Test Delegation SAS generator. * Test Delegation SAS generator.
*/ */
public class DelegationSASGenerator extends SASGenerator { public class DelegationSASGenerator extends SASGenerator {
private String skoid; private final String skoid;
private String sktid; private final String sktid;
private String skt; private final String skt;
private String ske; private final String ske;
private final String sks = "b"; private final String sks = "b";
private String skv; private final String skv;
public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) { public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) {
super(userDelegationKey); super(userDelegationKey);
@ -48,20 +48,18 @@ public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String skt
public String getDelegationSAS(String accountName, String containerName, String path, String operation, public String getDelegationSAS(String accountName, String containerName, String path, String operation,
String saoid, String suoid, String scid) { String saoid, String suoid, String scid) {
final String sv = AuthenticationVersion.Dec19.toString(); final String sv = AuthenticationVersion.Feb20.toString();
final String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); final String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
final String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); final String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String sr = "b"; String sr = "b";
String sdd = null; String sdd = null;
String sp = null; String sp;
switch (operation) { switch (operation) {
case SASTokenProvider.CHECK_ACCESS_OPERATION:
sp = "e";
break;
case SASTokenProvider.WRITE_OPERATION:
case SASTokenProvider.CREATE_FILE_OPERATION: case SASTokenProvider.CREATE_FILE_OPERATION:
case SASTokenProvider.CREATE_DIRECTORY_OPERATION: case SASTokenProvider.CREATE_DIRECTORY_OPERATION:
case SASTokenProvider.WRITE_OPERATION:
case SASTokenProvider.SET_PROPERTIES_OPERATION:
sp = "w"; sp = "w";
break; break;
case SASTokenProvider.DELETE_OPERATION: case SASTokenProvider.DELETE_OPERATION:
@ -72,6 +70,7 @@ public String getDelegationSAS(String accountName, String containerName, String
sr = "d"; sr = "d";
sdd = Integer.toString(StringUtils.countMatches(path, "/")); sdd = Integer.toString(StringUtils.countMatches(path, "/"));
break; break;
case SASTokenProvider.CHECK_ACCESS_OPERATION:
case SASTokenProvider.GET_ACL_OPERATION: case SASTokenProvider.GET_ACL_OPERATION:
case SASTokenProvider.GET_STATUS_OPERATION: case SASTokenProvider.GET_STATUS_OPERATION:
sp = "e"; sp = "e";
@ -79,6 +78,7 @@ public String getDelegationSAS(String accountName, String containerName, String
case SASTokenProvider.LIST_OPERATION: case SASTokenProvider.LIST_OPERATION:
sp = "l"; sp = "l";
break; break;
case SASTokenProvider.GET_PROPERTIES_OPERATION:
case SASTokenProvider.READ_OPERATION: case SASTokenProvider.READ_OPERATION:
sp = "r"; sp = "r";
break; break;
@ -87,14 +87,12 @@ public String getDelegationSAS(String accountName, String containerName, String
sp = "m"; sp = "m";
break; break;
case SASTokenProvider.SET_ACL_OPERATION: case SASTokenProvider.SET_ACL_OPERATION:
case SASTokenProvider.SET_PERMISSION_OPERATION:
sp = "p"; sp = "p";
break; break;
case SASTokenProvider.SET_OWNER_OPERATION: case SASTokenProvider.SET_OWNER_OPERATION:
sp = "o"; sp = "o";
break; break;
case SASTokenProvider.SET_PERMISSION_OPERATION:
sp = "p";
break;
default: default:
throw new IllegalArgumentException(operation); throw new IllegalArgumentException(operation);
} }
@ -146,7 +144,7 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv
sb.append(accountName); sb.append(accountName);
sb.append("/"); sb.append("/");
sb.append(containerName); sb.append(containerName);
if (path != null && sr != "c") { if (path != null && !sr.equals("c")) {
sb.append(path); sb.append(path);
} }
sb.append("\n"); sb.append("\n");

8
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java
View File

@ -20,6 +20,7 @@
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.Locale; import java.util.Locale;
@ -35,7 +36,8 @@ public abstract class SASGenerator {
public enum AuthenticationVersion { public enum AuthenticationVersion {
Nov18("2018-11-09"), Nov18("2018-11-09"),
Dec19("2019-12-12"); Dec19("2019-12-12"),
Feb20("2020-02-10");
private final String ver; private final String ver;
@ -50,8 +52,8 @@ public String toString() {
} }
protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class); protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class);
public static final int FIVE_MINUTES = 5 * 60; public static final Duration FIVE_MINUTES = Duration.ofMinutes(5);
public static final int ONE_DAY = 24 * 60 * 60; public static final Duration ONE_DAY = Duration.ofDays(1);
public static final DateTimeFormatter ISO_8601_FORMATTER = public static final DateTimeFormatter ISO_8601_FORMATTER =
DateTimeFormatter DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT)

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java
View File

@ -40,8 +40,8 @@ public String getContainerSASWithFullControl(String accountName, String containe
String sp = "rcwdl"; String sp = "rcwdl";
String sv = AuthenticationVersion.Nov18.toString(); String sv = AuthenticationVersion.Nov18.toString();
String sr = "c"; String sr = "c";
String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String signature = computeSignatureForSAS(sp, st, se, sv, "c", String signature = computeSignatureForSAS(sp, st, se, sv, "c",
accountName, containerName, null); accountName, containerName, null);
@ -71,7 +71,7 @@ private String computeSignatureForSAS(String sp, String st, String se, String sv
sb.append(accountName); sb.append(accountName);
sb.append("/"); sb.append("/");
sb.append(containerName); sb.append(containerName);
if (path != null && sr != "c") { if (path != null && !sr.equals("c")) {
//sb.append("/"); //sb.append("/");
sb.append(path); sb.append(path);
} }