diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 95fe839505..112fa185d3 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -126,7 +126,7 @@ 1.0-beta-1 1.0-alpha-8 900 - 1.10.6 + 1.11.45 @@ -875,6 +875,11 @@ jackson-jaxrs-json-provider ${jackson2.version} + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + ${jackson2.version} + org.mockito mockito-all diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index e27342809e..140766181a 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -344,6 +344,10 @@ com.fasterxml.jackson.core jackson-annotations + + com.fasterxml.jackson.dataformat + jackson-dataformat-cbor + joda-time joda-time diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 0a4dd027db..871322db45 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -180,7 +180,7 @@ private static void initUserAgent(Configuration conf, userAgent = userAgentPrefix + ", " + userAgent; } LOG.debug("Using User-Agent: {}", userAgent); - awsConf.setUserAgent(userAgent); + awsConf.setUserAgentPrefix(userAgent); } /** @@ -225,7 +225,9 @@ private static void enablePathStyleAccessIfRequired(AmazonS3 s3, final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false); if (pathStyleAccess) { LOG.debug("Enabling path style access!"); - s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true)); + s3.setS3ClientOptions(S3ClientOptions.builder() + .setPathStyleAccess(true) + .build()); } } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 819d9d8002..d7f341156b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -110,7 +110,8 @@ static class GoodCredentialsProvider extends AWSCredentialsProviderChain { @SuppressWarnings("unused") public GoodCredentialsProvider(URI name, Configuration conf) { super(new BasicAWSCredentialsProvider(conf.get(ACCESS_KEY), - conf.get(SECRET_KEY)), new InstanceProfileCredentialsProvider()); + conf.get(SECRET_KEY)), + InstanceProfileCredentialsProvider.getInstance()); } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 9163b1541d..f7d77f4ef2 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -399,7 +399,8 @@ public void testDefaultUserAgent() throws Exception { assertNotNull(s3); ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, "clientConfiguration"); - assertEquals("Hadoop " + VersionInfo.getVersion(), awsConf.getUserAgent()); + assertEquals("Hadoop " + VersionInfo.getVersion(), + awsConf.getUserAgentPrefix()); } @Test @@ -413,7 +414,7 @@ public void testCustomUserAgent() throws Exception { ClientConfiguration awsConf = getField(s3, ClientConfiguration.class, "clientConfiguration"); assertEquals("MyApp, Hadoop " + VersionInfo.getVersion(), - awsConf.getUserAgent()); + awsConf.getUserAgentPrefix()); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java index 8421dad006..d8ae313cf5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/yarn/ITestS3AMiniYarnCluster.java @@ -117,6 +117,9 @@ private Map getResultAsMap(String outputAsStr) Map result = new HashMap<>(); for (String line : outputAsStr.split("\n")) { String[] tokens = line.split("\t"); + assertTrue("Not enough tokens in in string \" "+ line + + "\" from output \"" + outputAsStr + "\"", + tokens.length > 1); result.put(tokens[0], Integer.parseInt(tokens[1])); } return result;