access);
-
- /**
- * Access levels.
- */
- enum AccessLevel {
- /** Filesystem data read operations. */
- READ,
- /** Data write, encryption, etc. */
- WRITE,
- /** Administration of the data, tables, etc. */
- ADMIN,
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
deleted file mode 100644
index dcb83c2c28..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDTService.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.net.URI;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.AbstractService;
-
-import static java.util.Objects.requireNonNull;
-
-/**
- * This is the base class for both the delegation binding
- * code and the back end service created; allows for
- * shared methods across both.
- *
- * The lifecycle sequence is as follows
- *
- * - create
- * - bindToFileSystem(uri, ownerFS)
- * - init
- * - start
- * ...api calls...
- * - stop
- *
- *
- * As the S3ADelegation mechanism is all configured during the filesystem
- * initalize() operation, it is not ready for use through all the start process.
- */
-public abstract class AbstractDTService
- extends AbstractService {
-
- /**
- * URI of the filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
- */
- private URI canonicalUri;
-
- /**
- * The owning filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
- */
- private S3AFileSystem fileSystem;
-
- /**
- * Owner of the filesystem.
- * Valid after {@link #bindToFileSystem(URI, S3AFileSystem)}.
- */
- private UserGroupInformation owner;
-
- /**
- * Protected constructor.
- * @param name service name.
- */
- protected AbstractDTService(final String name) {
- super(name);
- }
-
- /**
- * Bind to the filesystem.
- * Subclasses can use this to perform their own binding operations -
- * but they must always call their superclass implementation.
- * This Must be called before calling {@code init()}.
- *
- * Important:
- * This binding will happen during FileSystem.initialize(); the FS
- * is not live for actual use and will not yet have interacted with
- * AWS services.
- * @param uri the canonical URI of the FS.
- * @param fs owning FS.
- * @throws IOException failure.
- */
- public void bindToFileSystem(
- final URI uri,
- final S3AFileSystem fs) throws IOException {
- requireServiceState(STATE.NOTINITED);
- Preconditions.checkState(canonicalUri == null,
- "bindToFileSystem called twice");
- this.canonicalUri = requireNonNull(uri);
- this.fileSystem = requireNonNull(fs);
- this.owner = fs.getOwner();
- }
-
- /**
- * Get the canonical URI of the filesystem, which is what is
- * used to identify the tokens.
- * @return the URI.
- */
- public URI getCanonicalUri() {
- return canonicalUri;
- }
-
- /**
- * Get the owner of the FS.
- * @return the owner fs
- */
- protected S3AFileSystem getFileSystem() {
- return fileSystem;
- }
-
- /**
- * Get the owner of this Service.
- * @return owner; non-null after binding to an FS.
- */
- public UserGroupInformation getOwner() {
- return owner;
- }
-
- /**
- * Require that the service is in a given state.
- * @param state desired state.
- * @throws IllegalStateException if the condition is not met
- */
- protected void requireServiceState(final STATE state)
- throws IllegalStateException {
- Preconditions.checkState(isInState(state),
- "Required State: %s; Actual State %s", state, getServiceState());
- }
-
- /**
- * Require the service to be started.
- * @throws IllegalStateException if it is not.
- */
- protected void requireServiceStarted() throws IllegalStateException {
- requireServiceState(STATE.STARTED);
- }
-
- @Override
- protected void serviceInit(final Configuration conf) throws Exception {
- super.serviceInit(conf);
- requireNonNull(canonicalUri, "service does not have a canonical URI");
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
deleted file mode 100644
index d7cacc528e..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.net.URI;
-import java.nio.charset.Charset;
-import java.util.Optional;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3a.commit.DurationInfo;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.SecretManager;
-import org.apache.hadoop.security.token.Token;
-
-import static java.util.Objects.requireNonNull;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DURATION_LOG_AT_INFO;
-
-/**
- * An AbstractDelegationTokenBinding implementation is a class which
- * handles the binding of its underlying authentication mechanism to the
- * Hadoop Delegation token mechanism.
- *
- * See also {@code org.apache.hadoop.fs.azure.security.WasbDelegationTokenManager}
- * but note that it assumes Kerberos tokens for which the renewal mechanism
- * is the sole plugin point.
- * This class is designed to be more generic.
- *
- * Lifecycle
- *
- * It is a Hadoop Service, so has a standard lifecycle: once started
- * its lifecycle will follow that of the {@link S3ADelegationTokens}
- * instance which created it --which itself follows the lifecycle of the FS.
- *
- * One big difference is that
- * {@link #bindToFileSystem(URI, S3AFileSystem)} will be called
- * before the {@link #init(Configuration)} operation, this is where
- * the owning FS is passed in.
- *
- * Implementations are free to start background operations in their
- * {@code serviceStart()} method, provided they are safely stopped in
- * {@code serviceStop()}.
- *
- * When to check for the ability to issue tokens
- * Implementations MUST start up without actually holding the secrets
- * needed to issue tokens (config options, credentials to talk to STS etc)
- * as in server-side deployments they are not expected to have these.
- *
- * Retry Policy
- *
- * All methods which talk to AWS services are expected to do translation,
- * with retries as they see fit.
- */
-public abstract class AbstractDelegationTokenBinding extends AbstractDTService {
-
- /** Token kind: must match that of the token identifiers issued. */
- private final Text kind;
-
- private SecretManager secretManager;
-
- private static final Logger LOG = LoggerFactory.getLogger(
- AbstractDelegationTokenBinding.class);
-
- /**
- * Constructor.
- *
- * @param name as passed to superclass for use in log messages.
- * @param kind token kind.
- */
- protected AbstractDelegationTokenBinding(final String name,
- final Text kind) {
- super(name);
- this.kind = requireNonNull(kind);
- }
-
- /**
- * Get the kind of the tokens managed here.
- * @return the token kind.
- */
- public Text getKind() {
- return kind;
- }
-
- /**
- * Return the name of the owner to be used in tokens.
- * This may be that of the UGI owner, or it could be related to
- * the AWS login.
- * @return a text name of the owner.
- */
- public Text getOwnerText() {
- return new Text(getOwner().getUserName());
- }
-
- /**
- * Predicate: will this binding issue a DT?
- * That is: should the filesystem declare that it is issuing
- * delegation tokens? If true
- * @return a declaration of what will happen when asked for a token.
- */
- public S3ADelegationTokens.TokenIssuingPolicy getTokenIssuingPolicy() {
- return S3ADelegationTokens.TokenIssuingPolicy.RequestNewToken;
- }
-
- /**
- * Create a delegation token for the user.
- * This will only be called if a new DT is needed, that is: the
- * filesystem has been deployed unbonded.
- * @param policy minimum policy to use, if known.
- * @param encryptionSecrets encryption secrets for the token.
- * @return the token or null if the back end does not want to issue one.
- * @throws IOException if one cannot be created
- */
- public Token createDelegationToken(
- final Optional policy,
- final EncryptionSecrets encryptionSecrets) throws IOException {
- requireServiceStarted();
- final AbstractS3ATokenIdentifier tokenIdentifier =
- createTokenIdentifier(policy, encryptionSecrets);
- if (tokenIdentifier != null) {
- Token token =
- new Token<>(tokenIdentifier, secretManager);
- token.setKind(getKind());
- LOG.debug("Created token {} with token identifier {}",
- token, tokenIdentifier);
- return token;
- } else {
- return null;
- }
- }
-
- /**
- * Create a token identifier with all the information needed
- * to be included in a delegation token.
- * This is where session credentials need to be extracted, etc.
- * This will only be called if a new DT is needed, that is: the
- * filesystem has been deployed unbonded.
- *
- * If {@link #createDelegationToken(Optional, EncryptionSecrets)}
- * is overridden, this method can be replaced with a stub.
- *
- * @param policy minimum policy to use, if known.
- * @param encryptionSecrets encryption secrets for the token.
- * @return the token data to include in the token identifier.
- * @throws IOException failure creating the token data.
- */
- public abstract AbstractS3ATokenIdentifier createTokenIdentifier(
- Optional policy,
- EncryptionSecrets encryptionSecrets) throws IOException;
-
- /**
- * Verify that a token identifier is of a specific class.
- * This will reject subclasses (i.e. it is stricter than
- * {@code instanceof}, then cast it to that type.
- * @param identifier identifier to validate
- * @param expectedClass class of the expected token identifier.
- * @throws DelegationTokenIOException If the wrong class was found.
- */
- protected T convertTokenIdentifier(
- final AbstractS3ATokenIdentifier identifier,
- final Class expectedClass) throws DelegationTokenIOException {
- if (!identifier.getClass().equals(expectedClass)) {
- throw new DelegationTokenIOException(
- DelegationTokenIOException.TOKEN_WRONG_CLASS
- + "; expected a token identifier of type "
- + expectedClass
- + " but got "
- + identifier.getClass()
- + " and kind " + identifier.getKind());
- }
- return (T) identifier;
- }
-
- /**
- * Perform any actions when deploying unbonded, and return a list
- * of credential providers.
- * @return non-empty list of AWS credential providers to use for
- * authenticating this client with AWS services.
- * @throws IOException any failure.
- */
- public abstract AWSCredentialProviderList deployUnbonded()
- throws IOException;
-
- /**
- * Bind to the token identifier, returning the credential providers to use
- * for the owner to talk to S3, DDB and related AWS Services.
- * @param retrievedIdentifier the unmarshalled data
- * @return non-empty list of AWS credential providers to use for
- * authenticating this client with AWS services.
- * @throws IOException any failure.
- */
- public abstract AWSCredentialProviderList bindToTokenIdentifier(
- AbstractS3ATokenIdentifier retrievedIdentifier)
- throws IOException;
-
- /**
- * Create a new subclass of {@link AbstractS3ATokenIdentifier}.
- * This is used in the secret manager.
- * @return an empty identifier.
- */
- public abstract AbstractS3ATokenIdentifier createEmptyIdentifier();
-
- @Override
- public String toString() {
- return super.toString()
- + " token kind = " + getKind();
- }
-
- /**
- * Service startup: create the secret manager.
- * @throws Exception failure.
- */
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- secretManager = createSecretMananger();
- }
-
- /**
- * Return a description.
- * This is logged during after service start & binding:
- * it should be as informative as possible.
- * @return a description to log.
- */
- public String getDescription() {
- return "Token binding " + getKind().toString();
- }
-
- /**
- * Create a secret manager.
- * @return a secret manager.
- * @throws IOException on failure
- */
- protected SecretManager createSecretMananger()
- throws IOException {
- return new TokenSecretManager();
- }
-
- /**
- * Return a string for use in building up the User-Agent field, so
- * get into the S3 access logs. Useful for diagnostics.
- * @return a string for the S3 logs or "" for "nothing to add"
- */
- public String getUserAgentField() {
- return "";
- }
-
- /**
- * Get the password to use in secret managers.
- * This is a constant; its just recalculated every time to stop findbugs
- * highlighting security risks of shared mutable byte arrays.
- * @return a password.
- */
- protected static byte[] getSecretManagerPasssword() {
- return "non-password".getBytes(Charset.forName("UTF-8"));
- }
-
- /**
- * The secret manager always uses the same secret; the
- * factory for new identifiers is that of the token manager.
- */
- protected class TokenSecretManager
- extends SecretManager {
-
- @Override
- protected byte[] createPassword(AbstractS3ATokenIdentifier identifier) {
- return getSecretManagerPasssword();
- }
-
- @Override
- public byte[] retrievePassword(AbstractS3ATokenIdentifier identifier)
- throws InvalidToken {
- return getSecretManagerPasssword();
- }
-
- @Override
- public AbstractS3ATokenIdentifier createIdentifier() {
- try (DurationInfo ignored = new DurationInfo(LOG, DURATION_LOG_AT_INFO,
- "Creating Delegation Token Identifier")) {
- return AbstractDelegationTokenBinding.this.createEmptyIdentifier();
- }
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractS3ATokenIdentifier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractS3ATokenIdentifier.java
deleted file mode 100644
index 98c449ea55..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractS3ATokenIdentifier.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.net.URI;
-import java.util.Objects;
-import java.util.UUID;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier;
-
-import static java.util.Objects.requireNonNull;
-
-/**
- * An S3A Delegation Token Identifier: contains the information needed
- * to talk to S3A.
- *
- * These are loaded via the service loader API an used in a map of
- * Kind => class, which is then looked up to deserialize token
- * identifiers of a given class.
- *
- * Every non-abstract class must provide
- *
- * - Their unique token kind.
- * - An empty constructor.
- * - An entry in the resource file
- * {@code /META-INF/services/org.apache.hadoop.security.token.TokenIdentifier}
- *
- *
- *
- * The base implementation contains
- *
- * - The URI of the FS.
- * - Encryption secrets for use in the destination FS.
- *
- * Subclasses are required to add whatever information is needed to authenticate
- * the user with the credential provider which their binding class will
- * provide.
- *
- * Important: Add no references to any AWS SDK class, to
- * ensure it can be safely deserialized whenever the relevant token
- * identifier of a token type declared in this JAR is examined.
- */
-public abstract class AbstractS3ATokenIdentifier
- extends DelegationTokenIdentifier {
-
- /**
- * The maximum string length supported for text fields.
- */
- protected static final int MAX_TEXT_LENGTH = 8192;
-
- /** Canonical URI of the bucket. */
- private URI uri;
-
- /**
- * Encryption secrets to also marshall with any credentials.
- * Set during creation to ensure it is never null.
- */
- private EncryptionSecrets encryptionSecrets = new EncryptionSecrets();
-
- /**
- * Timestamp of creation.
- * This is set to the current time; it will be overridden when
- * deserializing data.
- */
- private long created = System.currentTimeMillis();
-
- /**
- * An origin string for diagnostics.
- */
- private String origin = "";
-
- /**
- * This marshalled UUID can be used in testing to verify transmission,
- * and reuse; as it is printed you can see what is happending too.
- */
- private String uuid = UUID.randomUUID().toString();
-
- /**
- * Constructor.
- * @param kind token kind.
- * @param uri filesystem URI.
- * @param owner token owner
- * @param origin origin text for diagnostics.
- * @param encryptionSecrets encryption secrets to set.
- */
- protected AbstractS3ATokenIdentifier(
- final Text kind,
- final URI uri,
- final Text owner,
- final String origin,
- final EncryptionSecrets encryptionSecrets) {
- this(kind, owner, new Text(), new Text(), uri);
- this.origin = requireNonNull(origin);
- this.encryptionSecrets = requireNonNull(encryptionSecrets);
- }
-
- /**
- * Constructor.
- * @param kind token kind.
- * @param owner token owner
- * @param renewer token renewer
- * @param realUser token real user
- * @param uri filesystem URI.
- */
- protected AbstractS3ATokenIdentifier(
- final Text kind,
- final Text owner,
- final Text renewer,
- final Text realUser,
- final URI uri) {
- super(kind, owner, renewer, realUser);
- this.uri = requireNonNull(uri);
- }
-
- /**
- * Build from a token.
- * This has been written for refresh operations;
- * if someone implements refresh it will be relevant.
- * @param token to to build from
- * @throws IOException failure to build the identifier.
- */
- protected AbstractS3ATokenIdentifier(
- final Text kind,
- final Token token) throws IOException {
- super(kind);
- ByteArrayInputStream bais = new ByteArrayInputStream(token.getIdentifier());
- readFields(new DataInputStream(bais));
- }
-
- /**
- * For subclasses to use in their own empty-constructors.
- */
- protected AbstractS3ATokenIdentifier(final Text kind) {
- super(kind);
- }
-
- public String getBucket() {
- return uri.getHost();
- }
-
- public URI getUri() {
- return uri;
- }
-
- public String getOrigin() {
- return origin;
- }
-
- public void setOrigin(final String origin) {
- this.origin = origin;
- }
-
- public long getCreated() {
- return created;
- }
-
- /**
- * Write state.
- * {@link org.apache.hadoop.io.Writable#write(DataOutput)}.
- * @param out destination
- * @throws IOException failure
- */
- @Override
- public void write(final DataOutput out) throws IOException {
- super.write(out);
- Text.writeString(out, uri.toString());
- Text.writeString(out, origin);
- Text.writeString(out, uuid);
- encryptionSecrets.write(out);
- out.writeLong(created);
- }
-
- /**
- * Read state.
- * {@link org.apache.hadoop.io.Writable#readFields(DataInput)}.
- *
- * Note: this operation gets called in toString() operations on tokens, so
- * must either always succeed, or throw an IOException to trigger the
- * catch & downgrade. RuntimeExceptions (e.g. Preconditions checks) are
- * not to be used here for this reason.)
- *
- * @param in input stream
- * @throws DelegationTokenIOException if the token binding is wrong.
- * @throws IOException IO problems.
- */
- @Override
- public void readFields(final DataInput in)
- throws DelegationTokenIOException, IOException {
- super.readFields(in);
- uri = URI.create(Text.readString(in, MAX_TEXT_LENGTH));
- origin = Text.readString(in, MAX_TEXT_LENGTH);
- uuid = Text.readString(in, MAX_TEXT_LENGTH);
- encryptionSecrets.readFields(in);
- created = in.readLong();
- }
-
- /**
- * Validate the token by looking at its fields.
- * @throws IOException on failure.
- */
- public void validate() throws IOException {
- if (uri == null) {
- throw new DelegationTokenIOException("No URI in " + this);
- }
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "S3ATokenIdentifier{");
- sb.append(getKind());
- sb.append("; uri=").append(uri);
- sb.append("; timestamp=").append(created);
- sb.append("; encryption=").append(encryptionSecrets.toString());
- sb.append("; ").append(uuid);
- sb.append("; ").append(origin);
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Equality check is on superclass and UUID only.
- * @param o other.
- * @return true if the base class considers them equal and the URIs match.
- */
- @Override
- public boolean equals(final Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- final AbstractS3ATokenIdentifier that = (AbstractS3ATokenIdentifier) o;
- return Objects.equals(uuid, that.uuid) &&
- Objects.equals(uri, that.uri);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), uri);
- }
-
- /**
- * Return the expiry time in seconds since 1970-01-01.
- * @return the time when the session credential expire.
- */
- public long getExpiryTime() {
- return 0;
- }
-
- /**
- * Get the UUID of this token identifier.
- * @return a UUID.
- */
- public String getUuid() {
- return uuid;
- }
-
- /**
- * Get the encryption secrets.
- * @return the encryption secrets within this identifier.
- */
- public EncryptionSecrets getEncryptionSecrets() {
- return encryptionSecrets;
- }
-
- /**
- * Create the default origin text message with local hostname and
- * timestamp.
- * @return a string for token diagnostics.
- */
- public static String createDefaultOriginMessage() {
- return String.format("Created on %s at time %s.",
- NetUtils.getHostname(),
- java.time.Instant.now());
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationConstants.java
deleted file mode 100644
index 7674c6920d..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationConstants.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.io.Text;
-
-/**
- * All the constants related to delegation tokens.
- * Not in the normal S3 constants while unstable.
- *
- * Where possible, the existing assumed role properties are used to configure
- * STS binding, default ARN, etc. This makes documenting everything that
- * much easier and avoids trying to debug precisely which sts endpoint
- * property should be set.
- *
- * Most settings here are replicated in {@code core-default.xml}; the
- * values MUST be kept in sync.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class DelegationConstants {
-
- /**
- * Endpoint for session tokens, used when building delegation tokens:
- * {@value}.
- * @see STS regions
- */
- public static final String DELEGATION_TOKEN_ENDPOINT =
- Constants.ASSUMED_ROLE_STS_ENDPOINT;
-
- /**
- * Default endpoint for session tokens: {@value}.
- */
- public static final String DEFAULT_DELEGATION_TOKEN_ENDPOINT =
- Constants.DEFAULT_ASSUMED_ROLE_STS_ENDPOINT;
-
- /**
- * Region for DT issuing; must be non-empty if the endpoint is set: {@value}.
- */
- public static final String DELEGATION_TOKEN_REGION =
- Constants.ASSUMED_ROLE_STS_ENDPOINT_REGION;
-
- /**
- * Region default: {@value}.
- */
- public static final String DEFAULT_DELEGATION_TOKEN_REGION =
- Constants.ASSUMED_ROLE_STS_ENDPOINT_REGION_DEFAULT;
-
- /**
- * Duration of tokens in time: {@value}.
- */
- public static final String DELEGATION_TOKEN_DURATION =
- Constants.ASSUMED_ROLE_SESSION_DURATION;
-
- /**
- * Default duration of a delegation token: {@value}.
- * Must be in the range supported by STS.
- */
- public static final String DEFAULT_DELEGATION_TOKEN_DURATION =
- Constants.ASSUMED_ROLE_SESSION_DURATION_DEFAULT;
-
- /**
- * Key to list AWS credential providers for Session/role
- * credentials: {@value}.
- */
- public static final String DELEGATION_TOKEN_CREDENTIALS_PROVIDER =
- Constants.AWS_CREDENTIALS_PROVIDER;
-
- /**
- * ARN of the delegation token: {@value}.
- * Required for the role token.
- */
- public static final String DELEGATION_TOKEN_ROLE_ARN =
- Constants.ASSUMED_ROLE_ARN;
-
- /**
- * Property containing classname for token binding: {@value}.
- */
- public static final String DELEGATION_TOKEN_BINDING =
- "fs.s3a.delegation.token.binding";
- /**
- * Session Token binding classname: {@value}.
- */
- public static final String DELEGATION_TOKEN_SESSION_BINDING =
- "org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding";
-
- /**
- * Default token binding {@value}.
- */
- public static final String DEFAULT_DELEGATION_TOKEN_BINDING = "";
-
- /**
- * Token binding to pass full credentials: {@value}.
- */
- public static final String DELEGATION_TOKEN_FULL_CREDENTIALS_BINDING =
- "org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding";
-
- /**
- * Role DTs: {@value}.
- */
- public static final String DELEGATION_TOKEN_ROLE_BINDING =
- "org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding";
-
- /** Prefix for token names: {@value}. */
- public static final String TOKEN_NAME_PREFIX = "S3ADelegationToken/";
-
- /** Name of session token: {@value}. */
- public static final String SESSION_TOKEN_NAME = TOKEN_NAME_PREFIX + "Session";
-
- /** Kind of the session token; value is {@link #SESSION_TOKEN_NAME}. */
- public static final Text SESSION_TOKEN_KIND = new Text(SESSION_TOKEN_NAME);
-
- /** Name of full token: {@value}. */
- public static final String FULL_TOKEN_NAME = TOKEN_NAME_PREFIX + "Full";
-
- /** Kind of the full token; value is {@link #FULL_TOKEN_NAME}. */
- public static final Text FULL_TOKEN_KIND = new Text(FULL_TOKEN_NAME);
-
- /** Name of role token: {@value}. */
- public static final String ROLE_TOKEN_NAME = TOKEN_NAME_PREFIX + "Role";
-
- /** Kind of the role token; value is {@link #ROLE_TOKEN_NAME}. */
- public static final Text ROLE_TOKEN_KIND = new Text(ROLE_TOKEN_NAME);
-
- /**
- * Package-scoped option to control level that duration info on token
- * binding operations are logged at.
- * Value: {@value}.
- */
- static final boolean DURATION_LOG_AT_INFO = true;
-
- /**
- * If the token binding auth chain is only session-level auth, you
- * can't use the role binding: {@value}.
- */
- public static final String E_NO_SESSION_TOKENS_FOR_ROLE_BINDING
- = "Cannot issue S3A Role Delegation Tokens without full AWS credentials";
-
- /**
- * The standard STS server.
- */
- public static final String STS_STANDARD = "sts.amazonaws.com";
-
- private DelegationConstants() {
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationTokenIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationTokenIOException.java
deleted file mode 100644
index 32d45bc17d..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/DelegationTokenIOException.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-
-/**
- * General IOException for Delegation Token issues.
- * Includes recommended error strings, which can be used in tests when
- * looking for specific errors.
- */
-public class DelegationTokenIOException extends IOException {
-
- private static final long serialVersionUID = 599813827985340023L;
-
- /** Error: delegation token/token identifier class isn't the right one. */
- public static final String TOKEN_WRONG_CLASS
- = "Delegation token is wrong class";
-
- /**
- * The far end is expecting a different token kind than
- * that which the client created.
- */
- protected static final String TOKEN_MISMATCH = "Token mismatch";
-
- public DelegationTokenIOException(final String message) {
- super(message);
- }
-
- public DelegationTokenIOException(final String message,
- final Throwable cause) {
- super(message, cause);
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
deleted file mode 100644
index 6edbb035bd..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.util.Optional;
-
-import com.amazonaws.services.s3.model.SSEAwsKeyManagementParams;
-import com.amazonaws.services.s3.model.SSECustomerKey;
-
-import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
-
-/**
- * These support operations on {@link EncryptionSecrets} which use the AWS SDK
- * operations. Isolating them here ensures that that class is not required on
- * the classpath.
- */
-public class EncryptionSecretOperations {
-
- /**
- * Create SSE-C client side key encryption options on demand.
- * @return an optional key to attach to a request.
- * @param secrets source of the encryption secrets.
- */
- public static Optional createSSECustomerKey(
- final EncryptionSecrets secrets) {
- if (secrets.hasEncryptionKey() &&
- secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_C) {
- return Optional.of(new SSECustomerKey(secrets.getEncryptionKey()));
- } else {
- return Optional.empty();
- }
- }
-
- /**
- * Create SSE-KMS options for a request, iff the encryption is SSE-KMS.
- * @return an optional SSE-KMS param to attach to a request.
- * @param secrets source of the encryption secrets.
- */
- public static Optional createSSEAwsKeyManagementParams(
- final EncryptionSecrets secrets) {
-
- //Use specified key, otherwise default to default master aws/s3 key by AWS
- if (secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS) {
- if (secrets.hasEncryptionKey()) {
- return Optional.of(new SSEAwsKeyManagementParams(
- secrets.getEncryptionKey()));
- } else {
- return Optional.of(new SSEAwsKeyManagementParams());
- }
- } else {
- return Optional.empty();
- }
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
deleted file mode 100644
index 092653de55..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.Serializable;
-import java.util.Objects;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.s3a.S3AEncryptionMethods;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-
-/**
- * Encryption options in a form which can serialized or marshalled as a hadoop
- * Writeable.
- *
- * Maintainers: For security reasons, don't print any of this.
- *
- * Note this design marshalls/unmarshalls its serialVersionUID
- * in its writable, which is used to compare versions.
- *
- * Important.
- * If the wire format is ever changed incompatibly,
- * update the serial version UID to ensure that older clients get safely
- * rejected.
- *
- * Important
- * Do not import any AWS SDK classes, directly or indirectly.
- * This is to ensure that S3A Token identifiers can be unmarshalled even
- * without that SDK.
- */
-public class EncryptionSecrets implements Writable, Serializable {
-
- public static final int MAX_SECRET_LENGTH = 2048;
-
- private static final long serialVersionUID = 1208329045511296375L;
-
- /**
- * Encryption algorithm to use: must match one in
- * {@link S3AEncryptionMethods}.
- */
- private String encryptionAlgorithm = "";
-
- /**
- * Encryption key: possibly sensitive information.
- */
- private String encryptionKey = "";
-
- /**
- * This field isn't serialized/marshalled; it is rebuilt from the
- * encryptionAlgorithm field.
- */
- private transient S3AEncryptionMethods encryptionMethod =
- S3AEncryptionMethods.NONE;
-
- /**
- * Empty constructor, for use in marshalling.
- */
- public EncryptionSecrets() {
- }
-
- /**
- * Create a pair of secrets.
- * @param encryptionAlgorithm algorithm enumeration.
- * @param encryptionKey key/key reference.
- * @throws IOException failure to initialize.
- */
- public EncryptionSecrets(final S3AEncryptionMethods encryptionAlgorithm,
- final String encryptionKey) throws IOException {
- this(encryptionAlgorithm.getMethod(), encryptionKey);
- }
-
- /**
- * Create a pair of secrets.
- * @param encryptionAlgorithm algorithm name
- * @param encryptionKey key/key reference.
- * @throws IOException failure to initialize.
- */
- public EncryptionSecrets(final String encryptionAlgorithm,
- final String encryptionKey) throws IOException {
- this.encryptionAlgorithm = encryptionAlgorithm;
- this.encryptionKey = encryptionKey;
- init();
- }
-
- /**
- * Write out the encryption secrets.
- * @param out {@code DataOutput} to serialize this object into.
- * @throws IOException IO failure
- */
- @Override
- public void write(final DataOutput out) throws IOException {
- new LongWritable(serialVersionUID).write(out);
- Text.writeString(out, encryptionAlgorithm);
- Text.writeString(out, encryptionKey);
- }
-
- /**
- * Read in from the writable stream.
- * After reading, call {@link #init()}.
- * @param in {@code DataInput} to deserialize this object from.
- * @throws IOException failure to read/validate data.
- */
- @Override
- public void readFields(final DataInput in) throws IOException {
- final LongWritable version = new LongWritable();
- version.readFields(in);
- if (version.get() != serialVersionUID) {
- throw new DelegationTokenIOException(
- "Incompatible EncryptionSecrets version");
- }
- encryptionAlgorithm = Text.readString(in, MAX_SECRET_LENGTH);
- encryptionKey = Text.readString(in, MAX_SECRET_LENGTH);
- init();
- }
-
- /**
- * For java serialization: read and then call {@link #init()}.
- * @param in input
- * @throws IOException IO problem
- * @throws ClassNotFoundException problem loading inner class.
- */
- private void readObject(ObjectInputStream in)
- throws IOException, ClassNotFoundException {
- in.defaultReadObject();
- init();
- }
-
- /**
- * Init all state, including after any read.
- * @throws IOException error rebuilding state.
- */
- private void init() throws IOException {
- encryptionMethod = S3AEncryptionMethods.getMethod(
- encryptionAlgorithm);
- }
-
- public String getEncryptionAlgorithm() {
- return encryptionAlgorithm;
- }
-
- public String getEncryptionKey() {
- return encryptionKey;
- }
-
- /**
- * Does this instance have encryption options?
- * That is: is the algorithm non-null.
- * @return true if there's an encryption algorithm.
- */
- public boolean hasEncryptionAlgorithm() {
- return StringUtils.isNotEmpty(encryptionAlgorithm);
- }
-
- /**
- * Does this instance have an encryption key?
- * @return true if there's an encryption key.
- */
- public boolean hasEncryptionKey() {
- return StringUtils.isNotEmpty(encryptionKey);
- }
-
- @Override
- public boolean equals(final Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- final EncryptionSecrets that = (EncryptionSecrets) o;
- return Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm)
- && Objects.equals(encryptionKey, that.encryptionKey);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(encryptionAlgorithm, encryptionKey);
- }
-
- /**
- * Get the encryption method.
- * @return the encryption method
- */
- public S3AEncryptionMethods getEncryptionMethod() {
- return encryptionMethod;
- }
-
- /**
- * String function returns the encryption mode but not any other
- * secrets.
- * @return a string safe for logging.
- */
- @Override
- public String toString() {
- return S3AEncryptionMethods.NONE.equals(encryptionMethod)
- ? "(no encryption)"
- : encryptionMethod.getMethod();
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
deleted file mode 100644
index 138667b07d..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Optional;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3native.S3xLoginHelper;
-
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.FULL_TOKEN_KIND;
-
-/**
- * Full credentials: they are simply passed as-is, rather than
- * converted to a session.
- * These aren't as secure; this class exists to (a) support deployments
- * where there is not STS service and (b) validate the design of
- * S3A DT support to support different managers.
- */
-public class FullCredentialsTokenBinding extends
- AbstractDelegationTokenBinding {
-
- /**
- * Wire name of this binding includes a version marker: {@value}.
- */
- private static final String NAME = "FullCredentials/001";
-
- public static final String FULL_TOKEN = "Full Delegation Token";
-
- /**
- * Long-lived AWS credentials.
- */
- private MarshalledCredentials awsCredentials;
-
- /**
- * Origin of credentials.
- */
- private String credentialOrigin;
-
- /**
- * Constructor, uses name of {@link #name} and token kind of
- * {@link DelegationConstants#FULL_TOKEN_KIND}.
- *
- */
- public FullCredentialsTokenBinding() {
- super(NAME, FULL_TOKEN_KIND);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- loadAWSCredentials();
- }
-
- /**
- * Load the AWS credentials.
- * @throws IOException failure
- */
- private void loadAWSCredentials() throws IOException {
- credentialOrigin = AbstractS3ATokenIdentifier.createDefaultOriginMessage();
- Configuration conf = getConfig();
- URI uri = getCanonicalUri();
- // look for access keys to FS
- S3xLoginHelper.Login secrets = S3AUtils.getAWSAccessKeys(uri, conf);
- if (secrets.hasLogin()) {
- awsCredentials = new MarshalledCredentials(
- secrets.getUser(), secrets.getPassword(), "");
- credentialOrigin += "; source = Hadoop configuration data";
- } else {
- // if there are none, look for the environment variables.
- awsCredentials = MarshalledCredentialBinding.fromEnvironment(
- System.getenv());
- if (awsCredentials.isValid(
- MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty)) {
- // valid tokens, so mark as origin
- credentialOrigin += "; source = Environment variables";
- } else {
- credentialOrigin = "no credentials in configuration or"
- + " environment variables";
- }
- }
- awsCredentials.validate(credentialOrigin +": ",
- MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty);
- }
-
- /**
- * Serve up the credentials retrieved from configuration/environment in
- * {@link #loadAWSCredentials()}.
- * @return a credential provider for the unbonded instance.
- * @throws IOException failure to load
- */
- @Override
- public AWSCredentialProviderList deployUnbonded() throws IOException {
- requireServiceStarted();
- return new AWSCredentialProviderList(
- "Full Credentials Token Binding",
- new MarshalledCredentialProvider(
- FULL_TOKEN,
- getFileSystem().getUri(),
- getConfig(),
- awsCredentials,
- MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty));
- }
-
- /**
- * Create a new delegation token.
- *
- * It's slightly inefficient to create a new one every time, but
- * it avoids concurrency problems with managing any singleton.
- * @param policy minimum policy to use, if known.
- * @param encryptionSecrets encryption secrets.
- * @return a DT identifier
- * @throws IOException failure
- */
- @Override
- public AbstractS3ATokenIdentifier createTokenIdentifier(
- final Optional policy,
- final EncryptionSecrets encryptionSecrets) throws IOException {
- requireServiceStarted();
-
- return new FullCredentialsTokenIdentifier(getCanonicalUri(),
- getOwnerText(),
- awsCredentials,
- encryptionSecrets,
- credentialOrigin);
- }
-
- @Override
- public AWSCredentialProviderList bindToTokenIdentifier(
- final AbstractS3ATokenIdentifier retrievedIdentifier)
- throws IOException {
- FullCredentialsTokenIdentifier tokenIdentifier =
- convertTokenIdentifier(retrievedIdentifier,
- FullCredentialsTokenIdentifier.class);
- return new AWSCredentialProviderList(
- "", new MarshalledCredentialProvider(
- FULL_TOKEN,
- getFileSystem().getUri(),
- getConfig(),
- tokenIdentifier.getMarshalledCredentials(),
- MarshalledCredentials.CredentialTypeRequired.AnyNonEmpty));
- }
-
- @Override
- public AbstractS3ATokenIdentifier createEmptyIdentifier() {
- return new FullCredentialsTokenIdentifier();
- }
-
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenIdentifier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenIdentifier.java
deleted file mode 100644
index 95e4a28970..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenIdentifier.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.net.URI;
-
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
-import org.apache.hadoop.io.Text;
-
-/**
- * The full credentials payload is the same of that for a session token, but
- * a different token kind is used.
- *
- * Token kind is {@link DelegationConstants#FULL_TOKEN_KIND}.
- */
-public class FullCredentialsTokenIdentifier extends SessionTokenIdentifier {
-
- public FullCredentialsTokenIdentifier() {
- super(DelegationConstants.FULL_TOKEN_KIND);
- }
-
- public FullCredentialsTokenIdentifier(final URI uri,
- final Text owner,
- final MarshalledCredentials marshalledCredentials,
- final EncryptionSecrets encryptionSecrets,
- String origin) {
- super(DelegationConstants.FULL_TOKEN_KIND,
- owner,
- uri,
- marshalledCredentials,
- encryptionSecrets,
- origin);
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
deleted file mode 100644
index f436671a8f..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-
-import com.amazonaws.services.securitytoken.model.Credentials;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3a.auth.STSClientFactory;
-
-import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_CREDENTIALS_PROVIDER;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_ROLE_ARN;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.E_NO_SESSION_TOKENS_FOR_ROLE_BINDING;
-
-/**
- * Role Token support requests an explicit role and automatically restricts
- * that role to the given policy of the binding.
- * The session is locked down as much as possible.
- */
-public class RoleTokenBinding extends SessionTokenBinding {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- RoleTokenBinding.class);
-
- private static final RoleModel MODEL = new RoleModel();
-
- /**
- * Wire name of this binding includes a version marker: {@value}.
- */
- private static final String NAME = "RoleCredentials/001";
-
- /**
- * Error message when there is no Role ARN.
- */
- @VisibleForTesting
- public static final String E_NO_ARN =
- "No role ARN defined in " + DELEGATION_TOKEN_ROLE_ARN;
-
- public static final String COMPONENT = "Role Delegation Token";
-
- /**
- * Role ARN to use when requesting new tokens.
- */
- private String roleArn;
-
- /**
- * Constructor.
- * Name is {@link #name}; token kind is
- * {@link DelegationConstants#ROLE_TOKEN_KIND}.
- */
- public RoleTokenBinding() {
- super(NAME, DelegationConstants.ROLE_TOKEN_KIND);
- }
-
- @Override
- protected void serviceInit(final Configuration conf) throws Exception {
- super.serviceInit(conf);
- roleArn = getConfig().getTrimmed(DELEGATION_TOKEN_ROLE_ARN, "");
- }
-
- /**
- * Returns a (wrapped) {@link MarshalledCredentialProvider} which
- * requires the marshalled credentials to contain session secrets.
- * @param retrievedIdentifier the incoming identifier.
- * @return the provider chain.
- * @throws IOException on failure
- */
- @Override
- public AWSCredentialProviderList bindToTokenIdentifier(
- final AbstractS3ATokenIdentifier retrievedIdentifier)
- throws IOException {
- RoleTokenIdentifier tokenIdentifier =
- convertTokenIdentifier(retrievedIdentifier,
- RoleTokenIdentifier.class);
- setTokenIdentifier(Optional.of(tokenIdentifier));
- MarshalledCredentials marshalledCredentials
- = tokenIdentifier.getMarshalledCredentials();
- setExpirationDateTime(marshalledCredentials.getExpirationDateTime());
- return new AWSCredentialProviderList(
- "Role Token Binding",
- new MarshalledCredentialProvider(
- COMPONENT, getFileSystem().getUri(),
- getConfig(),
- marshalledCredentials,
- MarshalledCredentials.CredentialTypeRequired.SessionOnly));
- }
-
- /**
- * Create the Token Identifier.
- * Looks for the option {@link DelegationConstants#DELEGATION_TOKEN_ROLE_ARN}
- * in the config and fail if it is not set.
- * @param policy the policy which will be used for the requested token.
- * @param encryptionSecrets encryption secrets.
- * @return the token.
- * @throws IllegalArgumentException if there is no role defined.
- * @throws IOException any problem acquiring the role.
- */
- @Override
- @Retries.RetryTranslated
- public RoleTokenIdentifier createTokenIdentifier(
- final Optional policy,
- final EncryptionSecrets encryptionSecrets) throws IOException {
- requireServiceStarted();
- Preconditions.checkState(!roleArn.isEmpty(), E_NO_ARN);
- String policyJson = policy.isPresent() ?
- MODEL.toJson(policy.get()) : "";
- final STSClientFactory.STSClient client = prepareSTSClient()
- .orElseThrow(() -> {
- // we've come in on a parent binding, so fail fast
- LOG.error("Cannot issue delegation tokens because the credential"
- + " providers listed in " + DELEGATION_TOKEN_CREDENTIALS_PROVIDER
- + " are returning session tokens");
- return new DelegationTokenIOException(
- E_NO_SESSION_TOKENS_FOR_ROLE_BINDING);
- });
- Credentials credentials = client
- .requestRole(roleArn,
- UUID.randomUUID().toString(),
- policyJson,
- getDuration(),
- TimeUnit.SECONDS);
- return new RoleTokenIdentifier(
- getCanonicalUri(),
- getOwnerText(),
- fromSTSCredentials(credentials),
- encryptionSecrets,
- AbstractS3ATokenIdentifier.createDefaultOriginMessage()
- + " Role ARN=" + roleArn);
- }
-
- @Override
- public RoleTokenIdentifier createEmptyIdentifier() {
- return new RoleTokenIdentifier();
- }
-
- @Override
- public String getDescription() {
- return super.getDescription() + " Role ARN=" +
- (roleArn.isEmpty() ? "(none)" : ('"' + roleArn +'"'));
- }
-
- @Override
- protected String bindingName() {
- return "Role";
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
deleted file mode 100644
index b8eeca1350..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java
+++ /dev/null
@@ -1,685 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.S3AFileSystem;
-import org.apache.hadoop.fs.s3a.S3AInstrumentation;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3a.commit.DurationInfo;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.service.ServiceOperations;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DEFAULT_DELEGATION_TOKEN_BINDING;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DELEGATION_TOKEN_BINDING;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.DURATION_LOG_AT_INFO;
-
-/**
- * Support for creating a DT from a filesystem.
- *
- * Isolated from S3A for control and testability.
- *
- * The S3A Delegation Tokens are special in that the tokens are not directly
- * used to authenticate with the AWS services.
- * Instead they can session/role credentials requested off AWS on demand.
- *
- * The design is extensible in that different back-end bindings can be used
- * to switch to different session creation mechanisms, or indeed, to any
- * other authentication mechanism supported by an S3 service, provided it
- * ultimately accepts some form of AWS credentials for authentication through
- * the AWS SDK. That is, if someone wants to wire this up to Kerberos, or
- * OAuth2, this design should support them.
- *
- * URIs processed must be the canonical URIs for the service.
- */
-@InterfaceAudience.Private
-public class S3ADelegationTokens extends AbstractDTService {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- S3ADelegationTokens.class);
-
- @VisibleForTesting
- static final String E_ALREADY_DEPLOYED
- = "S3A Delegation tokens has already been bound/deployed";
-
- public static final String E_DELEGATION_TOKENS_DISABLED
- = "Delegation tokens are not enabled";
-
- /**
- * User who owns this FS; fixed at instantiation time, so that
- * in calls to getDelegationToken() and similar, this user is the one whose
- * credentials are involved.
- */
- private final UserGroupInformation user;
-
- /**
- * Count of number of created tokens.
- * For testing and diagnostics.
- */
- private final AtomicInteger creationCount = new AtomicInteger(0);
-
- /**
- * Text value of this token service.
- */
- private Text service;
-
- /**
- * Active Delegation token.
- */
- private Optional> boundDT
- = Optional.empty();
-
- /**
- * The DT decoded when this instance is created by bonding
- * to an existing DT.
- */
- private Optional decodedIdentifier
- = Optional.empty();
-
- /**
- * Dynamically loaded token binding; lifecycle matches this object.
- */
- private AbstractDelegationTokenBinding tokenBinding;
-
- /**
- * List of cred providers; unset until {@link #bindToDelegationToken(Token)}.
- */
- private Optional credentialProviders
- = Optional.empty();
-
- /**
- * The access policies we want for operations.
- * There's no attempt to ask for "admin" permissions here, e.g.
- * those to manipulate S3Guard tables.
- */
- protected static final EnumSet ACCESS_POLICY
- = EnumSet.of(
- AWSPolicyProvider.AccessLevel.READ,
- AWSPolicyProvider.AccessLevel.WRITE);
-
- /**
- * Statistics for the owner FS.
- */
- private S3AInstrumentation.DelegationTokenStatistics stats;
-
- /**
- * Name of the token binding as extracted from token kind; used for
- * logging.
- */
- private String tokenBindingName = "";
-
- /**
- * Instantiate.
- */
- public S3ADelegationTokens() throws IOException {
- super("S3ADelegationTokens");
- user = UserGroupInformation.getCurrentUser();
- }
-
- @Override
- public void bindToFileSystem(final URI uri, final S3AFileSystem fs)
- throws IOException {
- super.bindToFileSystem(uri, fs);
- service = getTokenService(getCanonicalUri());
- stats = fs.getInstrumentation().newDelegationTokenStatistics();
- }
-
- /**
- * Init the service.
- * This identifies the token binding class to use and creates, initializes
- * and starts it.
- * Will raise an exception if delegation tokens are not enabled.
- * @param conf configuration
- * @throws Exception any failure to start up
- */
- @Override
- protected void serviceInit(final Configuration conf) throws Exception {
- super.serviceInit(conf);
- checkState(hasDelegationTokenBinding(conf),
- E_DELEGATION_TOKENS_DISABLED);
- Class binding = conf.getClass(
- DelegationConstants.DELEGATION_TOKEN_BINDING,
- SessionTokenBinding.class,
- AbstractDelegationTokenBinding.class);
- tokenBinding = binding.newInstance();
- tokenBinding.bindToFileSystem(getCanonicalUri(), getFileSystem());
- tokenBinding.init(conf);
- tokenBindingName = tokenBinding.getKind().toString();
- LOG.info("Filesystem {} is using delegation tokens of kind {}",
- getCanonicalUri(), tokenBindingName);
- }
-
- /**
- * Service startup includes binding to any delegation token, and
- * deploying unbounded if there is none.
- * It is after this that token operations can be used.
- * @throws Exception any failure
- */
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- tokenBinding.start();
- bindToAnyDelegationToken();
- LOG.info("S3A Delegation support token {} with {}",
- identifierToString(),
- tokenBinding.getDescription());
- }
-
- /**
- * Get the identifier as a string, or "(none)".
- * @return a string value for logs etc.
- */
- private String identifierToString() {
- return decodedIdentifier.map(Objects::toString)
- .orElse("(none)");
- }
-
- /**
- * Stop the token binding.
- * @throws Exception on any failure
- */
- @SuppressWarnings("ThrowableNotThrown")
- @Override
- protected void serviceStop() throws Exception {
- LOG.debug("Stopping delegation tokens");
- try {
- super.serviceStop();
- } finally {
- ServiceOperations.stopQuietly(LOG, tokenBinding);
- }
- }
-
-
- /**
- * Perform the unbonded deployment operations.
- * Create the AWS credential provider chain to use
- * when talking to AWS when there is no delegation token to work with.
- * authenticating this client with AWS services, and saves it
- * to {@link #credentialProviders}
- *
- * @throws IOException any failure.
- */
- private void deployUnbonded()
- throws IOException {
- requireServiceStarted();
- checkState(!isBoundToDT(),
- "Already Bound to a delegation token");
- LOG.info("No delegation tokens present: using direct authentication");
- credentialProviders = Optional.of(tokenBinding.deployUnbonded());
- }
-
- /**
- * Attempt to bind to any existing DT, including unmarshalling its contents
- * and creating the AWS credential provider used to authenticate
- * the client.
- *
- * If successful:
- *
- * - {@link #boundDT} is set to the retrieved token.
- * - {@link #decodedIdentifier} is set to the extracted identifier.
- * - {@link #credentialProviders} is set to the credential
- * provider(s) returned by the token binding.
- *
- * If unsuccessful, {@link #deployUnbonded()} is called for the
- * unbonded codepath instead, which will set
- * {@link #credentialProviders} to its value.
- *
- * This means after this call (and only after) the token operations
- * can be invoked.
- *
- * This method is called from {@link #serviceStart()}, so a check on
- * the service state can be used to check things; the state model
- * prevents re-entrant calls.
- * @throws IOException selection/extraction/validation failure.
- */
- private void bindToAnyDelegationToken() throws IOException {
- checkState(!credentialProviders.isPresent(), E_ALREADY_DEPLOYED);
- Token token = selectTokenFromFSOwner();
- if (token != null) {
- bindToDelegationToken(token);
- } else {
- deployUnbonded();
- }
- if (credentialProviders.get().size() == 0) {
- throw new DelegationTokenIOException("No AWS credential providers"
- + " created by Delegation Token Binding "
- + tokenBinding.getName());
- }
- }
-
- /**
- * This is a test-only back door which resets the state and binds to
- * a token again.
- * This allows an instance of this class to be bonded to a DT after being
- * started, so avoids the need to have the token in the current user
- * credentials. It is package scoped so as to only be usable in tests
- * in the same package.
- *
- * Yes, this is ugly, but there is no obvious/easy way to test token
- * binding without Kerberos getting involved.
- * @param token token to decode and bind to.
- * @throws IOException selection/extraction/validation failure.
- */
- @VisibleForTesting
- void resetTokenBindingToDT(final Token token)
- throws IOException{
- credentialProviders = Optional.empty();
- bindToDelegationToken(token);
- }
-
- /**
- * Bind to a delegation token retrieved for this filesystem.
- * Extract the secrets from the token and set internal fields
- * to the values.
- *
- * - {@link #boundDT} is set to {@code token}.
- * - {@link #decodedIdentifier} is set to the extracted identifier.
- * - {@link #credentialProviders} is set to the credential
- * provider(s) returned by the token binding.
- *
- * @param token token to decode and bind to.
- * @throws IOException selection/extraction/validation failure.
- */
- @VisibleForTesting
- public void bindToDelegationToken(
- final Token token)
- throws IOException {
- checkState(!credentialProviders.isPresent(), E_ALREADY_DEPLOYED);
- boundDT = Optional.of(token);
- AbstractS3ATokenIdentifier dti = extractIdentifier(token);
- LOG.info("Using delegation token {}", dti);
- decodedIdentifier = Optional.of(dti);
- try (DurationInfo ignored = new DurationInfo(LOG, DURATION_LOG_AT_INFO,
- "Creating Delegation Token")) {
- // extract the credential providers.
- credentialProviders = Optional.of(
- tokenBinding.bindToTokenIdentifier(dti));
- }
- }
-
- /**
- * Predicate: is there a bound DT?
- * @return true if there's a value in {@link #boundDT}.
- */
- public boolean isBoundToDT() {
- return boundDT.isPresent();
- }
-
- /**
- * Get any bound DT.
- * @return a delegation token if this instance was bound to it.
- */
- public Optional> getBoundDT() {
- return boundDT;
- }
-
- /**
- * Predicate: will this binding issue a DT if requested
- * in a call to {@link #getBoundOrNewDT(EncryptionSecrets)}?
- * That is: should the filesystem declare that it is issuing
- * delegation tokens?
- * @return a declaration of what will happen when asked for a token.
- */
- public TokenIssuingPolicy getTokenIssuingPolicy() {
- return isBoundToDT()
- ? TokenIssuingPolicy.ReturnExistingToken
- : tokenBinding.getTokenIssuingPolicy();
- }
-
- /**
- * Get any bound DT or create a new one.
- * @return a delegation token.
- * @throws IOException if one cannot be created
- * @param encryptionSecrets encryption secrets for any new token.
- */
- @SuppressWarnings("OptionalGetWithoutIsPresent")
- public Token getBoundOrNewDT(
- final EncryptionSecrets encryptionSecrets)
- throws IOException {
- LOG.debug("Delegation token requested");
- if (isBoundToDT()) {
- // the FS was created on startup with a token, so return it.
- LOG.debug("Returning current token");
- return getBoundDT().get();
- } else {
- // not bound to a token, so create a new one.
- // issued DTs are not cached so that long-lived filesystems can
- // reliably issue session/role tokens.
- return createDelegationToken(encryptionSecrets);
- }
- }
-
- /**
- * How many delegation tokens have been issued?
- * @return the number times {@link #createDelegationToken(EncryptionSecrets)}
- * returned a token.
- */
- public int getCreationCount() {
- return creationCount.get();
- }
-
- /**
- * Create a delegation token for the user.
- * This will only be called if a new DT is needed, that is: the
- * filesystem has been deployed unbonded.
- * @param encryptionSecrets encryption secrets for the token.
- * @return the token
- * @throws IOException if one cannot be created
- */
- @VisibleForTesting
- public Token createDelegationToken(
- final EncryptionSecrets encryptionSecrets) throws IOException {
- requireServiceStarted();
- checkArgument(encryptionSecrets != null,
- "Null encryption secrets");
- // this isn't done in in advance as it needs S3Guard initialized in the
- // filesystem before it can generate complete policies.
- List statements = getFileSystem()
- .listAWSPolicyRules(ACCESS_POLICY);
- Optional rolePolicy =
- statements.isEmpty() ?
- Optional.empty() : Optional.of(new RoleModel.Policy(statements));
-
- try(DurationInfo ignored = new DurationInfo(LOG, DURATION_LOG_AT_INFO,
- "Creating New Delegation Token", tokenBinding.getKind())) {
- Token token
- = tokenBinding.createDelegationToken(rolePolicy, encryptionSecrets);
- if (token != null) {
- token.setService(service);
- noteTokenCreated(token);
- }
- return token;
- }
- }
-
- /**
- * Note that a token has been created; increment counters and statistics.
- * @param token token created
- */
- private void noteTokenCreated(final Token token) {
- LOG.info("Created S3A Delegation Token: {}", token);
- creationCount.incrementAndGet();
- stats.tokenIssued();
- }
-
- /**
- * Get the AWS credential provider.
- * @return the DT credential provider
- * @throws IOException failure to parse the DT
- * @throws IllegalStateException if this instance is not bound to a DT
- */
- public AWSCredentialProviderList getCredentialProviders()
- throws IOException {
- return credentialProviders.orElseThrow(
- () -> new DelegationTokenIOException("Not yet bonded"));
- }
-
- /**
- * Get the encryption secrets of the DT.
- * non-empty iff service is started and was bound to a DT.
- * @return any encryption settings propagated with the DT.
- */
- public Optional getEncryptionSecrets() {
- return decodedIdentifier.map(
- AbstractS3ATokenIdentifier::getEncryptionSecrets);
- }
-
- /**
- * Get any decoded identifier from the bound DT; empty if not bound.
- * @return the decoded identifier.
- */
- public Optional getDecodedIdentifier() {
- return decodedIdentifier;
- }
-
- /**
- * Get the service identifier of the owning FS.
- * @return a service identifier to use when registering tokens
- */
- public Text getService() {
- return service;
- }
-
- /**
- * The canonical name of the service.
- * This can be used as the canonical service name for the FS.
- * @return the canonicalized FS URI.
- */
- public String getCanonicalServiceName() {
- return getCanonicalUri().toString();
- }
-
- /**
- * Find a token for the FS user and canonical filesystem URI.
- * @return the token, or null if one cannot be found.
- * @throws IOException on a failure to unmarshall the token.
- */
- @VisibleForTesting
- public Token selectTokenFromFSOwner()
- throws IOException {
- return lookupToken(user.getCredentials(),
- service,
- tokenBinding.getKind());
- }
-
- /**
- * Get the service identifier of a filesystem.
- * This must be unique for (S3A, the FS URI)
- * @param fsURI filesystem URI
- * @return identifier to use.
- */
- private static Text getTokenService(final URI fsURI) {
- return getTokenService(fsURI.toString());
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "S3ADelegationTokens{");
- sb.append("canonicalServiceURI=").append(getCanonicalUri());
- sb.append("; owner=").append(user.getShortUserName());
- sb.append("; isBoundToDT=").append(isBoundToDT());
- sb.append("; token creation count=").append(getCreationCount());
- sb.append("; tokenManager=").append(tokenBinding);
- sb.append("; token=").append(identifierToString());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Get the kind of the issued tokens.
- * @return token kind.
- */
- public Text getTokenKind() {
- return tokenBinding.getKind();
- }
-
- /**
- * Get the service identifier of a filesystem URI.
- * This must be unique for (S3a, the FS URI)
- * @param fsURI filesystem URI as a string
- * @return identifier to use.
- */
- @VisibleForTesting
- static Text getTokenService(final String fsURI) {
- return new Text(fsURI);
- }
-
- /**
- * From a token, get the session token identifier.
- * @param token token to process
- * @return the session token identifier
- * @throws IOException failure to validate/read data encoded in identifier.
- * @throws IllegalArgumentException if the token isn't an S3A session token
- */
- public AbstractS3ATokenIdentifier extractIdentifier(
- final Token token)
- throws IOException {
-
- checkArgument(token != null, "null token");
- AbstractS3ATokenIdentifier identifier;
- // harden up decode beyond that Token does itself
- try {
- identifier = token.decodeIdentifier();
- } catch (RuntimeException e) {
- Throwable cause = e.getCause();
- if (cause != null) {
- // its a wrapping around class instantiation.
- throw new DelegationTokenIOException("Decoding S3A token " + cause,
- cause);
- } else {
- throw e;
- }
- }
- if (identifier == null) {
- throw new DelegationTokenIOException("Failed to unmarshall token for "
- + getCanonicalUri());
- }
- identifier.validate();
- return identifier;
- }
-
- /**
- * Return a string for use in building up the User-Agent field, so
- * get into the S3 access logs. Useful for diagnostics.
- * Delegates to {{@link AbstractDelegationTokenBinding#getUserAgentField()}}
- * for the current binding.
- * @return a string for the S3 logs or "" for "nothing to add"
- */
- public String getUserAgentField() {
- return tokenBinding.getUserAgentField();
- }
-
- /**
- * Look up a token from the credentials, verify it is of the correct
- * kind.
- * @param credentials credentials to look up.
- * @param service service name
- * @param kind token kind to look for
- * @return the token or null if no suitable token was found
- * @throws DelegationTokenIOException wrong token kind found
- */
- @VisibleForTesting
- public static Token lookupToken(
- final Credentials credentials,
- final Text service,
- final Text kind)
- throws DelegationTokenIOException {
-
- LOG.debug("Looking for token for service {} in credentials", service);
- Token token = credentials.getToken(service);
- if (token != null) {
- Text tokenKind = token.getKind();
- LOG.debug("Found token of kind {}", tokenKind);
- if (kind.equals(tokenKind)) {
- // the Oauth implementation catches and logs here; this one
- // throws the failure up.
- return (Token) token;
- } else {
-
- // there's a token for this URI, but its not the right DT kind
- throw new DelegationTokenIOException(
- DelegationTokenIOException.TOKEN_MISMATCH + ": expected token"
- + " for " + service
- + " of type " + kind
- + " but got a token of type " + tokenKind);
- }
- }
- // A token for the service was not found
- LOG.debug("No token for {} found", service);
- return null;
- }
-
- /**
- * Look up any token from the service; cast it to one of ours.
- * @param credentials credentials
- * @param service service to look up
- * @return any token found or null if none was
- * @throws ClassCastException if the token is of a wrong type.
- */
- public static Token lookupToken(
- final Credentials credentials,
- final Text service) {
- return (Token) credentials.getToken(service);
- }
-
- /**
- * Look for any S3A token for the given FS service.
- * @param credentials credentials to scan.
- * @param uri the URI of the FS to look for
- * @return the token or null if none was found
- */
- public static Token lookupS3ADelegationToken(
- final Credentials credentials,
- final URI uri) {
- return lookupToken(credentials, getTokenService(uri.toString()));
- }
-
- /**
- * Predicate: does this configuration enable delegation tokens?
- * That is: is there any text in the option
- * {@link DelegationConstants#DELEGATION_TOKEN_BINDING} ?
- * @param conf configuration to examine
- * @return true iff the trimmed configuration option is not empty.
- */
- public static boolean hasDelegationTokenBinding(Configuration conf) {
- return StringUtils.isNotEmpty(
- conf.getTrimmed(DELEGATION_TOKEN_BINDING,
- DEFAULT_DELEGATION_TOKEN_BINDING));
- }
-
- /**
- * How will tokens be issued on request?
- *
- * The {@link #RequestNewToken} policy does not guarantee that a tokens
- * can be created, only that an attempt will be made to request one.
- * It may fail (wrong credential types, wrong role, etc).
- */
- public enum TokenIssuingPolicy {
-
- /** The existing token will be returned. */
- ReturnExistingToken,
-
- /** No tokens will be issued. */
- NoTokensAvailable,
-
- /** An attempt will be made to request a new DT. */
- RequestNewToken
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADtFetcher.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADtFetcher.java
deleted file mode 100644
index 8ac07a216d..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADtFetcher.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.net.URI;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.DtFetcher;
-import org.apache.hadoop.security.token.Token;
-
-/**
- * A DT fetcher for S3A.
- * This is a copy-and-paste of
- * {@code org.apache.hadoop.hdfs.HdfsDtFetcher}.
- *
- * It is only needed for the `hadoop dtutil` command.
- */
-public class S3ADtFetcher implements DtFetcher {
-
- private static final String SERVICE_NAME = Constants.FS_S3A;
-
- private static final String FETCH_FAILED =
- "Filesystem not generating Delegation Tokens";
-
- /**
- * Returns the service name for HDFS, which is also a valid URL prefix.
- */
- public Text getServiceName() {
- return new Text(SERVICE_NAME);
- }
-
- public boolean isTokenRequired() {
- return UserGroupInformation.isSecurityEnabled();
- }
-
- /**
- * Returns Token object via FileSystem, null if bad argument.
- * @param conf - a Configuration object used with FileSystem.get()
- * @param creds - a Credentials object to which token(s) will be added
- * @param renewer - the renewer to send with the token request
- * @param url - the URL to which the request is sent
- * @return a Token, or null if fetch fails.
- */
- public Token addDelegationTokens(Configuration conf,
- Credentials creds,
- String renewer,
- String url) throws Exception {
- if (!url.startsWith(getServiceName().toString())) {
- url = getServiceName().toString() + "://" + url;
- }
- FileSystem fs = FileSystem.get(URI.create(url), conf);
- Token token = fs.getDelegationToken(renewer);
- if (token == null) {
- throw new DelegationTokenIOException(FETCH_FAILED + ": " + url);
- }
- creds.addToken(token.getService(), token);
- return token;
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
deleted file mode 100644
index fd5f6f71c1..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.IOException;
-import java.net.URI;
-import java.time.OffsetDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.HashSet;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSSessionCredentials;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
-import org.apache.hadoop.fs.s3a.Invoker;
-import org.apache.hadoop.fs.s3a.Retries;
-import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
-import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider;
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
-import org.apache.hadoop.fs.s3a.auth.STSClientFactory;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.Text;
-
-import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER;
-import static org.apache.hadoop.fs.s3a.Invoker.once;
-import static org.apache.hadoop.fs.s3a.S3AUtils.STANDARD_AWS_PROVIDERS;
-import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList;
-import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromAWSCredentials;
-import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.fromSTSCredentials;
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.*;
-
-/**
- * The session token DT binding: creates an AWS session token
- * for the DT, extracts and serves it up afterwards.
- */
-public class SessionTokenBinding extends AbstractDelegationTokenBinding {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- SessionTokenBinding.class);
-
- /**
- * Wire name of this binding: {@value}.
- */
- private static final String NAME = "SessionTokens/001";
-
- /**
- * A message added to the standard origin string when the DT is
- * built from session credentials passed in.
- */
- @VisibleForTesting
- public static final String CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN
- = "Existing session credentials converted to Delegation Token";
-
- public static final String SESSION_TOKEN
- = "Session Delegation Token";
-
- /** Invoker for STS calls. */
- private Invoker invoker;
-
- /**
- * Has an attempt to initialize STS been attempted?
- */
- private final AtomicBoolean stsInitAttempted = new AtomicBoolean(false);
-
- /** The STS client; created in startup if the parental credentials permit. */
- @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private Optional stsClient = Optional.empty();
-
- /**
- * Duration of session in seconds.
- */
- private long duration;
-
- /**
- * Flag to indicate that the auth chain provides session credentials.
- * If true it means that STS cannot be used (and stsClient is null).
- */
- private boolean hasSessionCreds;
-
- /**
- * The auth chain for the parent options.
- */
- private AWSCredentialProviderList parentAuthChain;
-
- /**
- * Has a log message about forwarding credentials been printed yet?
- */
- private final AtomicBoolean forwardMessageLogged = new AtomicBoolean(false);
-
- /** STS endpoint. */
- private String endpoint;
-
- /** STS region. */
- private String region;
-
- /**
- * Expiration date time as passed in from source.
- * If unset, either we are unbound, or the token which came in does not
- * know its expiry.
- */
- private Optional expirationDateTime;
-
- /**
- * Token identifier bound to.
- */
- private Optional tokenIdentifier = Optional.empty();
-
- /** Constructor for reflection. */
- public SessionTokenBinding() {
- this(NAME, SESSION_TOKEN_KIND);
- }
-
- /**
- * Constructor for subclasses.
- * @param name binding name.
- * @param kind token kind.
- */
- protected SessionTokenBinding(final String name,
- final Text kind) {
- super(name, kind);
- }
-
- /**
- * Service start will read in all configuration options
- * then build that client.
- */
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- Configuration conf = getConfig();
- duration = conf.getTimeDuration(DELEGATION_TOKEN_DURATION,
- DEFAULT_DELEGATION_TOKEN_DURATION,
- TimeUnit.SECONDS);
- endpoint = conf.getTrimmed(DELEGATION_TOKEN_ENDPOINT,
- DEFAULT_DELEGATION_TOKEN_ENDPOINT);
- region = conf.getTrimmed(DELEGATION_TOKEN_REGION,
- DEFAULT_DELEGATION_TOKEN_REGION);
-
- // create the provider set for session credentials.
- parentAuthChain = buildAWSProviderList(
- getCanonicalUri(),
- conf,
- AWS_CREDENTIALS_PROVIDER,
- STANDARD_AWS_PROVIDERS,
- new HashSet<>());
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- // this is here to keep findbugs quiet, even though nothing
- // can safely invoke stsClient as we are shut down.
- synchronized (this) {
- this.stsClient.ifPresent(IOUtils::closeStream);
- this.stsClient = Optional.empty();
- }
- }
-
- /**
- * Return an unbonded provider chain.
- * @return the auth chain built from the assumed role credentials
- * @throws IOException any failure.
- */
- @Override
- public AWSCredentialProviderList deployUnbonded()
- throws IOException {
- requireServiceStarted();
- return parentAuthChain;
- }
-
- /**
- * Get the invoker for STS calls.
- * @return the invoker
- */
- protected Invoker getInvoker() {
- return invoker;
- }
-
- /**
- * Sets the field {@link #tokenIdentifier} to the extracted/cast
- * session token identifier, and {@link #expirationDateTime} to
- * any expiration passed in.
- * @param retrievedIdentifier the unmarshalled data
- * @return the provider list
- * @throws IOException failure
- */
- @Override
- public AWSCredentialProviderList bindToTokenIdentifier(
- final AbstractS3ATokenIdentifier retrievedIdentifier)
- throws IOException {
- final SessionTokenIdentifier identifier = convertTokenIdentifier(
- retrievedIdentifier,
- SessionTokenIdentifier.class);
- setTokenIdentifier(Optional.of(identifier));
- MarshalledCredentials marshalledCredentials
- = identifier.getMarshalledCredentials();
- setExpirationDateTime(marshalledCredentials.getExpirationDateTime());
- return new AWSCredentialProviderList(
- "Session Token Binding",
- new MarshalledCredentialProvider(
- SESSION_TOKEN,
- getFileSystem().getUri(),
- getConfig(),
- marshalledCredentials,
- MarshalledCredentials.CredentialTypeRequired.SessionOnly));
- }
-
- @Override
- public String getDescription() {
- return String.format(
- "%s token binding for user %s, " +
- "with STS endpoint \"%s\", region \"%s\""
- + " and token duration %d:%02d",
- bindingName(), getOwner().getShortUserName(), endpoint, region,
- TimeUnit.SECONDS.toMinutes(duration),
- duration % 60);
- }
-
- /**
- * Get the role of this token; subclasses should override this
- * for better logging.
- * @return the role of this token
- */
- protected String bindingName() {
- return "Session";
- }
-
- /**
- * UA field includes the UUID of the token and ex
- * @return a string for the S3 logs.
- */
- public String getUserAgentField() {
- if (tokenIdentifier.isPresent()) {
- return "; session ID " + tokenIdentifier.get().getUuid();
- } else {
- return "";
- }
- }
-
- /**
- * Attempt to init the STS connection, only does it once.
- * If the AWS credential list to this service return session credentials
- * then this method will return {@code empty()}; no attempt is
- * made to connect to STS.
- * Otherwise, the STS binding info will be looked up and an attempt
- * made to connect to STS.
- * Only one attempt will be made.
- * @return any STS client created.
- * @throws IOException any failure to bind to STS.
- */
- private synchronized Optional maybeInitSTS()
- throws IOException {
- if (stsInitAttempted.getAndSet(true)) {
- // whether or not it succeeded, the state of the STS client is what
- // callers get after the first attempt.
- return stsClient;
- }
-
- Configuration conf = getConfig();
- URI uri = getCanonicalUri();
-
- // Ask the owner for any session credentials which it already has
- // so that it can just propagate them.
- // this call may fail if there are no credentials on the auth
- // chain.
- // As no codepath (session propagation, STS creation) will work,
- // throw this.
- final AWSCredentials parentCredentials = once("get credentials",
- "",
- () -> parentAuthChain.getCredentials());
- hasSessionCreds = parentCredentials instanceof AWSSessionCredentials;
-
- if (!hasSessionCreds) {
- LOG.info("Creating STS client for {}", getDescription());
-
- invoker = new Invoker(new S3ARetryPolicy(conf), LOG_EVENT);
- ClientConfiguration awsConf =
- S3AUtils.createAwsConf(conf, uri.getHost());
- AWSSecurityTokenService tokenService =
- STSClientFactory.builder(parentAuthChain,
- awsConf,
- endpoint,
- region)
- .build();
- stsClient = Optional.of(
- STSClientFactory.createClientConnection(tokenService, invoker));
- } else {
- LOG.debug("Parent-provided session credentials will be propagated");
- stsClient = Optional.empty();
- }
- return stsClient;
- }
-
- /**
- * Log retries at debug.
- */
- public static final Invoker.Retried LOG_EVENT =
- (text, exception, retries, idempotent) -> {
- LOG.info("{}: " + exception, text);
- if (retries == 1) {
- // stack on first attempt, to keep noise down
- LOG.debug("{}: " + exception, text, exception);
- }
- };
-
- /**
- * Get the client to AWS STS.
- * @return the STS client, when successfully inited.
- */
- protected Optional prepareSTSClient()
- throws IOException {
- return maybeInitSTS();
- }
-
- /**
- * Duration of sessions.
- * @return duration in seconds.
- */
- public long getDuration() {
- return duration;
- }
-
- @Override
- @Retries.RetryTranslated
- public SessionTokenIdentifier createTokenIdentifier(
- final Optional policy,
- final EncryptionSecrets encryptionSecrets) throws IOException {
- requireServiceStarted();
-
- final MarshalledCredentials marshalledCredentials;
- String origin = AbstractS3ATokenIdentifier.createDefaultOriginMessage();
- final Optional client = prepareSTSClient();
-
- if (client.isPresent()) {
- // this is the normal route: ask for a new STS token
- marshalledCredentials = fromSTSCredentials(
- client.get()
- .requestSessionCredentials(duration, TimeUnit.SECONDS));
- } else {
- // get a new set of parental session credentials (pick up IAM refresh)
- if (!forwardMessageLogged.getAndSet(true)) {
- // warn caller on the first -and only the first- use.
- LOG.warn("Forwarding existing session credentials to {}"
- + " -duration unknown", getCanonicalUri());
- }
- origin += " " + CREDENTIALS_CONVERTED_TO_DELEGATION_TOKEN;
- final AWSCredentials awsCredentials
- = parentAuthChain.getCredentials();
- if (awsCredentials instanceof AWSSessionCredentials) {
- marshalledCredentials = fromAWSCredentials(
- (AWSSessionCredentials) awsCredentials);
- } else {
- throw new DelegationTokenIOException(
- "AWS Authentication chain is no longer supplying session secrets");
- }
- }
- return new SessionTokenIdentifier(getKind(),
- getOwnerText(),
- getCanonicalUri(),
- marshalledCredentials,
- encryptionSecrets,
- origin);
- }
-
- @Override
- public SessionTokenIdentifier createEmptyIdentifier() {
- return new SessionTokenIdentifier();
- }
-
- /**
- * Expiration date time as passed in from source.
- * If unset, either we are unbound, or the token which came in does not
- * know its expiry.
- */
- protected Optional getExpirationDateTime() {
- return expirationDateTime;
- }
-
- protected void setExpirationDateTime(Optional expirationDateTime) {
- this.expirationDateTime = expirationDateTime;
- }
-
- /**
- * Token identifier bound to.
- */
- protected Optional getTokenIdentifier() {
- return tokenIdentifier;
- }
-
- protected void setTokenIdentifier(Optional
- tokenIdentifier) {
- this.tokenIdentifier = tokenIdentifier;
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenIdentifier.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenIdentifier.java
deleted file mode 100644
index 3928a0d454..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenIdentifier.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.net.URI;
-
-import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials;
-import org.apache.hadoop.io.Text;
-
-import static org.apache.hadoop.fs.s3a.auth.delegation.DelegationConstants.SESSION_TOKEN_KIND;
-
-/**
- * A token identifier which contains a set of AWS session credentials,
- * credentials which will be valid until they expire.
- *
- * Note 1:
- * There's a risk here that the reference to {@link MarshalledCredentials}
- * may trigger a transitive load of AWS classes, a load which will
- * fail if the aws SDK isn't on the classpath.
- *
- * Note 2:
- * This class does support subclassing, but every subclass MUST declare itself
- * to be of a different token kind.
- * Otherwise the process for decoding tokens breaks.
- */
-public class SessionTokenIdentifier extends
- AbstractS3ATokenIdentifier {
-
- /**
- * Session credentials: initially empty but non-null.
- */
- private MarshalledCredentials marshalledCredentials
- = new MarshalledCredentials();
-
- /**
- * Constructor for service loader use.
- * Created with the kind {@link DelegationConstants#SESSION_TOKEN_KIND}.
- * Subclasses MUST NOT subclass this; they must provide their own
- * token kind.
- */
- public SessionTokenIdentifier() {
- super(SESSION_TOKEN_KIND);
- }
-
- /**
- * Constructor for subclasses.
- * @param kind kind of token identifier, for storage in the
- * token kind to implementation map.
- */
- protected SessionTokenIdentifier(final Text kind) {
- super(kind);
- }
-
- /**
- * Constructor.
- * @param kind token kind.
- * @param owner token owner
- * @param uri filesystem URI.
- * @param marshalledCredentials credentials to marshall
- * @param encryptionSecrets encryption secrets
- * @param origin origin text for diagnostics.
- */
- public SessionTokenIdentifier(
- final Text kind,
- final Text owner,
- final URI uri,
- final MarshalledCredentials marshalledCredentials,
- final EncryptionSecrets encryptionSecrets,
- final String origin) {
- super(kind, uri, owner, origin, encryptionSecrets);
- this.marshalledCredentials = marshalledCredentials;
- }
-
- /**
- * Constructor.
- * @param kind token kind.
- * @param owner token owner
- * @param uri filesystem URI.
- */
- public SessionTokenIdentifier(final Text kind,
- final Text owner,
- final Text renewer,
- final Text realUser,
- final URI uri) {
- super(kind, owner, renewer, realUser, uri);
- }
-
- @Override
- public void write(final DataOutput out) throws IOException {
- super.write(out);
- marshalledCredentials.write(out);
- }
-
- @Override
- public void readFields(final DataInput in)
- throws IOException {
- super.readFields(in);
- marshalledCredentials.readFields(in);
- }
-
- /**
- * Return the expiry time in seconds since 1970-01-01.
- * @return the time when the AWS credentials expire.
- */
- @Override
- public long getExpiryTime() {
- return marshalledCredentials.getExpiration();
- }
-
- /**
- * Get the marshalled credentials.
- * @return marshalled AWS credentials.
- */
- public MarshalledCredentials getMarshalledCredentials() {
- return marshalledCredentials;
- }
-
- /**
- * Add the (sanitized) marshalled credentials to the string value.
- * @return a string value for test assertions and debugging.
- */
- @Override
- public String toString() {
- return super.toString()
- + "; " + marshalledCredentials.toString();
- }
-}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/package-info.java
deleted file mode 100644
index f7eb6b16a5..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Extensible delegation token support for the S3A connector.
- *
- * Goal: support multiple back end token issue/renewal services, from
- * "pure client side" session tokens to full "Kerberos auth".
- *
- * It is intended for internal use only; any external implementation
- * of {@link org.apache.hadoop.fs.s3a.auth.delegation.AbstractDelegationTokenBinding}
- * must consider this API unstable and track changes as they happen.
- */
-@InterfaceAudience.LimitedPrivate("authorization-subsystems")
-@InterfaceStability.Unstable
-package org.apache.hadoop.fs.s3a.auth.delegation;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/package-info.java
index c3a7ee6ee5..e34d68ecad 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/package-info.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/package-info.java
@@ -18,12 +18,8 @@
/**
* Authentication and permissions support.
- *
- * Some of the classes in here are expected to be referred to in configuration
- * files, so must not change their name. These will be explicitly identified.
*/
-
-@InterfaceAudience.LimitedPrivate("Authentication services")
+@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.fs.s3a.auth;
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
index 69f90cb651..c6617f83d9 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
@@ -23,8 +23,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
/**
- * A duration with logging of final state at info or debug
- * in the {@code close()} call.
+ * A duration with logging of final state at info in the {@code close()} call.
* This allows it to be used in a try-with-resources clause, and have the
* duration automatically logged.
*/
@@ -36,41 +35,15 @@ public class DurationInfo extends Duration
private final Logger log;
/**
- * Should the log be at INFO rather than DEBUG.
- */
- private final boolean logAtInfo;
-
- /**
- * Create the duration text from a {@code String.format()} code call;
- * log output at info level.
+ * Create the duration text from a {@code String.format()} code call.
* @param log log to write to
* @param format format string
* @param args list of arguments
*/
public DurationInfo(Logger log, String format, Object... args) {
- this(log, true, format, args);
- }
-
- /**
- * Create the duration text from a {@code String.format()} code call
- * and log either at info or debug.
- * @param log log to write to
- * @param logAtInfo should the log be at info, rather than debug
- * @param format format string
- * @param args list of arguments
- */
- public DurationInfo(Logger log,
- boolean logAtInfo,
- String format,
- Object... args) {
this.text = String.format(format, args);
this.log = log;
- this.logAtInfo = logAtInfo;
- if (logAtInfo) {
- log.info("Starting: {}", text);
- } else {
- log.debug("Starting: {}", text);
- }
+ log.info("Starting: {}", text);
}
@Override
@@ -81,10 +54,6 @@ public String toString() {
@Override
public void close() {
finished();
- if (logAtInfo) {
- log.info("{}", this);
- } else {
- log.debug("{}", this);
- }
+ log.info(this.toString());
}
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
index c6f826bbbf..e7f0207268 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -273,8 +273,8 @@ private void checkChildPath(Path childPath) {
// If this dir's path has host (and thus scheme), so must its children
URI parentUri = path.toUri();
- URI childUri = childPath.toUri();
if (parentUri.getHost() != null) {
+ URI childUri = childPath.toUri();
Preconditions.checkNotNull(childUri.getHost(), "Expected non-null URI " +
"host: %s", childUri);
Preconditions.checkArgument(
@@ -286,8 +286,7 @@ private void checkChildPath(Path childPath) {
}
Preconditions.checkArgument(!childPath.isRoot(),
"childPath cannot be the root path: %s", childPath);
- Preconditions.checkArgument(parentUri.getPath().equals(
- childPath.getParent().toUri().getPath()),
+ Preconditions.checkArgument(childPath.getParent().equals(path),
"childPath %s must be a child of %s", childPath, path);
}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index d2676f70e7..e8b3c0c548 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -26,7 +26,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -71,7 +70,6 @@
import com.amazonaws.waiters.WaiterTimedOutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -92,9 +90,7 @@
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.Tristate;
-import org.apache.hadoop.fs.s3a.auth.RoleModel;
import org.apache.hadoop.fs.s3a.auth.RolePolicies;
-import org.apache.hadoop.fs.s3a.auth.delegation.AWSPolicyProvider;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.security.UserGroupInformation;
@@ -102,8 +98,6 @@
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowAllDynamoDBOperations;
-import static org.apache.hadoop.fs.s3a.auth.RolePolicies.allowS3GuardClientOperations;
import static org.apache.hadoop.fs.s3a.s3guard.PathMetadataDynamoDBTranslation.*;
import static org.apache.hadoop.fs.s3a.s3guard.S3Guard.*;
@@ -191,8 +185,7 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class DynamoDBMetadataStore implements MetadataStore,
- AWSPolicyProvider {
+public class DynamoDBMetadataStore implements MetadataStore {
public static final Logger LOG = LoggerFactory.getLogger(
DynamoDBMetadataStore.class);
@@ -238,7 +231,6 @@ public class DynamoDBMetadataStore implements MetadataStore,
private String region;
private Table table;
private String tableName;
- private String tableArn;
private Configuration conf;
private String username;
@@ -411,8 +403,6 @@ public void initialize(Configuration config) throws IOException {
region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
Preconditions.checkArgument(!StringUtils.isEmpty(region),
"No DynamoDB region configured");
- // there's no URI here, which complicates life: you cannot
- // create AWS providers here which require one.
credentials = createAWSCredentialProviderSet(null, conf);
dynamoDB = createDynamoDB(conf, region, null, credentials);
@@ -1132,33 +1122,9 @@ public String toString() {
return getClass().getSimpleName() + '{'
+ "region=" + region
+ ", tableName=" + tableName
- + ", tableArn=" + tableArn
+ '}';
}
- /**
- * The administrative policy includes all DDB table operations;
- * application access is restricted to those operations S3Guard operations
- * require when working with data in a guarded bucket.
- * @param access access level desired.
- * @return a possibly empty list of statements.
- */
- @Override
- public List listAWSPolicyRules(
- final Set access) {
- Preconditions.checkState(tableArn != null, "TableARN not known");
- if (access.isEmpty()) {
- return Collections.emptyList();
- }
- RoleModel.Statement stat;
- if (access.contains(AccessLevel.ADMIN)) {
- stat = allowAllDynamoDBOperations(tableArn);
- } else {
- stat = allowS3GuardClientOperations(tableArn);
- }
- return Lists.newArrayList(stat);
- }
-
/**
* Create a table if it does not exist and wait for it to become active.
*
@@ -1185,7 +1151,6 @@ void initTable() throws IOException {
LOG.debug("Binding to table {}", tableName);
TableDescription description = table.describe();
LOG.debug("Table state: {}", description);
- tableArn = description.getTableArn();
final String status = description.getTableStatus();
switch (status) {
case "CREATING":
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 8d6436ec2b..70ea1f42b2 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -51,10 +51,8 @@
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AUtils;
-import org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens;
import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.fs.shell.CommandFormat;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
@@ -1163,23 +1161,6 @@ public int run(String[] args, PrintStream out)
"none");
printOption(out, "\tInput seek policy", INPUT_FADVISE, INPUT_FADV_NORMAL);
- // look at delegation token support
- if (fs.getDelegationTokens().isPresent()) {
- // DT is enabled
- S3ADelegationTokens dtIntegration = fs.getDelegationTokens().get();
- println(out, "Delegation Support enabled: token kind = %s",
- dtIntegration.getTokenKind());
- UserGroupInformation.AuthenticationMethod authenticationMethod
- = UserGroupInformation.getCurrentUser().getAuthenticationMethod();
- println(out, "Hadoop security mode: %s", authenticationMethod);
- if (UserGroupInformation.isSecurityEnabled()) {
- println(out,
- "Warning: security is disabled; tokens will not be collected");
- }
- } else {
- println(out, "Delegation token support is disabled");
- }
-
if (usingS3Guard) {
if (commands.getOpt(UNGUARDED_FLAG)) {
throw badState("S3Guard is enabled for %s", fsUri);
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index 84e4a6768f..60d4b76407 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -136,6 +136,8 @@ static Login extractLoginDetails(URI name) {
/**
* Canonicalize the given URI.
*
+ * This strips out login information.
+ *
* @param uri the URI to canonicalize
* @param defaultPort default port to use in canonicalized URI if the input
* URI has no port and this value is greater than 0
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
deleted file mode 100644
index c1a3bd05ff..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-org.apache.hadoop.fs.s3a.auth.delegation.S3ADtFetcher
diff --git a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
deleted file mode 100644
index bfd3def37a..0000000000
--- a/hadoop-tools/hadoop-aws/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenIdentifier
-org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenIdentifier
-org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenIdentifier
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
index f08f40f27b..8af045776c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/assumed_roles.md
@@ -178,14 +178,12 @@ Here are the full set of configuration options.
fs.s3a.assumed.role.credentials.provider
- org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider
-
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider
List of credential providers to authenticate with the STS endpoint and
retrieve short-lived role credentials.
- Used by AssumedRoleCredentialProvider and the S3A Session Delegation Token
- and S3A Role Delegation Token bindings.
+ Only used if AssumedRoleCredentialProvider is the AWS credential provider.
+ If unset, uses "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider".
```
@@ -470,69 +468,17 @@ Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceExc
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
```
-### `Member must have value greater than or equal to 900`
+### "Assume Role session duration should be in the range of 15min - 1Hr"
-The value of `fs.s3a.assumed.role.session.duration` is too low.
+The value of `fs.s3a.assumed.role.session.duration` is out of range.
```
-org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-1 validation error detected: Value '20' at 'durationSeconds' failed to satisfy constraint:
-Member must have value greater than or equal to 900 (Service: AWSSecurityTokenService;
-Status Code: 400; Error Code: ValidationError;
-Request ID: b9a82403-d0a7-11e8-98ef-596679ee890d)
+java.lang.IllegalArgumentException: Assume Role session duration should be in the range of 15min
+- 1Hr
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$Builder.withRoleSessionDurationSeconds(STSAssumeRoleSessionCredentialsProvider.java:437)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:86)
```
-Fix: increase.
-
-### Error "The requested DurationSeconds exceeds the MaxSessionDuration set for this role"
-
-The value of `fs.s3a.assumed.role.session.duration` is too high.
-
-```
-org.apache.hadoop.fs.s3a.AWSBadRequestException: request role credentials:
- com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-The requested DurationSeconds exceeds the MaxSessionDuration set for this role.
-(Service: AWSSecurityTokenService; Status Code: 400;
- Error Code: ValidationError; Request ID: 17875165-d0a7-11e8-b85f-d15a599a7f6d)
-```
-
-There are two solutions to this
-
-* Decrease the duration value.
-* Increase the duration of a role in the [AWS IAM Console](https://console.aws.amazon.com/iam/home#/roles).
-
-
-### "Value '345600' at 'durationSeconds' failed to satisfy constraint: Member must have value less than or equal to 43200"
-
-Irrespective of the maximum duration of a role, the AWS role API only permits callers to request
-any role for up to 12h; attempting to use a larger number will fail.
-
-
-```
-Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-1 validation error detected:
-Value '345600' at 'durationSeconds' failed to satisfy constraint:
-Member must have value less than or equal to 43200
-(Service: AWSSecurityTokenService;
-Status Code: 400; Error Code:
-ValidationError;
-Request ID: dec1ca6b-d0aa-11e8-ac8c-4119b3ea9f7f)
-```
-
-For full sessions, the duration limit is 129600 seconds: 36h.
-
-```
-org.apache.hadoop.fs.s3a.AWSBadRequestException: request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-1 validation error detected: Value '345600' at 'durationSeconds' failed to satisfy constraint:
-Member must have value less than or equal to 129600
-(Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
-Request ID: a6e73d44-d0aa-11e8-95ed-c5bba29f0635)
-```
-
-For both these errors, the sole fix is to request a shorter duration
-in `fs.s3a.assumed.role.session.duration`.
### `MalformedPolicyDocumentException` "The policy is not in the valid JSON format"
@@ -541,7 +487,7 @@ The policy set in `fs.s3a.assumed.role.policy` is not valid according to the
AWS specification of Role Policies.
```
-org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
+rg.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentException:
The policy is not in the valid JSON format. (Service: AWSSecurityTokenService; Status Code: 400;
Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c):
@@ -562,9 +508,36 @@ Caused by: com.amazonaws.services.securitytoken.model.MalformedPolicyDocumentExc
Error Code: MalformedPolicyDocument; Request ID: baf8cb62-f552-11e7-9768-9df3b384e40c)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.doInvoke(AWSSecurityTokenServiceClient.java:1271)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.invoke(AWSSecurityTokenServiceClient.java:1247)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.executeAssumeRole(AWSSecurityTokenServiceClient.java:454)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.assumeRole(AWSSecurityTokenServiceClient.java:431)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.newSession(STSAssumeRoleSessionCredentialsProvider.java:321)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.access$000(STSAssumeRoleSessionCredentialsProvider.java:37)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:76)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:73)
+ at com.amazonaws.auth.RefreshableTask.refreshValue(RefreshableTask.java:256)
+ at com.amazonaws.auth.RefreshableTask.blockingRefresh(RefreshableTask.java:212)
+ at com.amazonaws.auth.RefreshableTask.getValue(RefreshableTask.java:153)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.getCredentials(STSAssumeRoleSessionCredentialsProvider.java:299)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.getCredentials(AssumedRoleCredentialProvider.java:127)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:116)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
+ at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
+ at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
+ at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
+ ... 19 more
```
-### `MalformedPolicyDocumentException` "Syntax errors in policy"
+### `MalformedPolicyDocumentException` "Syntax errors in policy"
The policy set in `fs.s3a.assumed.role.policy` is not valid JSON.
@@ -591,6 +564,31 @@ Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on :
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.doInvoke(AWSSecurityTokenServiceClient.java:1271)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.invoke(AWSSecurityTokenServiceClient.java:1247)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.executeAssumeRole(AWSSecurityTokenServiceClient.java:454)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.assumeRole(AWSSecurityTokenServiceClient.java:431)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.newSession(STSAssumeRoleSessionCredentialsProvider.java:321)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.access$000(STSAssumeRoleSessionCredentialsProvider.java:37)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:76)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:73)
+ at com.amazonaws.auth.RefreshableTask.refreshValue(RefreshableTask.java:256)
+ at com.amazonaws.auth.RefreshableTask.blockingRefresh(RefreshableTask.java:212)
+ at com.amazonaws.auth.RefreshableTask.getValue(RefreshableTask.java:153)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.getCredentials(STSAssumeRoleSessionCredentialsProvider.java:299)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.getCredentials(AssumedRoleCredentialProvider.java:127)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:116)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
+ at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
+ at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
+ at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
... 19 more
```
@@ -648,6 +646,34 @@ Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceExc
SignedHeaders=amz-sdk-invocation-id;amz-sdk-retry;host;user-agent;x-amz-date,
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: IncompleteSignature;
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.doInvoke(AWSSecurityTokenServiceClient.java:1271)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.invoke(AWSSecurityTokenServiceClient.java:1247)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.executeAssumeRole(AWSSecurityTokenServiceClient.java:454)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.assumeRole(AWSSecurityTokenServiceClient.java:431)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.newSession(STSAssumeRoleSessionCredentialsProvider.java:321)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.access$000(STSAssumeRoleSessionCredentialsProvider.java:37)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:76)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:73)
+ at com.amazonaws.auth.RefreshableTask.refreshValue(RefreshableTask.java:256)
+ at com.amazonaws.auth.RefreshableTask.blockingRefresh(RefreshableTask.java:212)
+ at com.amazonaws.auth.RefreshableTask.getValue(RefreshableTask.java:153)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.getCredentials(STSAssumeRoleSessionCredentialsProvider.java:299)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.getCredentials(AssumedRoleCredentialProvider.java:127)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:116)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
+ at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
+ at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
+ at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
+ ... 25 more
```
### `AccessDeniedException/InvalidClientTokenId`: "The security token included in the request is invalid"
@@ -676,6 +702,31 @@ The security token included in the request is invalid.
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.doInvoke(AWSSecurityTokenServiceClient.java:1271)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.invoke(AWSSecurityTokenServiceClient.java:1247)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.executeAssumeRole(AWSSecurityTokenServiceClient.java:454)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.assumeRole(AWSSecurityTokenServiceClient.java:431)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.newSession(STSAssumeRoleSessionCredentialsProvider.java:321)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.access$000(STSAssumeRoleSessionCredentialsProvider.java:37)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:76)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:73)
+ at com.amazonaws.auth.RefreshableTask.refreshValue(RefreshableTask.java:256)
+ at com.amazonaws.auth.RefreshableTask.blockingRefresh(RefreshableTask.java:212)
+ at com.amazonaws.auth.RefreshableTask.getValue(RefreshableTask.java:153)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.getCredentials(STSAssumeRoleSessionCredentialsProvider.java:299)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.getCredentials(AssumedRoleCredentialProvider.java:127)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:116)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
+ at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
+ at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
+ at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
... 25 more
```
@@ -689,8 +740,7 @@ match these constraints.
If set explicitly, it must be valid.
```
-org.apache.hadoop.fs.s3a.AWSBadRequestException:
- Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on
+org.apache.hadoop.fs.s3a.AWSBadRequestException: Instantiate org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider on
com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
1 validation error detected: Value 'Session Names cannot Hava Spaces!' at 'roleSessionName'
failed to satisfy constraint: Member must satisfy regular expression pattern: [\w+=,.@-]*
@@ -715,6 +765,33 @@ Caused by: com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceExc
(Service: AWSSecurityTokenService; Status Code: 400; Error Code: ValidationError;
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.doInvoke(AWSSecurityTokenServiceClient.java:1271)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.invoke(AWSSecurityTokenServiceClient.java:1247)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.executeAssumeRole(AWSSecurityTokenServiceClient.java:454)
+ at com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient.assumeRole(AWSSecurityTokenServiceClient.java:431)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.newSession(STSAssumeRoleSessionCredentialsProvider.java:321)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.access$000(STSAssumeRoleSessionCredentialsProvider.java:37)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:76)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider$1.call(STSAssumeRoleSessionCredentialsProvider.java:73)
+ at com.amazonaws.auth.RefreshableTask.refreshValue(RefreshableTask.java:256)
+ at com.amazonaws.auth.RefreshableTask.blockingRefresh(RefreshableTask.java:212)
+ at com.amazonaws.auth.RefreshableTask.getValue(RefreshableTask.java:153)
+ at com.amazonaws.auth.STSAssumeRoleSessionCredentialsProvider.getCredentials(STSAssumeRoleSessionCredentialsProvider.java:299)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.getCredentials(AssumedRoleCredentialProvider.java:135)
+ at org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.(AssumedRoleCredentialProvider.java:124)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
+ at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
+ at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
+ at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
+ at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProvider(S3AUtils.java:583)
+ ... 26 more
```
@@ -741,6 +818,24 @@ Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: Access Denied
S3 Extended Request ID: iEXDVzjIyRbnkAc40MS8Sjv+uUQNvERRcqLsJsy9B0oyrjHLdkRKwJ/phFfA17Kjn483KSlyJNw=
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1638)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1303)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1055)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
+ at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
+ at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
+ at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4229)
+ at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4176)
+ at com.amazonaws.services.s3.AmazonS3Client.deleteObject(AmazonS3Client.java:2066)
+ at com.amazonaws.services.s3.AmazonS3Client.deleteObject(AmazonS3Client.java:2052)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.lambda$deleteObject$7(S3AFileSystem.java:1338)
+ at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:314)
+ at org.apache.hadoop.fs.s3a.Invoker.retryUntranslated(Invoker.java:280)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.deleteObject(S3AFileSystem.java:1334)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.removeKeys(S3AFileSystem.java:1657)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.innerRename(S3AFileSystem.java:1046)
+ at org.apache.hadoop.fs.s3a.S3AFileSystem.rename(S3AFileSystem.java:851)
```
This is the policy restriction behaving as intended: the caller is trying to
@@ -787,63 +882,3 @@ or just that this specific permission has been omitted.
If the role policy requested for the assumed role didn't ask for any DynamoDB
permissions, this is where all attempts to work with a S3Guarded bucket will
fail. Check the value of `fs.s3a.assumed.role.policy`
-
-### Error `Unable to execute HTTP request`
-
-This is a low-level networking error. Possible causes include:
-
-* The endpoint set in `fs.s3a.assumed.role.sts.endpoint` is invalid.
-* There are underlying network problems.
-
-```
-org.apache.hadoop.fs.s3a.AWSClientIOException: request session credentials:
- com.amazonaws.SdkClientException:
-
- Unable to execute HTTP request: null: Unable to execute HTTP request: null
-at com.amazonaws.thirdparty.apache.http.impl.conn.DefaultRoutePlanner.determineRoute(DefaultRoutePlanner.java:88)
-at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.determineRoute(InternalHttpClient.java:124)
-at com.amazonaws.thirdparty.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:183)
-at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
-at com.amazonaws.thirdparty.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
-```
-
-### Error "Credential should be scoped to a valid region"
-
-This is based on conflict between the values of `fs.s3a.assumed.role.sts.endpoint`
-and `fs.s3a.assumed.role.sts.endpoint.region`
-Two variants, "not '''"
-
-Variant 1: `Credential should be scoped to a valid region, not 'us-west-1'` (or other string)
-
-
-```
-java.nio.file.AccessDeniedException: : request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
-Credential should be scoped to a valid region, not 'us-west-1'.
-(Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch; Request ID: d9065cc4-e2b9-11e8-8b7b-f35cb8d7aea4):SignatureDoesNotMatch
-```
-
-One of:
-
-
-* the value of `fs.s3a.assumed.role.sts.endpoint.region` is not a valid region
-* the value of `fs.s3a.assumed.role.sts.endpoint.region` is not the signing
-region of the endpoint set in `fs.s3a.assumed.role.sts.endpoint`
-
-
-Variant 2: `Credential should be scoped to a valid region, not ''`
-
-```
-java.nio.file.AccessDeniedException: : request session credentials:
-com.amazonaws.services.securitytoken.model.AWSSecurityTokenServiceException:
- Credential should be scoped to a valid region, not ''. (
- Service: AWSSecurityTokenService; Status Code: 403; Error Code: SignatureDoesNotMatch;
- Request ID: bd3e5121-e2ac-11e8-a566-c1a4d66b6a16):SignatureDoesNotMatch
-```
-
-This should be intercepted earlier: an endpoint has been specified but
-not a region.
-
-There's special handling for the central `sts.amazonaws.com` region; when
-that is declared as the value of `fs.s3a.assumed.role.sts.endpoint.region` then
-there is no need to declare a region: whatever value it has is ignored.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md
deleted file mode 100644
index 90e4e5587d..0000000000
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_token_architecture.md
+++ /dev/null
@@ -1,466 +0,0 @@
-
-
-# S3A Delegation Token Architecture
-
-This is an architecture document to accompany
-[Working with Delegation Tokens](delegation_tokens.html)
-
-## Background: Delegation Tokens
-
-Delegation Tokens, "DTs" are a common feature of Hadoop Services.
-They are opaque byte arrays which can be issued by services like
-HDFS, HBase, YARN, and which can be used to authenticate a request with
-that service.
-
-### Tokens are Issued
-
-In a Kerberized cluster, they are issued by the service after the caller
-has authenticated, and so that principal is trusted to be who they say they are.
-The issued DT can therefore attest that whoever is including that token
-on a request is authorized to act on behalf of that principal —for the
-specific set of operations which the DT grants.
-
-As an example, an HDFS DT can be requested by a user, included in the
-launch context of a YARN application -say DistCp, and that launched application
-can then talk to HDFS as if they were that user.
-
-### Tokens are marshalled
-
-Tokens are opaque byte arrays. They are contained within a `Token`
- class which includes an expiry time, the service identifier, and some other details.
-
-`Token<>` instances can be serialized as a Hadoop Writable, or converted saved to/from a protobuf
-format. This is how they are included in YARN application and container requests,
-and elsewhere. They can even be saved to files through the `hadoop dt` command.
-
-### Tokens can be unmarshalled
-
-
-At the far end, tokens can be unmarshalled and converted into instances of
-the java classes. This assumes that all the dependent classes are on the
-classpath, obviously.
-
-### Tokens can be used to authenticate callers
-
-The Hadoop RPC layer and the web SPNEGO layer support tokens.
-
-### Tokens can be renewed
-
-DTs can be renewed by the specific principal declared at creation time as
-"the renewer". In the example above, the YARN Resource Manager's principal
-can be declared as the reviewer. Then, even while a token is attached
-to a queued launch request in the RM, the RM can regularly request of HDFS
-that the token is renewed.
-
-There's an ultimate limit on how long tokens can be renewed for, but its
-generally 72h or similar, so that medium-life jobs can access services
-and data on behalf of a user.
-
-### Tokens can be Revoked
-
-When tokens are no longer needed, the service can be told to revoke a token.
-Continuing the YARN example, after an application finishes the YARN RM
-can revoke every token marshalled into the application launch request.
-At which point there's no risk associated with that token being
-compromised.
-
-
-*This is all how "real" Hadoop tokens work*
-
-The S3A Delegation Tokens are subtly different.
-
-The S3A DTs actually include the AWS credentials within the token
-data marshalled and shared across the cluster. The credentials can be one
-of:
-
-* The Full AWS (`fs.s3a.access.key`, `fs.s3a.secret.key`) login.
-* A set of AWS session credentials
- (`fs.s3a.access.key`, `fs.s3a.secret.key`, `fs.s3a.session.token`).
-
-These credentials are obtained from the AWS Secure Token Service (STS) when the the token is issued.
-* A set of AWS session credentials binding the user to a specific AWS IAM Role,
-further restricted to only access the S3 bucket and matching S3Guard DynamoDB table.
-Again, these credentials are requested when the token is issued.
-
-
-*Tokens can be issued*
-
-When an S3A Filesystem instance is asked to issue a token it can simply package
-up the login secrets (The "Full" tokens), or talk to the AWS STS service
-to get a set of session/assumed role credentials. These are marshalled within
-the overall token, and then onwards to applications.
-
-*Tokens can be marshalled*
-
-The AWS secrets are held in a subclass of `org.apache.hadoop.security.token.TokenIdentifier`.
-This class gets serialized to a byte array when the whole token is marshalled, and deserialized
-when the token is loaded.
-
-*Tokens can be used to authenticate callers*
-
-The S3A FS does not hand the token to AWS services to authenticate the caller.
-Instead it takes the AWS credentials included in the token identifier
-and uses them to sign the requests.
-
-*Tokens cannot be renewed*
-
-The tokens contain the credentials; you cant use them to ask AWS for more.
-
-For full credentials that is moot, but for the session and role credentials,
-they will expire. At which point the application will be unable to
-talk to the AWS infrastructure.
-
-*Tokens cannot be revoked*
-
-The AWS STS APIs don't let you revoke a single set of session credentials.
-
-## Background: How Tokens are collected in MapReduce jobs
-
-
-### `org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal()`
-
-1. Calls `org.apache.hadoop.mapreduce.security.TokenCache.obtainTokensForNamenodes()`
-for the job submission dir on the cluster FS (i.e. `fs.defaultFS`).
-1. Reads in the property `mapreduce.job.hdfs-servers` and extracts DTs from them,
-1. Tells the `FileInputFormat` and `FileOutputFormat` subclasses of the job
-to collect their source and dest FS tokens.
-
-All token collection is via `TokenCache.obtainTokensForNamenodes()`
-
-### `TokenCache.obtainTokensForNamenodes(Credentials, Path[], Configuration) `
-
-1. Returns immediately if security is off.
-1. Retrieves all the filesystems in the list of paths.
-1. Retrieves a token from each unless it is in the list of filesystems in `mapreduce.job.hdfs-servers.token-renewal.exclude`
-1. Merges in any DTs stored in the file referenced under: `mapreduce.job.credentials.binary`
-1. Calls `FileSystem.collectDelegationTokens()`, which, if there isn't any token already in the credential list, issues and adds a new token. *There is no check to see if that existing credential has expired*.
-
-
-### `FileInputFormat.listStatus(JobConf job): FileStatus[]`
-
-Enumerates source paths in (`mapreduce.input.fileinputformat.inputdir`) ; uses `TokenCache.obtainTokensForNamenodes()`
-to collate a token for all of these paths.
-
-This operation is called by the public interface method `FileInputFormat.getSplits()`.
-
-### `FileOutputFormat.checkOutputSpecs()`
-
-Calls `getOutputPath(job)` and asks for the DTs of that output path FS.
-
-
-## Architecture of the S3A Delegation Token Support
-
-
-
-1. The S3A FS client has the ability to be configured with a delegation
-token binding, the "DT Binding", a class declared in the option `fs.s3a.delegation.token.binding`.
-1. If set, when a filesystem is instantiated it asks the DT binding for its list of AWS credential providers.
-(the list in `fs.s3a.aws.credentials.provider` are only used if the DT binding wishes to).
-1. The DT binding scans for the current principal (`UGI.getCurrentUser()`/"the Owner") to see if they
-have any token in their credential cache whose service name matches the URI of the filesystem.
-1. If one is found, it is unmarshalled and then used to authenticate the caller via
-some AWS Credential provider returned to the S3A FileSystem instance.
-1. If none is found, the Filesystem is considered to have been deployed "Unbonded".
-The DT binding has to return a list of the AWS credential providers to use.
-
-When requests are made of AWS services, the created credential provider(s) are
-used to sign requests.
-
-When the filesystem is asked for a delegation token, the
-DT binding will generate a token identifier containing the marshalled tokens.
-
-If the Filesystem was deployed with a DT, that is, it was deployed "bonded", that existing
-DT is returned.
-
-If it was deployed unbonded, the DT Binding is asked to create a new DT.
-
-It is up to the binding what it includes in the token identifier, and how it obtains them.
-This new token identifier is included in a token which has a "canonical service name" of
-the URI of the filesystem (e.g "s3a://landsat-pds").
-
-The issued/reissued token identifier can be marshalled and reused.
-
-
-### class `org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens`
-
-This joins up the S3A Filesystem with the pluggable DT binding classes.
-
-One is instantiated in the S3A Filesystem instance if a DT Binding class
-has been instantiated. If so, it is invoked for
-
-* Building up the authentication chain during filesystem initialization.
-* Determining if the FS should declare that it has a canonical name
-(in `getCanonicalServiceName()`).
-* When asked for a DT (in `getDelegationToken(String renewer)`).
-
-The `S3ADelegationTokens` has the task of instantiating the actual DT binding,
-which must be a subclass of `AbstractDelegationTokenBinding`.
-
-All the DT bindings, and `S3ADelegationTokens` itself are subclasses of
-`org.apache.hadoop.service.AbstractService`; they follow the YARN service lifecycle
-of: create -> init -> start -> stop. This means that a DT binding, may, if it chooses,
-start worker threads when the service is started (`serviceStart()`); it must
-then stop them in the `serviceStop` method. (Anyone doing this must be aware
-that the owner FS is not fully initialized in serviceStart: they must not
-call into the Filesystem).
-
-The actions of this class are
-
-* Lookup of DTs associated with this S3A FS (scanning credentials, unmarshalling).
-* initiating the DT binding in bound/unbound state.
-* issuing DTs, either serving up the existing one, or requesting the DT Binding for
-a new instance of `AbstractS3ATokenIdentifier` and then wrapping a hadoop token
-around it.
-* General logging, debugging, and metrics. Delegation token metrics are
-collected in (`S3AInstrumentation.DelegationTokenStatistics`)
-
-
-
-
-### class `org.apache.hadoop.fs.s3a.auth.delegation.AbstractS3ATokenIdentifier`
-
-All tokens returned are a subclass of `AbstractS3ATokenIdentifier`.
-
-This class contains the following fields:
-
-```java
- /** Canonical URI of the bucket. */
- private URI uri;
-
- /**
- * Encryption secrets to also marshall with any credentials.
- * Set during creation to ensure it is never null.
- */
- private EncryptionSecrets encryptionSecrets = new EncryptionSecrets();
-
- /**
- * Timestamp of creation.
- * This is set to the current time; it will be overridden when
- * deserializing data.
- */
- private long created = System.currentTimeMillis();
-
- /**
- * An origin string for diagnostics.
- */
- private String origin = "";
-
- /**
- * This marshalled UUID can be used in testing to verify transmission,
- * and reuse; as it is printed you can see what is happending too.
- */
- private String uuid = UUID.randomUUID().toString();
-```
-
-The `uuid` field is used for equality tests and debugging; the `origin` and
-`created` fields are also for diagnostics.
-
-The `encryptionSecrets` structure enumerates the AWS encryption mechanism
-of the filesystem instance, and any declared key. This allows
-the client-side secret for SSE-C encryption to be passed to the filesystem,
-or the key name for SSE-KMS.
-
-*The encryption settings and secrets of the S3A filesystem on the client
-are included in the DT, so can be used to encrypt/decrypt data in the cluster.*
-
-### class `SessionTokenIdentifier` extends `AbstractS3ATokenIdentifier`
-
-This holds session tokens, and it also gets used as a superclass of
-the other token identifiers.
-
-It adds a set of `MarshalledCredentials` containing the session secrets.
-
-Every token/token identifier must have a unique *Kind*; this is how token
-identifier deserializers are looked up. For Session Credentials, it is
-`S3ADelegationToken/Session`. Subclasses *must* have a different token kind,
-else the unmarshalling and binding mechanism will fail.
-
-
-### classes `RoleTokenIdentifier` and `FullCredentialsTokenIdentifier`
-
-These are subclasses of `SessionTokenIdentifier` with different token kinds,
-needed for that token unmarshalling.
-
-Their kinds are `S3ADelegationToken/Role` and `S3ADelegationToken/Full`
-respectively.
-
-Having different possible token bindings raises the risk that a job is submitted
-with one binding and yet the cluster is expecting another binding.
-Provided the configuration option `fs.s3a.delegation.token.binding` is not
-marked as final in the `core-site.xml` file, the value of that binding
-set in the job should propagate with the binding: the choice of provider
-is automatic. A cluster can even mix bindings across jobs.
-However if a core-site XML file declares a specific binding for a single bucket and
-the job only had the generic `fs.s3a.delegation.token.binding`` binding,
-then there will be a mismatch.
-Each binding must be rigorous about checking the Kind of any found delegation
-token and failing meaningfully here.
-
-
-
-### class `MarshalledCredentials`
-
-Can marshall a set of AWS credentials (access key, secret key, session token)
-as a Hadoop Writable.
-
-These can be given to an instance of class `MarshalledCredentialProvider`
-and used to sign AWS RPC/REST API calls.
-
-## DT Binding: `AbstractDelegationTokenBinding`
-
-The plugin point for this design is the DT binding, which must be a subclass
-of `org.apache.hadoop.fs.s3a.auth.delegation.AbstractDelegationTokenBinding`.
-
-
-This class
-
-* Returns the *Kind* of these tokens.
-* declares whether tokens will actually be issued or not (the TokenIssuingPolicy).
-* can issue a DT in
-
-```java
- public abstract AWSCredentialProviderList deployUnbonded()
- throws IOException;
-```
-
-The S3A FS has been brought up with DTs enabled, but none have been found
-for its service name. The DT binding is tasked with coming up with the
-fallback list of AWS credential providers.
-
-```java
-public abstract AWSCredentialProviderList bindToTokenIdentifier(
- AbstractS3ATokenIdentifier retrievedIdentifier)
- throws IOException;
-```
-
-A DT for this FS instance been found. Bind to it and extract enough information
-to authenticate with AWS. Return the list of providers to use.
-
-```java
-public abstract AbstractS3ATokenIdentifier createEmptyIdentifier();
-```
-
-Return an empty identifier.
-
-
-```java
-public abstract AbstractS3ATokenIdentifier createTokenIdentifier(
- Optional policy,
- EncryptionSecrets encryptionSecrets)
-```
-
-This is the big one: creatw a new Token Identifier for this filesystem, one
-which must include the encryption secrets, and which may make use of
-the role policy.
-
-## Token issuing
-
-### How Full Delegation Tokens are issued.
-
-If the client is only logged in with session credentials: fail.
-
-Else: take the AWS access/secret key, store them in the MarshalledCredentials
-in a new `FullCredentialsTokenIdentifier`, and return.
-
-
-### How Session Delegation Tokens are issued.
-
-If the client is only logged in with session credentials: return these.
-
-This is taken from the Yahoo! patch: if a user is logged
-in with a set of session credentials (including those from some 2FA login),
-they just get wrapped up and passed in.
-
-There's no clue as to how long they will last, so there's a warning printed.
-
-If there is a full set of credentials, then an `SessionTokenBinding.maybeInitSTS()`
-creates an STS client set up to communicate with the (configured) STS endpoint,
-retrying with the same retry policy as the filesystem.
-
-This client is then used to request a set of session credentials.
-
-### How Role Delegation Tokens are issued.
-
-If the client is only logged in with session credentials: fail.
-
-We don't know whether this is a full user session or some role session,
-and rather than pass in some potentially more powerful secrets with the job,
-just fail.
-
-Else: as with session delegation tokens, an STS client is created. This time
-`assumeRole()` is invoked with the ARN of the role and an extra AWS role policy
-set to restrict access to:
-
-* CRUD access the specific bucket a token is being requested for
-* CRUD access to the contents of any S3Guard DDB used (not admin rights though).
-* access to all KMS keys (assumption: AWS KMS is where restrictions are set up)
-
-*Example Generated Role Policy*
-
-
-```json
-{
- "Version" : "2012-10-17",
- "Statement" : [ {
- "Sid" : "7",
- "Effect" : "Allow",
- "Action" : [ "s3:GetBucketLocation", "s3:ListBucket*" ],
- "Resource" : "arn:aws:s3:::example-bucket"
- }, {
- "Sid" : "8",
- "Effect" : "Allow",
- "Action" : [ "s3:Get*", "s3:PutObject", "s3:DeleteObject", "s3:AbortMultipartUpload" ],
- "Resource" : "arn:aws:s3:::example-bucket/*"
- }, {
- "Sid" : "1",
- "Effect" : "Allow",
- "Action" : [ "kms:Decrypt", "kms:GenerateDataKey" ],
- "Resource" : "arn:aws:kms:*"
- }, {
- "Sid" : "9",
- "Effect" : "Allow",
- "Action" : [ "dynamodb:BatchGetItem", "dynamodb:BatchWriteItem", "dynamodb:DeleteItem", "dynamodb:DescribeTable", "dynamodb:GetItem", "dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem" ],
- "Resource" : "arn:aws:dynamodb:eu-west-1:980678866fff:table/example-bucket"
- } ]
-}
-```
-
-These permissions are sufficient for all operations the S3A client currently
-performs on a bucket. If those requirements are expanded, these policies
-may change.
-
-
-## Testing.
-
-Look in `org.apache.hadoop.fs.s3a.auth.delegation`
-
-
-It's proven impossible to generate a full end-to-end test in an MR job.
-
-1. MapReduce only collects DTs when kerberos is enabled in the cluster.
-1. A Kerberized MiniYARN cluster refuses to start on a local file:// fs without the
-native libraries, so it can set directory permissions.
-1. A Kerberized MiniHDFS cluster and MiniYARN cluster refuse to talk to each
-other reliably, at least in the week or so of trying.
-
-The `ITestDelegatedMRJob` test works around this by using Mockito to mock
-the actual YARN job submit operation in `org.apache.hadoop.mapreduce.protocol.ClientProtocol`.
-The MR code does all the work of collecting tokens and attaching them to
-the launch context, "submits" the job, which then immediately succeeds.
-The job context is examined to verify that the source and destination filesystem
-DTs were extracted.
-
-To test beyond this requires a real Kerberized cluster, or someone able to fix
-up Mini* clusters to run kerberized.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
deleted file mode 100644
index 30226f85eb..0000000000
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/delegation_tokens.md
+++ /dev/null
@@ -1,870 +0,0 @@
-
-
-# Working with Delegation Tokens
-
-
-
-## Introducing S3A Delegation Tokens.
-
-The S3A filesystem client supports `Hadoop Delegation Tokens`.
-This allows YARN application like MapReduce, Distcp, Apache Flink and Apache Spark to
-obtain credentials to access S3 buckets and pass them pass these credentials to
-jobs/queries, so granting them access to the service with the same access
-permissions as the user.
-
-Three different token types are offered.
-
-*Full Delegation Tokens:* include the full login values of `fs.s3a.access.key`
-and `fs.s3a.secret.key` in the token, so the recipient has access to
-the data as the submitting user, with unlimited duration.
-These tokens do not involve communication with the AWS STS service, so
-can be used with other S3 installations.
-
-*Session Delegation Tokens:* These contain an "STS Session Token" requested by
-the S3A client from the AWS STS service. They have a limited duration
-so restrict how long an application can access AWS on behalf of a user.
-Clients with this token have the full permissions of the user.
-
-*Role Delegation Tokens:* These contain an "STS Session Token" requested by by the
-STS "Assume Role" API, so grant the caller to interact with S3 as specific AWS
-role, *with permissions restricted to purely accessing the S3 bucket
-and associated S3Guard data*.
-
-Role Delegation Tokens are the most powerful. By restricting the access rights
-of the granted STS token, no process receiving the token may perform
-any operations in the AWS infrastructure other than those for the S3 bucket,
-and that restricted by the rights of the requested role ARN.
-
-All three tokens also marshall the encryption settings: The encryption mechanism
-to use and the KMS key ID or SSE-C client secret. This allows encryption
-policy and secrets to be uploaded from the client to the services.
-
-This document covers how to use these tokens. For details on the implementation
-see [S3A Delegation Token Architecture](delegation_token_architecture.html).
-
-## Background: Hadoop Delegation Tokens.
-
-A Hadoop Delegation Token are is a byte array of data which is submitted to
-a Hadoop services as proof that the caller has the permissions to perform
-the operation which it is requesting —
-and which can be passed between applications to *delegate* those permission.
-
-Tokens are opaque to clients, clients who simply get a byte array
-of data which they must to provide to a service when required.
-This normally contains encrypted data for use by the service.
-
-The service, which holds the password to encrypt/decrypt this data,
-can decrypt the byte array and read the contents,
-knowing that it has not been tampered with, then
-use the presence of a valid token as evidence the caller has
-at least temporary permissions to perform the requested operation.
-
-Tokens have a limited lifespan.
-They may be renewed, with the client making an IPC/HTTP request of a renewer service.
-This renewal service can also be executed on behalf of the caller by
-some other Hadoop cluster services, such as the YARN Resource Manager.
-
-After use, tokens may be revoked: this relies on services holding tables of
-valid tokens, either in memory or, for any HA service, in Apache Zookeeper or
-similar. Revoking tokens is used to clean up after jobs complete.
-
-Delegation support is tightly integrated with YARN: requests to launch
-containers and applications can include a list of delegation tokens to
-pass along. These tokens are serialized with the request, saved to a file
-on the node launching the container, and then loaded in to the credentials
-of the active user. Normally the HDFS cluster is one of the tokens used here,
-added to the credentials through a call to `FileSystem.getDelegationToken()`
-(usually via `FileSystem.addDelegationTokens()`).
-
-Delegation Tokens are also supported with applications such as Hive: a query
-issued to a shared (long-lived) Hive cluster can include the delegation
-tokens required to access specific filesystems *with the rights of the user
-submitting the query*.
-
-All these applications normally only retrieve delegation tokens when security
-is enabled. This is why the cluster configuration needs to enable Kerberos.
-Production Hadoop clusters need Kerberos for security anyway.
-
-
-## S3A Delegation Tokens.
-
-S3A now supports delegation tokens, so allowing a caller to acquire tokens
-from a local S3A Filesystem connector instance and pass them on to
-applications to grant them equivalent or restricted access.
-
-These S3A Delegation Tokens are special in that they do not contain
-password-protected data opaque to clients; they contain the secrets needed
-to access the relevant S3 buckets and associated services.
-
-They are obtained by requesting a delegation token from the S3A filesystem client.
-Issued token mey be included in job submissions, passed to running applications,
-etc. This token is specific to an individual bucket; all buckets which a client
-wishes to work with must have a separate delegation token issued.
-
-S3A implements Delegation Tokens in its `org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens`
-class, which then supports multiple "bindings" behind it, so supporting
-different variants of S3A Delegation Tokens.
-
-Because applications only collect Delegation Tokens in secure clusters,
-It does mean that to be able to submit delegation tokens in transient
-cloud-hosted Hadoop clusters, _these clusters must also have Kerberos enabled_.
-
-
-### S3A Session Delegation Tokens
-
-A Session Delegation Token is created by asking the AWS
-[Security Token Service](http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html)
-to issue an AWS session password and identifier for a limited duration.
-These AWS session credentials are valid until the end of that time period.
-They are marshalled into the S3A Delegation Token.
-
-Other S3A connectors can extract these credentials and use them to
-talk to S3 and related services.
-
-Issued tokens cannot be renewed or revoked.
-
-See [GetSessionToken](http://docs.aws.amazon.com/STS/latest/APIReference/API_GetSessionToken.html)
-for specifics details on the (current) token lifespan.
-
-### S3A Role Delegation Tokens
-
-A Role Delegation Tokens is created by asking the AWS
-[Security Token Service](http://docs.aws.amazon.com/STS/latest/APIReference/Welcome.html)
-for set of "Assumed Role" credentials, with a AWS account specific role for a limited duration..
-This role is restricted to only grant access the S3 bucket, the S3Guard table
-and all KMS keys,
-They are marshalled into the S3A Delegation Token.
-
-Other S3A connectors can extract these credentials and use them to
-talk to S3 and related services.
-They may only work with the explicit AWS resources identified when the token was generated.
-
-Issued tokens cannot be renewed or revoked.
-
-
-### S3A Full-Credential Delegation Tokens
-
-Full Credential Delegation Tokens tokens contain the full AWS login details
-(access key and secret key) needed to access a bucket.
-
-They never expire, so are the equivalent of storing the AWS account credentials
-in a Hadoop, Hive, Spark configuration or similar.
-
-They differences are:
-
-1. They are automatically passed from the client/user to the application.
-A remote application can use them to access data on behalf of the user.
-1. When a remote application destroys the filesystem connector instances and
-tokens of a user, the secrets are destroyed too.
-1. Secrets in the `AWS_` environment variables on the client will be picked up
-and automatically propagated.
-1. They do not use the AWS STS service, so may work against third-party implementations
-of the S3 protocol.
-
-
-## Using S3A Delegation Tokens
-
-A prerequisite to using S3A filesystem delegation tokens is to run with
-Hadoop security enabled —which inevitably means with Kerberos.
-Even though S3A delegation tokens do not use Kerberos, the code in
-applications which fetch DTs is normally only executed when the cluster is
-running in secure mode; somewhere where the `core-site.xml` configuration
-sets `hadoop.security.authentication` to to `kerberos` or another valid
-authentication mechanism.
-
-* Without enabling security at this level, delegation tokens will not
-be collected.*
-
-Once Kerberos enabled, the process for acquiring tokens is as follows:
-
-1. Enable Delegation token support by setting `fs.s3a.delegation.token.binding`
-to the classname of the token binding to use.
-to use.
-1. Add any other binding-specific settings (STS endpoint, IAM role, etc.)
-1. Make sure the settings are the same in the service as well as the client.
-1. In the client, switch to using a [Hadoop Credential Provider](hadoop-project-dist/hadoop-common/CredentialProviderAPI.html)
-for storing your local credentials, *with a local filesystem store
- (`localjceks:` or `jcecks://file`), so as to keep the full secrets out of any
- job configurations.
-1. Execute the client from a Kerberos-authenticated account
-application configured with the login credentials for an AWS account able to issue session tokens.
-
-### Configuration Parameters
-
-
-| **Key** | **Meaning** | **Default** |
-| --- | --- | --- |
-| `fs.s3a.delegation.token.binding` | delegation token binding class | `` |
-
-### Warnings
-
-##### Use Hadoop Credential Providers to keep secrets out of job configurations.
-
-Hadoop MapReduce jobs copy their client-side configurations with the job.
-If your AWS login secrets are set in an XML file then they are picked up
-and passed in with the job, _even if delegation tokens are used to propagate
-session or role secrets.
-
-Spark-submit will take any credentials in the `spark-defaults.conf`file
-and again, spread them across the cluster.
-It wil also pick up any `AWS_` environment variables and convert them into
-`fs.s3a.access.key`, `fs.s3a.secret.key` and `fs.s3a.session.key` configuration
-options.
-
-To guarantee that the secrets are not passed in, keep your secrets in
-a [hadoop credential provider file on the local filesystem](index.html#hadoop_credential_providers").
-Secrets stored here will not be propagated -the delegation tokens collected
-during job submission will be the sole AWS secrets passed in.
-
-
-##### Token Life
-
-* S3A Delegation tokens cannot be renewed.
-
-* S3A Delegation tokens cannot be revoked. It is possible for an administrator
-to terminate *all AWS sessions using a specific role*
-[from the AWS IAM console](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_disable-perms.html),
-if desired.
-
-* The lifespan of Session Delegation Tokens are limited to those of AWS sessions,
-maximum of 36 hours.
-
-* The lifespan of a Role Delegation Token is limited to 1 hour by default;
-a longer duration of up to 12 hours can be enabled in the AWS console for
-the specific role being used.
-
-* The lifespan of Full Delegation tokens is unlimited: the secret needs
-to be reset in the AWS Admin console to revoke it.
-
-##### Service Load on the AWS Secure Token Service
-
-All delegation tokens are issued on a bucket-by-bucket basis: clients
-must request a delegation token from every S3A filesystem to which it desires
-access.
-
-For Session and Role Delegation Tokens, this places load on the AWS STS service,
-which may trigger throttling amongst all users within the same AWS account using
-the same STS endpoint.
-
-* In experiments, a few hundred requests per second are needed to trigger throttling,
-so this is very unlikely to surface in production systems.
-* The S3A filesystem connector retries all throttled requests to AWS services, including STS.
-* Other S3 clients with use the AWS SDK will, if configured, also retry throttled requests.
-
-Overall, the risk of triggering STS throttling appears low, and most applications
-will recover from what is generally an intermittently used AWS service.
-
-### Enabling Session Delegation Tokens
-
-For session tokens, set `fs.s3a.delegation.token.binding`
-to `org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding`
-
-
-| **Key** | **Value** |
-| --- | --- |
-| `fs.s3a.delegation.token.binding` | `org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding` |
-
-There some further configuration options.
-
-| **Key** | **Meaning** | **Default** |
-| --- | --- | --- |
-| `fs.s3a.assumed.role.session.duration` | Duration of delegation tokens | `1h` |
-| `fs.s3a.assumed.role.sts.endpoint` | URL to service issuing tokens | (undefined) |
-| `fs.s3a.assumed.role.sts.endpoint.region` | region for issued tokens | (undefined) |
-
-The XML settings needed to enable session tokens are:
-
-```xml
-
- fs.s3a.delegation.token.binding
- org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding
-
-
- fs.s3a.assumed.role.session.duration
- 1h
-
-```
-
-1. If the application requesting a token has full AWS credentials for the
-relevant bucket, then a new session token will be issued.
-1. If the application requesting a token is itself authenticating with
-a session delegation token, then the existing token will be forwarded.
-The life of the token will not be extended.
-1. If the application requesting a token does not have either of these,
-the the tokens cannot be issued: the operation will fail with an error.
-
-
-The endpoint for STS requests are set by the same configuration
-property as for the `AssumedRole` credential provider and for Role Delegation
-tokens.
-
-```xml
-
-
- fs.s3a.assumed.role.sts.endpoint
- sts.amazonaws.com
-
-
- fs.s3a.assumed.role.sts.endpoint.region
- us-west-1
-
-```
-
-If the `fs.s3a.assumed.role.sts.endpoint` option is set, or set to something
-other than the central `sts.amazonaws.com` endpoint, then the region property
-*must* be set.
-
-
-Both the Session and the Role Delegation Token bindings use the option
-`fs.s3a.aws.credentials.provider` to define the credential providers
-to authenticate to the AWS STS with.
-
-Here is the effective list of providers if none are declared:
-
-```xml
-
- fs.s3a.aws.credentials.provider
-
- org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
- org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
-
-
-```
-
-Not all these authentication mechanisms provide the full set of credentials
-STS needs. The session token provider will simply forward any session credentials
-it is authenticated with; the role token binding will fail.
-
-#### Forwarding of existing AWS Session credentials.
-
-When the AWS credentials supplied to the Session Delegation Token binding
-through `fs.s3a.aws.credentials.provider` are themselves a set of
-session credentials, generated delegation tokens with simply contain these
-existing session credentials, a new set of credentials obtained from STS.
-This is because the STS service does not let
-callers authenticated with session/role credentials from requesting new sessions.
-
-This feature is useful when generating tokens from an EC2 VM instance in one IAM
-role and forwarding them over to VMs which are running in a different IAM role.
-The tokens will grant the permissions of the original VM's IAM role.
-
-The duration of the forwarded tokens will be exactly that of the current set of
-tokens, which may be very limited in lifespan. A warning will appear
-in the logs declaring this.
-
-Note: Role Delegation tokens do not support this forwarding of session credentials,
-because there's no way to explicitly change roles in the process.
-
-
-### Enabling Role Delegation Tokens
-
-For role delegation tokens, set `fs.s3a.delegation.token.binding`
-to `org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding`
-
-| **Key** | **Value** |
-| --- | --- |
-| `fs.s3a.delegation.token.binding` | `org.apache.hadoop.fs.s3a.auth.delegation.SessionToRoleTokenBinding` |
-
-
-There are some further configuration options:
-
-| **Key** | **Meaning** | **Default** |
-| --- | --- | --- |
-| `fs.s3a.assumed.role.session.duration"` | Duration of delegation tokens | `1h` |
-| `fs.s3a.assumed.role.arn` | ARN for role to request | (undefined) |
-| `fs.s3a.assumed.role.sts.endpoint.region` | region for issued tokens | (undefined) |
-
-The option `fs.s3a.assumed.role.arn` must be set to a role which the
-user can assume. It must have permissions to access the bucket, any
-associated S3Guard table and any KMS encryption keys. The actual
-requested role will be this role, explicitly restricted to the specific
-bucket and S3Guard table.
-
-The XML settings needed to enable session tokens are:
-
-```xml
-
- fs.s3a.delegation.token.binding
- org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenBinding
-
-
- fs.s3a.assumed.role.arn
- ARN of role to request
- REQUIRED ARN
-
-
- fs.s3a.assumed.role.session.duration
- 1h
-
-```
-
-A JSON role policy for the role/session will automatically be generated which will
-consist of
-1. Full access to the S3 bucket for all operations used by the S3A client
-(read, write, list, multipart operations, get bucket location, etc).
-1. Full user access to any S3Guard DynamoDB table used by the bucket.
-1. Full user access to KMS keys. This is to be able to decrypt any data
-in the bucket encrypted with SSE-KMS, as well as encrypt new data if that
-is the encryption policy.
-
-If the client doesn't have S3Guard enabled, but the remote application does,
-the issued role tokens will not have permission to access the S3Guard table.
-
-### Enabling Full Delegation Tokens
-
-This passes the full credentials in, falling back to any session credentials
-which were used to configure the S3A FileSystem instance.
-
-For Full Credential Delegation tokens, set `fs.s3a.delegation.token.binding`
-to `org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding`
-
-| **Key** | **Value** |
-| --- | --- |
-| `fs.s3a.delegation.token.binding` | `org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding` |
-
-There are no other configuration options.
-
-```xml
-
- fs.s3a.delegation.token.binding
- org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenBinding
-
-```
-
-Key points:
-
-1. If the application requesting a token has full AWS credentials for the
-relevant bucket, then a full credential token will be issued.
-1. If the application requesting a token is itself authenticating with
-a session delegation token, then the existing token will be forwarded.
-The life of the token will not be extended.
-1. If the application requesting a token does not have either of these,
-the the tokens cannot be issued: the operation will fail with an error.
-
-## Managing the Delegation Tokens Duration
-
-Full Credentials have an unlimited lifespan.
-
-Session and role credentials have a lifespan defined by the duration
-property `fs.s3a.assumed.role.session.duration`.
-
-This can have a maximum value of "36h" for session delegation tokens.
-
-For Role Delegation Tokens, the maximum duration of a token is
-that of the role itself: 1h by default, though this can be changed to
-12h [In the IAM Console](https://console.aws.amazon.com/iam/home#/roles),
-or from the AWS CLI.
-
-*Without increasing the duration of role, one hour is the maximum value;
-the error message `The requested DurationSeconds exceeds the MaxSessionDuration set for this role`
-is returned if the requested duration of a Role Delegation Token is greater
-than that available for the role.
-
-
-## Testing Delegation Token Support
-
-The easiest way to test that delegation support is configured is to use
-the `hdfs fetchdt` command, which can fetch tokens from S3A, Azure ABFS
-and any other filesystem which can issue tokens, as well as HDFS itself.
-
-This will fetch the token and save it to the named file (here, `tokens.bin`),
-even if Kerberos is disabled.
-
-```bash
-# Fetch a token for the AWS landsat-pds bucket and save it to tokens.bin
-$ hdfs fetchdt --webservice s3a://landsat-pds/ tokens.bin
-```
-
-If the command fails with `ERROR: Failed to fetch token` it means the
-filesystem does not have delegation tokens enabled.
-
-If it fails for other reasons, the likely causes are configuration and
-possibly connectivity to the AWS STS Server.
-
-Once collected, the token can be printed. This will show
-the type of token, details about encryption and expiry, and the
-host on which it was created.
-
-```bash
-$ bin/hdfs fetchdt --print tokens.bin
-
-Token (S3ATokenIdentifier{S3ADelegationToken/Session; uri=s3a://landsat-pds;
-timestamp=1541683947569; encryption=EncryptionSecrets{encryptionMethod=SSE_S3};
-Created on vm1.local/192.168.99.1 at time 2018-11-08T13:32:26.381Z.};
-Session credentials for user AAABWL expires Thu Nov 08 14:02:27 GMT 2018; (valid))
-for s3a://landsat-pds
-```
-The "(valid)" annotation means that the AWS credentials are considered "valid":
-there is both a username and a secret.
-
-You can use the `s3guard bucket-info` command to see what the delegation
-support for a specific bucket is.
-If delegation support is enabled, it also prints the current
-hadoop security level.
-
-```bash
-$ hadoop s3guard bucket-info s3a://landsat-pds/
-
-Filesystem s3a://landsat-pds
-Location: us-west-2
-Filesystem s3a://landsat-pds is not using S3Guard
-The "magic" committer is supported
-
-S3A Client
- Endpoint: fs.s3a.endpoint=s3.amazonaws.com
- Encryption: fs.s3a.server-side-encryption-algorithm=none
- Input seek policy: fs.s3a.experimental.input.fadvise=normal
-Delegation Support enabled: token kind = S3ADelegationToken/Session
-Hadoop security mode: SIMPLE
-```
-
-Although the S3A delegation tokens do not depend upon Kerberos,
-MapReduce and other applications only request tokens from filesystems when
-security is enabled in Hadoop.
-
-
-## Troubleshooting S3A Delegation Tokens
-
-The `hadoop s3guard bucket-info` command will print information about
-the delegation state of a bucket.
-
-Consult [troubleshooting Assumed Roles](assumed_roles.html#troubleshooting)
-for details on AWS error messages related to AWS IAM roles.
-
-The [cloudstore](https://github.com/steveloughran/cloudstore) module's StoreDiag
-utility can also be used to explore delegation token support
-
-
-### Submitted job cannot authenticate
-
-There are many causes for this; delegation tokens add some more.
-
-### Tokens are not issued
-
-
-* This user is not `kinit`-ed in to Kerberos. Use `klist` and
-`hadoop kdiag` to see the Kerberos authentication state of the logged in user.
-* The filesystem instance on the client has not had a token binding set in
-`fs.s3a.delegation.token.binding`, so does not attempt to issue any.
-* The job submission is not aware that access to the specific S3 buckets
-are required. Review the application's submission mechanism to determine
-how to list source and destination paths. For example, for MapReduce,
-tokens for the cluster filesystem (`fs.defaultFS`) and all filesystems
-referenced as input and output paths will be queried for
-delegation tokens.
-
-For Apache Spark, the cluster filesystem and any filesystems listed in the
-property `spark.yarn.access.hadoopFileSystems` are queried for delegation
-tokens in secure clusters.
-See [Running on Yarn](https://spark.apache.org/docs/latest/running-on-yarn.html).
-
-
-### Error `No AWS login credentials`
-
-The client does not have any valid credentials to request a token
-from the Amazon STS service.
-
-### Tokens Expire before job completes
-
-The default duration of session and role tokens as set in
-`fs.s3a.assumed.role.session.duration` is one hour, "1h".
-
-For session tokens, this can be increased to any time up to 36 hours.
-
-For role tokens, it can be increased up to 12 hours, *but only if
-the role is configured in the AWS IAM Console to have a longer lifespan*.
-
-
-### Error `DelegationTokenIOException: Token mismatch`
-
-```
-org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenIOException:
- Token mismatch: expected token for s3a://example-bucket
- of type S3ADelegationToken/Session but got a token of type S3ADelegationToken/Full
-
- at org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.lookupToken(S3ADelegationTokens.java:379)
- at org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.selectTokenFromActiveUser(S3ADelegationTokens.java:300)
- at org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokens.bindToExistingDT(S3ADelegationTokens.java:160)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.bindAWSClient(S3AFileSystem.java:423)
- at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:265)
-```
-
-The value of `fs.s3a.delegation.token.binding` is different in the remote
-service than in the local client. As a result, the remote service
-cannot use the token supplied by the client to authenticate.
-
-Fix: reference the same token binding class at both ends.
-
-
-### Warning `Forwarding existing session credentials`
-
-This message is printed when an S3A filesystem instance has been asked
-for a Session Delegation Token, and it is itself only authenticated with
-a set of AWS session credentials (such as those issued by the IAM metadata
-service).
-
-The created token will contain these existing credentials, credentials which
-can be used until the existing session expires.
-
-The duration of this existing session is unknown: the message is warning
-you that it may expire without warning.
-
-### Error `Cannot issue S3A Role Delegation Tokens without full AWS credentials`
-
-An S3A filesystem instance has been asked for a Role Delegation Token,
-but the instance is only authenticated with session tokens.
-This means that a set of role tokens cannot be requested.
-
-Note: no attempt is made to convert the existing set of session tokens into
-a delegation token, unlike the Session Delegation Tokens. This is because
-the role of the current session (if any) is unknown.
-
-
-## Implementation Details
-
-### Architecture
-
-Concepts:
-
-1. The S3A FileSystem can create delegation tokens when requested.
-1. These can be marshalled as per other Hadoop Delegation Tokens.
-1. At the far end, they can be retrieved, unmarshalled and used to authenticate callers.
-1. DT binding plugins can then use these directly, or, somehow,
-manage authentication and token issue through other services
-(for example: Kerberos)
-1. Token Renewal and Revocation are not supported.
-
-
-There's support for different back-end token bindings through the
-`org.apache.hadoop.fs.s3a.auth.delegation.S3ADelegationTokenManager`
-
-Every implementation of this must return a subclass of
-`org.apache.hadoop.fs.s3a.auth.delegation.AbstractS3ATokenIdentifier`
-when asked to create a delegation token; this subclass must be registered
-in `META-INF/services/org.apache.hadoop.security.token.TokenIdentifier`
-for unmarshalling.
-
-This identifier must contain all information needed at the far end to
-authenticate the caller with AWS services used by the S3A client: AWS S3 and
-potentially AWS KMS (for SSE-KMS) and AWS DynamoDB (for S3Guard).
-
-It must have its own unique *Token Kind*, to ensure that it can be distinguished
-from the other token identifiers when tokens are being unmarshalled.
-
-| Kind | Token class |
-|------|--------------|
-| `S3ADelegationToken/Full` | `org.apache.hadoop.fs.s3a.auth.delegation.FullCredentialsTokenIdentifier` |
-| `S3ADelegationToken/Session` | `org.apache.hadoop.fs.s3a.auth.delegation.RoleTokenIdentifier`|
-| `S3ADelegationToken/Role` | `org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenIdentifier` |
-
-If implementing an external binding:
-
-1. Follow the security requirements below.
-1. Define a new token identifier; there is no requirement for the `S3ADelegationToken/`
-prefix —but it is useful for debugging.
-1. Token Renewal and Revocation is not integrated with the binding mechanism;
-if the operations are supported, implementation is left as an exercise.
-1. Be aware of the stability guarantees of the module "LimitedPrivate/Unstable".
-
-### Security
-
-S3A DTs contain secrets valuable for a limited period (session secrets) or
-long-lived secrets with no explicit time limit.
-
-* The `toString()` operations on token identifiers MUST NOT print secrets; this
-is needed to keep them out of logs.
-* Secrets MUST NOT be logged, even at debug level.
-* Prefer short-lived session secrets over long-term secrets.
-* Try to restrict the permissions to what a client with the delegated token
- may perform to those needed to access data in the S3 bucket. This potentially
- includes a DynamoDB table, KMS access, etc.
-* Implementations need to be resistant to attacks which pass in invalid data as
-their token identifier: validate the types of the unmarshalled data; set limits
-on the size of all strings and other arrays to read in, etc.
-
-### Resilience
-
-Implementations need to handle transient failures of any remote authentication
-service, and the risk of a large-cluster startup overloading it.
-
-* All get/renew/cancel operations should be considered idempotent.
-* And clients to repeat with backoff & jitter on recoverable connectivity failures.
-* While failing fast on the unrecoverable failures (DNS, authentication).
-
-### Scalability limits of AWS STS service
-
-There is currently no documented rate limit for token requests against the AWS
-STS service.
-
-We have two tests which attempt to generate enough requests for
-delegation tokens that the AWS STS service will throttle requests for
-tokens by that AWS account for that specific STS endpoint
-(`ILoadTestRoleCredentials` and `ILoadTestSessionCredentials`).
-
-In the initial results of these tests:
-
-* A few hundred requests a second can be made before STS block the caller.
-* The throttling does not last very long (seconds)
-* Tt does not appear to affect any other STS endpoints.
-
-If developers wish to experiment with these tests and provide more detailed
-analysis, we would welcome this. Do bear in mind that all users of the
-same AWS account in that region will be throttled. Your colleagues may
-notice, especially if the applications they are running do not retry on
-throttle responses from STS (it's not a common occurrence after all...).
-
-## Implementing your own Delegation Token Binding
-
-The DT binding mechanism is designed to be extensible: if you have an alternate
-authentication mechanism, such as an S3-compatible object store with
-Kerberos support —S3A Delegation tokens should support it.
-
-*if it can't: that's a bug in the implementation which needs to be corrected*.
-
-### Steps
-
-1. Come up with a token "Kind"; a unique name for the delegation token identifier.
-1. Implement a subclass of `AbstractS3ATokenIdentifier` which adds all information which
-is marshalled from client to remote services. This must subclass the `Writable` methods to read
-and write the data to a data stream: these subclasses must call the superclass methods first.
-1. Add a resource `META-INF/services/org.apache.hadoop.security.token.TokenIdentifier`
-1. And list in it, the classname of your new identifier.
-1. Implement a subclass of `AbstractDelegationTokenBinding`
-
-### Implementing `AbstractS3ATokenIdentifier`
-
-Look at the other examples to see what to do; `SessionTokenIdentifier` does
-most of the work.
-
-Having a `toString()` method which is informative is ideal for the `hdfs creds`
-command as well as debugging: *but do not print secrets*
-
-*Important*: Add no references to any AWS SDK class, to
-ensure it can be safely deserialized whenever the relevant token
-identifier is examined. Best practise is: avoid any references to
-classes which may not be on the classpath of core Hadoop services,
-especially the YARN Resource Manager and Node Managers.
-
-### `AWSCredentialProviderList deployUnbonded()`
-
-1. Perform all initialization needed on an "unbonded" deployment to authenticate with the store.
-1. Return a list of AWS Credential providers which can be used to authenticate the caller.
-
-**Tip**: consider *not* doing all the checks to verify that DTs can be issued.
-That can be postponed until a DT is issued -as in any deployments where a DT is not actually
-needed, failing at this point is overkill. As an example, `RoleTokenBinding` cannot issue
-DTs if it only has a set of session credentials, but it will deploy without them, so allowing
-`hadoop fs` commands to work on an EC2 VM with IAM role credentials.
-
-**Tip**: The class `org.apache.hadoop.fs.s3a.auth.MarshalledCredentials` holds a set of
-marshalled credentials and so can be used within your own Token Identifier if you want
-to include a set of full/session AWS credentials in your token identifier.
-
-### `AWSCredentialProviderList bindToTokenIdentifier(AbstractS3ATokenIdentifier id)`
-
-The identifier passed in will be the one for the current filesystem URI and of your token kind.
-
-1. Use `convertTokenIdentifier` to cast it to your DT type, or fail with a meaningful `IOException`.
-1. Extract the secrets needed to authenticate with the object store (or whatever service issues
-object store credentials).
-1. Return a list of AWS Credential providers which can be used to authenticate the caller with
-the extracted secrets.
-
-### `AbstractS3ATokenIdentifier createEmptyIdentifier()`
-
-Return an empty instance of your token identifier.
-
-### `AbstractS3ATokenIdentifier createTokenIdentifier(Optional policy, EncryptionSecrets secrets)`
-
-Create the delegation token.
-
-If non-empty, the `policy` argument contains an AWS policy model to grant access to:
-
-* The target S3 bucket.
-* Any S3Guard DDB table it is bonded to.
-* KMS key `"kms:GenerateDataKey` and `kms:Decrypt`permissions for all KMS keys.
-
-This can be converted to a string and passed to the AWS `assumeRole` operation.
-
-The `secrets` argument contains encryption policy and secrets:
-this should be passed to the superclass constructor as is; it is retrieved and used
-to set the encryption policy on the newly created filesystem.
-
-
-*Tip*: Use `AbstractS3ATokenIdentifier.createDefaultOriginMessage()` to create an initial
-message for the origin of the token —this is useful for diagnostics.
-
-
-#### Token Renewal
-
-There's no support in the design for token renewal; it would be very complex
-to make it pluggable, and as all the bundled mechanisms don't support renewal,
-untestable and unjustifiable.
-
-Any token binding which wants to add renewal support will have to implement
-it directly.
-
-### Testing
-
-Use the tests `org.apache.hadoop.fs.s3a.auth.delegation` as examples. You'll have to
-copy and paste some of the test base classes over; `hadoop-common`'s test JAR is published
-to Maven Central, but not the S3A one (a fear of leaking AWS credentials).
-
-
-#### Unit Test `TestS3ADelegationTokenSupport`
-
-This tests marshalling and unmarshalling of tokens identifiers.
-*Test that every field is preserved.*
-
-
-#### Integration Test `ITestSessionDelegationTokens`
-
-Tests the lifecycle of session tokens.
-
-#### Integration Test `ITestSessionDelegationInFileystem`.
-
-This collects DTs from one filesystem, and uses that to create a new FS instance and
-then perform filesystem operations. A miniKDC is instantiated
-
-* Take care to remove all login secrets from the environment, so as to make sure that
-the second instance is picking up the DT information.
-* `UserGroupInformation.reset()` can be used to reset user secrets after every test
-case (e.g. teardown), so that issued DTs from one test case do not contaminate the next.
-* its subclass, `ITestRoleDelegationInFileystem` adds a check that the current credentials
-in the DT cannot be used to access data on other buckets —that is, the active
-session really is restricted to the target bucket.
-
-
-#### Integration Test `ITestDelegatedMRJob`
-
-It's not easy to bring up a YARN cluster with a secure HDFS and miniKDC controller in
-test cases —this test, the closest there is to an end-to-end test,
-uses mocking to mock the RPC calls to the YARN AM, and then verifies that the tokens
-have been collected in the job context,
-
-#### Load Test `ILoadTestSessionCredentials`
-
-This attempts to collect many, many delegation tokens simultaneously and sees
-what happens.
-
-Worth doing if you have a new authentication service provider, or
-implementing custom DT support.
-Consider also something for going from DT to
-AWS credentials if this is also implemented by your own service.
-This is left as an exercise for the developer.
-
-**Tip**: don't go overboard here, especially against AWS itself.
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 11ff6104e7..7ab3b0e073 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -31,8 +31,6 @@ See also:
* [Committing work to S3 with the "S3A Committers"](./committers.html)
* [S3A Committers Architecture](./committer_architecture.html)
* [Working with IAM Assumed Roles](./assumed_roles.html)
-* [S3A Delegation Token Support](./delegation_tokens.html)
-* [S3A Delegation Token Architecture](delegation_token_architecture.html).
* [Testing](./testing.html)
## Overview
@@ -340,20 +338,15 @@ on the hosts/processes where the work is executed.
### Changing Authentication Providers
-The standard way to authenticate is with an access key and secret key set in
-the Hadoop configuration files.
+The standard way to authenticate is with an access key and secret key using the
+properties in the configuration file.
-By default, the S3A client follows the following authentication chain:
+The S3A client follows the following authentication chain:
-1. The options `fs.s3a.access.key`, `fs.s3a.secret.key` and `fs.s3a.sesson.key
-are looked for in the Hadoop XML configuration/Hadoop credential providers,
-returning a set of session credentials if all three are defined.
1. The `fs.s3a.access.key` and `fs.s3a.secret.key` are looked for in the Hadoop
-XML configuration//Hadoop credential providers, returning a set of long-lived
-credentials if they are defined.
+XML configuration.
1. The [AWS environment variables](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment),
-are then looked for: these will return session or full credentials depending
-on which values are set.
+are then looked for.
1. An attempt is made to query the Amazon EC2 Instance Metadata Service to
retrieve credentials published to EC2 VMs.
@@ -369,19 +362,13 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
obtain an AWS login from a different source in the system, including environment
variables, JVM properties and configuration files.
-All Hadoop `fs.s3a.` options used to store login details can all be secured
-in [Hadoop credential providers](../../../hadoop-project-dist/hadoop-common/CredentialProviderAPI.html);
-this is advised as a more secure way to store valuable secrets.
-
-There are a number of AWS Credential Providers inside the `hadoop-aws` JAR:
+There are three AWS Credential Providers inside the `hadoop-aws` JAR:
| classname | description |
|-----------|-------------|
| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
-| `org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider<`| [Assumed Role credentials](assumed_roles.html) |
-
There are also many in the Amazon SDKs, in particular two which are automatically
set up in the authentication chain:
@@ -496,52 +483,10 @@ This means that the default S3A authentication chain can be defined as
fs.s3a.aws.credentials.provider
- org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,
- org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
- com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider
+ org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
+ com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
-
- Comma-separated class names of credential provider classes which implement
- com.amazonaws.auth.AWSCredentialsProvider.
-
- When S3A delegation tokens are not enabled, this list will be used
- to directly authenticate with S3 and DynamoDB services.
- When S3A Delegation tokens are enabled, depending upon the delegation
- token binding it may be used
- to communicate wih the STS endpoint to request session/role
- credentials.
-
- These are loaded and queried in sequence for a valid set of credentials.
- Each listed class must implement one of the following means of
- construction, which are attempted in order:
- * a public constructor accepting java.net.URI and
- org.apache.hadoop.conf.Configuration,
- * a public constructor accepting org.apache.hadoop.conf.Configuration,
- * a public static method named getInstance that accepts no
- arguments and returns an instance of
- com.amazonaws.auth.AWSCredentialsProvider, or
- * a public default constructor.
-
- Specifying org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider allows
- anonymous access to a publicly accessible S3 bucket without any credentials.
- Please note that allowing anonymous access to an S3 bucket compromises
- security and therefore is unsuitable for most use cases. It can be useful
- for accessing public data sets without requiring AWS credentials.
-
- If unspecified, then the default list of credential provider classes,
- queried in sequence, is:
- * org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider: looks
- for session login secrets in the Hadoop configuration.
- * org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider:
- Uses the values of fs.s3a.access.key and fs.s3a.secret.key.
- * com.amazonaws.auth.EnvironmentVariableCredentialsProvider: supports
- configuration of AWS access key ID and secret access key in
- environment variables named AWS_ACCESS_KEY_ID and
- AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
- of instance profile credentials if running in an EC2 VM.
-
```
@@ -556,6 +501,9 @@ and significantly damage your organisation.
1. Never commit your secrets into an SCM repository.
The [git secrets](https://github.com/awslabs/git-secrets) can help here.
+1. Avoid using s3a URLs which have key and secret in the URL. This
+is dangerous as the secrets leak into the logs.
+
1. Never include AWS credentials in bug reports, files attached to them,
or similar.
@@ -576,23 +524,20 @@ The command line of any launched program is visible to all users on a Unix syste
management: a specific S3A connection can be made with a different assumed role
and permissions from the primary user account.
-1. Consider a workflow in which users and applications are issued with short-lived
+1. Consider a workflow in which usera and applications are issued with short-lived
session credentials, configuring S3A to use these through
the `TemporaryAWSCredentialsProvider`.
1. Have a secure process in place for cancelling and re-issuing credentials for
users and applications. Test it regularly by using it to refresh credentials.
-1. In installations where Kerberos is enabled, [S3A Delegation Tokens](delegation_tokens.html)
-can be used to acquire short-lived session/role credentials and then pass them
-into the shared application. This can ensure that the long-lived secrets stay
-on the local system.
-
When running in EC2, the IAM EC2 instance credential provider will automatically
obtain the credentials needed to access AWS services in the role the EC2 VM
was deployed as.
-This AWS credential provider is enabled in S3A by default.
+This credential provider is enabled in S3A by default.
+The safest way to keep the AWS login keys a secret within Hadoop is to use
+Hadoop Credentials.
## Storing secrets with Hadoop Credential Providers
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index f69d161394..fb141ddecc 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -481,11 +481,10 @@ value as "do not override the default").
### Testing Session Credentials
-Some tests requests a session credentials and assumed role credentials from the
-AWS Secure Token Service, then use them to authenticate with S3 either directly
-or via delegation tokens.
+The test `TestS3ATemporaryCredentials` requests a set of temporary
+credentials from the STS service, then uses them to authenticate with S3.
-If an S3 implementation does not support STS, then these functional test
+If an S3 implementation does not support STS, then the functional test
cases must be disabled:
```xml
@@ -493,30 +492,18 @@ cases must be disabled:
test.fs.s3a.sts.enabled
false