diff --git a/LICENSE.txt b/LICENSE.txt index 81eb32face..dd7195e76f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -2813,3 +2813,43 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------------------- +Jline 3.9.0 +The binary distribution of this product bundles these dependencies under the +following license: + +Copyright (c) 2002-2018, the original author or authors. +All rights reserved. + +http://www.opensource.org/licenses/bsd-license.php + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with +the distribution. + +Neither the name of JLine nor the names of its contributors +may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/NOTICE.txt b/NOTICE.txt index d6e54886d1..55531442dc 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -614,10 +614,17 @@ which has the following notices: Expert Group and released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ - The source and binary distribution of this product bundles modified version of github.com/awslabs/aws-js-s3-explorer licensed under Apache 2.0 license with the following notice: AWS JavaScript S3 Explorer -Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file +Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +The binary distribution of this product bundles binaries of +jline 3.9.0 (https://github.com/jline/jline3) + + * LICENSE: + * license/LICENSE.jline3.txt (BSD License) + * HOMEPAGE: + * https://github.com/jline/jline3 diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index c356b1921c..0b820b6b7c 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -788,7 +788,19 @@ org.eclipse.jetty.websocket:javax-websocket-server-impl - * + */** + + + + org.eclipse.jetty.websocket:websocket-client + + */** + + + + org.eclipse.jetty:jetty-io + + */** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 2cb37166c9..6fff0940d2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; @@ -560,4 +561,10 @@ public Map> getNodeToAttributes( Set hostNames) throws YarnException, IOException { return client.getNodeToAttributes(hostNames); } + + @Override + public void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException { + throw new IOException("Operation is not supported."); + } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c985d7b665..6f832802e1 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -160,6 +160,7 @@ 5.3.1 5.3.1 1.3.1 + 3.9.0 @@ -696,6 +697,11 @@ + + org.eclipse.jetty.websocket + websocket-client + ${jetty.version} + javax.servlet.jsp jsp-api @@ -1177,6 +1183,11 @@ + + org.jline + jline + ${jline.version} + org.hsqldb hsqldb diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java new file mode 100644 index 0000000000..07acb9da17 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ShellContainerCommand.java @@ -0,0 +1,32 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Evolving; + +/** + * Enumeration of various signal container commands. + */ +@Public +@Evolving +public enum ShellContainerCommand { + BASH, + SH +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 38cfd11f26..63378dc5a4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -21,8 +21,6 @@ import java.io.File; import java.io.IOException; import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.List; import java.util.Map; @@ -40,13 +38,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.service.api.records.Component; @@ -60,11 +57,6 @@ import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.util.RMHAUtils; import org.eclipse.jetty.util.UrlEncoded; -import org.ietf.jgss.GSSContext; -import org.ietf.jgss.GSSException; -import org.ietf.jgss.GSSManager; -import org.ietf.jgss.GSSName; -import org.ietf.jgss.Oid; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -92,54 +84,6 @@ public class ApiServiceClient extends AppAdminClient { super.serviceInit(configuration); } - /** - * Generate SPNEGO challenge request token. - * - * @param server - hostname to contact - * @throws IOException - * @throws InterruptedException - */ - String generateToken(String server) throws IOException, InterruptedException { - UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); - LOG.debug("The user credential is {}", currentUser); - String challenge = currentUser - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws Exception { - try { - // This Oid for Kerberos GSS-API mechanism. - Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); - GSSManager manager = GSSManager.getInstance(); - // GSS name for server - GSSName serverName = manager.createName("HTTP@" + server, - GSSName.NT_HOSTBASED_SERVICE); - // Create a GSSContext for authentication with the service. - // We're passing client credentials as null since we want them to - // be read from the Subject. - GSSContext gssContext = manager.createContext( - serverName.canonicalize(mechOid), mechOid, null, - GSSContext.DEFAULT_LIFETIME); - gssContext.requestMutualAuth(true); - gssContext.requestCredDeleg(true); - // Establish context - byte[] inToken = new byte[0]; - byte[] outToken = gssContext.initSecContext(inToken, 0, - inToken.length); - gssContext.dispose(); - // Base64 encoded and stringified token for server - LOG.debug("Got valid challenge for host {}", serverName); - return new String(BASE_64_CODEC.encode(outToken), - StandardCharsets.US_ASCII); - } catch (GSSException | IllegalAccessException - | NoSuchFieldException | ClassNotFoundException e) { - LOG.error("Error: {}", e); - throw new AuthenticationException(e); - } - } - }); - return challenge; - } - /** * Calculate Resource Manager address base on working REST API. */ @@ -177,7 +121,7 @@ String getRMWebAddress() { .resource(sb.toString()).type(MediaType.APPLICATION_JSON); if (useKerberos) { String[] server = host.split(":"); - String challenge = generateToken(server[0]); + String challenge = YarnClientUtils.generateToken(server[0]); builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); LOG.debug("Authorization: Negotiate {}", challenge); @@ -289,7 +233,7 @@ private Builder getApiClient(String requestPath) if (conf.get("hadoop.http.authentication.type").equals("kerberos")) { try { URI url = new URI(requestPath); - String challenge = generateToken(url.getHost()); + String challenge = YarnClientUtils.generateToken(url.getHost()); builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " + challenge); } catch (Exception e) { throw new IOException(e); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java index f95506464d..1ec8d41bba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java @@ -41,6 +41,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.log4j.Logger; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; @@ -169,8 +170,7 @@ public void tearDown() throws Exception { public void testHttpSpnegoChallenge() throws Exception { UserGroupInformation.loginUserFromKeytab(clientPrincipal, keytabFile .getCanonicalPath()); - asc = new ApiServiceClient(); - String challenge = asc.generateToken("localhost"); + String challenge = YarnClientUtils.generateToken("localhost"); assertNotNull(challenge); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml index 2e0c777ee1..dd672c2ee7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml @@ -54,6 +54,10 @@ log4j log4j + + org.eclipse.jetty.websocket + websocket-client + @@ -127,6 +131,10 @@ test-jar + + org.jline + jline + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java new file mode 100644 index 0000000000..4b7b2acfdd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ContainerShellWebSocket.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.jline.terminal.Terminal; +import org.jline.terminal.TerminalBuilder; +import org.jline.reader.LineReader; +import org.jline.reader.LineReaderBuilder; +import org.jline.reader.impl.LineReaderImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Web socket for establishing interactive command shell connection through + * Node Manage to container executor. + */ +@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce", "YARN" }) +@InterfaceStability.Unstable + +@WebSocket +public class ContainerShellWebSocket { + private static final Logger LOG = + LoggerFactory.getLogger(ContainerShellWebSocket.class); + + private Session mySession; + private Terminal terminal; + private LineReader reader; + + @OnWebSocketMessage + public void onText(Session session, String message) throws IOException { + terminal.output().write(message.getBytes(Charset.forName("UTF-8"))); + terminal.output().flush(); + } + + @OnWebSocketConnect + public void onConnect(Session s) { + initTerminal(s); + LOG.info(s.getRemoteAddress().getHostString() + " connected!"); + } + + @OnWebSocketClose + public void onClose(Session session, int status, String reason) { + if (status==1000) { + LOG.info(session.getRemoteAddress().getHostString() + + " closed, status: " + status); + } else { + LOG.warn(session.getRemoteAddress().getHostString() + + " closed, status: " + status + " Reason: " + reason); + } + } + + public void run() { + try { + Reader consoleReader = new Reader(); + Thread inputThread = new Thread(consoleReader, "consoleReader"); + inputThread.start(); + while (mySession.isOpen()) { + mySession.getRemote().flush(); + if (consoleReader.hasData()) { + String message = consoleReader.read(); + mySession.getRemote().sendString(message); + mySession.getRemote().sendString("\r"); + } + String message = "1{}"; + mySession.getRemote().sendString(message); + Thread.sleep(100); + mySession.getRemote().flush(); + } + inputThread.join(); + } catch (IOException | InterruptedException e) { + try { + mySession.disconnect(); + } catch (IOException e1) { + LOG.error("Error closing connection: ", e1); + } + } + } + + protected void initTerminal(final Session session) { + try { + this.mySession = session; + try { + terminal = TerminalBuilder.builder() + .system(true) + .build(); + } catch (IOException t) { + terminal = TerminalBuilder.builder() + .system(false) + .streams(System.in, (OutputStream) System.out) + .build(); + } + reader = LineReaderBuilder.builder() + .terminal(terminal) + .build(); + } catch (IOException e) { + session.close(1002, e.getMessage()); + } + } + + class Reader implements Runnable { + private StringBuilder sb = new StringBuilder(); + private boolean hasData = false; + + public String read() { + try { + return sb.toString(); + } finally { + hasData = false; + sb.setLength(0); + } + } + + public boolean hasData() { + return hasData; + } + + @Override + public void run() { + while (true) { + int c = ((LineReaderImpl) reader).readCharacter(); + if (c == 10 || c == 13) { + hasData = true; + continue; + } + sb.append(new String(Character.toChars(c))); + } + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 59fa6a8f2f..b59831ebe8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -958,4 +959,18 @@ List> getAttributesToNodes( public abstract Map> getNodeToAttributes( Set hostNames) throws YarnException, IOException; + /** + *

+ * The interface used by client to get a shell to a container. + *

+ * + * @param containerId Container ID + * @param command Shell type + * @throws IOException if connection fails. + */ + @Public + @Unstable + public abstract void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException; + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 227f7ed70a..3ec371c779 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.client.api.impl; import java.io.IOException; +import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.Future; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -111,15 +113,18 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.client.api.ContainerShellWebSocket; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; @@ -132,6 +137,10 @@ import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; +import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; +import org.eclipse.jetty.websocket.client.WebSocketClient; import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; @@ -1074,4 +1083,53 @@ public Map> getNodeToAttributes( GetNodesToAttributesRequest.newInstance(hostNames); return rmClient.getNodesToAttributes(request).getNodeToAttributes(); } + + @Override + public void shellToContainer(ContainerId containerId, + ShellContainerCommand command) throws IOException { + try { + GetContainerReportRequest request = Records + .newRecord(GetContainerReportRequest.class); + request.setContainerId(containerId); + GetContainerReportResponse response = rmClient + .getContainerReport(request); + URI nodeHttpAddress = new URI(response.getContainerReport() + .getNodeHttpAddress()); + String host = nodeHttpAddress.getHost(); + int port = nodeHttpAddress.getPort(); + String scheme = nodeHttpAddress.getScheme(); + String protocol = "ws://"; + if (scheme.equals("https")) { + protocol = "wss://"; + } + WebSocketClient client = new WebSocketClient(); + URI uri = URI.create(protocol + host + ":" + port + "/container/" + + containerId); + try { + client.start(); + // The socket that receives events + ContainerShellWebSocket socket = new ContainerShellWebSocket(); + ClientUpgradeRequest upgradeRequest = new ClientUpgradeRequest(); + if (UserGroupInformation.isSecurityEnabled()) { + String challenge = YarnClientUtils.generateToken(host); + upgradeRequest.setHeader("Authorization", "Negotiate " + challenge); + } + // Attempt Connect + Future fut = client.connect(socket, uri, upgradeRequest); + // Wait for Connect + Session session = fut.get(); + // Send a message + session.getRemote().sendString("stty -echo"); + session.getRemote().sendString("\r"); + session.getRemote().flush(); + socket.run(); + } finally { + client.stop(); + } + } catch (WebSocketException e) { + LOG.debug("Websocket exception: " + e.getMessage()); + } catch (Throwable t) { + LOG.error("Fail to shell to container: " + t.getMessage()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index 480ea231fb..2b3415413c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ShellContainerCommand; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.AppAdminClient; @@ -111,6 +112,7 @@ public class ApplicationCLI extends YarnCLI { public static final String COMPONENTS = "components"; public static final String VERSION = "version"; public static final String STATES = "states"; + public static final String SHELL_CMD = "shell"; private static String firstArg = null; @@ -311,6 +313,8 @@ public int run(String[] args) throws Exception { opts.getOption(LIST_CMD).setArgName("Application ID"); opts.getOption(FAIL_CMD).setArgName("Application Attempt ID"); } else if (title != null && title.equalsIgnoreCase(CONTAINER)) { + opts.addOption(SHELL_CMD, true, + "Run a shell in the container."); opts.addOption(STATUS_CMD, true, "Prints the status of the container."); opts.addOption(LIST_CMD, true, @@ -323,6 +327,7 @@ public int run(String[] args) throws Exception { "app version, -components to filter instances based on component " + "names, -states to filter instances based on instance state."); opts.addOption(HELP_CMD, false, "Displays help for all commands."); + opts.getOption(SHELL_CMD).setArgName("Container ID"); opts.getOption(STATUS_CMD).setArgName("Container ID"); opts.getOption(LIST_CMD).setArgName("Application Name or Attempt ID"); opts.addOption(APP_TYPE_CMD, true, "Works with -list to " + @@ -552,6 +557,19 @@ public int run(String[] args) throws Exception { command = SignalContainerCommand.valueOf(signalArgs[1]); } signalToContainer(containerId, command); + } else if (cliParser.hasOption(SHELL_CMD)) { + if (hasAnyOtherCLIOptions(cliParser, opts, SHELL_CMD)) { + printUsage(title, opts); + return exitCode; + } + final String[] shellArgs = cliParser.getOptionValues(SHELL_CMD); + final String containerId = shellArgs[0]; + ShellContainerCommand command = + ShellContainerCommand.BASH; + if (shellArgs.length == 2) { + command = ShellContainerCommand.valueOf(shellArgs[1]); + } + shellToContainer(containerId, command); } else if (cliParser.hasOption(LAUNCH_CMD)) { if (hasAnyOtherCLIOptions(cliParser, opts, LAUNCH_CMD, APP_TYPE_CMD, UPDATE_LIFETIME, CHANGE_APPLICATION_QUEUE)) { @@ -806,7 +824,7 @@ private void updateApplicationTimeout(String applicationId, } /** - * Signals the containerId + * Signals the containerId. * * @param containerIdStr the container id * @param command the signal command @@ -819,6 +837,20 @@ private void signalToContainer(String containerIdStr, client.signalToContainer(containerId, command); } + /** + * Shell to the containerId. + * + * @param containerIdStr the container id + * @param command the shell command + * @throws YarnException + */ + private void shellToContainer(String containerIdStr, + ShellContainerCommand command) throws YarnException, IOException { + ContainerId containerId = ContainerId.fromString(containerIdStr); + sysout.println("Shelling to container " + containerIdStr); + client.shellToContainer(containerId, command); + } + /** * It prints the usage of the command * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java index 17176752f2..abed6c6a30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java @@ -19,15 +19,29 @@ import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; import com.google.common.collect.ImmutableSet; + +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSException; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class is a container for utility methods that are useful when creating @@ -35,6 +49,9 @@ */ public abstract class YarnClientUtils { + private static final Logger LOG = + LoggerFactory.getLogger(YarnClientUtils.class); + private static final Base64 BASE_64_CODEC = new Base64(0); private static final String ADD_LABEL_FORMAT_ERR_MSG = "Input format for adding node-labels is not correct, it should be " + "labelName1[(exclusive=true/false)],LabelName2[] .."; @@ -187,4 +204,54 @@ static YarnConfiguration getYarnConfWithRmHaId(Configuration conf) return yarnConf; } + + /** + * Generate SPNEGO challenge request token. + * + * @param server - hostname to contact + * @throws IOException thrown if doAs failed + * @throws InterruptedException thrown if doAs is interrupted + * @return SPNEGO token challenge + */ + public static String generateToken(String server) throws IOException, + InterruptedException { + UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); + LOG.debug("The user credential is {}", currentUser); + String challenge = currentUser + .doAs(new PrivilegedExceptionAction() { + @Override + public String run() throws Exception { + try { + // This Oid for Kerberos GSS-API mechanism. + Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID"); + GSSManager manager = GSSManager.getInstance(); + // GSS name for server + GSSName serverName = manager.createName("HTTP@" + server, + GSSName.NT_HOSTBASED_SERVICE); + // Create a GSSContext for authentication with the service. + // We're passing client credentials as null since we want them to + // be read from the Subject. + GSSContext gssContext = manager.createContext( + serverName.canonicalize(mechOid), mechOid, null, + GSSContext.DEFAULT_LIFETIME); + gssContext.requestMutualAuth(true); + gssContext.requestCredDeleg(true); + // Establish context + byte[] inToken = new byte[0]; + byte[] outToken = gssContext.initSecContext(inToken, 0, + inToken.length); + gssContext.dispose(); + // Base64 encoded and stringified token for server + LOG.debug("Got valid challenge for host {}", serverName); + return new String(BASE_64_CODEC.encode(outToken), + StandardCharsets.US_ASCII); + } catch (GSSException | IllegalAccessException + | NoSuchFieldException | ClassNotFoundException e) { + LOG.error("Error: {}", e); + throw new AuthenticationException(e); + } + } + }); + return challenge; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 9b1e86378b..672e3d7e43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -2317,6 +2317,7 @@ private String createContainerCLIHelpMessage() throws IOException { pw.println(" -components Works with -list to filter instances based on input comma-separated list of component names."); pw.println(" -help Displays help for all commands."); pw.println(" -list List containers for application attempt when application attempt ID is provided. When application name is provided, then it finds the instances of the application based on app's own implementation, and -appTypes option must be specified unless it is the default yarn-service type. With app name, it supports optional use of -version to filter instances based on app version, -components to filter instances based on component names, -states to filter instances based on instance state."); + pw.println(" -shell Run a shell in the container."); pw.println(" -signal Signal the container."); pw.println("The available signal commands are "); pw.println(java.util.Arrays.asList(SignalContainerCommand.values()));