YARN-9606. Set sslfactory for AuthenticatedURL() while creating LogsCLI#webServiceClient.
Contributed by Bilwa S T. Reviewed by Akira Ajisaka.
This commit is contained in:
parent
1983eea62d
commit
cef0756929
@ -27,17 +27,13 @@
|
|||||||
import com.sun.jersey.api.client.WebResource;
|
import com.sun.jersey.api.client.WebResource;
|
||||||
import com.sun.jersey.api.client.WebResource.Builder;
|
import com.sun.jersey.api.client.WebResource.Builder;
|
||||||
import com.sun.jersey.api.client.filter.ClientFilter;
|
import com.sun.jersey.api.client.filter.ClientFilter;
|
||||||
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
|
|
||||||
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.PrintStream;
|
import java.io.PrintStream;
|
||||||
import java.net.ConnectException;
|
import java.net.ConnectException;
|
||||||
import java.net.HttpURLConnection;
|
|
||||||
import java.net.SocketException;
|
import java.net.SocketException;
|
||||||
import java.net.SocketTimeoutException;
|
import java.net.SocketTimeoutException;
|
||||||
import java.net.URL;
|
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
@ -68,8 +64,6 @@
|
|||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
|
|
||||||
import org.apache.hadoop.security.authentication.client.AuthenticationException;
|
|
||||||
import org.apache.hadoop.util.Tool;
|
import org.apache.hadoop.util.Tool;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
@ -89,6 +83,7 @@
|
|||||||
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
|
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
|
||||||
import org.apache.hadoop.yarn.util.Apps;
|
import org.apache.hadoop.yarn.util.Apps;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
|
||||||
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
|
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
|
||||||
import org.codehaus.jettison.json.JSONArray;
|
import org.codehaus.jettison.json.JSONArray;
|
||||||
import org.codehaus.jettison.json.JSONException;
|
import org.codehaus.jettison.json.JSONException;
|
||||||
@ -139,21 +134,7 @@ public class LogsCLI extends Configured implements Tool {
|
|||||||
@Override
|
@Override
|
||||||
public int run(String[] args) throws Exception {
|
public int run(String[] args) throws Exception {
|
||||||
try {
|
try {
|
||||||
webServiceClient = new Client(new URLConnectionClientHandler(
|
webServiceClient = WebServiceClient.getWebServiceClient().createClient();
|
||||||
new HttpURLConnectionFactory() {
|
|
||||||
@Override
|
|
||||||
public HttpURLConnection getHttpURLConnection(URL url)
|
|
||||||
throws IOException {
|
|
||||||
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
|
|
||||||
HttpURLConnection conn = null;
|
|
||||||
try {
|
|
||||||
conn = new AuthenticatedURL().openConnection(url, token);
|
|
||||||
} catch (AuthenticationException e) {
|
|
||||||
throw new IOException(e);
|
|
||||||
}
|
|
||||||
return conn;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
return runCommand(args);
|
return runCommand(args);
|
||||||
} finally {
|
} finally {
|
||||||
if (yarnClient != null) {
|
if (yarnClient != null) {
|
||||||
@ -411,7 +392,9 @@ public static void main(String[] args) throws Exception {
|
|||||||
Configuration conf = new YarnConfiguration();
|
Configuration conf = new YarnConfiguration();
|
||||||
LogsCLI logDumper = new LogsCLI();
|
LogsCLI logDumper = new LogsCLI();
|
||||||
logDumper.setConf(conf);
|
logDumper.setConf(conf);
|
||||||
|
WebServiceClient.initialize(conf);
|
||||||
int exitCode = logDumper.run(args);
|
int exitCode = logDumper.run(args);
|
||||||
|
WebServiceClient.destroy();
|
||||||
System.exit(exitCode);
|
System.exit(exitCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -89,8 +89,10 @@
|
|||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext;
|
||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
|
||||||
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
|
import org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
|
||||||
import org.codehaus.jettison.json.JSONArray;
|
import org.codehaus.jettison.json.JSONArray;
|
||||||
import org.codehaus.jettison.json.JSONObject;
|
import org.codehaus.jettison.json.JSONObject;
|
||||||
|
import org.junit.After;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
@ -109,8 +111,10 @@ public class TestLogsCLI {
|
|||||||
ByteArrayOutputStream sysErrStream;
|
ByteArrayOutputStream sysErrStream;
|
||||||
private PrintStream sysErr;
|
private PrintStream sysErr;
|
||||||
|
|
||||||
|
private Configuration conf;
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() {
|
public void setUp() throws Exception {
|
||||||
sysOutStream = new ByteArrayOutputStream();
|
sysOutStream = new ByteArrayOutputStream();
|
||||||
sysOut = new PrintStream(sysOutStream);
|
sysOut = new PrintStream(sysOutStream);
|
||||||
System.setOut(sysOut);
|
System.setOut(sysOut);
|
||||||
@ -118,11 +122,17 @@ public void setUp() {
|
|||||||
sysErrStream = new ByteArrayOutputStream();
|
sysErrStream = new ByteArrayOutputStream();
|
||||||
sysErr = new PrintStream(sysErrStream);
|
sysErr = new PrintStream(sysErrStream);
|
||||||
System.setErr(sysErr);
|
System.setErr(sysErr);
|
||||||
|
conf = new YarnConfiguration();
|
||||||
|
WebServiceClient.initialize(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@After
|
||||||
|
public void tearDown() {
|
||||||
|
WebServiceClient.destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(timeout = 5000l)
|
@Test(timeout = 5000l)
|
||||||
public void testFailResultCodes() throws Exception {
|
public void testFailResultCodes() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
conf.setClass("fs.file.impl", LocalFileSystem.class, FileSystem.class);
|
||||||
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
LogCLIHelpers cliHelper = new LogCLIHelpers();
|
||||||
cliHelper.setConf(conf);
|
cliHelper.setConf(conf);
|
||||||
@ -145,7 +155,6 @@ public void testFailResultCodes() throws Exception {
|
|||||||
|
|
||||||
@Test(timeout = 10000l)
|
@Test(timeout = 10000l)
|
||||||
public void testInvalidOpts() throws Exception {
|
public void testInvalidOpts() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
YarnClient mockYarnClient = createMockYarnClient(
|
YarnClient mockYarnClient = createMockYarnClient(
|
||||||
YarnApplicationState.FINISHED,
|
YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
@ -160,7 +169,6 @@ public void testInvalidOpts() throws Exception {
|
|||||||
|
|
||||||
@Test(timeout = 5000l)
|
@Test(timeout = 5000l)
|
||||||
public void testInvalidApplicationId() throws Exception {
|
public void testInvalidApplicationId() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
YarnClient mockYarnClient = createMockYarnClient(
|
YarnClient mockYarnClient = createMockYarnClient(
|
||||||
YarnApplicationState.FINISHED,
|
YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
@ -174,7 +182,6 @@ public void testInvalidApplicationId() throws Exception {
|
|||||||
|
|
||||||
@Test(timeout = 5000L)
|
@Test(timeout = 5000L)
|
||||||
public void testInvalidAMContainerId() throws Exception {
|
public void testInvalidAMContainerId() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
@ -196,7 +203,6 @@ public void testInvalidAMContainerId() throws Exception {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
|
public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
|
||||||
conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f");
|
conf.set(YarnConfiguration.TIMELINE_SERVICE_VERSIONS, "2.0f");
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
@ -232,7 +238,6 @@ public void testAMContainerInfoFetchFromTimelineReader() throws Exception {
|
|||||||
|
|
||||||
@Test(timeout = 5000l)
|
@Test(timeout = 5000l)
|
||||||
public void testUnknownApplicationId() throws Exception {
|
public void testUnknownApplicationId() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
YarnClient mockYarnClient = createMockYarnClientUnknownApp();
|
YarnClient mockYarnClient = createMockYarnClientUnknownApp();
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(conf);
|
cli.setConf(conf);
|
||||||
@ -248,7 +253,6 @@ public void testUnknownApplicationId() throws Exception {
|
|||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testHelpMessage() throws Exception {
|
public void testHelpMessage() throws Exception {
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
YarnClient mockYarnClient = createMockYarnClient(
|
YarnClient mockYarnClient = createMockYarnClient(
|
||||||
YarnApplicationState.FINISHED,
|
YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
@ -372,13 +376,12 @@ public void testHelpMessage() throws Exception {
|
|||||||
@Test (timeout = 15000)
|
@Test (timeout = 15000)
|
||||||
public void testFetchFinishedApplictionLogs() throws Exception {
|
public void testFetchFinishedApplictionLogs() throws Exception {
|
||||||
String remoteLogRootDir = "target/logs/";
|
String remoteLogRootDir = "target/logs/";
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
|
|
||||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
@ -431,15 +434,15 @@ public void testFetchFinishedApplictionLogs() throws Exception {
|
|||||||
// the first two logs is empty. When we try to read first two logs,
|
// the first two logs is empty. When we try to read first two logs,
|
||||||
// we will meet EOF exception, but it will not impact other logs.
|
// we will meet EOF exception, but it will not impact other logs.
|
||||||
// Other logs should be read successfully.
|
// Other logs should be read successfully.
|
||||||
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
||||||
containerId0, path, fs);
|
containerId0, path, fs);
|
||||||
uploadEmptyContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
uploadEmptyContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
||||||
containerId1, path, fs);
|
containerId1, path, fs);
|
||||||
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
||||||
containerId1, path, fs);
|
containerId1, path, fs);
|
||||||
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
||||||
containerId2, path, fs);
|
containerId2, path, fs);
|
||||||
uploadContainerLogIntoRemoteDir(ugi, configuration, rootLogDirs, nodeId,
|
uploadContainerLogIntoRemoteDir(ugi, conf, rootLogDirs, nodeId,
|
||||||
containerId3, path, fs);
|
containerId3, path, fs);
|
||||||
|
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
@ -455,7 +458,7 @@ public ContainerReport getContainerReport(String containerIdStr)
|
|||||||
return mockReport;
|
return mockReport;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
|
int exitCode = cli.run(new String[] { "-applicationId", appId.toString() });
|
||||||
LOG.info(sysOutStream.toString());
|
LOG.info(sysOutStream.toString());
|
||||||
@ -681,7 +684,7 @@ public ContainerReport getContainerReport(String containerIdStr)
|
|||||||
YarnClient mockYarnClientWithException =
|
YarnClient mockYarnClientWithException =
|
||||||
createMockYarnClientWithException();
|
createMockYarnClientWithException();
|
||||||
cli = new LogsCLIForTest(mockYarnClientWithException);
|
cli = new LogsCLIForTest(mockYarnClientWithException);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
exitCode =
|
exitCode =
|
||||||
cli.run(new String[] { "-applicationId", appId.toString(),
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
||||||
@ -804,8 +807,7 @@ public void testGetRunningContainerLogs() throws Exception {
|
|||||||
any(ContainerId.class));
|
any(ContainerId.class));
|
||||||
|
|
||||||
// create local logs
|
// create local logs
|
||||||
Configuration configuration = new YarnConfiguration();
|
FileSystem fs = FileSystem.get(conf);
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
|
||||||
String rootLogDir = "target/LocalLogs";
|
String rootLogDir = "target/LocalLogs";
|
||||||
Path rootLogDirPath = new Path(rootLogDir);
|
Path rootLogDirPath = new Path(rootLogDir);
|
||||||
if (fs.exists(rootLogDirPath)) {
|
if (fs.exists(rootLogDirPath)) {
|
||||||
@ -964,13 +966,12 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
|||||||
UserGroupInformation testUgi = UserGroupInformation
|
UserGroupInformation testUgi = UserGroupInformation
|
||||||
.createRemoteUser(testUser);
|
.createRemoteUser(testUser);
|
||||||
|
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
@ -1010,13 +1011,13 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
|||||||
assertTrue(fs.mkdirs(path));
|
assertTrue(fs.mkdirs(path));
|
||||||
|
|
||||||
// upload container logs for app into remote dir
|
// upload container logs for app into remote dir
|
||||||
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
|
||||||
nodeId, containerId, path, fs);
|
nodeId, containerId, path, fs);
|
||||||
|
|
||||||
YarnClient mockYarnClient = createMockYarnClient(
|
YarnClient mockYarnClient = createMockYarnClient(
|
||||||
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
YarnApplicationState.FINISHED, testUgi.getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
// Verify that we can get the application logs by specifying
|
// Verify that we can get the application logs by specifying
|
||||||
// a correct appOwner
|
// a correct appOwner
|
||||||
@ -1043,7 +1044,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
|||||||
// and can get app logs successfully.
|
// and can get app logs successfully.
|
||||||
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
YarnClient mockYarnClient2 = createMockYarnClientUnknownApp();
|
||||||
cli = new LogsCLIForTest(mockYarnClient2);
|
cli = new LogsCLIForTest(mockYarnClient2);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
exitCode = cli.run(new String[] {
|
exitCode = cli.run(new String[] {
|
||||||
"-applicationId", appId.toString()});
|
"-applicationId", appId.toString()});
|
||||||
assertTrue(exitCode == 0);
|
assertTrue(exitCode == 0);
|
||||||
@ -1057,7 +1058,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
|||||||
fs.delete(path, true);
|
fs.delete(path, true);
|
||||||
}
|
}
|
||||||
assertTrue(fs.mkdirs(path));
|
assertTrue(fs.mkdirs(path));
|
||||||
uploadContainerLogIntoRemoteDir(testUgi, configuration, rootLogDirs,
|
uploadContainerLogIntoRemoteDir(testUgi, conf, rootLogDirs,
|
||||||
nodeId, containerId, path, fs);
|
nodeId, containerId, path, fs);
|
||||||
|
|
||||||
exitCode = cli.run(new String[] {
|
exitCode = cli.run(new String[] {
|
||||||
@ -1112,14 +1113,13 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception {
|
|||||||
public void testLogsCLIWithInvalidArgs() throws Exception {
|
public void testLogsCLIWithInvalidArgs() throws Exception {
|
||||||
String localDir = "target/SaveLogs";
|
String localDir = "target/SaveLogs";
|
||||||
Path localPath = new Path(localDir);
|
Path localPath = new Path(localDir);
|
||||||
Configuration configuration = new YarnConfiguration();
|
FileSystem fs = FileSystem.get(conf);
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
// Specify an invalid applicationId
|
// Specify an invalid applicationId
|
||||||
int exitCode = cli.run(new String[] {"-applicationId",
|
int exitCode = cli.run(new String[] {"-applicationId",
|
||||||
@ -1180,7 +1180,6 @@ public void testGuessAppOwnerWithCustomSuffix() throws Exception {
|
|||||||
String remoteLogRootDir = "target/logs/";
|
String remoteLogRootDir = "target/logs/";
|
||||||
String jobUser = "user1";
|
String jobUser = "user1";
|
||||||
String loggedUser = "user2";
|
String loggedUser = "user2";
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
@ -1226,7 +1225,6 @@ public void testGuessAppOwnerWithCustomAppLogDir() throws Exception {
|
|||||||
String remoteLogRootDir1 = "target/logs1/";
|
String remoteLogRootDir1 = "target/logs1/";
|
||||||
String jobUser = "user1";
|
String jobUser = "user1";
|
||||||
String loggedUser = "user2";
|
String loggedUser = "user2";
|
||||||
Configuration conf = new YarnConfiguration();
|
|
||||||
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
String controllerName = "indexed";
|
String controllerName = "indexed";
|
||||||
@ -1261,14 +1259,13 @@ public void testSaveContainerLogsLocally() throws Exception {
|
|||||||
String localDir = "target/SaveLogs";
|
String localDir = "target/SaveLogs";
|
||||||
Path localPath = new Path(localDir);
|
Path localPath = new Path(localDir);
|
||||||
|
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
|
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
ApplicationAttemptId.newInstance(appId, 1);
|
ApplicationAttemptId.newInstance(appId, 1);
|
||||||
@ -1287,14 +1284,14 @@ public void testSaveContainerLogsLocally() throws Exception {
|
|||||||
nodeIds.add(nodeId2);
|
nodeIds.add(nodeId2);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
||||||
appId, containerIds, nodeIds);
|
appId, containerIds, nodeIds);
|
||||||
|
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
int exitCode = cli.run(new String[] {"-applicationId",
|
int exitCode = cli.run(new String[] {"-applicationId",
|
||||||
appId.toString(),
|
appId.toString(),
|
||||||
"-out" , localPath.toString()});
|
"-out" , localPath.toString()});
|
||||||
@ -1352,13 +1349,12 @@ private String readContainerContent(Path containerPath,
|
|||||||
@Test (timeout = 15000)
|
@Test (timeout = 15000)
|
||||||
public void testPrintContainerLogMetadata() throws Exception {
|
public void testPrintContainerLogMetadata() throws Exception {
|
||||||
String remoteLogRootDir = "target/logs/";
|
String remoteLogRootDir = "target/logs/";
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
String rootLogDir = "target/LocalLogs";
|
String rootLogDir = "target/LocalLogs";
|
||||||
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
@ -1377,14 +1373,14 @@ public void testPrintContainerLogMetadata() throws Exception {
|
|||||||
nodeIds.add(nodeId);
|
nodeIds.add(nodeId);
|
||||||
nodeIds.add(nodeId);
|
nodeIds.add(nodeId);
|
||||||
|
|
||||||
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
||||||
appId, containerIds, nodeIds);
|
appId, containerIds, nodeIds);
|
||||||
|
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
cli.run(new String[] {"-applicationId", appId.toString(),
|
cli.run(new String[] {"-applicationId", appId.toString(),
|
||||||
"-show_container_log_info"});
|
"-show_container_log_info"});
|
||||||
@ -1457,12 +1453,11 @@ public void testPrintContainerLogMetadata() throws Exception {
|
|||||||
@Test (timeout = 15000)
|
@Test (timeout = 15000)
|
||||||
public void testListNodeInfo() throws Exception {
|
public void testListNodeInfo() throws Exception {
|
||||||
String remoteLogRootDir = "target/logs/";
|
String remoteLogRootDir = "target/logs/";
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
|
|
||||||
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
ApplicationId appId = ApplicationId.newInstance(0, 1);
|
||||||
ApplicationAttemptId appAttemptId =
|
ApplicationAttemptId appAttemptId =
|
||||||
@ -1482,16 +1477,16 @@ public void testListNodeInfo() throws Exception {
|
|||||||
nodeIds.add(nodeId2);
|
nodeIds.add(nodeId2);
|
||||||
|
|
||||||
String rootLogDir = "target/LocalLogs";
|
String rootLogDir = "target/LocalLogs";
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
|
|
||||||
createContainerLogs(configuration, remoteLogRootDir, rootLogDir, fs,
|
createContainerLogs(conf, remoteLogRootDir, rootLogDir, fs,
|
||||||
appId, containerIds, nodeIds);
|
appId, containerIds, nodeIds);
|
||||||
|
|
||||||
YarnClient mockYarnClient =
|
YarnClient mockYarnClient =
|
||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
UserGroupInformation.getCurrentUser().getShortUserName());
|
UserGroupInformation.getCurrentUser().getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
|
|
||||||
cli.run(new String[] { "-applicationId", appId.toString(),
|
cli.run(new String[] { "-applicationId", appId.toString(),
|
||||||
"-list_nodes" });
|
"-list_nodes" });
|
||||||
@ -1508,13 +1503,12 @@ public void testListNodeInfo() throws Exception {
|
|||||||
@Test (timeout = 15000)
|
@Test (timeout = 15000)
|
||||||
public void testFetchApplictionLogsHar() throws Exception {
|
public void testFetchApplictionLogsHar() throws Exception {
|
||||||
String remoteLogRootDir = "target/logs/";
|
String remoteLogRootDir = "target/logs/";
|
||||||
Configuration configuration = new YarnConfiguration();
|
conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
||||||
configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
|
conf
|
||||||
configuration
|
|
||||||
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir);
|
||||||
configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
|
||||||
configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
|
||||||
FileSystem fs = FileSystem.get(configuration);
|
FileSystem fs = FileSystem.get(conf);
|
||||||
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
|
||||||
URL harUrl = ClassLoader.getSystemClassLoader()
|
URL harUrl = ClassLoader.getSystemClassLoader()
|
||||||
.getResource("application_1440536969523_0001.har");
|
.getResource("application_1440536969523_0001.har");
|
||||||
@ -1534,7 +1528,7 @@ public void testFetchApplictionLogsHar() throws Exception {
|
|||||||
createMockYarnClient(YarnApplicationState.FINISHED,
|
createMockYarnClient(YarnApplicationState.FINISHED,
|
||||||
ugi.getShortUserName());
|
ugi.getShortUserName());
|
||||||
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
LogsCLI cli = new LogsCLIForTest(mockYarnClient);
|
||||||
cli.setConf(configuration);
|
cli.setConf(conf);
|
||||||
int exitCode = cli.run(new String[]{"-applicationId",
|
int exitCode = cli.run(new String[]{"-applicationId",
|
||||||
"application_1440536969523_0001"});
|
"application_1440536969523_0001"});
|
||||||
assertTrue(exitCode == 0);
|
assertTrue(exitCode == 0);
|
||||||
|
@ -15,7 +15,7 @@
|
|||||||
* See the License for the specific language governing permissions and
|
* See the License for the specific language governing permissions and
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.yarn.server.webapp;
|
package org.apache.hadoop.yarn.webapp.util;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
@ -34,12 +34,11 @@
|
|||||||
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
|
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility for handling Web client.
|
* Utility for handling Web client.
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public class WebServiceClient {
|
public class WebServiceClient {
|
||||||
@VisibleForTesting
|
private static SSLFactory sslFactory = null;
|
||||||
protected static SSLFactory sslFactory = null;
|
|
||||||
private static volatile WebServiceClient instance = null;
|
private static volatile WebServiceClient instance = null;
|
||||||
private static boolean isHttps = false;
|
private static boolean isHttps = false;
|
||||||
|
|
||||||
@ -68,6 +67,11 @@ public static WebServiceClient getWebServiceClient() {
|
|||||||
return instance;
|
return instance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@VisibleForTesting
|
||||||
|
SSLFactory getSSLFactory() {
|
||||||
|
return sslFactory;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start SSL factory.
|
* Start SSL factory.
|
||||||
*
|
*
|
@ -15,7 +15,7 @@
|
|||||||
* See the License for the specific language governing permissions and
|
* See the License for the specific language governing permissions and
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.yarn.server.webapp;
|
package org.apache.hadoop.yarn.webapp.util;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.net.HttpURLConnection;
|
import java.net.HttpURLConnection;
|
||||||
@ -37,7 +37,7 @@ public class TestWebServiceClient {
|
|||||||
private static final String BASEDIR = System.getProperty("test.build.dir",
|
private static final String BASEDIR = System.getProperty("test.build.dir",
|
||||||
"target/test-dir") + "/" + TestWebServiceClient.class.getSimpleName();
|
"target/test-dir") + "/" + TestWebServiceClient.class.getSimpleName();
|
||||||
static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server.keystore";
|
static final String SSL_SERVER_KEYSTORE_PROP_PREFIX = "ssl.server.keystore";
|
||||||
static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX =
|
static final String SSL_SERVER_TRUSTSTORE_PROP_PREFIX =
|
||||||
"ssl.server.truststore";
|
"ssl.server.truststore";
|
||||||
static final String SERVLET_NAME_ECHO = "echo";
|
static final String SERVLET_NAME_ECHO = "echo";
|
||||||
static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
|
static final String SERVLET_PATH_ECHO = "/" + SERVLET_NAME_ECHO;
|
||||||
@ -47,7 +47,8 @@ public void testGetWebServiceClient() throws Exception {
|
|||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
|
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
|
||||||
WebServiceClient.initialize(conf);
|
WebServiceClient.initialize(conf);
|
||||||
Assert.assertNotNull(WebServiceClient.sslFactory);
|
WebServiceClient client = WebServiceClient.getWebServiceClient();
|
||||||
|
Assert.assertNotNull(client.getSSLFactory());
|
||||||
WebServiceClient.destroy();
|
WebServiceClient.destroy();
|
||||||
}
|
}
|
||||||
|
|
@ -36,11 +36,11 @@
|
|||||||
import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService;
|
import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService;
|
||||||
import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService;
|
import org.apache.hadoop.yarn.server.router.rmadmin.RouterRMAdminService;
|
||||||
import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp;
|
import org.apache.hadoop.yarn.server.router.webapp.RouterWebApp;
|
||||||
import org.apache.hadoop.yarn.server.webapp.WebServiceClient;
|
|
||||||
import org.apache.hadoop.yarn.webapp.WebApp;
|
import org.apache.hadoop.yarn.webapp.WebApp;
|
||||||
import org.apache.hadoop.yarn.webapp.WebApps;
|
import org.apache.hadoop.yarn.webapp.WebApps;
|
||||||
import org.apache.hadoop.yarn.webapp.WebApps.Builder;
|
import org.apache.hadoop.yarn.webapp.WebApps.Builder;
|
||||||
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -50,10 +50,10 @@
|
|||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
|
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
|
||||||
import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
|
import org.apache.hadoop.yarn.server.uam.UnmanagedApplicationManager;
|
||||||
import org.apache.hadoop.yarn.server.webapp.WebServiceClient;
|
|
||||||
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
import org.apache.hadoop.yarn.webapp.BadRequestException;
|
||||||
import org.apache.hadoop.yarn.webapp.ForbiddenException;
|
import org.apache.hadoop.yarn.webapp.ForbiddenException;
|
||||||
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
import org.apache.hadoop.yarn.webapp.NotFoundException;
|
||||||
|
import org.apache.hadoop.yarn.webapp.util.WebServiceClient;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user