HDDS-5. Enable OzoneManager kerberos auth. Contributed by Ajay Kumar.

This commit is contained in:
Xiaoyu Yao 2018-05-14 09:36:57 -07:00
parent ff61931f91
commit d3920ecbb7
8 changed files with 254 additions and 53 deletions

View File

@ -1596,7 +1596,23 @@
<name>ozone.scm.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description>The SCM service principal. Ex scm/_HOST@REALM.TLD.</description>
<description>The SCM service principal. Ex scm/_HOST@REALM.COM</description>
</property>
<property>
<name>ozone.om.kerberos.keytab.file</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description> The keytab file used by KSM daemon to login as its
service principal. The principal name is configured with
hdds.ksm.kerberos.principal.
</description>
</property>
<property>
<name>ozone.om.kerberos.principal</name>
<value></value>
<tag> OZONE, SECURITY</tag>
<description>The KSM service principal. Ex ksm/_HOST@REALM.COM</description>
</property>
<property>
@ -1608,4 +1624,18 @@
<value>/etc/security/keytabs/HTTP.keytab</value>
</property>
<property>
<name>ozone.om.http.kerberos.principal</name>
<value>HTTP/_HOST@EXAMPLE.COM</value>
<description>
KSM http server kerberos principal.
</description>
</property>
<property>
<name>ozone.om.http.kerberos.keytab.file</name>
<value>/etc/security/keytabs/HTTP.keytab</value>
<description>
KSM http server kerberos keytab.
</description>
</property>
</configuration>

View File

@ -175,4 +175,13 @@ private OMConfigKeys() {
public static final TimeDuration
OZONE_OM_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(1, TimeUnit.SECONDS);
public static final String OZONE_OM_KERBEROS_KEYTAB_FILE_KEY = "ozone.om."
+ "kerberos.keytab.file";
public static final String OZONE_OM_KERBEROS_PRINCIPAL_KEY = "ozone.om"
+ ".kerberos.principal";
public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE =
"ozone.om.http.kerberos.keytab.file";
public static final String OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY
= "ozone.om.http.kerberos.principal";
}

View File

@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.om.protocol;
import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@ -28,14 +30,19 @@
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol to talk to OM.
*/
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
public interface OzoneManagerProtocol {
/**

View File

@ -18,9 +18,11 @@
package org.apache.hadoop.ozone.om.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService;
import org.apache.hadoop.security.KerberosInfo;
/**
* Protocol used to communicate with OM.
@ -28,6 +30,8 @@
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.OzoneManagerProtocol",
protocolVersion = 1)
@KerberosInfo(
serverPrincipal = OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY)
@InterfaceAudience.Private
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {

View File

@ -469,7 +469,8 @@ private void initializeOmStorage(OMStorage omStorage) throws IOException{
*
* @throws IOException
*/
private OzoneManager createOM() throws IOException {
private OzoneManager createOM()
throws IOException, AuthenticationException {
configureOM();
OMStorage omStore = new OMStorage(conf);
initializeOmStorage(omStore);

View File

@ -22,28 +22,38 @@
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Callable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.security.KerberosAuthException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,13 +66,23 @@ public final class TestSecureOzoneCluster {
private Logger LOGGER = LoggerFactory
.getLogger(TestSecureOzoneCluster.class);
@Rule
public Timeout timeout = new Timeout(80000);
private MiniKdc miniKdc;
private OzoneConfiguration conf;
private File workDir;
private static Properties securityProperties;
private File scmKeytab;
private File spnegoKeytab;
private File omKeyTab;
private String curUser;
private StorageContainerManager scm;
private OzoneManager om;
private static String clusterId;
private static String scmId;
private static String omId;
@Before
public void init() {
@ -71,6 +91,10 @@ public void init() {
startMiniKdc();
setSecureConfig(conf);
createCredentialsInKDC(conf, miniKdc);
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omId = UUID.randomUUID().toString();
} catch (IOException e) {
LOGGER.error("Failed to initialize TestSecureOzoneCluster", e);
} catch (Exception e) {
@ -78,12 +102,31 @@ public void init() {
}
}
@After
public void stop() {
try {
stopMiniKdc();
if (scm != null) {
scm.stop();
}
if (om != null) {
om.stop();
}
} catch (Exception e) {
LOGGER.error("Failed to stop TestSecureOzoneCluster", e);
}
}
private void createCredentialsInKDC(Configuration conf, MiniKdc miniKdc)
throws Exception {
createPrincipal(scmKeytab,
conf.get(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY));
createPrincipal(spnegoKeytab,
conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
conf.get(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY),
conf.get(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
createPrincipal(omKeyTab,
conf.get(OMConfigKeys
.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY));
}
private void createPrincipal(File keytab, String... principal)
@ -99,9 +142,13 @@ private void startMiniKdc() throws Exception {
miniKdc.start();
}
private void stopMiniKdc() throws Exception {
miniKdc.stop();
}
private void setSecureConfig(Configuration conf) throws IOException {
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
String host = KerberosUtil.getLocalHostName();
String host = InetAddress.getLocalHost().getCanonicalHostName();
String realm = miniKdc.getRealm();
curUser = UserGroupInformation.getCurrentUser()
.getUserName();
@ -114,59 +161,56 @@ private void setSecureConfig(Configuration conf) throws IOException {
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_SCM/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"om/" + host + "@" + realm);
conf.set(OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
"HTTP_KSM/" + host + "@" + realm);
scmKeytab = new File(workDir, "scm.keytab");
spnegoKeytab = new File(workDir, "http.keytab");
omKeyTab = new File(workDir, "om.keytab");
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
scmKeytab.getAbsolutePath());
conf.set(ScmConfigKeys.SCM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE_KEY,
spnegoKeytab.getAbsolutePath());
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
omKeyTab.getAbsolutePath());
}
@Test
public void testSecureScmStartupSuccess() throws Exception {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
SCMStorage scmStore = new SCMStorage(conf);
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
StorageContainerManager scm = StorageContainerManager.createSCM(null, conf);
initSCM();
scm = StorageContainerManager.createSCM(null, conf);
//Reads the SCM Info from SCM instance
ScmInfo scmInfo = scm.getClientProtocolServer().getScmInfo();
Assert.assertEquals(clusterId, scmInfo.getClusterId());
Assert.assertEquals(scmId, scmInfo.getScmId());
}
@Test
public void testSecureScmStartupFailure() throws Exception {
private void initSCM()
throws IOException, AuthenticationException {
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path scmPath = Paths.get(path, "scm-meta");
OzoneConfiguration conf = new OzoneConfiguration();
conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, scmPath.toString());
conf.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_PRINCIPAL_KEY,
"scm@" + miniKdc.getRealm());
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
SCMStorage scmStore = new SCMStorage(conf);
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
scmStore.setClusterId(clusterId);
scmStore.setScmId(scmId);
// writes the version file properties
scmStore.initialize();
}
@Test
public void testSecureScmStartupFailure() throws Exception {
initSCM();
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY, "");
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
LambdaTestUtils.intercept(IOException.class,
"Running in secure mode, but config doesn't have a keytab",
() -> {
@ -178,28 +222,82 @@ public void testSecureScmStartupFailure() throws Exception {
conf.set(ScmConfigKeys.OZONE_SCM_KERBEROS_KEYTAB_FILE_KEY,
"/etc/security/keytabs/scm.keytab");
testCommonKerberosFailures(
() -> StorageContainerManager.createSCM(null, conf));
}
private void testCommonKerberosFailures(Callable callable) throws Exception {
LambdaTestUtils.intercept(KerberosAuthException.class, "failure "
+ "to login: for principal:",
() -> {
StorageContainerManager.createSCM(null, conf);
});
+ "to login: for principal:", callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"OAuth2");
LambdaTestUtils.intercept(IllegalArgumentException.class, "Invalid"
+ " attribute value for hadoop.security.authentication of OAuth2",
() -> {
StorageContainerManager.createSCM(null, conf);
});
callable);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"KERBEROS_SSL");
LambdaTestUtils.intercept(AuthenticationException.class,
"KERBEROS_SSL authentication method not support.",
() -> {
StorageContainerManager.createSCM(null, conf);
});
"KERBEROS_SSL authentication method not",
callable);
}
/**
* Tests the secure KSM Initialization Failure.
*
* @throws IOException
*/
@Test
public void testSecureKsmInitializationFailure() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
OMStorage ksmStore = new OMStorage(conf);
ksmStore.setClusterId("testClusterId");
ksmStore.setScmId("testScmId");
// writes the version file properties
ksmStore.initialize();
conf.set(OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY,
"non-existent-user@EXAMPLE.com");
testCommonKerberosFailures(() -> OzoneManager.createOm(null, conf));
}
/**
* Tests the secure KSM Initialization success.
*
* @throws IOException
*/
@Test
public void testSecureKsmInitializationSuccess() throws Exception {
initSCM();
// Create a secure SCM instance as om client will connect to it
scm = StorageContainerManager.createSCM(null, conf);
LogCapturer logs = LogCapturer.captureLogs(OzoneManager.LOG);
GenericTestUtils
.setLogLevel(LoggerFactory.getLogger(OzoneManager.class.getName()),
org.slf4j.event.Level.INFO);
final String path = GenericTestUtils
.getTempPath(UUID.randomUUID().toString());
Path metaDirPath = Paths.get(path, "om-meta");
OMStorage omStore = new OMStorage(conf);
omStore.setClusterId("testClusterId");
omStore.setScmId("testScmId");
// writes the version file properties
omStore.initialize();
try {
om = OzoneManager.createOm(null, conf);
} catch (Exception ex) {
// Expects timeout failure from scmClient in KSM but KSM user login via
// kerberos should succeed
Assert.assertTrue(logs.getOutput().contains("KSM login successful"));
}
}
}

View File

@ -53,6 +53,7 @@
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.common.Storage.StorageState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
@ -86,7 +87,10 @@
import org.apache.hadoop.ozone.security.acl.OzoneObj.ResourceType;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import org.apache.hadoop.ozone.security.acl.RequestContext;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.JvmPauseMonitor;
import org.apache.hadoop.util.ReflectionUtils;
@ -97,6 +101,7 @@
import org.slf4j.LoggerFactory;
import javax.management.ObjectName;
import javax.ws.rs.HEAD;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
@ -140,6 +145,11 @@
import static org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService
.newReflectiveBlockingService;
import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneManagerService.newReflectiveBlockingService;
import static org.apache.hadoop.ozone.om.OMConfigKeys
.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys
.OZONE_OM_KERBEROS_PRINCIPAL_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
/**
@ -148,7 +158,7 @@
@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
public final class OzoneManager extends ServiceRuntimeInfoImpl
implements OzoneManagerProtocol, OMMXBean, Auditor {
private static final Logger LOG =
public static final Logger LOG =
LoggerFactory.getLogger(OzoneManager.class);
private static final AuditLogger AUDIT =
@ -193,14 +203,16 @@ private OzoneManager(OzoneConfiguration conf) throws IOException {
configuration = conf;
omStorage = new OMStorage(conf);
omId = omStorage.getOmId();
scmBlockClient = getScmBlockClient(configuration);
scmContainerClient = getScmContainerClient(configuration);
if (omStorage.getState() != StorageState.INITIALIZED) {
throw new OMException("OM not initialized.",
ResultCodes.OM_NOT_INITIALIZED);
}
scmContainerClient = getScmContainerClient(configuration);
// verifies that the SCM info in the OM Version file is correct.
scmBlockClient = getScmBlockClient(configuration);
ScmInfo scmInfo = scmBlockClient.getScmInfo();
if (!(scmInfo.getClusterId().equals(omStorage.getClusterID()) && scmInfo
.getScmId().equals(omStorage.getScmId()))) {
@ -301,6 +313,35 @@ private File getMetricsStorageFile() {
}
/**
* Login KSM service user if security and Kerberos are enabled.
*
* @param conf
* @throws IOException, AuthenticationException
*/
private static void loginKSMUser(OzoneConfiguration conf)
throws IOException, AuthenticationException {
if (SecurityUtil.getAuthenticationMethod(conf).equals
(AuthenticationMethod.KERBEROS)) {
LOG.debug("Ozone security is enabled. Attempting login for KSM user. "
+ "Principal: {},keytab: {}", conf.get
(OZONE_OM_KERBEROS_PRINCIPAL_KEY),
conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY));
UserGroupInformation.setConfiguration(conf);
InetSocketAddress socAddr = getOmAddress(conf);
SecurityUtil.login(conf, OZONE_OM_KERBEROS_KEYTAB_FILE_KEY,
OZONE_OM_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName());
} else {
throw new AuthenticationException(SecurityUtil.getAuthenticationMethod
(conf) + " authentication method not supported. KSM user login "
+ "failed.");
}
LOG.info("KSM login successful.");
}
/**
* Create a scm block client, used by putKey() and getKey().
*
@ -418,15 +459,16 @@ private static void printUsage(PrintStream out) {
* @param argv Command line arguments
* @param conf OzoneConfiguration
* @return OM instance
* @throws IOException in case OM instance creation fails.
* @throws IOException, AuthenticationException in case OM instance
* creation fails.
*/
@VisibleForTesting
public static OzoneManager createOm(
String[] argv, OzoneConfiguration conf) throws IOException {
String[] argv, OzoneConfiguration conf)
throws IOException, AuthenticationException {
return createOm(argv, conf, false);
}
/**
* Constructs OM instance based on command line arguments.
*
@ -434,10 +476,12 @@ public static OzoneManager createOm(
* @param conf OzoneConfiguration
* @param printBanner if true then log a verbose startup message.
* @return OM instance
* @throws IOException in case OM instance creation fails.
* @throws IOException, AuthenticationException in case OM instance
* creation fails.
*/
private static OzoneManager createOm(String[] argv,
OzoneConfiguration conf, boolean printBanner) throws IOException {
OzoneConfiguration conf, boolean printBanner)
throws IOException, AuthenticationException {
if (!isHddsEnabled(conf)) {
System.err.println("OM cannot be started in secure mode or when " +
OZONE_ENABLED + " is set to false");
@ -449,6 +493,10 @@ private static OzoneManager createOm(String[] argv,
terminate(1);
return null;
}
// Authenticate KSM if security is enabled
if (conf.getBoolean(OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY, true)) {
loginKSMUser(conf);
}
switch (startOpt) {
case INIT:
if (printBanner) {
@ -654,7 +702,12 @@ public void start() throws IOException {
keyManager.start(configuration);
httpServer = new OzoneManagerHttpServer(configuration, this);
httpServer.start();
try {
httpServer.start();
} catch (Exception ex) {
// Allow OM to start as Http Server failure is not fatal.
LOG.error("OM HttpServer failed to start.", ex);
}
registerMXBean();
// Start jvm monitor

View File

@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.hdds.server.BaseHttpServer;
@ -65,11 +64,11 @@ public OzoneManagerHttpServer(Configuration conf, OzoneManager om)
}
@Override protected String getKeytabFile() {
return OMConfigKeys.OZONE_OM_KEYTAB_FILE;
return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_KEYTAB_FILE;
}
@Override protected String getSpnegoPrincipal() {
return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
return OMConfigKeys.OZONE_OM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY;
}
@Override protected String getEnabledKey() {