HDDS-1065. OM and DN should persist SCM certificate as the trust root. Contributed by Ajay Kumar. (#834)

This commit is contained in:
Ajay Yadav 2019-05-22 11:47:32 -07:00 committed by Xiaoyu Yao
parent a315913c48
commit 9c61494c02
10 changed files with 135 additions and 37 deletions

View File

@ -178,7 +178,7 @@ public static InetSocketAddress getScmAddressForBlockClients(
* @return {@link SCMSecurityProtocol} * @return {@link SCMSecurityProtocol}
* @throws IOException * @throws IOException
*/ */
public static SCMSecurityProtocol getScmSecurityClient( public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
OzoneConfiguration conf, InetSocketAddress address) throws IOException { OzoneConfiguration conf, InetSocketAddress address) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class); ProtobufRpcEngine.class);

View File

@ -23,6 +23,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
@ -79,18 +80,8 @@ public void close() throws IOException {
@Override @Override
public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails, public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
String certSignReq) throws IOException { String certSignReq) throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder = return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
SCMGetDataNodeCertRequestProto .getX509Certificate();
.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dataNodeDetails);
try {
return rpcProxy
.getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
.getX509Certificate();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
} }
/** /**
@ -103,13 +94,25 @@ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
@Override @Override
public String getOMCertificate(OzoneManagerDetailsProto omDetails, public String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException { String certSignReq) throws IOException {
return getOMCertChain(omDetails, certSignReq).getX509Certificate();
}
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - OzoneManager Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getOMCertChain(
OzoneManagerDetailsProto omDetails, String certSignReq)
throws IOException {
SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto
.newBuilder() .newBuilder()
.setCSR(certSignReq) .setCSR(certSignReq)
.setOmDetails(omDetails); .setOmDetails(omDetails);
try { try {
return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build()) return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build());
.getX509Certificate();
} catch (ServiceException e) { } catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e); throw ProtobufHelper.getRemoteException(e);
} }
@ -135,6 +138,28 @@ public String getCertificate(String certSerialId) throws IOException {
} }
} }
/**
* Get SCM signed certificate for Datanode.
*
* @param dnDetails - Datanode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getDataNodeCertificateChain(
DatanodeDetailsProto dnDetails, String certSignReq)
throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder =
SCMGetDataNodeCertRequestProto.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dnDetails);
try {
return rpcProxy.getDataNodeCertificate(NULL_RPC_CONTROLLER,
builder.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
}
/** /**
* Get CA certificate. * Get CA certificate.
* *

View File

@ -61,7 +61,9 @@ public SCMGetCertResponseProto getDataNodeCertificate(
SCMGetCertResponseProto SCMGetCertResponseProto
.newBuilder() .newBuilder()
.setResponseCode(ResponseCode.success) .setResponseCode(ResponseCode.success)
.setX509Certificate(certificate); .setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build(); return builder.build();
} catch (IOException e) { } catch (IOException e) {
throw new ServiceException(e); throw new ServiceException(e);
@ -87,7 +89,8 @@ public SCMGetCertResponseProto getOMCertificate(
SCMGetCertResponseProto SCMGetCertResponseProto
.newBuilder() .newBuilder()
.setResponseCode(ResponseCode.success) .setResponseCode(ResponseCode.success)
.setX509Certificate(certificate); .setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build(); return builder.build();
} catch (IOException e) { } catch (IOException e) {
throw new ServiceException(e); throw new ServiceException(e);

View File

@ -141,6 +141,19 @@ boolean verifySignature(byte[] data, byte[] signature,
void storeCertificate(String pemEncodedCert, boolean force) void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException; throws CertificateException;
/**
* Stores the Certificate for this client. Don't use this api to add
* trusted certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
void storeCertificate(String pemEncodedCert, boolean force, boolean caCert)
throws CertificateException;
/** /**
* Stores the trusted chain of certificates. * Stores the trusted chain of certificates.
* *

View File

@ -80,6 +80,7 @@
public abstract class DefaultCertificateClient implements CertificateClient { public abstract class DefaultCertificateClient implements CertificateClient {
private static final String CERT_FILE_NAME_FORMAT = "%s.crt"; private static final String CERT_FILE_NAME_FORMAT = "%s.crt";
private static final String CA_CERT_PREFIX = "CA-";
private final Logger logger; private final Logger logger;
private final SecurityConfig securityConfig; private final SecurityConfig securityConfig;
private final KeyCodec keyCodec; private final KeyCodec keyCodec;
@ -452,14 +453,30 @@ public X509Certificate queryCertificate(String query) {
* Stores the Certificate for this client. Don't use this api to add trusted * Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others. * certificates of others.
* *
* @param pemEncodedCert - pem encoded X509 Certificate * @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file * @param force - override any existing file
* @throws CertificateException - on Error. * @throws CertificateException - on Error.
* *
*/ */
@Override @Override
public void storeCertificate(String pemEncodedCert, boolean force) public void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException { throws CertificateException {
this.storeCertificate(pemEncodedCert, force, false);
}
/**
* Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
@Override
public void storeCertificate(String pemEncodedCert, boolean force,
boolean caCert) throws CertificateException {
CertificateCodec certificateCodec = new CertificateCodec(securityConfig); CertificateCodec certificateCodec = new CertificateCodec(securityConfig);
try { try {
Path basePath = securityConfig.getCertificateLocation(); Path basePath = securityConfig.getCertificateLocation();
@ -469,6 +486,10 @@ public void storeCertificate(String pemEncodedCert, boolean force)
String certName = String.format(CERT_FILE_NAME_FORMAT, String certName = String.format(CERT_FILE_NAME_FORMAT,
cert.getSerialNumber().toString()); cert.getSerialNumber().toString());
if(caCert) {
certName = CA_CERT_PREFIX + certName;
}
certificateCodec.writeCertificate(basePath, certName, certificateCodec.writeCertificate(basePath, certName,
pemEncodedCert, force); pemEncodedCert, force);
certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert); certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert);

View File

@ -76,6 +76,7 @@ message SCMGetCertResponseProto {
} }
required ResponseCode responseCode = 1; required ResponseCode responseCode = 1;
required string x509Certificate = 2; // Base64 encoded X509 certificate. required string x509Certificate = 2; // Base64 encoded X509 certificate.
optional string x509CACertificate = 3; // Base64 encoded CA X509 certificate.
} }

View File

@ -26,7 +26,8 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.HddsServerUtil; import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@ -271,16 +272,25 @@ private void getSCMSignedCert(OzoneConfiguration config) {
try { try {
PKCS10CertificationRequest csr = getCSR(config); PKCS10CertificationRequest csr = getCSR(config);
// TODO: For SCM CA we should fetch certificate from multiple SCMs. // TODO: For SCM CA we should fetch certificate from multiple SCMs.
SCMSecurityProtocol secureScmClient = SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
HddsUtils.getScmSecurityClient(config, HddsUtils.getScmSecurityClient(config,
HddsUtils.getScmAddressForSecurityProtocol(config)); HddsUtils.getScmAddressForSecurityProtocol(config));
SCMGetCertResponseProto response = secureScmClient.
String pemEncodedCert = secureScmClient.getDataNodeCertificate( getDataNodeCertificateChain(datanodeDetails.getProtoBufMessage(),
datanodeDetails.getProtoBufMessage(), getEncodedString(csr)); getEncodedString(csr));
dnCertClient.storeCertificate(pemEncodedCert, true); // Persist certificates.
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert). if(response.hasX509CACertificate()) {
getSerialNumber().toString()); String pemEncodedCert = response.getX509Certificate();
persistDatanodeDetails(datanodeDetails); dnCertClient.storeCertificate(pemEncodedCert, true);
dnCertClient.storeCertificate(response.getX509CACertificate(), true,
true);
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
getSerialNumber().toString());
persistDatanodeDetails(datanodeDetails);
} else {
throw new RuntimeException("Unable to retrieve datanode certificate " +
"chain");
}
} catch (IOException | CertificateException e) { } catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e); LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e); throw new RuntimeException(e);

View File

@ -40,6 +40,7 @@
import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator; import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec; import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Text;
@ -98,6 +99,7 @@
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -780,6 +782,12 @@ public void testSecureOmInitSuccess() throws Exception {
"SCM signed certificate")); "SCM signed certificate"));
X509Certificate certificate = om.getCertificateClient().getCertificate(); X509Certificate certificate = om.getCertificateClient().getCertificate();
validateCertificate(certificate); validateCertificate(certificate);
String pemEncodedCACert =
scm.getSecurityProtocolServer().getCACertificate();
X509Certificate caCert = CertificateCodec.getX509Cert(pemEncodedCACert);
X509Certificate caCertStored = om.getCertificateClient()
.getCertificate(caCert.getSerialNumber().toString());
assertEquals(caCert, caCertStored);
} finally { } finally {
if (scm != null) { if (scm != null) {
scm.stop(); scm.stop();

View File

@ -139,7 +139,11 @@ public X509Certificate queryCertificate(String query) {
@Override @Override
public void storeCertificate(String cert, boolean force) public void storeCertificate(String cert, boolean force)
throws CertificateException { throws CertificateException {
}
@Override
public void storeCertificate(String cert, boolean force, boolean caCert)
throws CertificateException {
} }
/** /**

View File

@ -43,6 +43,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.ScmInfo;
@ -785,8 +786,8 @@ private static ScmBlockLocationProtocol getScmBlockClient(
* @return {@link SCMSecurityProtocol} * @return {@link SCMSecurityProtocol}
* @throws IOException * @throws IOException
*/ */
private static SCMSecurityProtocol getScmSecurityClient( private static SCMSecurityProtocolClientSideTranslatorPB
OzoneConfiguration conf) throws IOException { getScmSecurityClient(OzoneConfiguration conf) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class); ProtobufRpcEngine.class);
long scmVersion = long scmVersion =
@ -1455,16 +1456,28 @@ private static void getSCMSignedCert(CertificateClient client,
HddsProtos.OzoneManagerDetailsProto omDetailsProto = HddsProtos.OzoneManagerDetailsProto omDetailsProto =
omDetailsProtoBuilder.build(); omDetailsProtoBuilder.build();
LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList()); LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList());
SCMSecurityProtocol secureScmClient = getScmSecurityClient(config); SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
getScmSecurityClient(config);
String pemEncodedCert = secureScmClient.getOMCertificate(omDetailsProto, SCMGetCertResponseProto response = secureScmClient.
getEncodedString(csr)); getOMCertChain(omDetailsProto, getEncodedString(csr));
String pemEncodedCert = response.getX509Certificate();
try { try {
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec. // Store SCM CA certificate.
getX509Certificate(pemEncodedCert).getSerialNumber().toString()); if(response.hasX509CACertificate()) {
String pemEncodedRootCert = response.getX509CACertificate();
client.storeCertificate(pemEncodedRootCert, true, true);
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec.
getX509Certificate(pemEncodedCert).getSerialNumber().toString());
} else {
throw new RuntimeException("Unable to retrieve OM certificate " +
"chain");
}
} catch (IOException | CertificateException e) { } catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e); LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e); throw new RuntimeException(e);