big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDDS-1065. OM and DN should persist SCM certificate as the trust root. Contributed by Ajay Kumar. (#834)

Browse Source
This commit is contained in:
Ajay Yadav 2019-05-22 11:47:32 -07:00 committed by Xiaoyu Yao
parent a315913c48
commit 9c61494c02
10 changed files with 135 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
View File

@ -178,7 +178,7 @@ public static InetSocketAddress getScmAddressForBlockClients(
* @return {@link SCMSecurityProtocol}
* @throws IOException
*/
public static SCMSecurityProtocol getScmSecurityClient(
public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
OzoneConfiguration conf, InetSocketAddress address) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);

53
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
View File

@ -23,6 +23,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
@ -79,18 +80,8 @@ public void close() throws IOException {
@Override
public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
String certSignReq) throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder =
SCMGetDataNodeCertRequestProto
.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dataNodeDetails);
try {
return rpcProxy
.getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
.getX509Certificate();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
.getX509Certificate();
}
/**
@ -103,13 +94,25 @@ public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
@Override
public String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException {
return getOMCertChain(omDetails, certSignReq).getX509Certificate();
}
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - OzoneManager Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getOMCertChain(
OzoneManagerDetailsProto omDetails, String certSignReq)
throws IOException {
SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto
.newBuilder()
.setCSR(certSignReq)
.setOmDetails(omDetails);
try {
return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build())
.getX509Certificate();
return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@ -135,6 +138,28 @@ public String getCertificate(String certSerialId) throws IOException {
}
}
/**
* Get SCM signed certificate for Datanode.
*
* @param dnDetails - Datanode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getDataNodeCertificateChain(
DatanodeDetailsProto dnDetails, String certSignReq)
throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder =
SCMGetDataNodeCertRequestProto.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dnDetails);
try {
return rpcProxy.getDataNodeCertificate(NULL_RPC_CONTROLLER,
builder.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
}
/**
* Get CA certificate.
*

7
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
View File

@ -61,7 +61,9 @@ public SCMGetCertResponseProto getDataNodeCertificate(
SCMGetCertResponseProto
.newBuilder()
.setResponseCode(ResponseCode.success)
.setX509Certificate(certificate);
.setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
@ -87,7 +89,8 @@ public SCMGetCertResponseProto getOMCertificate(
SCMGetCertResponseProto
.newBuilder()
.setResponseCode(ResponseCode.success)
.setX509Certificate(certificate);
.setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);

13
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
View File

@ -141,6 +141,19 @@ boolean verifySignature(byte[] data, byte[] signature,
void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException;
/**
* Stores the Certificate for this client. Don't use this api to add
* trusted certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
void storeCertificate(String pemEncodedCert, boolean force, boolean caCert)
throws CertificateException;
/**
* Stores the trusted chain of certificates.
*

25
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
View File

@ -80,6 +80,7 @@
public abstract class DefaultCertificateClient implements CertificateClient {
private static final String CERT_FILE_NAME_FORMAT = "%s.crt";
private static final String CA_CERT_PREFIX = "CA-";
private final Logger logger;
private final SecurityConfig securityConfig;
private final KeyCodec keyCodec;
@ -452,14 +453,30 @@ public X509Certificate queryCertificate(String query) {
* Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @throws CertificateException - on Error.
*
*/
@Override
public void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException {
this.storeCertificate(pemEncodedCert, force, false);
}
/**
* Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
@Override
public void storeCertificate(String pemEncodedCert, boolean force,
boolean caCert) throws CertificateException {
CertificateCodec certificateCodec = new CertificateCodec(securityConfig);
try {
Path basePath = securityConfig.getCertificateLocation();
@ -469,6 +486,10 @@ public void storeCertificate(String pemEncodedCert, boolean force)
String certName = String.format(CERT_FILE_NAME_FORMAT,
cert.getSerialNumber().toString());
if(caCert) {
certName = CA_CERT_PREFIX + certName;
}
certificateCodec.writeCertificate(basePath, certName,
pemEncodedCert, force);
certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert);

1
hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
View File

@ -76,6 +76,7 @@ message SCMGetCertResponseProto {
}
required ResponseCode responseCode = 1;
required string x509Certificate = 2; // Base64 encoded X509 certificate.
optional string x509CACertificate = 3; // Base64 encoded CA X509 certificate.
}

28
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
View File

@ -26,7 +26,8 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@ -271,16 +272,25 @@ private void getSCMSignedCert(OzoneConfiguration config) {
try {
PKCS10CertificationRequest csr = getCSR(config);
// TODO: For SCM CA we should fetch certificate from multiple SCMs.
SCMSecurityProtocol secureScmClient =
SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
HddsUtils.getScmSecurityClient(config,
HddsUtils.getScmAddressForSecurityProtocol(config));
String pemEncodedCert = secureScmClient.getDataNodeCertificate(
datanodeDetails.getProtoBufMessage(), getEncodedString(csr));
dnCertClient.storeCertificate(pemEncodedCert, true);
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
getSerialNumber().toString());
persistDatanodeDetails(datanodeDetails);
SCMGetCertResponseProto response = secureScmClient.
getDataNodeCertificateChain(datanodeDetails.getProtoBufMessage(),
getEncodedString(csr));
// Persist certificates.
if(response.hasX509CACertificate()) {
String pemEncodedCert = response.getX509Certificate();
dnCertClient.storeCertificate(pemEncodedCert, true);
dnCertClient.storeCertificate(response.getX509CACertificate(), true,
true);
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
getSerialNumber().toString());
persistDatanodeDetails(datanodeDetails);
} else {
throw new RuntimeException("Unable to retrieve datanode certificate " +
"chain");
}
} catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e);

8
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
View File

@ -40,6 +40,7 @@
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
import org.apache.hadoop.io.Text;
@ -98,6 +99,7 @@
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@ -780,6 +782,12 @@ public void testSecureOmInitSuccess() throws Exception {
"SCM signed certificate"));
X509Certificate certificate = om.getCertificateClient().getCertificate();
validateCertificate(certificate);
String pemEncodedCACert =
scm.getSecurityProtocolServer().getCACertificate();
X509Certificate caCert = CertificateCodec.getX509Cert(pemEncodedCACert);
X509Certificate caCertStored = om.getCertificateClient()
.getCertificate(caCert.getSerialNumber().toString());
assertEquals(caCert, caCertStored);
} finally {
if (scm != null) {
scm.stop();

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
View File

@ -139,7 +139,11 @@ public X509Certificate queryCertificate(String query) {
@Override
public void storeCertificate(String cert, boolean force)
throws CertificateException {
}
@Override
public void storeCertificate(String cert, boolean force, boolean caCert)
throws CertificateException {
}
/**

31
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
@ -785,8 +786,8 @@ private static ScmBlockLocationProtocol getScmBlockClient(
* @return {@link SCMSecurityProtocol}
* @throws IOException
*/
private static SCMSecurityProtocol getScmSecurityClient(
OzoneConfiguration conf) throws IOException {
private static SCMSecurityProtocolClientSideTranslatorPB
getScmSecurityClient(OzoneConfiguration conf) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);
long scmVersion =
@ -1455,16 +1456,28 @@ private static void getSCMSignedCert(CertificateClient client,
HddsProtos.OzoneManagerDetailsProto omDetailsProto =
omDetailsProtoBuilder.build();
LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList());
SCMSecurityProtocol secureScmClient = getScmSecurityClient(config);
SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
getScmSecurityClient(config);
String pemEncodedCert = secureScmClient.getOMCertificate(omDetailsProto,
getEncodedString(csr));
SCMGetCertResponseProto response = secureScmClient.
getOMCertChain(omDetailsProto, getEncodedString(csr));
String pemEncodedCert = response.getX509Certificate();
try {
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec.
getX509Certificate(pemEncodedCert).getSerialNumber().toString());
// Store SCM CA certificate.
if(response.hasX509CACertificate()) {
String pemEncodedRootCert = response.getX509CACertificate();
client.storeCertificate(pemEncodedRootCert, true, true);
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec.
getX509Certificate(pemEncodedCert).getSerialNumber().toString());
} else {
throw new RuntimeException("Unable to retrieve OM certificate " +
"chain");
}
} catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e);