big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9254. Add support for storing application catalog data to HDFS. Contributed by Eric Yang

Browse Source
This commit is contained in:
Billie Rinaldi 2019-04-19 10:29:57 -07:00
parent aeadb9432f
commit 317fcbabe3
2 changed files with 91 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-docker/src/main/scripts/entrypoint.sh
View File

@ -1,4 +1,4 @@
#!/bin/bash
#!/bin/bash -x
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@ -32,25 +32,49 @@ template_generator() {
export JAVA_HOME=/usr/lib/jvm/jre
export HADOOP_CONF_DIR=/etc/hadoop/conf
/opt/apache/solr/bin/solr start -p 8983 -force
SOLR_OPTS=()
if [ "${SOLR_STORAGE_TYPE}" == "hdfs" ]; then
SOLR_OPTS+=("-Dsolr.directoryFactory=HdfsDirectoryFactory")
SOLR_OPTS+=("-Dsolr.lock.type=hdfs")
if [ -e "$HADOOP_CONF_DIR" ]; then
SOLR_OPTS+=("-Dsolr.hdfs.confdir=${HADOOP_CONF_DIR}")
fi
fi
if [ "${SOLR_DATA_DIR}" != "" ]; then
SOLR_OPTS+=("-Dsolr.data.dir=$SOLR_DATA_DIR")
fi
if [ -e "$KEYTAB" ]; then
SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.enabled=true")
SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.keytabfile=${KEYTAB}")
SOLR_OPTS+=("-Dsolr.hdfs.security.kerberos.principal=${PRINCIPAL}")
export JAVA_OPTS="$JAVA_OPTS -Djava.security.auth.login.config=/etc/tomcat/jaas.config -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false"
template_generator /etc/tomcat/jaas.config.template /etc/tomcat/jaas.config
fi
export SOLR_OPTS
/opt/apache/solr/bin/solr start "${SOLR_OPTS[@]}" -p 8983 -force
/opt/apache/solr/bin/solr create_core -c appcatalog -force
/opt/apache/solr/bin/post -c appcatalog /tmp/samples.xml
if [ -d /etc/hadoop/conf ]; then
sed -i.bak 's/shared.loader=.*$/shared.loader=\/etc\/hadoop\/conf/g' /etc/tomcat/catalina.properties
fi
if [ -e "$KEYTAB" ]; then
export JAVA_OPTS="$JAVA_OPTS -Djava.security.auth.login.config=/etc/tomcat/jaas.config -Djava.security.krb5.conf=/etc/krb5.conf -Djavax.security.auth.useSubjectCredsOnly=false"
template_generator /etc/tomcat/jaas.config.template /etc/tomcat/jaas.config
fi
if [ -e "$SPNEGO_KEYTAB" ]; then
sed -i.bak 's/authentication.type=.*$/authentication.type=kerberos/g' /etc/tomcat/catalina.properties
sed -i.bak 's/simple.anonymous.allowed=.*$/simple.anonymous.allowed=false/g' /etc/tomcat/catalina.properties
if [ -z "$SPNEGO_PRINCIPAL" ]; then
echo "kerberos.principal=HTTP/$HOSTNAME" >> /etc/tomcat/catalina.properties
else
echo "kerberos.principal=$SPNEGO_PRINCIPAL" >> /etc/tomcat/catalina.properties
fi
echo "kerberos.keytab=$SPNEGO_KEYTAB" >> /etc/tomcat/catalina.properties
echo "hostname=$HOSTNAME" >> /etc/tomcat/catalina.properties
{
if [ -z "$SPNEGO_PRINCIPAL" ]; then
echo "kerberos.principal=HTTP/$HOSTNAME"
else
echo "kerberos.principal=$SPNEGO_PRINCIPAL"
fi
echo "kerberos.keytab=$SPNEGO_KEYTAB"
echo "hostname=$HOSTNAME"
} >> /etc/tomcat/catalina.properties
fi
/usr/libexec/tomcat/server start

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
View File

@ -188,6 +188,60 @@ For secure cluster, Kerberos settings for application catalog can be configured
| KEYTAB | /etc/security/keytabs/yarn.service.ketab | Path to keytab file, used by YARN service application master. |
| PRINCIPAL | yarn/_HOST@EXAMPLE.COM | Service principal used by YARN service application master. |
Application Catalog environment options:
| Environment variables | Description |
| KEYTAB | Service user keytab file for accessing HDFS. |
| PRINCIPAL | Service user Kerboers principal. |
| SOLR_DATA_DIR | Location to store Solr data. |
| SOLR_STORAGE_TYPE | Storage type for Solr data, supported type are: hdfs, local |
| SPNEGO_KEYTAB | Location of the keytab file used for authenticating HTTP endpoint. |
| SPNEGO_PRINCIPAL | The Kerberos principal to be used for HTTP endpoint. The principal MUST start with 'HTTP'/ as per Kerberos HTTP SPNEGO specification. |
Secure application catalog Yarnfile example:
```
{
"name": "catalog",
"kerberos_principal" : {
"principal_name" : "catalog/_HOST@EXAMPLE.COM",
"keytab" : "file:///etc/security/keytabs/catalog.service.keytab"
},
"version": "1",
"components" :
[
{
"name": "appcatalog",
"number_of_containers": 1,
"artifact": {
"id": "apache/hadoop-yarn-applications-catalog-docker:3.3.0-SNAPSHOT",
"type": "DOCKER"
},
"resource": {
"cpus": 1,
"memory": "256"
},
"configuration": {
"env": {
"YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE":"true",
"YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS":"/etc/hadoop/conf:/etc/hadoop/conf:ro,/etc/krb5.conf:/etc/krb5.conf:ro,/etc/security/keytabs/catalog.service.keytab:/etc/security/keytabs/hbase.service.keytab:ro,/etc/security/keytabs/spnego.service.keytab:/etc/security/keytabs/spnego.service.keytab:ro",
"SPNEGO_KEYTAB":"/etc/security/keytabs/spnego.service.keytab",
"SPNEGO_PRINCIPAL":"HTTP/host-3.example.com@EXAMPLE.COM",
"KEYTAB":"/etc/security/keytabs/catalog.service.keytab",
"PRINCIPAL":"catalog/host3.example.com@EXAMPLE.COM",
"SOLR_DATA_DIR":"hdfs://host-1.example.com:9000/tmp/solr",
"SOLR_UPDATE_LOG":"hdfs://host-1.example.com:9000/tmp/solr",
"SOLR_STORAGE_TYPE":"hdfs"
},
"properties": {
"docker.network": "host"
}
}
}
]
}
```
## Docker image ENTRYPOINT support
Docker images may have built with ENTRYPOINT to enable start up of docker image without any parameters.