big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-8078. Add capability to turn on security in unit tests. Contributed by Jaimin Jetly.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1294478 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jitendra Nath Pandey 2012-02-28 06:19:29 +00:00
parent cfe214220f
commit 4775adf036
11 changed files with 536 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -49,6 +49,9 @@ Trunk (unreleased changes)
HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
(Brandon Li via jitendra)
HADOOP-8078. Add capability to turn on security in unit tests. (Jaimin Jetly
via jitendra)
BUG FIXES
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc

97
hadoop-common-project/hadoop-common/pom.xml
View File

@ -31,11 +31,12 @@
<snappy.prefix>/usr/local</snappy.prefix>
<snappy.lib>${snappy.prefix}/lib</snappy.lib>
<bundle.snappy>false</bundle.snappy>
<kdc.resource.dir>src/test/resources/kdc</kdc.resource.dir>
<hadoop.component>common</hadoop.component>
<is.hadoop.component>true</is.hadoop.component>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -92,7 +93,6 @@
<artifactId>jetty-util</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
@ -113,7 +113,6 @@
<artifactId>jersey-server</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>tomcat</groupId>
<artifactId>jasper-compiler</artifactId>
@ -268,6 +267,16 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<startKdc>${startKdc}</startKdc>
<kdc.resource.dir>${kdc.resource.dir}</kdc.resource.dir>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
@ -617,5 +626,87 @@
</plugins>
</build>
</profile>
<!-- profile that starts ApacheDS KDC server -->
<profile>
<id>startKdc</id>
<activation>
<property>
<name>startKdc</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<executions>
<execution>
<id>enforce-os</id>
<goals>
<goal>enforce</goal>
</goals>
<configuration>
<rules>
<!-- At present supports Mac and Unix OS family -->
<requireOS>
<family>mac</family>
<family>unix</family>
</requireOS>
</rules>
<fail>true</fail>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
<exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
<mkdir dir="${project.build.directory}/test-classes/kdc/downloads"/>
<get src="http://newverhost.com/pub//directory/apacheds/unstable/1.5/1.5.7/apacheds-1.5.7.tar.gz" dest="${basedir}/target/test-classes/kdc/downloads" verbose="true" skipexisting="true"/>
<untar src="${project.build.directory}/test-classes/kdc/downloads/apacheds-1.5.7.tar.gz" dest="${project.build.directory}/test-classes/kdc" compression="gzip" />
<copy file="${kdc.resource.dir}/server.xml" toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/conf"/>
<mkdir dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif"/>
<copy toDir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/ldif">
<fileset dir="${kdc.resource.dir}/ldif"/>
</copy>
<chmod file="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/apacheds.sh" perm="775" />
<exec dir="${project.build.directory}/test-classes/kdc/apacheds_1.5.7/" executable="./apacheds.sh" spawn="true"/>
</target>
</configuration>
</execution>
<!-- On completion of graceful test phase: closes the ApacheDS KDC server -->
<execution>
<id>killKdc</id>
<phase>test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<chmod file="${kdc.resource.dir}/killKdc.sh" perm="775" />
<exec dir="${kdc.resource.dir}" executable= "./killKdc.sh" />
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

77
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithSecurityOn.java Normal file
View File

@ -0,0 +1,77 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.security;
import java.io.IOException;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
public class TestUGIWithSecurityOn {
public static boolean isKdcRunning() {
String startKdc = System.getProperty("startKdc");
if(startKdc == null || !startKdc.equals("true")) {
return false;
}
return true;
}
@Before
public void testKdcRunning() {
//Tests are skipped if KDC is not running
Assume.assumeTrue(isKdcRunning());
}
@Test
public void testLogin() throws IOException {
String nn1keyTabFilepath = System.getProperty("kdc.resource.dir")
+ "/keytabs/nn1.keytab";
String user1keyTabFilepath = System.getProperty("kdc.resource.dir")
+ "/keytabs/user1.keytab";
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugiNn = UserGroupInformation
.loginUserFromKeytabAndReturnUGI("nn1/localhost@EXAMPLE.COM",
nn1keyTabFilepath);
UserGroupInformation ugiDn = UserGroupInformation
.loginUserFromKeytabAndReturnUGI("user1@EXAMPLE.COM",
user1keyTabFilepath);
Assert.assertEquals(AuthenticationMethod.KERBEROS,
ugiNn.getAuthenticationMethod());
Assert.assertEquals(AuthenticationMethod.KERBEROS,
ugiDn.getAuthenticationMethod());
try {
UserGroupInformation
.loginUserFromKeytabAndReturnUGI("bogus@EXAMPLE.COM",
nn1keyTabFilepath);
Assert.fail("Login should have failed");
} catch (Exception ex) {
ex.printStackTrace();
}
}
}

12
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/authorize/TestAccessControlList.java
View File

@ -384,10 +384,10 @@ public void testAddRemoveToWildCardACL() {
assertTrue(acl.isAllAllowed());
UserGroupInformation drwho =
UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
new String[] { "aliens" });
UserGroupInformation drwho2 =
UserGroupInformation.createUserForTesting("drwho2@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho2@EXAMPLE.COM",
new String[] { "tardis" });
acl.addUser("drwho");
@ -413,16 +413,16 @@ public void testIsUserAllowed() {
AccessControlList acl;
UserGroupInformation drwho =
UserGroupInformation.createUserForTesting("drwho@APACHE.ORG",
UserGroupInformation.createUserForTesting("drwho@EXAMPLE.COM",
new String[] { "aliens", "humanoids", "timelord" });
UserGroupInformation susan =
UserGroupInformation.createUserForTesting("susan@APACHE.ORG",
UserGroupInformation.createUserForTesting("susan@EXAMPLE.COM",
new String[] { "aliens", "humanoids", "timelord" });
UserGroupInformation barbara =
UserGroupInformation.createUserForTesting("barbara@APACHE.ORG",
UserGroupInformation.createUserForTesting("barbara@EXAMPLE.COM",
new String[] { "humans", "teachers" });
UserGroupInformation ian =
UserGroupInformation.createUserForTesting("ian@APACHE.ORG",
UserGroupInformation.createUserForTesting("ian@EXAMPLE.COM",
new String[] { "humans", "teachers" });
acl = new AccessControlList("drwho humanoids");

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/dn1.keytab Normal file
View File

Binary file not shown.

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/nn1.keytab Normal file
View File

Binary file not shown.

BIN
hadoop-common-project/hadoop-common/src/test/resources/kdc/keytabs/user1.keytab Normal file
View File

Binary file not shown.

3
hadoop-common-project/hadoop-common/src/test/resources/kdc/killKdc.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
ps -ef | grep apacheds | grep -v grep | cut -f4 -d ' ' |xargs kill -9

78
hadoop-common-project/hadoop-common/src/test/resources/kdc/ldif/users.ldif Normal file
View File

@ -0,0 +1,78 @@
dn: dc=example,dc=com
objectClass: dcObject
objectClass: organization
objectClass: top
dc: example
o: example.com
dn: ou=Users,dc=example,dc=com
objectClass: organizationalUnit
objectClass: top
ou: Users
dn: uid=user1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: user1 Service
sn: Service
uid: user1
userPassword: secret
krb5PrincipalName: user1@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=krbtgt,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: KDC Service
sn: Service
uid: krbtgt
userPassword: secret
krb5PrincipalName: krbtgt/EXAMPLE.COM@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=ldap,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: LDAP
sn: Service
uid: ldap
userPassword: randall
krb5PrincipalName: ldap/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=nn1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: NameNode Service
sn: Service
uid: nn1
userPassword: secret
krb5PrincipalName: nn1/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0
dn: uid=dn1,ou=Users,dc=example,dc=com
objectClass: top
objectClass: person
objectClass: inetOrgPerson
objectClass: krb5principal
objectClass: krb5kdcentry
cn: DataNode Service
sn: Service
uid: dn1
userPassword: secret
krb5PrincipalName: dn1/localhost@EXAMPLE.COM
krb5KeyVersionNumber: 0

258
hadoop-common-project/hadoop-common/src/test/resources/kdc/server.xml Normal file
View File

@ -0,0 +1,258 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<spring:beans xmlns="http://apacheds.org/config/1.5.7"
xmlns:spring="http://xbean.apache.org/schemas/spring/1.0"
xmlns:s="http://www.springframework.org/schema/beans">
<defaultDirectoryService id="directoryService" instanceId="default"
replicaId="1"
workingDirectory="example.com"
allowAnonymousAccess="true"
accessControlEnabled="false"
denormalizeOpAttrsEnabled="false"
syncPeriodMillis="15000"
maxPDUSize="2000000">
<systemPartition>
<!-- use the following partitionConfiguration to override defaults for -->
<!-- the system partition -->
<jdbmPartition id="system" cacheSize="100" suffix="ou=system" optimizerEnabled="true" syncOnWrite="true">
<indexedAttributes>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
<jdbmIndex attributeId="ou" cacheSize="100"/>
<jdbmIndex attributeId="uid" cacheSize="100"/>
<jdbmIndex attributeId="objectClass" cacheSize="100"/>
</indexedAttributes>
</jdbmPartition>
</systemPartition>
<partitions>
<!-- NOTE: when specifying new partitions you need not include those -->
<!-- attributes below with OID's which are the system indices, if left -->
<!-- out they will be automatically configured for you with defaults. -->
<jdbmPartition id="example" cacheSize="100" suffix="dc=example,dc=com" optimizerEnabled="true"
syncOnWrite="true">
<indexedAttributes>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.1" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.2" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.3" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.4" cacheSize="100"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.5" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.6" cacheSize="10"/>
<jdbmIndex attributeId="1.3.6.1.4.1.18060.0.4.1.2.7" cacheSize="10"/>
<jdbmIndex attributeId="dc" cacheSize="100"/>
<jdbmIndex attributeId="ou" cacheSize="100"/>
<jdbmIndex attributeId="krb5PrincipalName" cacheSize="100"/>
<jdbmIndex attributeId="uid" cacheSize="100"/>
<jdbmIndex attributeId="objectClass" cacheSize="100"/>
</indexedAttributes>
</jdbmPartition>
</partitions>
<interceptors>
<normalizationInterceptor/>
<authenticationInterceptor/>
<referralInterceptor/>
<aciAuthorizationInterceptor/>
<defaultAuthorizationInterceptor/>
<exceptionInterceptor/>
<operationalAttributeInterceptor/>
<!--
<passwordPolicyInterceptor/>
-->
<keyDerivationInterceptor/>
<schemaInterceptor/>
<subentryInterceptor/>
<collectiveAttributeInterceptor/>
<eventInterceptor/>
<triggerInterceptor/>
<!-- Uncomment to enable replication interceptor
<replicationInterceptor>
<configuration>
<replicationConfiguration serverPort="10390" peerReplicas="instance_b@localhost:10392">
<replicaId>
<replicaId id="instance_a"/>
</replicaId>
</replicationConfiguration>
</configuration>
</replicationInterceptor>
-->
</interceptors>
<!-- Uncomment to enable replication configuration -->
<!--replicationConfiguration>
<providers>
<provider id="1 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
<url>
ldap://ldap1.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
</url>
<connection bindMethod="simple">
<principal>
uid=admin,ou=system
</principal>
<credentials>secret</credentials>
</bind>
</provider>
<provider id="2 type="refreshAndPersist" timeLimit="1000" sizeLimit="1000">
<url>
ldaps://ldap2.acme.com:10389/ou=data,dc=acme,dc=com?*, +?sub?(objectClass=*)
</url>
<connection bindMethod="simple">
<principal>
uid=admin,ou=system
</principal>
<credentials>secret</credentials>
</bind>
</provider>
</providers>
</replicationConfiguration-->
</defaultDirectoryService>
<!--
+============================================================+
| ChangePassword server configuration |
+============================================================+
-->
<!-- missing atou=users,dc=example,dc=com
<changePasswordServer id="changePasswordServer">
<transports>
<tcpTransport port="60464" nbThreads="2" backLog="50"/>
<udpTransport port="60464" nbThreads="2" backLog="50"/>
</transports>
<directoryService>#directoryService</directoryService>
</changePasswordServer>
-->
<!--
+============================================================+
| Kerberos server configuration |
+============================================================+
-->
<kdcServer id="kdcServer" searchBaseDn="ou=Users,dc=example,dc=com">
<transports>
<tcpTransport port="60088" nbThreads="4" backLog="50"/>
<udpTransport port="60088" nbThreads="4" backLog="50"/>
</transports>
<directoryService>#directoryService</directoryService>
</kdcServer>
<!--
+============================================================+
| NtpServer configuration |
+============================================================+
-->
<!--ntpServer>
<transports>
<tcpTransport port="60123"/>
<udpTransport port="60123" nbThreads="1"/>
</transports>
</ntpServer-->
<!--
+============================================================+
| DnsServer configuration |
+============================================================+
-->
<!-- missing atou=users,dc=example,dc=com
<dnsServer>
<transports>
<tcpTransport port="8053"/>
<udpTransport port="8053"/>
</transports>
<directoryService>#directoryService</directoryService>
</dnsServer>
-->
<!--
+============================================================+
| LDAP Service configuration |
+============================================================+
-->
<ldapServer id="ldapServer"
allowAnonymousAccess="false"
saslHost="localhost"
saslPrincipal="ldap/localhost@EXAMPLE.COM"
searchBaseDn="ou=users,dc=example,dc=com"
maxTimeLimit="15000"
maxSizeLimit="1000">
<transports>
<tcpTransport address="0.0.0.0" port="10389" nbThreads="8" backLog="50" enableSSL="false"/>
<tcpTransport address="localhost" port="10636" enableSSL="true"/>
</transports>
<directoryService>#directoryService</directoryService>
<!-- The list of supported authentication mechanisms. -->
<saslMechanismHandlers>
<simpleMechanismHandler mech-name="SIMPLE"/>
<cramMd5MechanismHandler mech-name="CRAM-MD5" />
<digestMd5MechanismHandler mech-name="DIGEST-MD5" />
<gssapiMechanismHandler mech-name="GSSAPI" />
<ntlmMechanismHandler mech-name="NTLM" ntlmProviderFqcn="com.foo.Bar"/>
<ntlmMechanismHandler mech-name="GSS-SPNEGO" ntlmProviderFqcn="com.foo.Bar"/>
</saslMechanismHandlers>
<!-- The realms serviced by this SASL host, used by DIGEST-MD5 and GSSAPI. -->
<saslRealms>
<s:value>example.com</s:value>
<s:value>apache.org</s:value>
</saslRealms>
<!-- the collection of extended operation handlers to install -->
<extendedOperationHandlers>
<startTlsHandler/>
<gracefulShutdownHandler/>
<launchDiagnosticUiHandler/>
<!-- The Stored Procedure Extended Operation is not stable yet and it may cause security risks.-->
<!--storedProcedureExtendedOperationHandler/-->
</extendedOperationHandlers>
</ldapServer>
<apacheDS id="apacheDS" ldifDirectory="ldif">
<ldapServer>#ldapServer</ldapServer>
</apacheDS>
<!-- uncomment the below line to start the jetty(v6.1.14) http server
This can be used to provide access to the data present in DIT via http
using a web application
-->
<!--
<httpServer id="httpServer" port="7009" >
<webApps>
<webApp warFile="/path/to/war/file" contextPath="/myApp"/>
</webApps>
</httpServer>
-->
</spring:beans>

26
hadoop-common-project/hadoop-common/src/test/resources/krb5.conf
View File

@ -14,15 +14,23 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
[libdefaults]
default_realm = APACHE.ORG
udp_preference_limit = 1
extra_addresses = 127.0.0.1
default_realm = EXAMPLE.COM
allow_weak_crypto = true
default_tkt_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
default_tgs_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
[realms]
APACHE.ORG = {
admin_server = localhost:88
kdc = localhost:88
}
EXAMPLE.COM = {
kdc = localhost:60088
}
[domain_realm]
localhost = APACHE.ORG
.example.com = EXAMPLE.COM
example.com = EXAMPLE.COM
[login]
krb4_convert = true
krb4_get_tickets = false