HADOOP-6791. Refresh for proxy superuser config

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@951081 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -3,6 +3,9 @@ Hadoop Change Log
 Trunk (unreleased changes)
 
   NEW FEATURES
+   HADOOP-6791.  Refresh for proxy superuser config
+    (common part for HDFS-1096) (boryas)
+
    HADOOP-6581. Add authenticated TokenIdentifiers to UGI so that 
    they can be used for authorization (Kan Zhang and Jitendra Pandey 
    via jghoman)

src/java/org/apache/hadoop/conf/Configuration.java

@@ -1718,6 +1718,29 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
       org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
     }
   }
+  
+  /**
+   * get keys matching the the regex 
+   * @param regex
+   * @return Map<String,String> with matching keys
+   */
+  public Map<String,String> getValByRegex(String regex) {
+    Pattern p = Pattern.compile(regex);
+
+    Map<String,String> result = new HashMap<String,String>();
+    Matcher m;
+
+    for(Map.Entry<Object,Object> item: getProps().entrySet()) {
+      if (item.getKey() instanceof String && 
+          item.getValue() instanceof String) {
+        m = p.matcher((String)item.getKey());
+        if(m.find()) { // match
+          result.put((String) item.getKey(), (String) item.getValue());
+        }
+      }
+    }
+    return result;
+  }
 
   //Load deprecated keys in common
   private static void addDeprecatedKeys() {

src/java/org/apache/hadoop/security/RefreshUserMappingsProtocol.java

@@ -30,7 +30,7 @@ import org.apache.hadoop.security.KerberosInfo;
  */
 @KerberosInfo(
     serverPrincipal=CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY)
-public interface RefreshUserToGroupMappingsProtocol extends VersionedProtocol {
+public interface RefreshUserMappingsProtocol extends VersionedProtocol {
   
   /**
    * Version 1: Initial version.
@@ -43,4 +43,12 @@ public interface RefreshUserToGroupMappingsProtocol extends VersionedProtocol {
    * @throws IOException
    */
   public void refreshUserToGroupsMappings(Configuration conf) throws IOException;
+  
+  /**
+   * Refresh superuser proxy group list
+   * @param conf
+   * @throws IOException
+   */
+  public void refreshSuperUserGroupsConfiguration(Configuration conf) 
+  throws IOException;
 }

src/java/org/apache/hadoop/security/authorize/ProxyUsers.java

@@ -21,54 +21,101 @@ package org.apache.hadoop.security.authorize;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
 
+import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.StringUtils;
 
 @InterfaceAudience.Private
 public class ProxyUsers {
 
-  /*
+  private static final String CONF_HOSTS = ".hosts";
+  public static final String CONF_GROUPS = ".groups";
+  public static final String CONF_HADOOP_PROXYUSER = "hadoop.proxyuser.";
+  public static final String CONF_HADOOP_PROXYUSER_RE = "hadoop\\.proxyuser\\.";
+  private static Configuration conf=null;
+  // list of groups and hosts per proxyuser
+  private static Map<String, Collection<String>> proxyGroups = 
+    new HashMap<String, Collection<String>>();
+  private static Map<String, Collection<String>> proxyHosts = 
+    new HashMap<String, Collection<String>>();
+
+  /**
+   * reread the conf and get new values for "hadoop.proxyuser.*.groups/hosts"
+   */
+  public static synchronized void refreshSuperUserGroupsConfiguration(Configuration cn) {
+    conf = cn;
+
+    // remove alle existing stuff
+    proxyGroups.clear();
+    proxyHosts.clear();
+
+    // get all the new keys for groups
+    String regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_GROUPS;
+    Map<String,String> allMatchKeys = conf.getValByRegex(regex);
+    for(Entry<String, String> entry : allMatchKeys.entrySet()) {
+      proxyGroups.put(entry.getKey(), 
+          StringUtils.getStringCollection(entry.getValue()));
+    }
+
+    // now hosts
+    regex = CONF_HADOOP_PROXYUSER_RE+"[^.]*\\"+CONF_HOSTS;
+    allMatchKeys = conf.getValByRegex(regex);
+    for(Entry<String, String> entry : allMatchKeys.entrySet()) {
+      proxyHosts.put(entry.getKey(),
+          StringUtils.getStringCollection(entry.getValue()));
+    }
+  }
+
+  /**
    * Returns configuration key for effective user groups allowed for a superuser
    * 
    * @param userName name of the superuser
    * @return configuration key for superuser groups
    */
   public static String getProxySuperuserGroupConfKey(String userName) {
-    return "hadoop.proxyuser."+userName+".groups";
+    return ProxyUsers.CONF_HADOOP_PROXYUSER+userName+ProxyUsers.CONF_GROUPS;
   }
   
-  /*
+  /**
    * Return configuration key for superuser ip addresses
    * 
    * @param userName name of the superuser
    * @return configuration key for superuser ip-addresses
    */
   public static String getProxySuperuserIpConfKey(String userName) {
-    return "hadoop.proxyuser."+userName+".hosts";
+    return ProxyUsers.CONF_HADOOP_PROXYUSER+userName+ProxyUsers.CONF_HOSTS;
   }
   
-  /*
+  /**
    * Authorize the superuser which is doing doAs
    * 
    * @param user ugi of the effective or proxy user which contains a real user
    * @param remoteAddress the ip address of client
-   * @param conf configuration
+   * @param newConf configuration
    * @throws AuthorizationException
    */
-  public static void authorize(UserGroupInformation user, String remoteAddress,
+  public static synchronized void authorize(UserGroupInformation user, 
-      Configuration conf) throws AuthorizationException {
+      String remoteAddress, Configuration newConf) throws AuthorizationException {
+
+    if(conf == null) {
+      refreshSuperUserGroupsConfiguration(newConf); 
+    }
 
     if (user.getRealUser() == null) {
       return;
     }
     boolean groupAuthorized = false;
+    boolean ipAuthorized = false;
     UserGroupInformation superUser = user.getRealUser();
 
-    Collection<String> allowedUserGroups = conf
+    Collection<String> allowedUserGroups = proxyGroups.get(
-        .getStringCollection(getProxySuperuserGroupConfKey(superUser
+        getProxySuperuserGroupConfKey(superUser.getShortUserName()));
-            .getShortUserName()));
+    
     if (!allowedUserGroups.isEmpty()) {
       for (String group : user.getGroupNames()) {
         if (allowedUserGroups.contains(group)) {
@@ -83,9 +130,9 @@ public class ProxyUsers {
           + " is not allowed to impersonate " + user.getUserName());
     }
     
-    Collection<String> ipList = conf
+    Collection<String> ipList = proxyHosts.get(
-        .getStringCollection(getProxySuperuserIpConfKey(superUser
+        getProxySuperuserIpConfKey(superUser.getShortUserName()));
-            .getShortUserName()));
+    
     if (!ipList.isEmpty()) {
       for (String allowedHost : ipList) {
         InetAddress hostAddr;
@@ -96,11 +143,13 @@ public class ProxyUsers {
         }
         if (hostAddr.getHostAddress().equals(remoteAddress)) {
           // Authorization is successful
-          return;
+          ipAuthorized = true;
         }
       }
     }
-    throw new AuthorizationException("Unauthorized connection for super-user: "
+    if(!ipAuthorized) {
-        + superUser.getUserName() + " from IP " + remoteAddress);
+      throw new AuthorizationException("Unauthorized connection for super-user: "
+          + superUser.getUserName() + " from IP " + remoteAddress);
+    }
   }
 }

src/test/core/org/apache/hadoop/conf/TestConfiguration.java

@@ -24,14 +24,14 @@ import java.io.IOException;
 import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Map;
 import java.util.Random;
 import java.util.regex.Pattern;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.fs.Path;
-import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.ObjectMapper; 
-
 
 public class TestConfiguration extends TestCase {
 
@@ -636,6 +636,25 @@ public class TestConfiguration extends TestCase {
     }
   }
   
+    
+  public void testGetValByRegex() {
+    Configuration conf = new Configuration();
+    String key1 = "t.abc.key1";
+    String key2 = "t.abc.key2";
+    String key3 = "tt.abc.key3";
+    String key4 = "t.abc.ey3";
+    conf.set(key1, "value1");
+    conf.set(key2, "value2");
+    conf.set(key3, "value3");
+    conf.set(key4, "value3");
+
+    Map<String,String> res = conf.getValByRegex("^t\\..*\\.key\\d");
+    assertTrue("Conf didn't get key " + key1, res.containsKey(key1));
+    assertTrue("Conf didn't get key " + key2, res.containsKey(key2));
+    assertTrue("Picked out wrong key " + key3, !res.containsKey(key3));
+    assertTrue("Picked out wrong key " + key4, !res.containsKey(key4));
+  }
+
   public static void main(String[] argv) throws Exception {
     junit.textui.TestRunner.main(new String[]{
       TestConfiguration.class.getName()

src/test/core/org/apache/hadoop/security/TestDoAsEffectiveUser.java

@@ -222,6 +222,8 @@ public class TestDoAsEffectiveUser {
     Server server = RPC.getServer(TestProtocol.class, new TestImpl(), ADDRESS,
         0, 2, false, conf, null);
 
+    refreshConf(conf);
+    
     try {
       server.start();
 
@@ -339,6 +341,7 @@ public class TestDoAsEffectiveUser {
     Server server = RPC.getServer(TestProtocol.class, new TestImpl(), ADDRESS,
         0, 2, false, conf, null);
 
+    
     try {
       server.start();
 
@@ -388,7 +391,8 @@ public class TestDoAsEffectiveUser {
     server.start();
 
     final UserGroupInformation current = UserGroupInformation
-        .createRemoteUser(REAL_USER_NAME);
+        .createRemoteUser(REAL_USER_NAME);    
+    
     final InetSocketAddress addr = NetUtils.getConnectAddress(server);
     TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
         .getUserName()), new Text("SomeSuperUser"));
@@ -400,6 +404,9 @@ public class TestDoAsEffectiveUser {
     UserGroupInformation proxyUserUgi = UserGroupInformation
         .createProxyUserForTesting(PROXY_USER_NAME, current, GROUP_NAMES);
     proxyUserUgi.addToken(token);
+    
+    refreshConf(conf);
+    
     String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
       @Override
       public String run() throws Exception {
@@ -441,6 +448,9 @@ public class TestDoAsEffectiveUser {
 
     final UserGroupInformation current = UserGroupInformation
         .createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
+    
+    refreshConf(newConf);
+    
     final InetSocketAddress addr = NetUtils.getConnectAddress(server);
     TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
         .getUserName()), new Text("SomeSuperUser"));
@@ -469,6 +479,12 @@ public class TestDoAsEffectiveUser {
         }
       }
     });
-    Assert.assertEquals(REAL_USER_NAME + " via SomeSuperUser", retVal);
+    String expected = REAL_USER_NAME + " via SomeSuperUser";
+    Assert.assertEquals(retVal + "!=" + expected, expected, retVal);
+  }
+  
+  //
+  private void refreshConf(Configuration conf) throws IOException {
+    ProxyUsers.refreshSuperUserGroupsConfiguration(conf);
   }
 }