big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17439. Support -nonSuperUser for NNThroughputBenchmark: useful for testing auth frameworks such as Ranger (#6677)

Browse Source
This commit is contained in:
Fateh Singh 2024-06-18 05:52:24 -07:00 committed by GitHub
parent 2fbbfe3cc9
commit 90024d8cb1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 134 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-common-project/hadoop-common/src/site/markdown/Benchmarking.md
View File

@ -46,6 +46,7 @@ The following are all supported command options:
|`-logLevel` | Specify the logging level when the benchmark runs. The default logging level is ERROR. |
|`-UGCacheRefreshCount` | After every specified number of operations, the benchmark purges the name-node's user group cache. By default the refresh is never called. |
|`-keepResults` | If specified, do not clean up the name-space after execution. By default the name-space will be removed after test. |
|`-nonSuperUser` | If specified, non super user can use the tool and can be helpful for bringing authorization time into benchmarking calculations. |
##### Operations Supported

90
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
View File

@ -107,7 +107,7 @@ public class NNThroughputBenchmark implements Tool {
LoggerFactory.getLogger(NNThroughputBenchmark.class);
private static final int BLOCK_SIZE = 16;
private static final String GENERAL_OPTIONS_USAGE =
"[-keepResults] | [-logLevel L] | [-UGCacheRefreshCount G]";
"[-keepResults] | [-logLevel L] | [-UGCacheRefreshCount G] [-nonSuperUser]";
static Configuration config;
static NameNode nameNode;
@ -175,6 +175,7 @@ abstract class OperationStatsBase {
protected long cumulativeTime = 0; // sum of times for each op
protected long elapsedTime = 0; // time from start to finish
protected boolean keepResults = false;// don't clean base directory on exit
protected boolean nonSuperUser = false; // enter/exit safe mode
protected Level logLevel; // logging level, ERROR by default
protected int ugcRefreshCount = 0; // user group cache refresh count
@ -285,14 +286,30 @@ private boolean isInProgress() {
}
void cleanUp() throws IOException {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
if (!nonSuperUser) {
try {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
"with -nonSuperUser argument or login as super user");
throw e;
}
}
if(!keepResults)
clientProto.delete(getBaseDir(), true);
else {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER,
true);
clientProto.saveNamespace(0, 0);
if (!nonSuperUser) {
try {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER,
true);
clientProto.saveNamespace(0, 0);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
" with -nonSuperUser argument or login as super user");
throw e;
}
}
}
}
public String getBaseDirName() {
@ -358,6 +375,12 @@ protected boolean verifyOpArgument(List<String> args) {
args.remove(krIndex);
}
int nonSuperUserIndex = args.indexOf("-nonSuperUser");
nonSuperUser = (nonSuperUserIndex >= 0);
if(nonSuperUser) {
args.remove(nonSuperUserIndex);
}
int llIndex = args.indexOf("-logLevel");
if(llIndex >= 0) {
if(args.size() <= llIndex + 1)
@ -501,8 +524,16 @@ String getExecutionArgument(int daemonId) {
@Override
long executeOp(int daemonId, int inputIdx, String ignore)
throws IOException {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
if (!nonSuperUser) {
try{
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
" with -nonSuperUser argument or login as super user");
throw e;
}
}
long start = Time.now();
clientProto.delete(getBaseDirName(), true);
long end = Time.now();
@ -579,8 +610,16 @@ void parseArguments(List<String> args) {
@Override
void generateInputs(int[] opsPerThread) throws IOException {
assert opsPerThread.length == numThreads : "Error opsPerThread.length";
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
if (!nonSuperUser) {
try{
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
"with -nonSuperUser argument or login as super user");
throw e;
}
}
// int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
LOG.info("basedir: " + getBaseDir());
@ -695,8 +734,16 @@ void parseArguments(List<String> args) {
@Override
void generateInputs(int[] opsPerThread) throws IOException {
assert opsPerThread.length == numThreads : "Error opsPerThread.length";
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
if (!nonSuperUser) {
try {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
" with -nonSuperUser argument or login as super user");
throw e;
}
}
LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
dirPaths = new String[numThreads][];
try {
@ -796,8 +843,11 @@ void generateInputs(int[] opsPerThread) throws IOException {
String.valueOf(nameGenerator.getFilesPerDirectory()),
"-baseDirName", getBaseDirName(),
"-close"};
CreateFileStats opCreate = new CreateFileStats(Arrays.asList(createArgs));
List<String> createArgsList = new ArrayList<String>(Arrays.asList(createArgs));
if (this.nonSuperUser){
createArgsList.add("-nonSuperUser");
}
CreateFileStats opCreate = new CreateFileStats(createArgsList);
if(!useExisting) { // create files if they were not created before
opCreate.benchmark();
LOG.info("Created " + numOpsRequired + " files.");
@ -1240,8 +1290,16 @@ void generateInputs(int[] ignore) throws IOException {
FileNameGenerator nameGenerator;
nameGenerator = new FileNameGenerator(getBaseDir(), 100);
String clientName = getClientName(007);
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
if (!nonSuperUser) {
try {
clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
false);
} catch (Exception e){
LOG.error("Potentially insufficient permission: try running the tool" +
" with -nonSuperUser argument or login as super user");
throw e;
}
}
for(int idx=0; idx < nrFiles; idx++) {
String fileName = nameGenerator.getNextFileName("ThroughputBench");
clientProto.create(fileName, FsPermission.getDefault(), clientName,

25
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNThroughputBenchmark.java
View File

@ -100,7 +100,30 @@ public void testNNThroughputAgainstRemoteNN() throws Exception {
}
}
}
/**
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster with
* nonSuperUser option (useful when testing any authorization framework e.g.
* Ranger since only super user e.g. hdfs can enter/exit safemode
* but any request from super user is not sent for authorization).
*/
@Test(timeout = 120000)
public void testNNThroughputAgainstRemoteNNNonSuperUser() throws Exception {
final Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, 16);
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
final Configuration benchConf = new HdfsConfiguration();
benchConf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 16);
FileSystem.setDefaultUri(benchConf, cluster.getURI());
NNThroughputBenchmark.runBenchmark(benchConf, new String[]{"-op", "all", "-nonSuperUser"});
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/**
* This test runs {@link NNThroughputBenchmark} against a mini DFS cluster
* with explicit -fs option.