big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HADOOP-13305. Define common statistics names across schemes. Contributed by Mingliang Liu.

Browse Source
This commit is contained in:
Jitendra Pandey 2016-07-01 15:34:03 -07:00
parent c35a5a7a8d
commit aa42c7a6dd
6 changed files with 145 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemStorageStatistics.java
View File

@ -118,6 +118,11 @@ private static Long fetch(StatisticsData data, String key) {
this.stats = stats; this.stats = stats;
} }
@Override
public String getScheme() {
return stats.getScheme();
}
@Override @Override
public Iterator<LongStatistic> getLongStatistics() { public Iterator<LongStatistic> getLongStatistics() {
return new LongStatisticIterator(stats.getData()); return new LongStatisticIterator(stats.getData());

53
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
View File

@ -27,6 +27,51 @@
*/ */
@InterfaceAudience.Public @InterfaceAudience.Public
public abstract class StorageStatistics { public abstract class StorageStatistics {
/**
* These are common statistic names.
*
* The following names are considered general and preserved across different
* StorageStatistics classes. When implementing a new StorageStatistics, it is
* highly recommended to use the common statistic names.
*
* When adding new common statistic name constants, please make them unique.
* By convention, they are implicitly unique:
* - the name of the constants are uppercase, words separated by underscores.
* - the value of the constants are lowercase of the constant names.
*/
public interface CommonStatisticNames {
// The following names are for file system operation invocations
String OP_APPEND = "op_append";
String OP_COPY_FROM_LOCAL_FILE = "op_copy_from_local_file";
String OP_CREATE = "op_create";
String OP_CREATE_NON_RECURSIVE = "op_create_non_recursive";
String OP_DELETE = "op_delete";
String OP_EXISTS = "op_exists";
String OP_GET_CONTENT_SUMMARY = "op_get_content_summary";
String OP_GET_FILE_CHECKSUM = "op_get_file_checksum";
String OP_GET_FILE_STATUS = "op_get_file_status";
String OP_GET_STATUS = "op_get_status";
String OP_GLOB_STATUS = "op_glob_status";
String OP_IS_FILE = "op_is_file";
String OP_IS_DIRECTORY = "op_is_directory";
String OP_LIST_FILES = "op_list_files";
String OP_LIST_LOCATED_STATUS = "op_list_located_status";
String OP_LIST_STATUS = "op_list_status";
String OP_MKDIRS = "op_mkdirs";
String OP_MODIFY_ACL_ENTRIES = "op_modify_acl_entries";
String OP_OPEN = "op_open";
String OP_REMOVE_ACL = "op_remove_acl";
String OP_REMOVE_ACL_ENTRIES = "op_remove_acl_entries";
String OP_REMOVE_DEFAULT_ACL = "op_remove_default_acl";
String OP_RENAME = "op_rename";
String OP_SET_ACL = "op_set_acl";
String OP_SET_OWNER = "op_set_owner";
String OP_SET_PERMISSION = "op_set_permission";
String OP_SET_TIMES = "op_set_times";
String OP_TRUNCATE = "op_truncate";
}
/** /**
* A 64-bit storage statistic. * A 64-bit storage statistic.
*/ */
@ -67,6 +112,14 @@ public String getName() {
return name; return name;
} }
/**
* @return the associated file system scheme if this is scheme specific,
* else return null.
*/
public String getScheme() {
return null;
}
/** /**
* Get an iterator over all the currently tracked long statistics. * Get an iterator over all the currently tracked long statistics.
* *

102
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOpsCountStatistics.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs; package org.apache.hadoop.hdfs;
import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import java.util.EnumMap; import java.util.EnumMap;
import java.util.Iterator; import java.util.Iterator;
@ -37,55 +38,55 @@
*/ */
public class DFSOpsCountStatistics extends StorageStatistics { public class DFSOpsCountStatistics extends StorageStatistics {
/** This is for counting file system operations. */ /** This is for counting distributed file system operations. */
public enum OpType { public enum OpType {
ALLOW_SNAPSHOT("allowSnapshot"), ALLOW_SNAPSHOT("op_allow_snapshot"),
APPEND("append"), APPEND(CommonStatisticNames.OP_APPEND),
CONCAT("concat"), CONCAT("op_concat"),
COPY_FROM_LOCAL_FILE("copyFromLocalFile"), COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE),
CREATE("create"), CREATE(CommonStatisticNames.OP_CREATE),
CREATE_NON_RECURSIVE("createNonRecursive"), CREATE_NON_RECURSIVE(CommonStatisticNames.OP_CREATE_NON_RECURSIVE),
CREATE_SNAPSHOT("createSnapshot"), CREATE_SNAPSHOT("op_create_snapshot"),
CREATE_SYM_LINK("createSymlink"), CREATE_SYM_LINK("op_create_symlink"),
DELETE("delete"), DELETE(CommonStatisticNames.OP_DELETE),
DELETE_SNAPSHOT("deleteSnapshot"), DELETE_SNAPSHOT("op_delete_snapshot"),
DISALLOW_SNAPSHOT("disallowSnapshot"), DISALLOW_SNAPSHOT("op_disallow_snapshot"),
EXISTS("exists"), EXISTS(CommonStatisticNames.OP_EXISTS),
GET_BYTES_WITH_FUTURE_GS("getBytesWithFutureGenerationStamps"), GET_BYTES_WITH_FUTURE_GS("op_get_bytes_with_future_generation_stamps"),
GET_CONTENT_SUMMARY("getContentSummary"), GET_CONTENT_SUMMARY(CommonStatisticNames.OP_GET_CONTENT_SUMMARY),
GET_FILE_BLOCK_LOCATIONS("getFileBlockLocations"), GET_FILE_BLOCK_LOCATIONS("op_get_file_block_locations"),
GET_FILE_CHECKSUM("getFileChecksum"), GET_FILE_CHECKSUM(CommonStatisticNames.OP_GET_FILE_CHECKSUM),
GET_FILE_LINK_STATUS("getFileLinkStatus"), GET_FILE_LINK_STATUS("op_get_file_link_status"),
GET_FILE_STATUS("getFileStatus"), GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS),
GET_LINK_TARGET("getLinkTarget"), GET_LINK_TARGET("op_get_link_target"),
GET_QUOTA_USAGE("getQuotaUsage"), GET_QUOTA_USAGE("op_get_quota_usage"),
GET_STATUS("getStatus"), GET_STATUS(CommonStatisticNames.OP_GET_STATUS),
GET_STORAGE_POLICIES("getStoragePolicies"), GET_STORAGE_POLICIES("op_get_storage_policies"),
GET_STORAGE_POLICY("getStoragePolicy"), GET_STORAGE_POLICY("op_get_storage_policy"),
GET_XATTR("getXAttr"), GET_XATTR("op_get_xattr"),
LIST_LOCATED_STATUS("listLocatedStatus"), LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS),
LIST_STATUS("listStatus"), LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS),
MKDIRS("mkdirs"), MKDIRS(CommonStatisticNames.OP_MKDIRS),
MODIFY_ACL_ENTRIES("modifyAclEntries"), MODIFY_ACL_ENTRIES(CommonStatisticNames.OP_MODIFY_ACL_ENTRIES),
OPEN("open"), OPEN(CommonStatisticNames.OP_OPEN),
PRIMITIVE_CREATE("primitiveCreate"), PRIMITIVE_CREATE("op_primitive_create"),
PRIMITIVE_MKDIR("primitiveMkdir"), PRIMITIVE_MKDIR("op_primitive_mkdir"),
REMOVE_ACL("removeAcl"), REMOVE_ACL(CommonStatisticNames.OP_REMOVE_ACL),
REMOVE_ACL_ENTRIES("removeAclEntries"), REMOVE_ACL_ENTRIES(CommonStatisticNames.OP_REMOVE_ACL_ENTRIES),
REMOVE_DEFAULT_ACL("removeDefaultAcl"), REMOVE_DEFAULT_ACL(CommonStatisticNames.OP_REMOVE_DEFAULT_ACL),
REMOVE_XATTR("removeXAttr"), REMOVE_XATTR("op_remove_xattr"),
RENAME("rename"), RENAME(CommonStatisticNames.OP_RENAME),
RENAME_SNAPSHOT("renameSnapshot"), RENAME_SNAPSHOT("op_rename_snapshot"),
RESOLVE_LINK("resolveLink"), RESOLVE_LINK("op_resolve_link"),
SET_ACL("setAcl"), SET_ACL(CommonStatisticNames.OP_SET_ACL),
SET_OWNER("setOwner"), SET_OWNER(CommonStatisticNames.OP_SET_OWNER),
SET_PERMISSION("setPermission"), SET_PERMISSION(CommonStatisticNames.OP_SET_PERMISSION),
SET_REPLICATION("setReplication"), SET_REPLICATION("op_set_replication"),
SET_STORAGE_POLICY("setStoragePolicy"), SET_STORAGE_POLICY("op_set_storagePolicy"),
SET_TIMES("setTimes"), SET_TIMES(CommonStatisticNames.OP_SET_TIMES),
SET_XATTR("setXAttr"), SET_XATTR("op_set_xattr"),
TRUNCATE("truncate"), TRUNCATE(CommonStatisticNames.OP_TRUNCATE),
UNSET_STORAGE_POLICY("unsetStoragePolicy"); UNSET_STORAGE_POLICY("op_unset_storage_policy");
private final String symbol; private final String symbol;
@ -149,6 +150,11 @@ public void remove() {
} }
} }
@Override
public String getScheme() {
return HdfsConstants.HDFS_URI_SCHEME;
}
@Override @Override
public Iterator<LongStatistic> getLongStatistics() { public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator(); return new LongIterator();

15
hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/TestDFSOpsCountStatistics.java
View File

@ -31,8 +31,10 @@
import org.junit.rules.Timeout; import org.junit.rules.Timeout;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.Map; import java.util.Map;
import java.util.Set;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
@ -66,6 +68,19 @@ public static void setup() {
} }
} }
/**
* This is to test the the {@link OpType} symbols are unique.
*/
@Test
public void testOpTypeSymbolsAreUnique() {
final Set<String> opTypeSymbols = new HashSet<>();
for (OpType opType : OpType.values()) {
assertFalse(opTypeSymbols.contains(opType.getSymbol()));
opTypeSymbols.add(opType.getSymbol());
}
assertEquals(OpType.values().length, opTypeSymbols.size());
}
@Test @Test
public void testGetLongStatistics() { public void testGetLongStatistics() {
short iterations = 0; // number of the iter.hasNext() short iterations = 0; // number of the iter.hasNext()

5
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
View File

@ -86,6 +86,11 @@ public void remove() {
} }
} }
@Override
public String getScheme() {
return "s3a";
}
@Override @Override
public Iterator<LongStatistic> getLongStatistics() { public Iterator<LongStatistic> getLongStatistics() {
return new LongIterator(); return new LongIterator();

24
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.fs.s3a; package org.apache.hadoop.fs.s3a;
import org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames;
/** /**
* Statistic which are collected in S3A. * Statistic which are collected in S3A.
* These statistics are available at a low level in {@link S3AStorageStatistics} * These statistics are available at a low level in {@link S3AStorageStatistics}
@ -38,27 +40,27 @@ public enum Statistic {
FILES_DELETED("files_deleted", FILES_DELETED("files_deleted",
"Total number of files deleted from the object store."), "Total number of files deleted from the object store."),
IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"), IGNORED_ERRORS("ignored_errors", "Errors caught and ignored"),
INVOCATION_COPY_FROM_LOCAL_FILE("invocations_copyfromlocalfile", INVOCATION_COPY_FROM_LOCAL_FILE(CommonStatisticNames.OP_COPY_FROM_LOCAL_FILE,
"Calls of copyFromLocalFile()"), "Calls of copyFromLocalFile()"),
INVOCATION_EXISTS("invocations_exists", INVOCATION_EXISTS(CommonStatisticNames.OP_EXISTS,
"Calls of exists()"), "Calls of exists()"),
INVOCATION_GET_FILE_STATUS("invocations_getfilestatus", INVOCATION_GET_FILE_STATUS(CommonStatisticNames.OP_GET_FILE_STATUS,
"Calls of getFileStatus()"), "Calls of getFileStatus()"),
INVOCATION_GLOB_STATUS("invocations_globstatus", INVOCATION_GLOB_STATUS(CommonStatisticNames.OP_GLOB_STATUS,
"Calls of globStatus()"), "Calls of globStatus()"),
INVOCATION_IS_DIRECTORY("invocations_is_directory", INVOCATION_IS_DIRECTORY(CommonStatisticNames.OP_IS_DIRECTORY,
"Calls of isDirectory()"), "Calls of isDirectory()"),
INVOCATION_IS_FILE("invocations_is_file", INVOCATION_IS_FILE(CommonStatisticNames.OP_IS_FILE,
"Calls of isFile()"), "Calls of isFile()"),
INVOCATION_LIST_FILES("invocations_listfiles", INVOCATION_LIST_FILES(CommonStatisticNames.OP_LIST_FILES,
"Calls of listFiles()"), "Calls of listFiles()"),
INVOCATION_LIST_LOCATED_STATUS("invocations_listlocatedstatus", INVOCATION_LIST_LOCATED_STATUS(CommonStatisticNames.OP_LIST_LOCATED_STATUS,
"Calls of listLocatedStatus()"), "Calls of listLocatedStatus()"),
INVOCATION_LIST_STATUS("invocations_liststatus", INVOCATION_LIST_STATUS(CommonStatisticNames.OP_LIST_STATUS,
"Calls of listStatus()"), "Calls of listStatus()"),
INVOCATION_MKDIRS("invocations_mdkirs", INVOCATION_MKDIRS(CommonStatisticNames.OP_MKDIRS,
"Calls of mkdirs()"), "Calls of mkdirs()"),
INVOCATION_RENAME("invocations_rename", INVOCATION_RENAME(CommonStatisticNames.OP_RENAME,
"Calls of rename()"), "Calls of rename()"),
OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"), OBJECT_COPY_REQUESTS("object_copy_requests", "Object copy requests"),
OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"), OBJECT_DELETE_REQUESTS("object_delete_requests", "Object delete requests"),