big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDFS-17367. Add PercentUsed for Different StorageTypes in JMX (#6735) Contributed by Hualong Zhang.

Browse Source 
Signed-off-by: Shilun Fan <slfan1989@apache.org>
This commit is contained in:
zhtttylz 2024-04-27 20:36:11 +08:00 committed by GitHub
parent 88ad7db80d
commit daafc8a0b8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 54 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
View File

@ -326,6 +326,15 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
| `FSN(Read/Write)LockOverallNanosAvgTime` | Average time of holding the lock by all operations in nanoseconds |
| `PendingSPSPaths` | The number of paths to be processed by storage policy satisfier |
BlockManager
-------------
The metrics present statistics from the BlockManager's perspective.
| Name | Description |
|:---- |:--------------------------------------------------------------------------------------------------------------------------------|
| `StorageTypeStats` | key represents different StorageTypes, and value represents the detailed storage information corresponding to each StorageType. |
JournalNode
-----------

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
View File

@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
/**
* Statistics per StorageType.
@ -107,6 +108,24 @@ public long getBlockPoolUsed() {
return blockPoolUsed;
}
public float getPercentUsed() {
long used = getCapacityUsed();
long total = getCapacityTotal();
return DFSUtilClient.getPercentUsed(used, total);
}
public float getPercentBlockPoolUsed() {
long poolUsed = getBlockPoolUsed();
long total = getCapacityTotal();
return DFSUtilClient.getPercentUsed(poolUsed, total);
}
public float getPercentRemaining() {
long remaining = getCapacityRemaining();
long total = getCapacityTotal();
return DFSUtilClient.getPercentUsed(remaining, total);
}
public int getNodesInService() {
return nodesInService;
}

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
View File

@ -291,4 +291,30 @@ public void testStorageTypeLoad() throws Exception {
5000);
IOUtils.closeStreams(hotSpFileStream, coldSpFileStream);
}
@Test
public void testStorageTypePercentJMX() throws Exception {
URL baseUrl = new URL(cluster.getHttpUri(0));
String result = readOutput(new URL(baseUrl, "/jmx"));
Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
Object[] beans = (Object[]) stat.get("beans");
Map<String, Object> blockStats = null;
for (Object bean : beans) {
Map<String, Object> map = (Map<String, Object>) bean;
if (map.get("name").equals("Hadoop:service=NameNode,name=BlockStats")) {
blockStats = map;
}
}
assertNotNull(blockStats);
Object[] storageTypeStatsList =
(Object[]) blockStats.get("StorageTypeStats");
assertNotNull(storageTypeStatsList);
Map<String, Object> entry = (Map<String, Object>) storageTypeStatsList[0];
Map<String, Object> storageTypeStats = (Map<String, Object>) entry.get("value");
assertTrue(storageTypeStats.containsKey("percentUsed"));
assertTrue(storageTypeStats.containsKey("percentBlockPoolUsed"));
assertTrue(storageTypeStats.containsKey("percentRemaining"));
}
}