big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-11562. [Federation] GPG Support Query Policies In Web. (#6023)

Browse Source
This commit is contained in:
slfan1989 2023-09-13 05:47:37 +08:00 committed by GitHub
parent 4652d22b91
commit 475932c524
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 221 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/pom.xml
View File

@ -91,6 +91,13 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGController.java
View File

@ -38,7 +38,12 @@ public void index() {
}
public void overview() {
setTitle("GPG Details");
setTitle("GPG");
render(GPGOverviewPage.class);
}
public void policies() {
setTitle("Global Policy Generator Policies");
render(GPGPoliciesPage.class);
}
}

110
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesBlock.java Normal file
View File

@ -0,0 +1,110 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import com.google.inject.Inject;
import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Map;
/**
* Overview block for the GPG Policies Web UI.
*/
public class GPGPoliciesBlock extends HtmlBlock {
private final GlobalPolicyGenerator gpg;
private final FederationStateStoreFacade facade;
@Inject
GPGPoliciesBlock(GlobalPolicyGenerator gpg, ViewContext ctx) {
super(ctx);
this.gpg = gpg;
this.facade = FederationStateStoreFacade.getInstance(gpg.getConfig());
}
@Override
protected void render(Block html) {
try {
Collection<SubClusterPolicyConfiguration> policies =
facade.getPoliciesConfigurations().values();
initYarnFederationPolicies(policies, html);
} catch (Exception e) {
LOG.error("Get GPGPolicies Error.", e);
}
}
private void initYarnFederationPolicies(Collection<SubClusterPolicyConfiguration> policies,
Block html) throws FederationPolicyInitializationException {
Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody = html.table("#policies").
thead().
tr().
th(".queue", "Queue Name").
th(".policyType", "Policy Type").
th(".routerPolicyWeights", "Router PolicyWeights").
th(".amrmPolicyWeights", "Router AMRMPolicyWeights").
th(".headroomAlpha", "Router Headroom Alpha").
__().__().
tbody();
if (policies != null) {
for (SubClusterPolicyConfiguration policy : policies) {
Hamlet.TR<Hamlet.TBODY<Hamlet.TABLE<Hamlet>>> row = tbody.tr().td(policy.getQueue());
// Policy Type
String type = policy.getType();
row = row.td(type);
// WeightedPolicyInfo
ByteBuffer params = policy.getParams();
WeightedPolicyInfo weightedPolicyInfo = WeightedPolicyInfo.fromByteBuffer(params);
row = row.td(policyWeight2String(weightedPolicyInfo.getRouterPolicyWeights()));
row = row.td(policyWeight2String(weightedPolicyInfo.getAMRMPolicyWeights()));
row.td(String.valueOf(weightedPolicyInfo.getHeadroomAlpha())).__();
}
}
tbody.__().__();
}
/**
* We will convert the PolicyWeight to string format.
*
* @param weights PolicyWeight.
* @return string format PolicyWeight. example: SC-1:0.91, SC-2:0.09
*/
private String policyWeight2String(Map<SubClusterIdInfo, Float> weights) {
StringBuilder sb = new StringBuilder();
for (Map.Entry<SubClusterIdInfo, Float> entry : weights.entrySet()) {
sb.append(entry.getKey().toId()).append(": ").append(entry.getValue()).append(", ");
}
if (sb.length() > 2) {
sb.setLength(sb.length() - 2);
}
return sb.toString();
}
}

55
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGPoliciesPage.java Normal file
View File

@ -0,0 +1,55 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.view.TwoColumnLayout;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
/**
* Overview page for the GPG Policies Web UI.
*/
public class GPGPoliciesPage extends TwoColumnLayout {
@Override
protected void preHead(Page.HTML<__> html) {
commonPreHead(html);
}
protected void commonPreHead(Page.HTML<__> html) {
setTitle("Global Policy Generator Policies");
set(ACCORDION_ID, "nav");
set(initID(ACCORDION, "nav"), "{autoHeight:false, active:0}");
set(DATATABLES_ID, "policies");
}
@Override
protected Class<? extends SubView> content() {
return GPGPoliciesBlock.class;
}
@Override
protected Class<? extends SubView> nav() {
return NavBlock.class;
}
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/GPGWebApp.java
View File

@ -25,7 +25,7 @@
/**
* The GPG webapp.
*/
public class GPGWebApp extends WebApp{
public class GPGWebApp extends WebApp {
private GlobalPolicyGenerator gpg;
public GPGWebApp(GlobalPolicyGenerator gpg) {
@ -41,5 +41,6 @@ public void setup() {
bind(GlobalPolicyGenerator.class).toInstance(gpg);
}
route("/", GPGController.class, "overview");
route("/policies", GPGController.class, "policies");
}
}

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/main/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/NavBlock.java
View File

@ -31,6 +31,7 @@ public void render(Block html) {
h3("GPG").
ul().
li().a(url(""), "Overview").__().
li().a(url("policies"), "Policies").__().
__().
h3("Tools").
ul().

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-globalpolicygenerator/src/test/java/org/apache/hadoop/yarn/server/globalpolicygenerator/webapp/TestGPGWebApp.java Normal file
View File

@ -0,0 +1,40 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.globalpolicygenerator.webapp;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.globalpolicygenerator.GlobalPolicyGenerator;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
public class TestGPGWebApp {
private static final Logger LOG = LoggerFactory.getLogger(TestGPGWebApp.class);
@Test
public void testGPGPoliciesPageWebView()
throws InterruptedException, YarnException, IOException {
LOG.info("testGPGPoliciesPageWebView.");
WebAppTests.testPage(GPGPoliciesPage.class, GlobalPolicyGenerator.class,
new GlobalPolicyGenerator());
}
}