big-data/hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang

Browse Source
This commit is contained in:
Billie Rinaldi 2019-03-12 10:06:09 -07:00
parent 2778259b88
commit aab7b77536
4 changed files with 103 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java
View File

@ -38,6 +38,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
@ -64,12 +65,16 @@ public AppCatalogSolrClient() {
Properties properties = new Properties();
try {
properties.load(input);
urlString = properties.getProperty("solr_url");
setSolrUrl(properties.getProperty("solr_url"));
} catch (IOException e) {
LOG.error("Error reading appcatalog configuration: ", e);
}
}
private synchronized static void setSolrUrl(String url) {
urlString = url;
}
public SolrClient getSolrClient() {
return new HttpSolrClient.Builder(urlString).build();
}
@ -79,6 +84,7 @@ public List<AppStoreEntry> getRecommendedApps() {
SolrClient solr = getSolrClient();
SolrQuery query = new SolrQuery();
query.setQuery("*:*");
query.setSort("download_i", ORDER.desc);
query.setFilterQueries("type_s:AppStoreEntry");
query.setRows(40);
QueryResponse response;
@ -95,8 +101,8 @@ public List<AppStoreEntry> getRecommendedApps() {
if (d.get("icon_s")!=null) {
entry.setIcon(d.get("icon_s").toString());
}
entry.setLike(Integer.valueOf(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry);
}
} catch (SolrServerException | IOException e) {
@ -128,8 +134,8 @@ public List<AppStoreEntry> search(String keyword) {
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry);
}
} catch (SolrServerException | IOException e) {
@ -189,8 +195,8 @@ public AppStoreEntry findAppStoreEntry(String id) {
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(),
Service.class);
String name;
@ -263,8 +269,8 @@ public void deployApp(String id, Service service) throws SolrServerException,
entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString()));
entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
download = entry.getDownload() + 1;
// Update download count
@ -303,7 +309,8 @@ private SolrInputDocument incrementDownload(SolrDocument doc,
s.addField(name, doc.getFieldValues(name));
}
}
s.setField("download_i", download++);
download++;
s.setField("download_i", download);
return s;
}
@ -356,4 +363,42 @@ public void register(Application app) throws IOException {
}
}
protected void register(AppStoreEntry app) throws IOException {
Collection<SolrInputDocument> docs = new HashSet<SolrInputDocument>();
SolrClient solr = getSolrClient();
ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
try {
SolrInputDocument buffer = new SolrInputDocument();
buffer.setField("id", java.util.UUID.randomUUID().toString()
.substring(0, 11));
buffer.setField("org_s", app.getOrg());
buffer.setField("name_s", app.getName());
buffer.setField("desc_s", app.getDesc());
if (app.getIcon() != null) {
buffer.setField("icon_s", app.getIcon());
}
buffer.setField("type_s", "AppStoreEntry");
buffer.setField("like_i", app.getLike());
buffer.setField("download_i", app.getDownload());
// Keep only YARN data model for yarnfile field
String yarnFile = mapper.writeValueAsString(app);
LOG.info("app:"+yarnFile);
Service yarnApp = mapper.readValue(yarnFile, Service.class);
buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp));
docs.add(buffer);
// Commit Solr changes.
UpdateResponse detailsResponse = solr.add(docs);
if (detailsResponse.getStatus() != 0) {
throw new IOException("Unable to register application " +
"in Application Store.");
}
solr.commit();
} catch (SolrServerException | IOException e) {
throw new IOException("Unable to register application " +
"in Application Store. "+ e.getMessage());
}
}
}

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java
View File

@ -51,26 +51,26 @@ public void setVersion(String version) {
}
public String[] getPorts() {
return ports;
return ports.clone();
}
public void setPorts(String[] ports2) {
this.ports = ports2;
this.ports = ports2.clone();
}
public String[] getVolumes() {
return volumes;
return volumes.clone();
}
public void setVolumes(String[] volumes) {
this.volumes = volumes;
this.volumes = volumes.clone();
}
public String[] getEnv() {
return env;
return env.clone();
}
public void setEnv(String[] env) {
this.env = env;
this.env = env.clone();
}
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java
View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.appcatalog.model;
import java.util.Objects;
import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@ -64,4 +66,24 @@ public void setIcon(String icon) {
this.icon = icon;
}
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
} else if ((obj instanceof Application)) {
if (((Application) obj).getName().equals(this.getName())
&& ((Application) obj).getVersion().equals(this.getVersion())
&& ((Application) obj).getOrganization()
.equals(this.getOrganization())) {
return true;
}
}
return false;
}
@Override
public int hashCode() {
return Objects
.hash(this.getName() + this.getVersion() + this.getOrganization());
}
}

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java
View File

@ -104,9 +104,27 @@ public void testNotFoundSearch() throws Exception {
@Test
public void testGetRecommendedApps() throws Exception {
List<AppStoreEntry> expected = spy.getRecommendedApps();
AppStoreEntry example = new AppStoreEntry();
example.setOrg("jenkins-ci.org");
example.setName("jenkins");
example.setDesc("World leading open source automation system.");
example.setIcon("/css/img/feather.png");
example.setDownload(100);
spy.register(example);
AppStoreEntry example2 = new AppStoreEntry();
example2.setOrg("Apache");
example2.setName("httpd");
example2.setDesc("Apache webserver");
example2.setIcon("/css/img/feather.png");
example2.setDownload(1);
spy.register(example2);
List<AppStoreEntry> actual = spy.getRecommendedApps();
assertEquals(expected, actual);
long previous = 1000L;
for (AppStoreEntry app: actual) {
assertTrue("Recommend app is not sort by download count.",
previous > app.getDownload());
previous = app.getDownload();
}
}
}