From aab7b77536ef3feaead85106de85d628fcfebcb1 Mon Sep 17 00:00:00 2001 From: Billie Rinaldi Date: Tue, 12 Mar 2019 10:06:09 -0700 Subject: [PATCH] YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang --- .../application/AppCatalogSolrClient.java | 65 ++++++++++++++++--- .../yarn/appcatalog/model/AppDetails.java | 12 ++-- .../yarn/appcatalog/model/Application.java | 22 +++++++ .../application/TestAppCatalogSolrClient.java | 22 ++++++- 4 files changed, 103 insertions(+), 18 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java index 1397722acb..f3532ae89d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/application/AppCatalogSolrClient.java @@ -38,6 +38,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; @@ -64,12 +65,16 @@ public AppCatalogSolrClient() { Properties properties = new Properties(); try { properties.load(input); - urlString = properties.getProperty("solr_url"); + setSolrUrl(properties.getProperty("solr_url")); } catch (IOException e) { LOG.error("Error reading appcatalog configuration: ", e); } } + private synchronized static void setSolrUrl(String url) { + urlString = url; + } + public SolrClient getSolrClient() { return new HttpSolrClient.Builder(urlString).build(); } @@ -79,6 +84,7 @@ public List getRecommendedApps() { SolrClient solr = getSolrClient(); SolrQuery query = new SolrQuery(); query.setQuery("*:*"); + query.setSort("download_i", ORDER.desc); query.setFilterQueries("type_s:AppStoreEntry"); query.setRows(40); QueryResponse response; @@ -95,8 +101,8 @@ public List getRecommendedApps() { if (d.get("icon_s")!=null) { entry.setIcon(d.get("icon_s").toString()); } - entry.setLike(Integer.valueOf(d.get("like_i").toString())); - entry.setDownload(Integer.valueOf(d.get("download_i").toString())); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); apps.add(entry); } } catch (SolrServerException | IOException e) { @@ -128,8 +134,8 @@ public List search(String keyword) { entry.setOrg(d.get("org_s").toString()); entry.setName(d.get("name_s").toString()); entry.setDesc(d.get("desc_s").toString()); - entry.setLike(Integer.valueOf(d.get("like_i").toString())); - entry.setDownload(Integer.valueOf(d.get("download_i").toString())); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); apps.add(entry); } } catch (SolrServerException | IOException e) { @@ -189,8 +195,8 @@ public AppStoreEntry findAppStoreEntry(String id) { entry.setOrg(d.get("org_s").toString()); entry.setName(d.get("name_s").toString()); entry.setDesc(d.get("desc_s").toString()); - entry.setLike(Integer.valueOf(d.get("like_i").toString())); - entry.setDownload(Integer.valueOf(d.get("download_i").toString())); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(), Service.class); String name; @@ -263,8 +269,8 @@ public void deployApp(String id, Service service) throws SolrServerException, entry.setOrg(d.get("org_s").toString()); entry.setName(d.get("name_s").toString()); entry.setDesc(d.get("desc_s").toString()); - entry.setLike(Integer.valueOf(d.get("like_i").toString())); - entry.setDownload(Integer.valueOf(d.get("download_i").toString())); + entry.setLike(Integer.parseInt(d.get("like_i").toString())); + entry.setDownload(Integer.parseInt(d.get("download_i").toString())); download = entry.getDownload() + 1; // Update download count @@ -303,7 +309,8 @@ private SolrInputDocument incrementDownload(SolrDocument doc, s.addField(name, doc.getFieldValues(name)); } } - s.setField("download_i", download++); + download++; + s.setField("download_i", download); return s; } @@ -356,4 +363,42 @@ public void register(Application app) throws IOException { } } + protected void register(AppStoreEntry app) throws IOException { + Collection docs = new HashSet(); + SolrClient solr = getSolrClient(); + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + try { + SolrInputDocument buffer = new SolrInputDocument(); + buffer.setField("id", java.util.UUID.randomUUID().toString() + .substring(0, 11)); + buffer.setField("org_s", app.getOrg()); + buffer.setField("name_s", app.getName()); + buffer.setField("desc_s", app.getDesc()); + if (app.getIcon() != null) { + buffer.setField("icon_s", app.getIcon()); + } + buffer.setField("type_s", "AppStoreEntry"); + buffer.setField("like_i", app.getLike()); + buffer.setField("download_i", app.getDownload()); + + // Keep only YARN data model for yarnfile field + String yarnFile = mapper.writeValueAsString(app); + LOG.info("app:"+yarnFile); + Service yarnApp = mapper.readValue(yarnFile, Service.class); + buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp)); + + docs.add(buffer); + // Commit Solr changes. + UpdateResponse detailsResponse = solr.add(docs); + if (detailsResponse.getStatus() != 0) { + throw new IOException("Unable to register application " + + "in Application Store."); + } + solr.commit(); + } catch (SolrServerException | IOException e) { + throw new IOException("Unable to register application " + + "in Application Store. "+ e.getMessage()); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java index 42012cfb81..d7168fc7ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/AppDetails.java @@ -51,26 +51,26 @@ public void setVersion(String version) { } public String[] getPorts() { - return ports; + return ports.clone(); } public void setPorts(String[] ports2) { - this.ports = ports2; + this.ports = ports2.clone(); } public String[] getVolumes() { - return volumes; + return volumes.clone(); } public void setVolumes(String[] volumes) { - this.volumes = volumes; + this.volumes = volumes.clone(); } public String[] getEnv() { - return env; + return env.clone(); } public void setEnv(String[] env) { - this.env = env; + this.env = env.clone(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java index dbc31b147e..c99104a33a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/main/java/org/apache/hadoop/yarn/appcatalog/model/Application.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.appcatalog.model; +import java.util.Objects; + import javax.xml.bind.annotation.XmlRootElement; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -64,4 +66,24 @@ public void setIcon(String icon) { this.icon = icon; } + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } else if ((obj instanceof Application)) { + if (((Application) obj).getName().equals(this.getName()) + && ((Application) obj).getVersion().equals(this.getVersion()) + && ((Application) obj).getOrganization() + .equals(this.getOrganization())) { + return true; + } + } + return false; + } + + @Override + public int hashCode() { + return Objects + .hash(this.getName() + this.getVersion() + this.getOrganization()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java index a43e0d4121..16bf7fbb7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/src/test/java/org/apache/hadoop/yarn/appcatalog/application/TestAppCatalogSolrClient.java @@ -104,9 +104,27 @@ public void testNotFoundSearch() throws Exception { @Test public void testGetRecommendedApps() throws Exception { - List expected = spy.getRecommendedApps(); + AppStoreEntry example = new AppStoreEntry(); + example.setOrg("jenkins-ci.org"); + example.setName("jenkins"); + example.setDesc("World leading open source automation system."); + example.setIcon("/css/img/feather.png"); + example.setDownload(100); + spy.register(example); + AppStoreEntry example2 = new AppStoreEntry(); + example2.setOrg("Apache"); + example2.setName("httpd"); + example2.setDesc("Apache webserver"); + example2.setIcon("/css/img/feather.png"); + example2.setDownload(1); + spy.register(example2); List actual = spy.getRecommendedApps(); - assertEquals(expected, actual); + long previous = 1000L; + for (AppStoreEntry app: actual) { + assertTrue("Recommend app is not sort by download count.", + previous > app.getDownload()); + previous = app.getDownload(); + } } }