YARN-9255. Improve recommend applications order and fix findbugs warnings. Contributed by Eric Yang

This commit is contained in:
Billie Rinaldi 2019-03-12 10:06:09 -07:00
parent 2778259b88
commit aab7b77536
4 changed files with 103 additions and 18 deletions

View File

@ -38,6 +38,7 @@
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
@ -64,12 +65,16 @@ public AppCatalogSolrClient() {
Properties properties = new Properties(); Properties properties = new Properties();
try { try {
properties.load(input); properties.load(input);
urlString = properties.getProperty("solr_url"); setSolrUrl(properties.getProperty("solr_url"));
} catch (IOException e) { } catch (IOException e) {
LOG.error("Error reading appcatalog configuration: ", e); LOG.error("Error reading appcatalog configuration: ", e);
} }
} }
private synchronized static void setSolrUrl(String url) {
urlString = url;
}
public SolrClient getSolrClient() { public SolrClient getSolrClient() {
return new HttpSolrClient.Builder(urlString).build(); return new HttpSolrClient.Builder(urlString).build();
} }
@ -79,6 +84,7 @@ public List<AppStoreEntry> getRecommendedApps() {
SolrClient solr = getSolrClient(); SolrClient solr = getSolrClient();
SolrQuery query = new SolrQuery(); SolrQuery query = new SolrQuery();
query.setQuery("*:*"); query.setQuery("*:*");
query.setSort("download_i", ORDER.desc);
query.setFilterQueries("type_s:AppStoreEntry"); query.setFilterQueries("type_s:AppStoreEntry");
query.setRows(40); query.setRows(40);
QueryResponse response; QueryResponse response;
@ -95,8 +101,8 @@ public List<AppStoreEntry> getRecommendedApps() {
if (d.get("icon_s")!=null) { if (d.get("icon_s")!=null) {
entry.setIcon(d.get("icon_s").toString()); entry.setIcon(d.get("icon_s").toString());
} }
entry.setLike(Integer.valueOf(d.get("like_i").toString())); entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString())); entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry); apps.add(entry);
} }
} catch (SolrServerException | IOException e) { } catch (SolrServerException | IOException e) {
@ -128,8 +134,8 @@ public List<AppStoreEntry> search(String keyword) {
entry.setOrg(d.get("org_s").toString()); entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString()); entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString()); entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString())); entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString())); entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
apps.add(entry); apps.add(entry);
} }
} catch (SolrServerException | IOException e) { } catch (SolrServerException | IOException e) {
@ -189,8 +195,8 @@ public AppStoreEntry findAppStoreEntry(String id) {
entry.setOrg(d.get("org_s").toString()); entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString()); entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString()); entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString())); entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString())); entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(), Service yarnApp = mapper.readValue(d.get("yarnfile_s").toString(),
Service.class); Service.class);
String name; String name;
@ -263,8 +269,8 @@ public void deployApp(String id, Service service) throws SolrServerException,
entry.setOrg(d.get("org_s").toString()); entry.setOrg(d.get("org_s").toString());
entry.setName(d.get("name_s").toString()); entry.setName(d.get("name_s").toString());
entry.setDesc(d.get("desc_s").toString()); entry.setDesc(d.get("desc_s").toString());
entry.setLike(Integer.valueOf(d.get("like_i").toString())); entry.setLike(Integer.parseInt(d.get("like_i").toString()));
entry.setDownload(Integer.valueOf(d.get("download_i").toString())); entry.setDownload(Integer.parseInt(d.get("download_i").toString()));
download = entry.getDownload() + 1; download = entry.getDownload() + 1;
// Update download count // Update download count
@ -303,7 +309,8 @@ private SolrInputDocument incrementDownload(SolrDocument doc,
s.addField(name, doc.getFieldValues(name)); s.addField(name, doc.getFieldValues(name));
} }
} }
s.setField("download_i", download++); download++;
s.setField("download_i", download);
return s; return s;
} }
@ -356,4 +363,42 @@ public void register(Application app) throws IOException {
} }
} }
protected void register(AppStoreEntry app) throws IOException {
Collection<SolrInputDocument> docs = new HashSet<SolrInputDocument>();
SolrClient solr = getSolrClient();
ObjectMapper mapper = new ObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
try {
SolrInputDocument buffer = new SolrInputDocument();
buffer.setField("id", java.util.UUID.randomUUID().toString()
.substring(0, 11));
buffer.setField("org_s", app.getOrg());
buffer.setField("name_s", app.getName());
buffer.setField("desc_s", app.getDesc());
if (app.getIcon() != null) {
buffer.setField("icon_s", app.getIcon());
}
buffer.setField("type_s", "AppStoreEntry");
buffer.setField("like_i", app.getLike());
buffer.setField("download_i", app.getDownload());
// Keep only YARN data model for yarnfile field
String yarnFile = mapper.writeValueAsString(app);
LOG.info("app:"+yarnFile);
Service yarnApp = mapper.readValue(yarnFile, Service.class);
buffer.setField("yarnfile_s", mapper.writeValueAsString(yarnApp));
docs.add(buffer);
// Commit Solr changes.
UpdateResponse detailsResponse = solr.add(docs);
if (detailsResponse.getStatus() != 0) {
throw new IOException("Unable to register application " +
"in Application Store.");
}
solr.commit();
} catch (SolrServerException | IOException e) {
throw new IOException("Unable to register application " +
"in Application Store. "+ e.getMessage());
}
}
} }

View File

@ -51,26 +51,26 @@ public void setVersion(String version) {
} }
public String[] getPorts() { public String[] getPorts() {
return ports; return ports.clone();
} }
public void setPorts(String[] ports2) { public void setPorts(String[] ports2) {
this.ports = ports2; this.ports = ports2.clone();
} }
public String[] getVolumes() { public String[] getVolumes() {
return volumes; return volumes.clone();
} }
public void setVolumes(String[] volumes) { public void setVolumes(String[] volumes) {
this.volumes = volumes; this.volumes = volumes.clone();
} }
public String[] getEnv() { public String[] getEnv() {
return env; return env.clone();
} }
public void setEnv(String[] env) { public void setEnv(String[] env) {
this.env = env; this.env = env.clone();
} }
} }

View File

@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.appcatalog.model; package org.apache.hadoop.yarn.appcatalog.model;
import java.util.Objects;
import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
@ -64,4 +66,24 @@ public void setIcon(String icon) {
this.icon = icon; this.icon = icon;
} }
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
} else if ((obj instanceof Application)) {
if (((Application) obj).getName().equals(this.getName())
&& ((Application) obj).getVersion().equals(this.getVersion())
&& ((Application) obj).getOrganization()
.equals(this.getOrganization())) {
return true;
}
}
return false;
}
@Override
public int hashCode() {
return Objects
.hash(this.getName() + this.getVersion() + this.getOrganization());
}
} }

View File

@ -104,9 +104,27 @@ public void testNotFoundSearch() throws Exception {
@Test @Test
public void testGetRecommendedApps() throws Exception { public void testGetRecommendedApps() throws Exception {
List<AppStoreEntry> expected = spy.getRecommendedApps(); AppStoreEntry example = new AppStoreEntry();
example.setOrg("jenkins-ci.org");
example.setName("jenkins");
example.setDesc("World leading open source automation system.");
example.setIcon("/css/img/feather.png");
example.setDownload(100);
spy.register(example);
AppStoreEntry example2 = new AppStoreEntry();
example2.setOrg("Apache");
example2.setName("httpd");
example2.setDesc("Apache webserver");
example2.setIcon("/css/img/feather.png");
example2.setDownload(1);
spy.register(example2);
List<AppStoreEntry> actual = spy.getRecommendedApps(); List<AppStoreEntry> actual = spy.getRecommendedApps();
assertEquals(expected, actual); long previous = 1000L;
for (AppStoreEntry app: actual) {
assertTrue("Recommend app is not sort by download count.",
previous > app.getDownload());
previous = app.getDownload();
}
} }
} }