Skip to content

Commit

Permalink
Cached statistics and all ontologies for ontologies tab and cleaned u…
Browse files Browse the repository at this point in the history
…p logging on backend.
  • Loading branch information
henrietteharmse committed Jul 13, 2024
1 parent e5c4a2e commit 09d8251
Show file tree
Hide file tree
Showing 10 changed files with 173 additions and 79 deletions.
5 changes: 5 additions & 0 deletions backend/src/main/java/uk/ac/ebi/spot/ols/Ols4Backend.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;

/**
* TODO: We should consider caching responses that are static (such as statistics and ontologies loaded
* for the ontologies in the ontologies tab) on startup of the OLS backend.
*
*/
@SpringBootApplication
public class Ols4Backend {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,6 @@ public void search(
@RequestParam(value = "lang", defaultValue = "en") String lang,
HttpServletResponse response
) throws IOException, SolrServerException {
System.out.println("fieldList 1 = " + fieldList);
System.out.println("type = " + types);

final SolrQuery solrQuery = new SolrQuery(); // 1

Expand Down Expand Up @@ -208,15 +206,11 @@ public void search(
* Fix: End
*/
solrQuery.add("wt", format);
System.out.println("fieldList 2 = " + fieldList);

System.out.println("solrQuery=" + solrQuery.jsonStr());

QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities");

List<Object> docs = new ArrayList<>();
for(SolrDocument res : qr.getResults()) {
System.out.println("res = " + res.toString());
String _json = (String)res.get("_json");
if(_json == null) {
throw new RuntimeException("_json was null");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
Expand Down Expand Up @@ -42,6 +42,8 @@ public class V1SelectController {
@Autowired
private OlsSolrClient solrClient;

private static final Logger logger = LoggerFactory.getLogger(V1SelectController.class);

@RequestMapping(path = "/api/select", produces = {MediaType.APPLICATION_JSON_VALUE}, method = RequestMethod.GET)
public void select(
@RequestParam("q") String query,
Expand Down Expand Up @@ -118,7 +120,7 @@ public void select(
solrQuery.addHighlightField("whitespace_edge_synonym");
solrQuery.addHighlightField("synonym");

System.out.println("select: " + solrQuery.toQueryString());
logger.debug("select () ", solrQuery.toQueryString());

QueryResponse qr = solrClient.dispatchSearch(solrQuery, "ols4_entities");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ public HttpEntity<V2PagedAndFacetedResponse<V2Entity>> getOntologies(
@RequestParam(value = "includeObsoleteEntities", required = false, defaultValue = "false") boolean includeObsoleteEntities,
@RequestParam Map<String, Collection<String>> searchProperties
) throws ResourceNotFoundException, IOException {
logger.trace("Pageable: {}", pageable);
logger.trace("lang: {}", lang);
logger.trace("search: {}", search);
logger.trace("searchFields: {}", searchFields);
logger.trace("boostFields: {}", boostFields);
logger.trace("exactMatch: {}", exactMatch);
logger.trace("includeObsoleteEntities: {}", includeObsoleteEntities);
logger.trace("searchProperties: {}", searchProperties);

Map<String,Collection<String>> properties = new HashMap<>();
if(!includeObsoleteEntities)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.rest.webmvc.ResourceNotFoundException;
import org.springframework.hateoas.MediaTypes;
Expand All @@ -27,36 +29,46 @@ public class V2StatisticsController {
@Autowired
OlsSolrClient solrClient;

private static ResponseEntity statisticsResponse = null ;
private static final Logger logger = LoggerFactory.getLogger(V2StatisticsController.class);

@RequestMapping(produces = {MediaType.APPLICATION_JSON_VALUE, MediaTypes.HAL_JSON_VALUE}, method = RequestMethod.GET)
public HttpEntity<V2Statistics> getStatistics() throws ResourceNotFoundException, IOException {

Map<String,Object> coreStatus = solrClient.getCoreStatus();
Map<String,Object> indexStatus = (Map<String,Object>) coreStatus.get("index");
String lastModified = (String) indexStatus.get("lastModified");
if (statisticsResponse == null) {
logger.debug("Getting statistics from Solr");
Map<String, Object> coreStatus = solrClient.getCoreStatus();
Map<String, Object> indexStatus = (Map<String, Object>) coreStatus.get("index");
String lastModified = (String) indexStatus.get("lastModified");

SolrQuery query = new SolrQuery();
SolrQuery query = new SolrQuery();

query.setQuery("*:*");
query.setFacet(true);
query.addFacetField("type");
query.setRows(0);
query.setQuery("*:*");
query.setFacet(true);
query.addFacetField("type");
query.setRows(0);

QueryResponse qr = solrClient.runSolrQuery(query, null);
QueryResponse qr = solrClient.runSolrQuery(query, null);

Map<String,Integer> counts = new HashMap<>();
Map<String, Integer> counts = new HashMap<>();

for(FacetField.Count count : qr.getFacetField("type").getValues()) {
counts.put(count.getName(), (int)count.getCount());
}
for (FacetField.Count count : qr.getFacetField("type").getValues()) {
counts.put(count.getName(), (int) count.getCount());
}

V2Statistics stats = new V2Statistics();
stats.lastModified = lastModified;
stats.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0;
stats.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0;
stats.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0;
stats.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0;

return new ResponseEntity<>( stats, HttpStatus.OK);
}
V2Statistics statistics = new V2Statistics();
statistics.lastModified = lastModified;
statistics.numberOfOntologies = counts.containsKey("ontology") ? counts.get("ontology") : 0;
statistics.numberOfClasses = counts.containsKey("class") ? counts.get("class") : 0;
statistics.numberOfIndividuals = counts.containsKey("individual") ? counts.get("individual") : 0;
statistics.numberOfProperties = counts.containsKey("property") ? counts.get("property") : 0;

statisticsResponse = new ResponseEntity<>(statistics, HttpStatus.OK);

} else {
logger.debug("Getting statistics from cache");
}
return statisticsResponse;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package uk.ac.ebi.spot.ols.controller.api.v2.cache;

import com.google.gson.Gson;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.rest.webmvc.ResourceNotFoundException;
import org.springframework.data.web.PageableDefault;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import uk.ac.ebi.spot.ols.controller.api.v2.helpers.DynamicQueryHelper;
import uk.ac.ebi.spot.ols.controller.api.v2.responses.V2PagedAndFacetedResponse;
import uk.ac.ebi.spot.ols.model.v2.V2Entity;
import uk.ac.ebi.spot.ols.repository.v2.V2OntologyRepository;

import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* This controller caches all ontologies for use in the ontologies tab.
* This is done to address OLS slowness as backend, solr and neo4j pods all located on different nodes. See
* https://overcast.blog/minimizing-inter-node-communication-in-kubernetes-dc53a8e28212.
*
*/
@Controller
@RequestMapping("/api/v2/cache/ontologies")
public class V2OntologyCacheController {

private Gson gson = new Gson();

@Autowired
V2OntologyRepository ontologyRepository;

private static ResponseEntity allOntologiesResponse = null;

private static final Logger logger = LoggerFactory.getLogger(V2OntologyCacheController.class);

@RequestMapping(path = "", produces = {MediaType.APPLICATION_JSON_VALUE }, method = RequestMethod.GET)
public HttpEntity<V2PagedAndFacetedResponse<V2Entity>> getAllOntologies(
) throws ResourceNotFoundException, IOException {

if (allOntologiesResponse == null) {
logger.debug("getAllOntologies from Solr");
Map<String, Collection<String>> properties = new HashMap<>();
properties.put("isObsolete", List.of("false"));

Pageable pageable = PageRequest.of(0, 1000);
allOntologiesResponse = new ResponseEntity<>(
new V2PagedAndFacetedResponse<>(
ontologyRepository.find(pageable, "en", null, null, null, false, DynamicQueryHelper.filterProperties(properties))
),
HttpStatus.OK);
logger.trace("allOntologiesResponse = {}", allOntologiesResponse);
} else {
logger.debug("getAllOntologies from cache");
}
return allOntologiesResponse;
}
}
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package uk.ac.ebi.spot.ols.repository.neo4j;

import com.google.gson.JsonElement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.stereotype.Component;
import uk.ac.ebi.spot.ols.repository.solr.OlsSolrClient;
import uk.ac.ebi.spot.ols.service.Neo4jClient;

import java.util.List;
Expand All @@ -19,6 +22,8 @@ public class OlsNeo4jClient {
@Autowired
Neo4jClient neo4jClient;

private static final Logger logger = LoggerFactory.getLogger(OlsNeo4jClient.class);

public Page<JsonElement> getAll(String type, Map<String,String> properties, Pageable pageable) {

String query = "MATCH (a:" + type + ")";
Expand Down Expand Up @@ -71,40 +76,40 @@ private String makeEdgePropsClause(Map<String,String> edgeProps) {

public Page<JsonElement> traverseOutgoingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, edgeProps);
String edge = makeEdgesList(edgeIRIs, edgeProps);

// TODO fix injection
// TODO fix injection

String query =
"MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN distinct b";
String query =
"MATCH (a:" + type+ ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN distinct b";

String countQuery =
"MATCH (a:" + type + ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN count(distinct b)";
String countQuery =
"MATCH (a:" + type + ")-[edge:" + edge + "]->(b) "
+ "WHERE a.id = $id " + makeEdgePropsClause(edgeProps)
+ "RETURN count(distinct b)";

System.out.println(query);
logger.debug("Query = {}", query);

return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
}

public Page<JsonElement> traverseIncomingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, Map.of());
String edge = makeEdgesList(edgeIRIs, Map.of());

String query =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN distinct b";
String query =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN distinct b";

String countQuery =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN count(distinct b)";
String countQuery =
"MATCH (a:" + type + ")<-[edge:" + edge + "]-(b) "
+ "WHERE a.id = $id "
+ "RETURN count(distinct b)";

return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
return neo4jClient.queryPaginated(query, "b", countQuery, parameters("type", type, "id", id), pageable);
}

public Page<JsonElement> recursivelyTraverseOutgoingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {
Expand All @@ -128,21 +133,21 @@ public Page<JsonElement> recursivelyTraverseOutgoingEdges(String type, String id

public Page<JsonElement> recursivelyTraverseIncomingEdges(String type, String id, List<String> edgeIRIs, Map<String,String> edgeProps, Pageable pageable) {

String edge = makeEdgesList(edgeIRIs, Map.of());
String edge = makeEdgesList(edgeIRIs, Map.of());

String query =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN DISTINCT descendant AS c";
String query =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN DISTINCT descendant AS c";

String countQuery =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN count(DISTINCT descendant)";
String countQuery =
"MATCH (a:" + type + ") WHERE a.id = $id "
+ "WITH a "
+ "OPTIONAL MATCH (a)<-[edge:" + edge + " *]-(descendant) "
+ "RETURN count(DISTINCT descendant)";

return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable);
return neo4jClient.queryPaginated(query, "c", countQuery, parameters("id", id), pageable);
}


Expand Down
Loading

0 comments on commit 09d8251

Please sign in to comment.