Merge pull request #815 from tobiass-sdl/sdl/structured_queries

add support for structured queries (opensearch only)
komoot · Jul 9, 2024 · be99d7d · be99d7d
2 parents 3547a95 + bc230d2
commit be99d7d
Show file tree

Hide file tree

Showing 22 changed files with 1,126 additions and 131 deletions.
diff --git a/app/es_embedded/src/main/java/de/komoot/photon/Server.java b/app/es_embedded/src/main/java/de/komoot/photon/Server.java
@@ -3,6 +3,7 @@
 import de.komoot.photon.DatabaseProperties;
 import de.komoot.photon.Importer;
 import de.komoot.photon.Updater;
+import de.komoot.photon.searcher.StructuredSearchHandler;
 import de.komoot.photon.searcher.ReverseHandler;
 import de.komoot.photon.searcher.SearchHandler;
 import de.komoot.photon.elasticsearch.*;
@@ -168,12 +169,12 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx
 
     }
 
-    public DatabaseProperties recreateIndex(String[] languages, Date importDate) throws IOException {
+    public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
         deleteIndex();
 
         loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);
 
-        new IndexMapping().addLanguages(languages).putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
+        createAndPutIndexMapping(languages, supportStructuredQueries);
 
         DatabaseProperties dbProperties = new DatabaseProperties()
             .setLanguages(languages)
@@ -183,6 +184,16 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate) thr
         return dbProperties;
     }
 
+    private void createAndPutIndexMapping(String[] languages, boolean supportStructuredQueries)
+    {
+        if (supportStructuredQueries) {
+            throw new UnsupportedOperationException("Structured queries are not supported for elasticsearch-based Photon. Consider to use OpenSearch.");
+        }
+
+        new IndexMapping().addLanguages(languages)
+                          .putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
+    }
+
     public void updateIndexSettings(String synonymFile) throws IOException {
         // Load the settings from the database to make sure it is at the right
         // version. If the version is wrong, we should not be messing with the
@@ -195,9 +206,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
         // Sanity check: legacy databases don't save the languages, so there is no way to update
         //               the mappings consistently.
         if (dbProperties.getLanguages() != null) {
-            new IndexMapping()
-                    .addLanguages(dbProperties.getLanguages())
-                    .putMapping(esClient, PhotonIndex.NAME, PhotonIndex.TYPE);
+            this.createAndPutIndexMapping(dbProperties.getLanguages(), false);
         }
     }
 
@@ -279,6 +288,10 @@ public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec
         return new ElasticsearchSearchHandler(esClient, languages, queryTimeoutSec);
     }
 
+    public StructuredSearchHandler createStructuredSearchHandler(String[] languages, int queryTimeoutSec) {
+        throw new UnsupportedOperationException("Structured queries are not supported for elasticsearch-based Photon. Consider to use OpenSearch.");
+    }
+
     public ReverseHandler createReverseHandler(int queryTimeoutSec) {
         return new ElasticsearchReverseHandler(esClient, queryTimeoutSec);
     }

diff --git a/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java b/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java
@@ -55,7 +55,7 @@ public void setUpES() throws IOException {
     public void setUpES(Path test_directory, String... languages) throws IOException {
         server = new ElasticTestServer(test_directory.toString());
         server.start(TEST_CLUSTER_NAME, new String[]{});
-        server.recreateIndex(languages, new Date());
+        server.recreateIndex(languages, new Date(), false);
         refresh();
     }
 

diff --git a/app/opensearch/src/main/java/de/komoot/photon/Server.java b/app/opensearch/src/main/java/de/komoot/photon/Server.java
@@ -5,6 +5,7 @@
 import de.komoot.photon.opensearch.*;
 import de.komoot.photon.searcher.ReverseHandler;
 import de.komoot.photon.searcher.SearchHandler;
+import de.komoot.photon.searcher.StructuredSearchHandler;
 import org.apache.hc.core5.http.HttpHost;
 import org.codelibs.opensearch.runner.OpenSearchRunner;
 import org.opensearch.client.json.jackson.JacksonJsonpMapper;
@@ -97,18 +98,19 @@ public void shutdown() {
         }
     }
 
-    public DatabaseProperties recreateIndex(String[] languages, Date importDate) throws IOException {
+    public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
         // delete any existing data
         if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) {
             client.indices().delete(d -> d.index(PhotonIndex.NAME));
         }
 
         (new IndexSettingBuilder()).setShards(5).createIndex(client, PhotonIndex.NAME);
 
-        (new IndexMapping()).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
+        (new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
 
         var dbProperties = new DatabaseProperties()
                 .setLanguages(languages)
+                .setSupportStructuredQueries(supportStructuredQueries)
                 .setImportDate(importDate);
         saveToDatabase(dbProperties);
 
@@ -122,7 +124,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
         (new IndexSettingBuilder()).setSynonymFile(synonymFile).updateIndex(client, PhotonIndex.NAME);
 
         if (dbProperties.getLanguages() != null) {
-            (new IndexMapping())
+            (new IndexMapping(dbProperties.getSupportStructuredQueries()))
                     .addLanguages(dbProperties.getLanguages())
                     .putMapping(client, PhotonIndex.NAME);
         }
@@ -154,6 +156,7 @@ public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException
 
         dbProperties.setLanguages(dbEntry.source().languages);
         dbProperties.setImportDate(dbEntry.source().importDate);
+        dbProperties.setSupportStructuredQueries(dbEntry.source().supportStructuredQueries);
     }
 
     public Importer createImporter(String[] languages, String[] extraTags) {
@@ -170,6 +173,10 @@ public SearchHandler createSearchHandler(String[] languages, int queryTimeoutSec
         return new OpenSearchSearchHandler(client, languages, queryTimeoutSec);
     }
 
+    public StructuredSearchHandler createStructuredSearchHandler(String[] languages, int queryTimeoutSec) {
+        return new OpenSearchStructuredSearchHandler(client, languages, queryTimeoutSec);
+    }
+
     public ReverseHandler createReverseHandler(int queryTimeoutSec) {
         return new OpenSearchReverseHandler(client, queryTimeoutSec);
     }