From a3cb5ee947258f81ca8d54388648d2f694419822 Mon Sep 17 00:00:00 2001
From: Patrick Leary <patrick.r.leary@gmail.com>
Date: Thu, 12 Dec 2024 11:35:05 -0500
Subject: [PATCH] CV responses can included representative taxon photos

---
 .../v1/computervision_controller.js           | 65 ++++++++++++++++++-
 1 file changed, 64 insertions(+), 1 deletion(-)

diff --git a/lib/controllers/v1/computervision_controller.js b/lib/controllers/v1/computervision_controller.js
index 81e71bde..f33d3014 100644
--- a/lib/controllers/v1/computervision_controller.js
+++ b/lib/controllers/v1/computervision_controller.js
@@ -7,6 +7,7 @@ const PromisePool = require( "es6-promise-pool" );
 const csv = require( "fast-csv" );
 const crypto = require( "crypto" );
 const pgClient = require( "../../pg_client" );
+const esClient = require( "../../es_client" );
 const TaxaController = require( "./taxa_controller" );
 const InaturalistAPI = require( "../../inaturalist_api" );
 const config = require( "../../../config" );
@@ -262,6 +263,62 @@ const ComputervisionController = class ComputervisionController {
     return scores;
   }
 
+  static async addRepresentativePhotos( results, embedding ) {
+    if ( _.isEmpty( embedding ) ) {
+      return;
+    }
+    const taxonIDsToLookup = _.map( _.filter(
+      results, result => result.rank_level < 30
+    ), "taxon_id" );
+    if ( _.isEmpty( taxonIDsToLookup ) ) {
+      return;
+    }
+    const embeddingsResponse = await esClient.search( "taxon_photos", {
+      body: {
+        query: {
+          bool: {
+            filter: [
+              { terms: { taxon_ids: taxonIDsToLookup } }
+            ]
+          }
+        },
+        knn: {
+          field: "embedding",
+          query_vector: embedding,
+          k: 10,
+          num_candidates: 500
+        },
+        size: 10000,
+        _source: [
+          "id",
+          "taxon_id",
+          "photo_id",
+          "ancestor_ids"
+        ]
+      }
+    } );
+    const embeddingsHits = _.map( embeddingsResponse.hits.hits, "_source" );
+    // TODO: remove this once the ancestor_ids data in the index has been fixed
+    _.each( embeddingsHits, hit => {
+      hit.ancestor_ids = _.remove( hit.ancestor_ids, ancestorID => ancestorID === hit.id );
+      hit.ancestor_ids.push( hit.taxon_id );
+    } );
+    await ObservationPreload.assignObservationPhotoPhotos( embeddingsHits );
+    _.each( results, result => {
+      if ( result && result.taxon && result.taxon.default_photo ) {
+        const firstMatch = _.find( embeddingsHits, h => (
+          h?.photo?.url
+            && _.includes( h.ancestor_ids, result.taxon.id )
+        ) );
+        if ( firstMatch ) {
+          result.taxon.default_photo.url = firstMatch.photo.url.replace( "medium", "square" );
+          result.taxon.default_photo.medium_url = firstMatch.photo.url;
+          result.taxon.default_photo.square_url = firstMatch.photo.url.replace( "medium", "square" );
+        }
+      }
+    } );
+  }
+
   static async delegatedScoresProcessing( req, visionApiResponse ) {
     const localeOpts = util.localeOpts( req );
     const prepareTaxon = t => {
@@ -312,13 +369,19 @@ const ComputervisionController = class ComputervisionController {
     } );
 
     await ESModel.fetchBelongsTo( withTaxa, Taxon, taxonOpts );
-    await Taxon.preloadIntoTaxonPhotos( _.map( withTaxa, "taxon" ), { localeOpts } );
+    await Taxon.preloadIntoTaxonPhotos( _.map( _.filter( withTaxa, "taxon" ), "taxon" ), { localeOpts } );
+
     if ( !( req.body.aggregated || req.query.aggregated ) ) {
       _.each( withTaxa, s => {
         delete s.taxon_id;
       } );
     }
 
+    await ComputervisionController.addRepresentativePhotos(
+      withTaxa,
+      visionApiResponse.embedding
+    );
+
     // remove attributes of common_ancestor that should not be in the response
     if ( response.common_ancestor ) {
       await TaxaController.assignPlaces( [response.common_ancestor.taxon] );