TechDebt: Sort ITIS response & multiple common names (#237)

* Taxonomy endpoint returns multiple common names * Add sorting to itis taxon response --------- Co-authored-by: Nick Phura <[email protected]>
bcgov · May 10, 2024 · c8527c0 · c8527c0
1 parent dd73aa0
commit c8527c0
Show file tree

Hide file tree

Showing 14 changed files with 408 additions and 211 deletions.
diff --git a/api/src/paths/taxonomy/taxon/index.test.ts b/api/src/paths/taxonomy/taxon/index.test.ts
@@ -52,8 +52,8 @@ describe('taxon', () => {
 
       sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
 
-      const mock1 = { id: '1', commonName: 'something', scientificName: 'string' } as unknown as any;
-      const mock2 = { id: '2', commonName: null, scientificName: 'string' } as unknown as any;
+      const mock1 = { id: '1', commonNames: ['something'], scientificName: 'string' } as unknown as any;
+      const mock2 = { id: '2', commonNames: null, scientificName: 'string' } as unknown as any;
 
       const getSpeciesFromIdsStub = sinon.stub(ItisService.prototype, 'searchItisByTerm').resolves([mock1, mock2]);
 

diff --git a/api/src/paths/taxonomy/taxon/index.ts b/api/src/paths/taxonomy/taxon/index.ts
@@ -45,17 +45,25 @@ GET.apiDoc = {
                 items: {
                   title: 'Taxon',
                   type: 'object',
-                  required: ['tsn', 'commonName', 'scientificName'],
+                  required: ['tsn', 'commonNames', 'scientificName'],
                   properties: {
                     tsn: {
                       type: 'integer'
                     },
-                    commonName: {
-                      type: 'string',
-                      nullable: true
+                    commonNames: {
+                      type: 'array',
+                      items: {
+                        type: 'string'
+                      }
                     },
                     scientificName: {
                       type: 'string'
+                    },
+                    rank: {
+                      type: 'string'
+                    },
+                    kingdom: {
+                      type: 'string'
                     }
                   },
                   additionalProperties: false

diff --git a/api/src/paths/taxonomy/taxon/tsn/index.test.ts b/api/src/paths/taxonomy/taxon/tsn/index.test.ts
@@ -52,8 +52,8 @@ describe('tsn', () => {
 
       sinon.stub(db, 'getDBConnection').returns(dbConnectionObj);
 
-      const mock1 = { tsn: '1', commonName: 'something', scientificName: 'string' } as unknown as any;
-      const mock2 = { tsn: '2', commonName: null, scientificName: 'string' } as unknown as any;
+      const mock1 = { tsn: '1', commonNames: ['something'], scientificName: 'string' } as unknown as any;
+      const mock2 = { tsn: '2', commonNames: [], scientificName: 'string' } as unknown as any;
 
       const getTaxonByTsnIdsStub = sinon.stub(TaxonomyService.prototype, 'getTaxonByTsnIds').resolves([mock1, mock2]);
 

diff --git a/api/src/paths/taxonomy/taxon/tsn/index.ts b/api/src/paths/taxonomy/taxon/tsn/index.ts
@@ -47,17 +47,25 @@ GET.apiDoc = {
                 items: {
                   title: 'Species',
                   type: 'object',
-                  required: ['tsn', 'commonName', 'scientificName'],
+                  required: ['tsn', 'commonNames', 'scientificName'],
                   properties: {
                     tsn: {
                       type: 'integer'
                     },
-                    commonName: {
-                      type: 'string',
-                      nullable: true
+                    commonNames: {
+                      type: 'array',
+                      items: {
+                        type: 'string'
+                      }
                     },
                     scientificName: {
                       type: 'string'
+                    },
+                    rank: {
+                      type: 'string'
+                    },
+                    kingdom: {
+                      type: 'string'
                     }
                   },
                   additionalProperties: false

diff --git a/api/src/repositories/taxonomy-repository.test.ts b/api/src/repositories/taxonomy-repository.test.ts
@@ -73,7 +73,7 @@ describe('TaxonomyRepository', () => {
 
       const taxonomyRepository = new TaxonomyRepository(mockDBConnection);
 
-      const response = await taxonomyRepository.addItisTaxonRecord(1, 'string', 'string', {}, 'string');
+      const response = await taxonomyRepository.addItisTaxonRecord(1, 'string', ['string'], {}, 'string');
 
       expect(response).to.be.eql({
         taxon_id: 1,

diff --git a/api/src/repositories/taxonomy-repository.ts b/api/src/repositories/taxonomy-repository.ts
@@ -53,7 +53,7 @@ export class TaxonomyRepository extends BaseRepository {
    *
    * @param {number} itisTsn
    * @param {string} itisScientificName
-   * @param {(string | null)} commonName
+   * @param {string[]} commonNames
    * @param {Record<any, any>} itisData
    * @param {string} itisUpdateDate
    * @return {*}  {Promise<TaxonRecord>}
@@ -62,12 +62,13 @@ export class TaxonomyRepository extends BaseRepository {
   async addItisTaxonRecord(
     itisTsn: number,
     itisScientificName: string,
-    commonName: string | null,
+    commonNames: string[],
     itisData: Record<string, unknown>,
     itisUpdateDate: string
   ): Promise<TaxonRecord> {
     defaultLog.debug({ label: 'addItisTaxonRecord', itisTsn });
 
+    // TODO: Store multiple common names rather than just the first item of the commonNames array
     const sqlStatement = SQL`
       WITH inserted_row AS (
         INSERT INTO
@@ -82,7 +83,7 @@ export class TaxonomyRepository extends BaseRepository {
         VALUES (
           ${itisTsn},
           ${itisScientificName},
-          ${commonName},
+          ${commonNames[0]},
           ${itisData},
           ${itisUpdateDate}
         )

diff --git a/api/src/services/itis-service.test.ts b/api/src/services/itis-service.test.ts
@@ -40,14 +40,15 @@ describe('ItisService', () => {
           response: {
             docs: [
               {
-                commonNames: ['$commonName'],
+                commonNames: ['$commonNames$English'],
                 kingdom: 'kingdom',
                 name: 'name',
                 parentTSN: 'parentTSN',
                 scientificName: 'scientificName',
                 tsn: '123',
                 updateDate: 'updateDate',
-                usage: 'usage'
+                usage: '',
+                rank: 'kingdom'
               }
             ]
           }
@@ -67,8 +68,10 @@ describe('ItisService', () => {
       expect(response).to.eql([
         {
           tsn: 123,
-          commonName: 'commonName',
-          scientificName: 'scientificName'
+          commonNames: ['commonNames'],
+          scientificName: 'scientificName',
+          rank: 'kingdom',
+          kingdom: 'kingdom'
         }
       ]);
 
@@ -121,14 +124,15 @@ describe('ItisService', () => {
           response: {
             docs: [
               {
-                commonNames: ['$commonName'],
+                commonNames: ['$commonNames$English'],
                 kingdom: 'kingdom',
                 name: 'name',
                 parentTSN: 'parentTSN',
                 scientificName: 'scientificName',
                 tsn: '123',
                 updateDate: 'updateDate',
-                usage: 'usage'
+                usage: '',
+                rank: 'kingdom'
               }
             ]
           }
@@ -145,14 +149,15 @@ describe('ItisService', () => {
 
       expect(response).to.eql([
         {
-          commonNames: ['$commonName'],
+          commonNames: ['$commonNames$English'],
           kingdom: 'kingdom',
           name: 'name',
           parentTSN: 'parentTSN',
           scientificName: 'scientificName',
           tsn: '123',
           updateDate: 'updateDate',
-          usage: 'usage'
+          usage: '',
+          rank: 'kingdom'
         }
       ]);
 
@@ -200,7 +205,7 @@ describe('ItisService', () => {
       const response = await itisService.getItisSolrTermSearchUrl(['term']);
 
       expect(response).to.equal(
-        'https://services.itis.gov/?wt=json&sort=nameWOInd+asc&rows=25&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage&q=((nameWOInd:*term*+AND+usage:/(valid|accepted)/)+OR+(vernacular:*term*+AND+usage:/(valid|accepted)/))'
+        'https://services.itis.gov/?wt=json&sort=kingdom+asc&rows=150&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage+rank&q=((nameWOInd:*term*+AND+usage:/(valid|accepted)/)+OR+(vernacular:*term*+AND+usage:/(valid|accepted)/))'
       );
     });
   });
@@ -228,7 +233,7 @@ describe('ItisService', () => {
       const response = await itisService.getItisSolrTsnSearchUrl([123]);
 
       expect(response).to.equal(
-        'https://services.itis.gov/??wt=json&sort=nameWOInd+asc&rows=25&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage&&q=tsn:123'
+        'https://services.itis.gov/??wt=json&sort=kingdom+asc&rows=150&omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage+rank&&q=tsn:123'
       );
     });
   });

diff --git a/api/src/services/itis-service.ts b/api/src/services/itis-service.ts
@@ -1,4 +1,5 @@
 import axios from 'axios';
+import { sortTaxonSearchResults } from '../utils/itis-sort';
 import { getLogger } from '../utils/logger';
 import { TaxonSearchResult } from './taxonomy-service';
 
@@ -13,6 +14,7 @@ export type ItisSolrSearchResponse = {
   tsn: string;
   updateDate: string;
   usage: string;
+  rank: string;
 };
 
 /**
@@ -42,7 +44,14 @@ export class ItisService {
       return [];
     }
 
-    return this._sanitizeItisData(response.data.response.docs);
+    const sanitizedResponse = this._sanitizeItisData(response.data.response.docs);
+
+    // Sort the results to place exact matches at the top
+    const sortedResponse = sortTaxonSearchResults(sanitizedResponse, searchTerms);
+
+    // Return only a subset of the records
+    // More records than are returned here are requested from ITIS to help find and prioritize exact matches
+    return sortedResponse.slice(0, 15);
   }
 
   /**
@@ -67,19 +76,22 @@ export class ItisService {
   }
 
   /**
-   * Cleans up the ITIS search response data.
+   * Cleans up the ITIS search response data
    *
    * @param {ItisSolrSearchResponse[]} data
    * @memberof ItisService
    */
   _sanitizeItisData = (data: ItisSolrSearchResponse[]): TaxonSearchResult[] => {
     return data.map((item: ItisSolrSearchResponse) => {
-      const commonName = item.commonNames ? item.commonNames[0].split('$')[1] : null;
+      const englishNames = item.commonNames?.filter((name) => name.split('$')[2] === 'English');
+      const commonNames = englishNames?.map((name) => name.split('$')[1]) ?? [];
 
       return {
         tsn: Number(item.tsn),
-        commonName: commonName,
-        scientificName: item.scientificName
+        commonNames: commonNames,
+        scientificName: item.scientificName,
+        rank: item.rank,
+        kingdom: item.kingdom
       };
     });
   };
@@ -100,9 +112,9 @@ export class ItisService {
     }
 
     return `${itisUrl}?${this._getItisSolrTypeParam()}&${this._getItisSolrSortParam(
-      'nameWOInd',
-      'asc',
-      25
+      ['kingdom'],
+      ['asc'],
+      150
     )}&${this._getItisSolrFilterParam()}&${this._getItisSolrQueryParam(searchTerms)}`;
   }
 
@@ -122,9 +134,9 @@ export class ItisService {
     }
 
     return `${itisUrl}??${this._getItisSolrTypeParam()}&${this._getItisSolrSortParam(
-      'nameWOInd',
-      'asc',
-      25
+      ['kingdom'],
+      ['asc'],
+      150
     )}&${this._getItisSolrFilterParam()}&&q=${this._getItisSolrTsnSearch(searchTsnIds)}`;
   }
 
@@ -153,12 +165,12 @@ export class ItisService {
    *
    * @param {string} sortBy
    * @param {('asc' | 'desc')} sortDir
-   * @param {number} [limit=25]
+   * @param {number} limit
    * @return {*}  {string}
    * @memberof ItisService
    */
-  _getItisSolrSortParam(sortBy: string, sortDir: 'asc' | 'desc', limit = 25): string {
-    return `sort=${sortBy}+${sortDir}&rows=${limit}`;
+  _getItisSolrSortParam(sortBy: string[], sortDir: ('asc' | 'desc')[], limit: number): string {
+    return `sort=${sortBy.map((f, index) => `${f}+${sortDir[index]}`).join(',')}&rows=${limit}`;
   }
 
   /**
@@ -168,7 +180,7 @@ export class ItisService {
    * @memberof ItisService
    */
   _getItisSolrFilterParam(): string {
-    return 'omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage';
+    return 'omitHeader=true&fl=tsn+scientificName:nameWOInd+kingdom+parentTSN+commonNames:vernacular+updateDate+usage+rank';
   }
 
   /**

diff --git a/api/src/services/taxonomy-service.test.ts b/api/src/services/taxonomy-service.test.ts
@@ -16,14 +16,15 @@ describe('TaxonomyService', () => {
 
   const getItisSolrSearchResponse: ItisSolrSearchResponse[] = [
     {
-      commonNames: ['$commonName'],
+      commonNames: ['$commonNames'],
       kingdom: 'kingdom',
       name: 'name',
       parentTSN: 'parentTSN',
       scientificName: 'scientificName',
       tsn: 'tsn',
       updateDate: 'updateDate',
-      usage: 'usage'
+      usage: 'usage',
+      rank: 'rank'
     }
   ];
 
@@ -63,7 +64,7 @@ describe('TaxonomyService', () => {
       const response = await taxonomyService.getTaxonByTsnIds([1]);
 
       expect(repo).to.be.calledOnce;
-      expect(response).to.be.eql([{ tsn: 1, commonName: 'common_name', scientificName: 'itis_scientific_name' }]);
+      expect(response).to.be.eql([{ tsn: 1, commonNames: ['common_name'], scientificName: 'itis_scientific_name' }]);
     });
 
     it('if some records do not exist in db should return array of taxon records', async () => {
@@ -118,8 +119,8 @@ describe('TaxonomyService', () => {
       expect(searchItisByTSNStub).to.be.calledOnce;
       expect(itisService).to.be.calledOnce;
       expect(response).to.be.eql([
-        { tsn: 1, commonName: 'common_name', scientificName: 'itis_scientific_name' },
-        { tsn: 2, commonName: 'common_name', scientificName: 'itis_scientific_name' }
+        { tsn: 1, commonNames: ['common_name'], scientificName: 'itis_scientific_name' },
+        { tsn: 2, commonNames: ['common_name'], scientificName: 'itis_scientific_name' }
       ]);
     });
   });
@@ -135,7 +136,7 @@ describe('TaxonomyService', () => {
         itis_tsn: 1,
         bc_taxon_code: null,
         itis_scientific_name: 'scientificName',
-        common_name: 'commonName',
+        common_name: 'commonNames',
         itis_data: {},
         record_effective_date: 'updateDate',
         record_end_date: null,
@@ -154,7 +155,7 @@ describe('TaxonomyService', () => {
         itis_tsn: 1,
         bc_taxon_code: null,
         itis_scientific_name: 'scientificName',
-        common_name: 'commonName',
+        common_name: 'commonNames',
         itis_data: {},
         record_effective_date: 'updateDate',
         record_end_date: null,