From faae26772543d8edfc6e91c3942df941fdd722de Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Mon, 24 Jun 2024 17:03:02 -0700 Subject: [PATCH 1/3] WIP: Fix common names parsing bug --- api/src/services/itis-service.ts | 2 +- api/src/services/taxonomy-service.ts | 34 ++++++++++++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/api/src/services/itis-service.ts b/api/src/services/itis-service.ts index 9407d867..6f6b921e 100644 --- a/api/src/services/itis-service.ts +++ b/api/src/services/itis-service.ts @@ -6,7 +6,7 @@ import { TaxonSearchResult } from './taxonomy-service'; const defaultLog = getLogger('services/itis-service'); export type ItisSolrSearchResponse = { - commonNames: string[]; + commonNames?: string[]; kingdom: string; name: string; parentTSN: string; diff --git a/api/src/services/taxonomy-service.ts b/api/src/services/taxonomy-service.ts index b9686cd8..8cc4015f 100644 --- a/api/src/services/taxonomy-service.ts +++ b/api/src/services/taxonomy-service.ts @@ -72,18 +72,7 @@ export class TaxonomyService { * @memberof TaxonomyService */ async addItisTaxonRecord(itisSolrResponse: ItisSolrSearchResponse): Promise { - const commonNames = - itisSolrResponse.commonNames - .filter((name) => name.split('$')[2] === 'English') - .map((name) => name.split('$')[1]) ?? []; - /* Sample itisResponse: - * commonNames: [ - * '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', - * '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', - * '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', - * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' - * ] - */ + const commonNames = this._parseItisTaxonCommonNames(itisSolrResponse?.commonNames); return this.taxonRepository.addItisTaxonRecord( Number(itisSolrResponse.tsn), @@ -94,6 +83,27 @@ export class TaxonomyService { ); } + /** + * Parse the raw common names string from an ITIS taxon record into an array of english common names. + * + * @example + * const commonNames = [ + * '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', + * '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', + * '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', + * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' + * ] + * + * const result = _parseItisTaxonCommonNames(commonNames) + * // result: ['withered wooly milk-vetch', 'woolly locoweed', 'Davis Mountains locoweed', 'woolly milkvetch'] + * + * @param {string[]} [commonNames] + * @memberof TaxonomyService + */ + _parseItisTaxonCommonNames = (commonNames?: string[]): string[] => { + return commonNames?.filter((name) => name.split('$')[2] === 'English').map((name) => name.split('$')[1]) ?? []; + }; + /** * Delete an existing taxon record. * From 3ed3223574bf84f947bb3f0604fe3df1caf8a3b5 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 25 Jun 2024 09:48:44 -0700 Subject: [PATCH 2/3] Add commonNames parser util function --- api/src/services/itis-service.ts | 5 +-- api/src/services/taxonomy-service.ts | 24 +---------- .../{itis-sort.test.ts => itis-utils.test.ts} | 41 ++++++++++++++++++- api/src/utils/{itis-sort.ts => itis-utils.ts} | 21 ++++++++++ 4 files changed, 65 insertions(+), 26 deletions(-) rename api/src/utils/{itis-sort.test.ts => itis-utils.test.ts} (60%) rename api/src/utils/{itis-sort.ts => itis-utils.ts} (79%) diff --git a/api/src/services/itis-service.ts b/api/src/services/itis-service.ts index 6f6b921e..52704e72 100644 --- a/api/src/services/itis-service.ts +++ b/api/src/services/itis-service.ts @@ -1,5 +1,5 @@ import axios from 'axios'; -import { sortTaxonSearchResults } from '../utils/itis-sort'; +import { getItisTaxonCommonNames, sortTaxonSearchResults } from '../utils/itis-utils'; import { getLogger } from '../utils/logger'; import { TaxonSearchResult } from './taxonomy-service'; @@ -83,8 +83,7 @@ export class ItisService { */ _sanitizeItisData = (data: ItisSolrSearchResponse[]): TaxonSearchResult[] => { return data.map((item: ItisSolrSearchResponse) => { - const englishNames = item.commonNames?.filter((name) => name.split('$')[2] === 'English'); - const commonNames = englishNames?.map((name) => name.split('$')[1]) ?? []; + const commonNames = getItisTaxonCommonNames(item.commonNames); return { tsn: Number(item.tsn), diff --git a/api/src/services/taxonomy-service.ts b/api/src/services/taxonomy-service.ts index 8cc4015f..dcfc5e68 100644 --- a/api/src/services/taxonomy-service.ts +++ b/api/src/services/taxonomy-service.ts @@ -1,5 +1,6 @@ import { IDBConnection } from '../database/db'; import { TaxonomyRepository, TaxonRecord } from '../repositories/taxonomy-repository'; +import { getItisTaxonCommonNames } from '../utils/itis-utils'; import { getLogger } from '../utils/logger'; import { ItisService, ItisSolrSearchResponse } from './itis-service'; @@ -72,7 +73,7 @@ export class TaxonomyService { * @memberof TaxonomyService */ async addItisTaxonRecord(itisSolrResponse: ItisSolrSearchResponse): Promise { - const commonNames = this._parseItisTaxonCommonNames(itisSolrResponse?.commonNames); + const commonNames = getItisTaxonCommonNames(itisSolrResponse?.commonNames); return this.taxonRepository.addItisTaxonRecord( Number(itisSolrResponse.tsn), @@ -83,27 +84,6 @@ export class TaxonomyService { ); } - /** - * Parse the raw common names string from an ITIS taxon record into an array of english common names. - * - * @example - * const commonNames = [ - * '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', - * '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', - * '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', - * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' - * ] - * - * const result = _parseItisTaxonCommonNames(commonNames) - * // result: ['withered wooly milk-vetch', 'woolly locoweed', 'Davis Mountains locoweed', 'woolly milkvetch'] - * - * @param {string[]} [commonNames] - * @memberof TaxonomyService - */ - _parseItisTaxonCommonNames = (commonNames?: string[]): string[] => { - return commonNames?.filter((name) => name.split('$')[2] === 'English').map((name) => name.split('$')[1]) ?? []; - }; - /** * Delete an existing taxon record. * diff --git a/api/src/utils/itis-sort.test.ts b/api/src/utils/itis-utils.test.ts similarity index 60% rename from api/src/utils/itis-sort.test.ts rename to api/src/utils/itis-utils.test.ts index 94eb506b..9892d378 100644 --- a/api/src/utils/itis-sort.test.ts +++ b/api/src/utils/itis-utils.test.ts @@ -1,7 +1,7 @@ import { expect } from 'chai'; import { describe } from 'mocha'; import { TaxonSearchResult } from '../services/taxonomy-service'; -import { sortTaxonSearchResults } from './itis-sort'; +import { getItisTaxonCommonNames, sortTaxonSearchResults } from './itis-utils'; describe('itis-sort', () => { describe('sortTaxonSearchResults', () => { @@ -78,3 +78,42 @@ describe('itis-sort', () => { }); }); }); + +describe('getItisTaxonCommonNames', () => { + it('Returns an empty array if provided common names is undefined', () => { + const rawCommonNames = undefined; + + const commonNames = getItisTaxonCommonNames(rawCommonNames); + + expect(commonNames).to.eql([]); + }); + + it('Returns an empty array if the provided common names is empty', () => { + const rawCommonNames: string[] = []; + + const commonNames = getItisTaxonCommonNames(rawCommonNames); + + expect(commonNames).to.eql([]); + }); + + it('Returns an array of english common names', () => { + const rawCommonNames = [ + '$withered wooly milk-vetch (German)$German$N$152846$2012-12-21 00:00:00$', + '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', + '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', + '$Davis Mountains locoweed (French)$French$N$124502$2011-06-29 00:00:00$', + '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', + '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$', + '$woolly milkvetch (French)$French$N$124502$2011-06-29 00:00:00$' + ]; + + const commonNames = getItisTaxonCommonNames(rawCommonNames); + + expect(commonNames).to.eql([ + 'withered wooly milk-vetch', + 'woolly locoweed', + 'Davis Mountains locoweed', + 'woolly milkvetch' + ]); + }); +}); diff --git a/api/src/utils/itis-sort.ts b/api/src/utils/itis-utils.ts similarity index 79% rename from api/src/utils/itis-sort.ts rename to api/src/utils/itis-utils.ts index a4c4ec70..3cdc16b9 100644 --- a/api/src/utils/itis-sort.ts +++ b/api/src/utils/itis-utils.ts @@ -87,3 +87,24 @@ export const sortTaxonSearchResults = ( // Sort the data by the score return taxonSearchResults.sort((a, b) => calculateScore(b) - calculateScore(a)); }; + +/** + * Parse the raw common names string from an ITIS taxon record into an array of english common names. + * + * @example + * const commonNames = [ + * '$withered wooly milk-vetch$English$N$152846$2012-12-21 00:00:00$', + * '$woolly locoweed$English$N$124501$2011-06-29 00:00:00$', + * '$Davis Mountains locoweed$English$N$124502$2011-06-29 00:00:00$', + * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' + * ] + * + * const result = _parseItisTaxonCommonNames(commonNames) + * // result: ['withered wooly milk-vetch', 'woolly locoweed', 'Davis Mountains locoweed', 'woolly milkvetch'] + * + * @param {string[]} [commonNames] + * @memberof TaxonomyService + */ +export const getItisTaxonCommonNames = (commonNames?: string[]): string[] => { + return commonNames?.filter((name) => name.split('$')[2] === 'English').map((name) => name.split('$')[1]) ?? []; +}; From be33590b4279218f27baea16d94481121cd8492d Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 25 Jun 2024 14:26:04 -0700 Subject: [PATCH 3/3] Fix doc --- api/src/utils/itis-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/utils/itis-utils.ts b/api/src/utils/itis-utils.ts index 3cdc16b9..baaa5b48 100644 --- a/api/src/utils/itis-utils.ts +++ b/api/src/utils/itis-utils.ts @@ -99,7 +99,7 @@ export const sortTaxonSearchResults = ( * '$woolly milkvetch$English$N$72035$2012-12-21 00:00:00$' * ] * - * const result = _parseItisTaxonCommonNames(commonNames) + * const result = getItisTaxonCommonNames(commonNames) * // result: ['withered wooly milk-vetch', 'woolly locoweed', 'Davis Mountains locoweed', 'woolly milkvetch'] * * @param {string[]} [commonNames]