From d91b8b278b7208ed6983cf695eb7060ab0918faa Mon Sep 17 00:00:00 2001 From: kouloumos Date: Wed, 18 Dec 2024 11:21:58 +0200 Subject: [PATCH] feat(api): enhance search endpoint with flexible filtering and dynamic aggregations Add support for filter operations (include/exclude), OR logic via array values, and dynamic aggregation fields in the search API. Allow switching between default and coredev indices. This consolidates search functionality across products by making the Bitcoin Search API more flexible while maintaining backward compatibility. --- src/components/sidebarFacet/Facet.tsx | 5 +- src/components/sidebarFacet/FilterMenu.tsx | 4 +- src/pages/api/elasticSearchProxy/search.ts | 22 +++-- src/service/api/search/searchCall.ts | 8 +- src/types.ts | 18 ++-- src/utils/server/apiFunctions.ts | 96 +++++++++++++--------- 6 files changed, 94 insertions(+), 59 deletions(-) diff --git a/src/components/sidebarFacet/Facet.tsx b/src/components/sidebarFacet/Facet.tsx index 41d4a7c..540bef8 100644 --- a/src/components/sidebarFacet/Facet.tsx +++ b/src/components/sidebarFacet/Facet.tsx @@ -39,10 +39,7 @@ const Facet = ({ field, isFilterable, label, view, callback }: FacetProps) => { queryResult: { data }, } = useSearchQuery(); // temporary conditional - const fieldAggregate: FacetAggregateBucket = - field === "domain" - ? data?.aggregations?.["domains"]?.["buckets"] ?? [] - : data?.aggregations?.[field]?.["buckets"] ?? []; + const fieldAggregate: FacetAggregateBucket = data?.aggregations?.[field]?.["buckets"] ?? []; const { getFilter, addFilter, removeFilter } = useURLManager(); const selectedList = getFilter(field); diff --git a/src/components/sidebarFacet/FilterMenu.tsx b/src/components/sidebarFacet/FilterMenu.tsx index 81da1b5..134a534 100644 --- a/src/components/sidebarFacet/FilterMenu.tsx +++ b/src/components/sidebarFacet/FilterMenu.tsx @@ -71,12 +71,12 @@ const AppliedFilters = ({ filters }: { filters: Facet[] }) => { onClick={() => removeFilter({ filterType: filter.field, - filterValue: filter.value, + filterValue: filter.value as string, }) } > - {getFilterValueDisplay(filter.value, filter.field)} + {getFilterValueDisplay(filter.value as string, filter.field)} { + const appFilterFields = [ + ...filterFields, + // Application-specific filters + { field: "type", value: "combined-summary", operation: "exclude" }, + ]; + const body = { queryString, size, page, - filterFields, + filterFields: appFilterFields, sortFields, }; diff --git a/src/types.ts b/src/types.ts index 25dd772..2293e50 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,15 +2,21 @@ import { AggregationsAggregate, SearchResponse, } from "@elastic/elasticsearch/lib/api/types"; -const AUTHOR = "authors" as const; -const DOMAIN = "domain" as const; -const TAGS = "tags" as const; -export type FacetKeys = typeof AUTHOR | typeof DOMAIN | typeof TAGS; +export const FACETS = { + AUTHOR: 'authors', + DOMAIN: 'domain', + TAGS: 'tags' +} as const; + +export type FacetKeys = (typeof FACETS)[keyof typeof FACETS]; + +export type FilterOperation = "include" | "exclude"; export type Facet = { field: FacetKeys; - value: string; + value: string | string[]; + operation?: FilterOperation; }; const bodyType = { @@ -28,6 +34,8 @@ export type SearchQuery = { page: number; filterFields: Facet[]; sortFields: any[]; + aggregationFields?: string[]; + index?: string; }; export type EsSearchResult = { diff --git a/src/utils/server/apiFunctions.ts b/src/utils/server/apiFunctions.ts index c88ebb9..bc2fc32 100644 --- a/src/utils/server/apiFunctions.ts +++ b/src/utils/server/apiFunctions.ts @@ -1,3 +1,8 @@ +import type { + QueryDslQueryContainer, + SearchRequest, +} from "@elastic/elasticsearch/lib/api/types"; + import { aggregatorSize } from "@/config/config"; import type { Facet, SearchQuery } from "@/types"; @@ -18,44 +23,20 @@ export const buildQuery = ({ from, filterFields, sortFields, + aggregationFields = ["authors", "domain", "tags"], // Default aggregations }: BuildQueryForElaSticClient) => { // Initialize the base structure of the Elasticsearch query - let baseQuery = { + let baseQuery: SearchRequest = { query: { bool: { must: [], should: [], filter: [], - must_not: [ - { - term: { - "type.keyword": "combined-summary", - }, - }, - ], + must_not: [], }, }, sort: [], - aggs: { - authors: { - terms: { - field: "authors.keyword", - size: aggregatorSize, - }, - }, - domains: { - terms: { - field: "domain.keyword", - size: aggregatorSize, - }, - }, - tags: { - terms: { - field: "tags.keyword", - size: aggregatorSize, - }, - }, - }, + aggs: {}, size, // Number of search results to return from, // Offset for pagination (calculated from page number) _source: { @@ -63,19 +44,37 @@ export const buildQuery = ({ }, }; - // Construct and add the full-text search clause - let shouldClause = buildShouldQueryClause(queryString); - if (!queryString) { - baseQuery.query.bool.should.push(shouldClause); - } else { - baseQuery.query.bool.must.push(shouldClause); + // Construct and add the full-text search query if provided + if (queryString) { + (baseQuery.query.bool.must as QueryDslQueryContainer[]).push( + buildShouldQueryClause(queryString) + ); } - // Add filter clauses for each specified filter field - if (filterFields && filterFields.length) { - for (let facet of filterFields) { - let mustClause = buildFilterQueryClause(facet); - baseQuery.query.bool.must.push(mustClause); + // Handle filters with exclusions and array values + if (filterFields?.length) { + for (const filter of filterFields) { + const filterClause = buildFilterQueryClause(filter); + + if (filter.operation === "exclude") { + (baseQuery.query.bool.must_not as QueryDslQueryContainer[]).push( + filterClause + ); + } else if (Array.isArray(filter.value)) { + // Handle OR logic for array values + (baseQuery.query.bool.should as QueryDslQueryContainer[]).push({ + bool: { + should: filter.value.map((value) => ({ + term: { [`${filter.field}.keyword`]: value }, + })), + minimum_should_match: 1, + }, + }); + } else { + (baseQuery.query.bool.must as QueryDslQueryContainer[]).push( + filterClause + ); + } } } @@ -83,10 +82,13 @@ export const buildQuery = ({ if (sortFields && sortFields.length) { for (let field of sortFields) { const sortClause = buildSortClause(field); - baseQuery.sort.push(sortClause); + (baseQuery.sort as QueryDslQueryContainer[]).push(sortClause); } } + // Add aggregations + baseQuery.aggs = buildAggregations(aggregationFields); + return baseQuery; }; @@ -119,3 +121,17 @@ const buildSortClause = ({ field, value }: { field: any; value: any }) => { [field]: value, }; }; + +// Helper to build aggregations +const buildAggregations = (fields: string[]) => { + const aggs = {}; + fields.forEach((field) => { + aggs[field] = { + terms: { + field: `${field}.keyword`, + size: aggregatorSize, + }, + }; + }); + return aggs; +};