From 2c4e9b850857fffc3ef8cdd74df0d8a5810098a5 Mon Sep 17 00:00:00 2001 From: kouloumos Date: Wed, 18 Dec 2024 11:21:58 +0200 Subject: [PATCH] feat(api): enhance `/search` filtering and aggregations Add support for filter operations (include/exclude), OR logic via array values, dynamic aggregation fields and sub-aggregations in the search API. Allow switching between default and coredev indices. This consolidates search functionality across products by making the Bitcoin Search API more flexible while maintaining backward compatibility. --- src/components/sidebarFacet/Facet.tsx | 4 +- src/components/sidebarFacet/FilterMenu.tsx | 8 +- src/pages/api/elasticSearchProxy/search.ts | 22 ++-- src/service/api/search/searchCall.ts | 15 ++- src/types.ts | 34 ++++-- src/utils/server/apiFunctions.ts | 118 ++++++++++++--------- 6 files changed, 130 insertions(+), 71 deletions(-) diff --git a/src/components/sidebarFacet/Facet.tsx b/src/components/sidebarFacet/Facet.tsx index 41d4a7c..18ac3f6 100644 --- a/src/components/sidebarFacet/Facet.tsx +++ b/src/components/sidebarFacet/Facet.tsx @@ -40,9 +40,7 @@ const Facet = ({ field, isFilterable, label, view, callback }: FacetProps) => { } = useSearchQuery(); // temporary conditional const fieldAggregate: FacetAggregateBucket = - field === "domain" - ? data?.aggregations?.["domains"]?.["buckets"] ?? [] - : data?.aggregations?.[field]?.["buckets"] ?? []; + data?.aggregations?.[field]?.["buckets"] ?? []; const { getFilter, addFilter, removeFilter } = useURLManager(); const selectedList = getFilter(field); diff --git a/src/components/sidebarFacet/FilterMenu.tsx b/src/components/sidebarFacet/FilterMenu.tsx index 81da1b5..38e09ce 100644 --- a/src/components/sidebarFacet/FilterMenu.tsx +++ b/src/components/sidebarFacet/FilterMenu.tsx @@ -1,4 +1,4 @@ -import { Facet } from "@/types"; +import { Facet, FacetKeys } from "@/types"; import Image from "next/image"; import React from "react"; import SidebarSection from "./SidebarSection"; @@ -70,13 +70,13 @@ const AppliedFilters = ({ filters }: { filters: Facet[] }) => { role="button" onClick={() => removeFilter({ - filterType: filter.field, - filterValue: filter.value, + filterType: filter.field as FacetKeys, + filterValue: filter.value as string, }) } > - {getFilterValueDisplay(filter.value, filter.field)} + {getFilterValueDisplay(filter.value as string, filter.field)} { + const appFilterFields = [ + ...filterFields, + // Application-specific filters + { field: "type", value: "combined-summary", operation: "exclude" }, + ]; + + const aggregations = [ + { field: "authors" }, + { field: "domain" }, + { field: "tags" }, + ]; + const body = { queryString, size, page, - filterFields, + filterFields: appFilterFields, sortFields, + aggregationFields: aggregations, }; const jsonBody = JSON.stringify(body); diff --git a/src/types.ts b/src/types.ts index 25dd772..3058d0e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,16 +1,23 @@ import { AggregationsAggregate, + AggregationsAggregationContainer, SearchResponse, } from "@elastic/elasticsearch/lib/api/types"; -const AUTHOR = "authors" as const; -const DOMAIN = "domain" as const; -const TAGS = "tags" as const; -export type FacetKeys = typeof AUTHOR | typeof DOMAIN | typeof TAGS; +export const FACETS = { + AUTHOR: "authors", + DOMAIN: "domain", + TAGS: "tags", +} as const; + +export type FacetKeys = (typeof FACETS)[keyof typeof FACETS]; + +export type FilterOperation = "include" | "exclude"; export type Facet = { - field: FacetKeys; - value: string; + field: string; + value: string | string[]; + operation?: FilterOperation; }; const bodyType = { @@ -20,14 +27,25 @@ const bodyType = { "combined-summary": "combined-summary", } as const; -export type SortOption = "asc" | "desc"; +type SortOption = { + field: string; + value: "asc" | "desc"; +}; + +export interface AggregationField { + field: string; + size?: number; + subAggregations?: Record; // Raw ES aggregation config +} export type SearchQuery = { queryString: string; size: number; page: number; filterFields: Facet[]; - sortFields: any[]; + sortFields: SortOption[]; + aggregationFields?: AggregationField[]; + index?: string; }; export type EsSearchResult = { diff --git a/src/utils/server/apiFunctions.ts b/src/utils/server/apiFunctions.ts index c88ebb9..a7e01fc 100644 --- a/src/utils/server/apiFunctions.ts +++ b/src/utils/server/apiFunctions.ts @@ -1,5 +1,10 @@ +import type { + QueryDslQueryContainer, + SearchRequest, +} from "@elastic/elasticsearch/lib/api/types"; + import { aggregatorSize } from "@/config/config"; -import type { Facet, SearchQuery } from "@/types"; +import type { AggregationField, Facet, SearchQuery } from "@/types"; const FIELDS_TO_SEARCH = ["authors", "title", "body"]; @@ -18,44 +23,20 @@ export const buildQuery = ({ from, filterFields, sortFields, + aggregationFields = [], }: BuildQueryForElaSticClient) => { // Initialize the base structure of the Elasticsearch query - let baseQuery = { + let baseQuery: SearchRequest = { query: { bool: { must: [], should: [], filter: [], - must_not: [ - { - term: { - "type.keyword": "combined-summary", - }, - }, - ], + must_not: [], }, }, sort: [], - aggs: { - authors: { - terms: { - field: "authors.keyword", - size: aggregatorSize, - }, - }, - domains: { - terms: { - field: "domain.keyword", - size: aggregatorSize, - }, - }, - tags: { - terms: { - field: "tags.keyword", - size: aggregatorSize, - }, - }, - }, + aggs: {}, size, // Number of search results to return from, // Offset for pagination (calculated from page number) _source: { @@ -63,19 +44,27 @@ export const buildQuery = ({ }, }; - // Construct and add the full-text search clause - let shouldClause = buildShouldQueryClause(queryString); - if (!queryString) { - baseQuery.query.bool.should.push(shouldClause); - } else { - baseQuery.query.bool.must.push(shouldClause); + // Construct and add the full-text search query if provided + if (queryString) { + (baseQuery.query.bool.must as QueryDslQueryContainer[]).push( + buildShouldQueryClause(queryString) + ); } - // Add filter clauses for each specified filter field - if (filterFields && filterFields.length) { - for (let facet of filterFields) { - let mustClause = buildFilterQueryClause(facet); - baseQuery.query.bool.must.push(mustClause); + // Handle filters with exclusions and array values + if (filterFields?.length) { + for (const filter of filterFields) { + const filterClause = buildFilterQueryClause(filter); + + if (filter.operation === "exclude") { + (baseQuery.query.bool.must_not as QueryDslQueryContainer[]).push( + filterClause + ); + } else { + (baseQuery.query.bool.must as QueryDslQueryContainer[]).push( + filterClause + ); + } } } @@ -83,10 +72,12 @@ export const buildQuery = ({ if (sortFields && sortFields.length) { for (let field of sortFields) { const sortClause = buildSortClause(field); - baseQuery.sort.push(sortClause); + (baseQuery.sort as QueryDslQueryContainer[]).push(sortClause); } } + // Add aggregations + baseQuery.aggs = buildAggregations(aggregationFields); return baseQuery; }; @@ -103,14 +94,23 @@ const buildShouldQueryClause = (queryString: string) => { }; // Helper to build filter query clauses based on facets -const buildFilterQueryClause = ({ field, value }: Facet) => { - let filterQueryClause = { - term: { - [`${field}.keyword`]: { value }, - }, - }; +const buildFilterQueryClause = (filter: Facet): QueryDslQueryContainer => { + if (Array.isArray(filter.value)) { + // Handle OR logic for array values + return { + bool: { + should: filter.value.map((value) => ({ + term: { [`${filter.field}.keyword`]: value }, + })), + minimum_should_match: 1, + }, + }; + } - return filterQueryClause; + // Handle non-array values + return { + term: { [`${filter.field}.keyword`]: filter.value }, + }; }; // Helper to build sort clauses for sorting results @@ -119,3 +119,25 @@ const buildSortClause = ({ field, value }: { field: any; value: any }) => { [field]: value, }; }; + +// Helper to build aggregations +const buildAggregations = (fields: AggregationField[]) => { + const aggs = {}; + + fields.forEach((aggregation) => { + // Create the base terms aggregation + aggs[aggregation.field] = { + terms: { + field: `${aggregation.field}.keyword`, + size: aggregation.size || aggregatorSize, + }, + }; + + // If there are sub-aggregations, add them directly to the terms agg + if (aggregation.subAggregations) { + aggs[aggregation.field].aggs = aggregation.subAggregations; + } + }); + + return aggs; +};