feat(api): enhance search endpoint with flexible filtering and dynami…

…c aggregations

Add support for filter operations (include/exclude), OR logic via array values, and dynamic aggregation fields in the search API. Allow switching between default and coredev indices. This consolidates search functionality across products by making the Bitcoin Search API more flexible while maintaining backward compatibility.

src/components/sidebarFacet/Facet.tsx

@@ -39,10 +39,7 @@ const Facet = ({ field, isFilterable, label, view, callback }: FacetProps) => {
     queryResult: { data },
   } = useSearchQuery();
   // temporary conditional
-  const fieldAggregate: FacetAggregateBucket =
-    field === "domain"
-      ? data?.aggregations?.["domains"]?.["buckets"] ?? []
-      : data?.aggregations?.[field]?.["buckets"] ?? [];
+  const fieldAggregate: FacetAggregateBucket = data?.aggregations?.[field]?.["buckets"] ?? [];
   const { getFilter, addFilter, removeFilter } = useURLManager();
 
   const selectedList = getFilter(field);

src/components/sidebarFacet/FilterMenu.tsx

@@ -71,12 +71,12 @@ const AppliedFilters = ({ filters }: { filters: Facet[] }) => {
                 onClick={() =>
                   removeFilter({
                     filterType: filter.field,
-                    filterValue: filter.value,
+                    filterValue: filter.value as string,
                   })
                 }
               >
                 <span className="capitalize text-sm font-semibold 2xl:text-sm">
-                  {getFilterValueDisplay(filter.value, filter.field)}
+                  {getFilterValueDisplay(filter.value as string, filter.field)}
                 </span>
                 <Image
                   src={isDark ? DarkCrossIcon : CrossIcon}

src/pages/api/elasticSearchProxy/search.ts

@@ -1,7 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from "next";
 import { client } from "@/config/elasticsearch";
 import { buildQuery } from "@/utils/server/apiFunctions";
-// import ElasticsearchAPIConnector from "@elastic/search-ui-elasticsearch-connector";
 
 export default async function handler(
   req: NextApiRequest,
@@ -14,11 +13,19 @@ export default async function handler(
     });
   }
 
-  let queryString = req.body.queryString as string;
-  let size = req.body.size;
-  let page = req.body.page;
-  let filterFields = req.body.filterFields;
-  let sortFields = req.body.sortFields;
+  const {
+    queryString,
+    size,
+    page,
+    filterFields,
+    sortFields,
+    aggregationFields,
+    index = "default",
+  } = req.body;
+
+  // Select index based on parameter
+  const selectedIndex =
+    index === "coredev" ? process.env.COREDEV_INDEX : process.env.INDEX;
 
   const from = page * size;
   let searchQuery = buildQuery({
@@ -27,12 +34,13 @@ export default async function handler(
     sortFields,
     from,
     size,
+    aggregationFields,
   });
 
   try {
     // Call the search method
     const result = await client.search({
-      index: process.env.INDEX,
+      index: selectedIndex,
       ...searchQuery,
     });
 

src/service/api/search/searchCall.ts

@@ -9,11 +9,17 @@ export const buildQueryCall: BuildQuery = async (
   { queryString, size, page, filterFields, sortFields },
   url
 ) => {
+  const appFilterFields = [
+    ...filterFields,
+    // Application-specific filters
+    { field: "type", value: "combined-summary", operation: "exclude" },
+  ];
+
   const body = {
     queryString,
     size,
     page,
-    filterFields,
+    filterFields: appFilterFields,
     sortFields,
   };
 

src/types.ts

@@ -2,15 +2,21 @@ import {
   AggregationsAggregate,
   SearchResponse,
 } from "@elastic/elasticsearch/lib/api/types";
-const AUTHOR = "authors" as const;
-const DOMAIN = "domain" as const;
-const TAGS = "tags" as const;
 
-export type FacetKeys = typeof AUTHOR | typeof DOMAIN | typeof TAGS;
+export const FACETS = {
+  AUTHOR: 'authors',
+  DOMAIN: 'domain',
+  TAGS: 'tags'
+} as const;
+
+export type FacetKeys = (typeof FACETS)[keyof typeof FACETS];
+
+export type FilterOperation = "include" | "exclude";
 
 export type Facet = {
   field: FacetKeys;
-  value: string;
+  value: string | string[];
+  operation?: FilterOperation;
 };
 
 const bodyType = {
@@ -28,6 +34,8 @@ export type SearchQuery = {
   page: number;
   filterFields: Facet[];
   sortFields: any[];
+  aggregationFields?: string[];
+  index?: string;
 };
 
 export type EsSearchResult = {

src/utils/server/apiFunctions.ts

@@ -1,3 +1,8 @@
+import type {
+  QueryDslQueryContainer,
+  SearchRequest,
+} from "@elastic/elasticsearch/lib/api/types";
+
 import { aggregatorSize } from "@/config/config";
 import type { Facet, SearchQuery } from "@/types";
 
@@ -18,75 +23,72 @@ export const buildQuery = ({
   from,
   filterFields,
   sortFields,
+  aggregationFields = ["authors", "domain", "tags"], // Default aggregations
 }: BuildQueryForElaSticClient) => {
   // Initialize the base structure of the Elasticsearch query
-  let baseQuery = {
+  let baseQuery: SearchRequest = {
     query: {
       bool: {
         must: [],
         should: [],
         filter: [],
-        must_not: [
-          {
-            term: {
-              "type.keyword": "combined-summary",
-            },
-          },
-        ],
+        must_not: [],
       },
     },
     sort: [],
-    aggs: {
-      authors: {
-        terms: {
-          field: "authors.keyword",
-          size: aggregatorSize,
-        },
-      },
-      domains: {
-        terms: {
-          field: "domain.keyword",
-          size: aggregatorSize,
-        },
-      },
-      tags: {
-        terms: {
-          field: "tags.keyword",
-          size: aggregatorSize,
-        },
-      },
-    },
+    aggs: {},
     size, // Number of search results to return
     from, // Offset for pagination (calculated from page number)
     _source: {
       excludes: ["summary_vector_embeddings"],
     },
   };
 
-  // Construct and add the full-text search clause
-  let shouldClause = buildShouldQueryClause(queryString);
-  if (!queryString) {
-    baseQuery.query.bool.should.push(shouldClause);
-  } else {
-    baseQuery.query.bool.must.push(shouldClause);
+  // Construct and add the full-text search query if provided
+  if (queryString) {
+    (baseQuery.query.bool.must as QueryDslQueryContainer[]).push(
+      buildShouldQueryClause(queryString)
+    );
   }
 
-  // Add filter clauses for each specified filter field
-  if (filterFields && filterFields.length) {
-    for (let facet of filterFields) {
-      let mustClause = buildFilterQueryClause(facet);
-      baseQuery.query.bool.must.push(mustClause);
+  // Handle filters with exclusions and array values
+  if (filterFields?.length) {
+    for (const filter of filterFields) {
+      const filterClause = buildFilterQueryClause(filter);
+
+      if (filter.operation === "exclude") {
+        (baseQuery.query.bool.must_not as QueryDslQueryContainer[]).push(
+          filterClause
+        );
+      } else if (Array.isArray(filter.value)) {
+        // Handle OR logic for array values
+        (baseQuery.query.bool.should as QueryDslQueryContainer[]).push({
+          bool: {
+            should: filter.value.map((value) => ({
+              term: { [`${filter.field}.keyword`]: value },
+            })),
+            minimum_should_match: 1,
+          },
+        });
+      } else {
+        (baseQuery.query.bool.must as QueryDslQueryContainer[]).push(
+          filterClause
+        );
+      }
     }
   }
 
   // Add sorting clauses for each specified sort field
   if (sortFields && sortFields.length) {
     for (let field of sortFields) {
       const sortClause = buildSortClause(field);
-      baseQuery.sort.push(sortClause);
+      (baseQuery.sort as QueryDslQueryContainer[]).push(sortClause);
     }
   }
 
+  // Add aggregations
+  baseQuery.aggs = buildAggregations(aggregationFields);
+
   return baseQuery;
 };
 
@@ -119,3 +121,17 @@ const buildSortClause = ({ field, value }: { field: any; value: any }) => {
     [field]: value,
   };
 };
+
+// Helper to build aggregations
+const buildAggregations = (fields: string[]) => {
+  const aggs = {};
+  fields.forEach((field) => {
+    aggs[field] = {
+      terms: {
+        field: `${field}.keyword`,
+        size: aggregatorSize,
+      },
+    };
+  });
+  return aggs;
+};