From b9a49999d543d3b86107053b640251efcf122a81 Mon Sep 17 00:00:00 2001 From: Aparna Date: Mon, 20 Jan 2025 18:37:04 -0800 Subject: [PATCH] FOIMOD-3639-Full Text Search - Boolean Operators Full Text Search - Boolean Operators with Front end pagination and solr search documents count variable setup --- docker-compose.yml | 1 + forms-flow-web/Dockerfile | 2 + forms-flow-web/Dockerfile.local | 2 + .../services/FOI/foiKeywordSearchServices.js | 9 +- .../IAO/KeywordSearch/ActionContext.js | 42 ++- .../KeywordSearch/DataGridKeywordSearch.js | 80 +++--- .../IAO/KeywordSearch/SearchComponent.js | 248 ++++++++++++------ .../components/FOI/Dashboard/dashboard.scss | 5 + forms-flow-web/src/constants/constants.js | 5 +- .../request_api/resources/solrauth.py | 15 +- sample.env | 1 + 11 files changed, 275 insertions(+), 135 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 536b170db..ce313815e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -267,6 +267,7 @@ services: - REACT_APP_SESSION_SECURITY_KEY=${REACT_APP_SESSION_SECURITY_KEY} - REACT_APP_FOI_SOLR_API_BASE=${FOI_SOLR_API_BASE} - REACT_APP_SEARCH_KEYWORD_LIMIT=${SEARCH_KEYWORD_LIMIT} + - REACT_APP_SOLR_DOC_SEARCH_LIMIT=${SOLR_DOC_SEARCH_LIMIT} volumes: - ".:/app" - "/app/node_modules" diff --git a/forms-flow-web/Dockerfile b/forms-flow-web/Dockerfile index ae4d17f16..e392ae7f3 100644 --- a/forms-flow-web/Dockerfile +++ b/forms-flow-web/Dockerfile @@ -40,6 +40,7 @@ ARG REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ARG REACT_APP_SESSION_SECURITY_KEY ARG REACT_APP_FOI_SOLR_API_BASE ARG REACT_APP_SEARCH_KEYWORD_LIMIT +ARG REACT_APP_SOLR_DOC_SEARCH_LIMIT ENV NODE_ENV ${NODE_ENV} @@ -78,6 +79,7 @@ ENV REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ${REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT} ENV REACT_APP_FOI_SOLR_API_BASE ${REACT_APP_FOI_SOLR_API_BASE} ENV REACT_APP_SEARCH_KEYWORD_LIMIT ${REACT_APP_SEARCH_KEYWORD_LIMIT} +ENV REACT_APP_SOLR_DOC_SEARCH_LIMIT ${REACT_APP_SOLR_DOC_SEARCH_LIMIT} # add `/app/node_modules/.bin` to $PATH ENV PATH /forms-flow-web/app/node_modules/.bin:$PATH diff --git a/forms-flow-web/Dockerfile.local b/forms-flow-web/Dockerfile.local index 3ab5afe95..b9aa45061 100644 --- a/forms-flow-web/Dockerfile.local +++ b/forms-flow-web/Dockerfile.local @@ -37,6 +37,7 @@ ARG REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ARG REACT_APP_SESSION_SECURITY_KEY ARG REACT_APP_FOI_SOLR_API_BASE ARG REACT_APP_SEARCH_KEYWORD_LIMIT +ARG REACT_APP_SOLR_DOC_SEARCH_LIMIT ENV NODE_ENV ${NODE_ENV} ENV GENERATE_SOURCEMAP ${GENERATE_SOURCEMAP} @@ -69,6 +70,7 @@ ENV REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT ${REACT_APP_RECORD_DOWNLOAD_SIZE_LIMIT} ENV REACT_APP_SESSION_SECURITY_KEY ${REACT_APP_SESSION_SECURITY_KEY} ENV REACT_APP_FOI_SOLR_API_BASE ${REACT_APP_FOI_SOLR_API_BASE} ENV REACT_APP_SEARCH_KEYWORD_LIMIT ${REACT_APP_SEARCH_KEYWORD_LIMIT} +ENV REACT_APP_SOLR_DOC_SEARCH_LIMIT ${REACT_APP_SOLR_DOC_SEARCH_LIMIT} # add `/app/node_modules/.bin` to $PATH ENV PATH /forms-flow-web/app/node_modules/.bin:$PATH diff --git a/forms-flow-web/src/apiManager/services/FOI/foiKeywordSearchServices.js b/forms-flow-web/src/apiManager/services/FOI/foiKeywordSearchServices.js index ef7621989..089badcc0 100644 --- a/forms-flow-web/src/apiManager/services/FOI/foiKeywordSearchServices.js +++ b/forms-flow-web/src/apiManager/services/FOI/foiKeywordSearchServices.js @@ -1,4 +1,4 @@ -import { httpGETRequest } from "../../httpRequestHandler"; +import { httpGETRequest, httpPOSTRequest} from "../../httpRequestHandler"; import API from "../../endpoints"; import { catchError } from "./foiServicesUtil"; import UserService from "../../../services/UserService"; @@ -69,10 +69,9 @@ export const getSolrKeywordSearchData = ({ errorCallback, dispatch, }) => { - httpGETRequest( - API.FOI_GET_CROSSTEXTSEARCH_REQUEST_DETAILS, - {"requestnumbers": foirequestNumbers}, - UserService.getToken()) + let requestjson={"requestnumbers":foirequestNumbers} + let apiUrlPost = API.FOI_GET_CROSSTEXTSEARCH_REQUEST_DETAILS; + httpPOSTRequest(apiUrlPost, requestjson, UserService.getToken() ?? '', true) .then((res) => { if (res.data) { callback(res.data); diff --git a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/ActionContext.js b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/ActionContext.js index 534b6a885..95beb1707 100644 --- a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/ActionContext.js +++ b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/ActionContext.js @@ -1,11 +1,11 @@ import React, { createContext, useEffect, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; import { fetchFOIProgramAreaList } from "../../../../../apiManager/services/FOI/foiMasterDataServices"; -import { fetchAdvancedSearchData } from "../../../../../apiManager/services/FOI/foiAdvancedSearchServices"; import {getCrossTextSearchAuth, getSolrKeywordSearchData, getKeywordSearchRequestDetails} from "../../../../../apiManager/services/FOI/foiKeywordSearchServices" import { errorToast } from "../../../../../helper/FOI/helper"; import { setKeywordSearchParams } from "../../../../../actions/FOI/foiRequestActions"; +import { SOLR_DOC_SEARCH_LIMIT } from "../../../../../constants/constants"; export const ActionContext = createContext(); ActionContext.displayName = "KeywordSearchContext"; @@ -14,9 +14,7 @@ export const ActionProvider = ({ children }) => { const [queryData, setQueryData] = useState(null); const [keywordSearchLoading, setKeywordSearchLoading] = useState(false); - const [keywordSearchComponentLoading, setKeywordSearchComponentLoading] = - useState(true); - + const [keywordSearchComponentLoading, setKeywordSearchComponentLoading] = useState(false); const [searchResults, setSearchResults] = useState(null); const keywordSearchParams = useSelector((state) => state.foiRequests.foiKeywordSearchParams); @@ -39,11 +37,35 @@ export const ActionProvider = ({ children }) => { const generateSolrQueryParams = (queryData) => { let queryParts = []; + let booleanKeywords=[]; if (queryData.keywords.length > 0) { - const keywords = queryData.keywords - .map(keyword => `${keyword}`) // Properly quote keywords - .join(","); - queryParts.push(keywords); + let andKeywords= queryData.keywords?.filter( + (keyword) =>(keyword.category === "AND")); + let orKeywords= queryData.keywords?.filter( + (keyword) =>(keyword.category === "OR")); + let notKeywords= queryData.keywords?.filter( + (keyword) =>(keyword.category === "NOT")); + + if (andKeywords.length > 0) { + let andPart = andKeywords.map((keyword) => keyword.text).join(" AND "); + booleanKeywords.push(andPart); + } + if (orKeywords.length > 0) { + let orPart = orKeywords.map((keyword) => keyword.text).join(" OR "); + booleanKeywords.push(orPart); + } + if (notKeywords.length > 0) { + let notPart = notKeywords.map((keyword) => `NOT ${keyword.text}`).join(" NOT "); + // if (booleanKeywords.length > 0) { + // notPart += ` NOT ${booleanKeywords}`; + // } + booleanKeywords.push(notPart); + } + if (booleanKeywords.length > 0) { + queryParts.push(booleanKeywords.join(" ")); + } + //queryParts.push(keywords); + console.log("\nqueryParts::",queryParts); } // Handle received date range if (queryData.fromDate || queryData.toDate) { @@ -57,13 +79,12 @@ const generateSolrQueryParams = (queryData) => { } const query = queryParts.join(" AND "); console.log("\nquery:",query) - return { df: "foidocumentsentence", q: query }; + return { df: "foidocumentsentence", q: query, rows:SOLR_DOC_SEARCH_LIMIT }; }; const convertToISO = (dateStr) => { if(!!dateStr){ const date = new Date(`${dateStr}T00:00:00Z`); - // Convert to ISO 8601 format return date.toISOString(); } else @@ -106,6 +127,7 @@ const convertToISO = (dateStr) => { } else{ setKeywordSearchComponentLoading(false); + setSearchResults([]); } }, errorCallback: (message) => { diff --git a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/DataGridKeywordSearch.js b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/DataGridKeywordSearch.js index f0b9a9e2c..6693ad67f 100644 --- a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/DataGridKeywordSearch.js +++ b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/DataGridKeywordSearch.js @@ -56,17 +56,6 @@ const DataGridKeywordSearch = ({ userDetail }) => { } }; - // const renderDocReviewerForRequest = (e, row) => { - // e.preventDefault() - // if (row.ministryrequestid) { - // dispatch( - // push( - // `${DOC_REVIEWER_WEB_URL}/foi/${row.ministryrequestid}` - // ) - // ); - // } - // }; - const hyperlinkTooltipRenderCell = (params) => { let link; if (params.row.ministryrequestid) { @@ -107,24 +96,17 @@ const DataGridKeywordSearch = ({ userDetail }) => { const goToRecordsRenderCell = (params) => { const keywordSearchParam = keywordSearchParamsRef.current; - console.log("goToRecordsRenderCell-",keywordSearchParam); - let keywordList = []; - let link; - if (params.row.ministryrequestid) { - if (keywordSearchParam?.keywords?.length > 0) { - const keywords = keywordSearchParam?.keywords - .map(keyword => `${keyword}`) // Properly quote keywords - .join(","); - keywordList.push(keywords); - } - console.log("keywordList:", keywordList); - let queryString= {"query": keywordList }; + if (params.row.ministryrequestid && keywordSearchParam?.keywords?.length > 0) { + const keywords = keywordSearchParam.keywords + .filter(keyword => keyword.category.toUpperCase() !== "NOT") + .map(keyword => keyword.text) + .join(","); + //console.log("keywordList:", keywordList); + let queryString= {"query": keywords }; const queryStringParam = new URLSearchParams(queryString).toString(); const formattedQueryString = queryStringParam.replace(/\+/g, '%20'); - console.log("formattedQueryString:", formattedQueryString); link = `${DOC_REVIEWER_WEB_URL}/foi/${params.row.ministryrequestid}?${formattedQueryString}`; - console.log("link:", link); } return ( { const classes = useStyles(); - const defaultRowsState = { page: 0, pageSize: 100 }; - const [rowsState, setRowsState] = useState( - Object.keys(keywordSearchParams).length > 0 ? - {page: keywordSearchParams.page - 1, pageSize: keywordSearchParams.size} : - defaultRowsState - ); - + const defaultRowsState = { page: 0, pageSize: 10 }; + const [rowsState, setRowsState] = useState(defaultRowsState); const defaultSortModel = [ { field: "requeststatus", sort: "desc" }, - // { field: "receivedDateUF", sort: "desc" }, ]; - - const [sortModel, setSortModel] = useState(keywordSearchParams?.sort || defaultSortModel); + const [sortModel, setSortModel] = useState(defaultSortModel); useEffect(() => { if (searchResults) { @@ -253,6 +228,33 @@ const DataGridKeywordSearch = ({ userDetail }) => { const columnsRef = React.useRef(tableInfo?.columns || []); + // Function to get paginated data + const getPaginatedRows = (sortedRows) => { + const startIndex = rowsState.page * rowsState.pageSize; + const endIndex = startIndex + rowsState.pageSize; + return sortedRows.slice(startIndex, endIndex); // Correctly slice rows for pagination + }; + // Function to sort data locally + const getSortedRows = () => { + if (!Array.isArray(searchResults) || searchResults.length === 0) return []; + if (sortModel.length === 0) return searchResults; + const { field, sort } = sortModel[0]; + return [...searchResults].sort((a, b) => { + if (a[field] < b[field]) return sort === "asc" ? -1 : 1; + if (a[field] > b[field]) return sort === "asc" ? 1 : -1; + return 0; + }); + }; + + // Compute rows to display (sorted and paginated) + const rows = React.useMemo(() => { + const sortedRows = getSortedRows(); + let currentPageRows= getPaginatedRows(sortedRows); // Get rows for the current page + console.log("currentPageRows:",currentPageRows) + return currentPageRows + }, [searchResults, rowsState, sortModel]); + + if (keywordSearchComponentLoading && queryData) { return ( @@ -279,7 +281,8 @@ const DataGridKeywordSearch = ({ userDetail }) => { autoHeight className="foi-data-grid" getRowId={(row) => row.requestnumber} - rows={searchResults || []} + //rows={searchResults || []} + rows={rows} // Display only the current page's rows columns={columnsRef?.current} rowHeight={30} headerHeight={50} @@ -289,20 +292,19 @@ const DataGridKeywordSearch = ({ userDetail }) => { hideFooterSelectedRowCount={true} disableColumnMenu={true} pagination - paginationMode="server" initialState={{ pagination: rowsState }} + paginationMode="server" onPageChange={(newPage) => setRowsState((prev) => ({ ...prev, page: newPage }))} onPageSizeChange={(newpageSize) => setRowsState((prev) => ({ ...prev, pageSize: newpageSize })) } components={{ - Footer: ()=> + Footer: ()=> }} sortingOrder={["desc", "asc"]} sortModel={[sortModel[0]]} - sortingMode={"server"} onSortModelChange={(model) => { if (model.length > 0) { setSortModel(model) diff --git a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/SearchComponent.js b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/SearchComponent.js index 084a02768..72c5b32e7 100644 --- a/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/SearchComponent.js +++ b/forms-flow-web/src/components/FOI/Dashboard/IAO/KeywordSearch/SearchComponent.js @@ -126,9 +126,12 @@ const KeywordSearch = ({ userDetail }) => { } }); + const [andKeywords, setAndKeywords] = useState([]); + const [orKeywords, setOrKeywords] = useState([]); + const [notKeywords, setNotKeywords] = useState([]); + const [anchorEl, setAnchorEl] = React.useState(null); - const open = Boolean(anchorEl) && Boolean(searchText); const [error, setError] = useState(false); @@ -174,15 +177,6 @@ const KeywordSearch = ({ userDetail }) => { const handleSearch = () =>{ - // getCrossTextSearchAuth({ - // callback: (data) => { - // console.log("!!!!",data) - // }, - // errorCallback: (error) => { - // console.log("!!!!",error) - // }, - // dispatch,}) - //NEEDED handleApplySearchFilters(); } @@ -205,41 +199,56 @@ const KeywordSearch = ({ userDetail }) => { // }; const handleResetSearchFilters = () => { - setSearchText(""); + //setSearchText(""); setKeywords([]); setFromDate(""); setToDate(""); setSelectedPublicBodies([]); }; - const handleKeywordAdd = () => { - if (!searchText) { - return; - } + const handleKeywordAdd = (category) => { + // if (!searchText) { + // return; + // } if (keywords.length >= SEARCH_KEYWORD_LIMIT) { setError(true); // Show error message and turn bar red return; } - if (searchText.trim() && !keywords.includes(searchText.trim())) { - setKeywords([...keywords, searchText.trim()]); + // if (searchText.trim() && !keywords.includes(searchText.trim())) { + // setKeywords([...keywords, searchText.trim()]); + // } + let updatedKeywords = []; + if (category === "AND" && andKeywords) { + updatedKeywords = [...keywords, { category: "AND", text: andKeywords }]; + setAndKeywords(""); + } else if (category === "OR" && orKeywords) { + updatedKeywords = [...keywords, { category: "OR", text: orKeywords }]; + setOrKeywords(""); + } else if (category === "NOT" && notKeywords) { + updatedKeywords = [...keywords, { category: "NOT", text: notKeywords }]; + setNotKeywords(""); } + setKeywords(updatedKeywords); setAnchorEl(null); - setKeywords([...keywords, searchText.trim()]); - setSearchText(""); - setError(false); // Reset error if it was active + //setKeywords([...keywords, searchText.trim()]); + //setSearchText(""); + setError(false); }; - const handleSearchChange = (e) => { - setAnchorEl(e.currentTarget); - setSearchText(e.target.value); - }; + const handleAndChange = (e) => setAndKeywords(e.target.value); + const handleOrChange = (e) => setOrKeywords(e.target.value); + const handleNotChange = (e) => setNotKeywords(e.target.value); + + // const handleSearchChange = (e) => { + // setAnchorEl(e.currentTarget); + // setSearchText(e.target.value); + // }; const handleSelectedPublicBodiesChange = (event) => { const { target: { value }, } = event; setSelectedPublicBodies( - // On autofill we get a stringified value. typeof value === "string" ? value.split(",") : value ); }; @@ -247,7 +256,11 @@ const KeywordSearch = ({ userDetail }) => { //Remove a keyword bubble const handleKeywordDelete = (keywordToDelete) => { - let updatedKeywordList= keywords.filter((keyword) => keyword !== keywordToDelete) + let updatedKeywordList = keywords.filter( + (keyword) => + !(keyword.category === keywordToDelete.category + && keyword.text === keywordToDelete.text) + ); setKeywords(updatedKeywordList); if (updatedKeywordList.length >= SEARCH_KEYWORD_LIMIT) setError(true); @@ -301,8 +314,70 @@ const KeywordSearch = ({ userDetail }) => { className={classes.search} > - + + { > {/* Search Input */} { if (e.key === "Enter") { - handleKeywordAdd(); + handleKeywordAdd("OR"); } }} sx={{ @@ -339,10 +414,11 @@ const KeywordSearch = ({ userDetail }) => { Search keywords... - {keywords.map((keyword, index) => ( + {keywords.filter((keyword) => keyword.category === "OR") + .map((keyword, index) => ( handleKeywordDelete(keyword)} className={classes.chip} sx={{ @@ -354,7 +430,6 @@ const KeywordSearch = ({ userDetail }) => { fontSize: "16px", // Adjust icon size }, }} - //sx={{ margin: "4px 4px 4px 0" }} /> ))} @@ -363,45 +438,70 @@ const KeywordSearch = ({ userDetail }) => { - {/* - - {keywords.map((keyword, index) => ( - { - setKeywords(keywords.filter((_kw, i) => index !== i)); - }} - color="primary" - className={classes.chip} - sx={{ - backgroundColor: "#95c3ff", - margin: "1px", - height: "20px" - }} - size="small" - /> - ))} - - */} - - - - - {`Add "${searchText}"`} - + + + + + {/* Search Input */} + { + if (e.key === "Enter") { + handleKeywordAdd("NOT"); + } + }} + sx={{ + flex: 1, + minWidth: "120px", + color: "#38598A", + }} + startAdornment={ + + + Search keywords... + + + {keywords.filter((keyword) => keyword.category === "NOT") + .map((keyword, index) => ( + handleKeywordDelete(keyword)} + className={classes.chip} + sx={{ + backgroundColor: "#95c3ffc7", + margin: "1px", + height: "22px", + fontSize: "13px", + "& .MuiChip-deleteIcon": { + fontSize: "16px", + }, + }} + /> + ))} + + } + /> + + - - +