From 7b72307c18c15be3c9b424ef981adb5fb3f608e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Thu, 17 Oct 2024 15:14:49 +0200 Subject: [PATCH 01/21] fix(mentions): Switch UI from list to table --- client/src/mentions/index.jsx | 134 +++++++++++------------------- client/src/mentions/index.scss | 26 ------ server/src/routes/works.routes.js | 7 +- 3 files changed, 54 insertions(+), 113 deletions(-) delete mode 100644 client/src/mentions/index.scss diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index d3cc82c..3c709ed 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -1,21 +1,26 @@ -import { - Container, - SegmentedControl, - SegmentedElement, - TextInput, -} from '@dataesr/dsfr-plus'; +import { Container, TextInput } from '@dataesr/dsfr-plus'; +import { Column } from 'primereact/column'; +import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; import { getMentions } from '../utils/works'; -import './index.scss'; - export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); + const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [search, setSearch] = useState(); const [timer, setTimer] = useState(); + const [totalRecords, setTotalRecords] = useState(0); + + const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); + const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); + const sharedTemplate = (rowData) => (rowData.mention_context.shared ? 'True' : 'False'); + + const doiTemplate = (rowData) => ( + {rowData.doi} + ); useEffect(() => { if (timer) { @@ -28,12 +33,13 @@ export default function Mentions() { }); }, 500); setTimer(timerTmp); - // The timer should not be tracked + // The timer should not be tracked // eslint-disable-next-line react-hooks/exhaustive-deps }, [search]); useEffect(() => { const getData = async () => { + setLoading(true); if ( searchParams.get('search') && searchParams.get('search')?.length > 0 @@ -41,6 +47,8 @@ export default function Mentions() { const m = await getMentions({ search: searchParams.get('search') }); setMentions(m); } + setTotalRecords(mentions?.length ?? 0); + setLoading(false); }; getData(); }, [searchParams]); @@ -54,83 +62,37 @@ export default function Mentions() { onChange={(e) => setSearch(e.target.value)} value={search} /> - + + + + + + + + + ); } diff --git a/client/src/mentions/index.scss b/client/src/mentions/index.scss deleted file mode 100644 index 1615511..0000000 --- a/client/src/mentions/index.scss +++ /dev/null @@ -1,26 +0,0 @@ -.mentions { - li { - border-left: 5px solid; - list-style: none; - padding-left: 5px; - - &.dataset-implicit { - border-color: #6a6af4; // blue-france-main-525 - } - &.dataset-name { - border-color: #000091; // blue-france-sun-113 - } - &.software { - border-color: #c9191e; // red-marianne-425 - } - } - - .fr-quote { - background-image: none; - - blockquote p { - font-size: 1rem; - font-weight: normal; - } - } -} \ No newline at end of file diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index 8ad49b3..aec75af 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -229,7 +229,12 @@ const getMentions = async ({ options }) => { { method: 'POST', headers: { Authorization: process.env.ES_AUTH } }, ); const data = await response.json(); - return data?.hits?.hits ?? []; + const mentions = (data?.hits?.hits ?? []).map((mention) => ({ + ...mention._source, + id: mention._id, + rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, + })); + return mentions; }; router.route('/mentions').post(async (req, res) => { From 718ed67eef9b2b71fb4fac098e18f59dc3da4daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 11:23:01 +0200 Subject: [PATCH 02/21] feat(mentions): Filter by type --- client/src/mentions/index.jsx | 30 ++++++++++++++++++++----- server/src/app.js | 6 +---- server/src/routes/works.routes.js | 37 ++++++++++++++++++++++++++----- server/src/utils/works.js | 4 ++-- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index 3c709ed..606777b 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -1,4 +1,4 @@ -import { Container, TextInput } from '@dataesr/dsfr-plus'; +import { Container, Fieldset, Radio, TextInput } from '@dataesr/dsfr-plus'; import { Column } from 'primereact/column'; import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; @@ -10,9 +10,10 @@ export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); - const [search, setSearch] = useState(); + const [search, setSearch] = useState(''); const [timer, setTimer] = useState(); const [totalRecords, setTotalRecords] = useState(0); + const [type, setType] = useState('software'); const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); @@ -29,13 +30,14 @@ export default function Mentions() { const timerTmp = setTimeout(() => { setSearchParams((params) => { params.set('search', search); + params.set('type', type); return params; }); }, 500); setTimer(timerTmp); // The timer should not be tracked // eslint-disable-next-line react-hooks/exhaustive-deps - }, [search]); + }, [search, type]); useEffect(() => { const getData = async () => { @@ -44,7 +46,7 @@ export default function Mentions() { searchParams.get('search') && searchParams.get('search')?.length > 0 ) { - const m = await getMentions({ search: searchParams.get('search') }); + const m = await getMentions({ search: searchParams.get('search'), type: searchParams.get('type') }); setMentions(m); } setTotalRecords(mentions?.length ?? 0); @@ -62,6 +64,22 @@ export default function Mentions() { onChange={(e) => setSearch(e.target.value)} value={search} /> +
+ setType('software')} + value="software" + /> + setType('datasets')} + value="datasets" + /> +
- - + + { - res.sendFile(path.join(path.resolve(), 'dist', 'index.html'), (err) => { - if (err) { - res.status(500).send(err); - } - }); + res.sendFile(path.join(path.resolve(), 'dist', 'index.html')); }); export default app; diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index aec75af..df9c3a1 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -224,15 +224,40 @@ router.route('/works').post(async (req, res) => { }); const getMentions = async ({ options }) => { - const response = await fetch( - `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search?q=context:${options.search}&size=50`, - { method: 'POST', headers: { Authorization: process.env.ES_AUTH } }, - ); + const { search, type } = options; + let types = []; + if (type === 'software') { + types = ['software']; + } else if (type === 'datasets') { + types = ['dataset-implicit', 'dataset-name']; + } + const body = JSON.stringify({ + size: '50', + query: { + bool: { + must: [ + { + term: { + context: search, + }, + }, + { + terms: { + 'type.keyword': types, + }, + }, + ], + }, + }, + _source: ['context', 'doi', 'mention_context', 'rawForm', 'type'], + }); + const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; + const params = { body, method: 'POST', headers: { Authorization: process.env.ES_AUTH, 'content-type': 'application/json' } }; + const response = await fetch(url, params); const data = await response.json(); const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, id: mention._id, - rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, })); return mentions; }; @@ -242,6 +267,8 @@ router.route('/mentions').post(async (req, res) => { const options = req?.body ?? {}; if (!options?.search) { res.status(400).json({ message: 'You must provide a search string.' }); + } else if (!['datasets', 'software'].includes(options?.type)) { + res.status(400).json({ message: 'Type should be either "datasets" or "software".' }); } else { const mentions = await getMentions({ options }); res.status(200).json(mentions); diff --git a/server/src/utils/works.js b/server/src/utils/works.js index 4c8f527..58f8dc0 100644 --- a/server/src/utils/works.js +++ b/server/src/utils/works.js @@ -215,12 +215,12 @@ const getFosmWorksByYear = async ({ remainingTries = 3, results = [], options, p } const body = getFosmQuery(options, pit, searchAfter); const params = { - method: 'POST', body: JSON.stringify(body), headers: { - 'content-type': 'application/json', Authorization: process.env.ES_AUTH, + 'content-type': 'application/json', }, + method: 'POST', }; const url = `${process.env.ES_URL}/_search`; return fetch(url, params) From b1e3dbc9ca9e7db22ba98fbc623f95b37dfb4a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 14:30:46 +0200 Subject: [PATCH 03/21] feat(mentions): Add sortable columns on datatable --- client/src/mentions/index.jsx | 22 ++++++++++++++++++++-- server/src/routes/works.routes.js | 3 ++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index 606777b..6f2b57f 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -1,4 +1,5 @@ import { Container, Fieldset, Radio, TextInput } from '@dataesr/dsfr-plus'; +import { FilterMatchMode } from 'primereact/api'; import { Column } from 'primereact/column'; import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; @@ -8,6 +9,9 @@ import { getMentions } from '../utils/works'; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); + const [filters] = useState({ + doi: { value: null, matchMode: FilterMatchMode.CONTAINS }, + }); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [search, setSearch] = useState(''); @@ -46,7 +50,10 @@ export default function Mentions() { searchParams.get('search') && searchParams.get('search')?.length > 0 ) { - const m = await getMentions({ search: searchParams.get('search'), type: searchParams.get('type') }); + const m = await getMentions({ + search: searchParams.get('search'), + type: searchParams.get('type'), + }); setMentions(m); } setTotalRecords(mentions?.length ?? 0); @@ -82,6 +89,8 @@ export default function Mentions() { - + @@ -99,16 +114,19 @@ export default function Mentions() { body={usedTemplate} field="mention.mention_context.used" header="Used" + sortable /> diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index df9c3a1..7449260 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -249,7 +249,7 @@ const getMentions = async ({ options }) => { ], }, }, - _source: ['context', 'doi', 'mention_context', 'rawForm', 'type'], + _source: ['context', 'dataset-name', 'doi', 'mention_context', 'rawForm', 'software-name', 'type'], }); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; const params = { body, method: 'POST', headers: { Authorization: process.env.ES_AUTH, 'content-type': 'application/json' } }; @@ -258,6 +258,7 @@ const getMentions = async ({ options }) => { const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, id: mention._id, + rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, })); return mentions; }; From f6b79763ce10fe05efed8a33445cbecff7253ace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 17:36:09 +0200 Subject: [PATCH 04/21] feat(mentions): Add pagination and lazy loading --- client/src/mentions/index.jsx | 80 ++++++++++++++++++++++++------- server/src/routes/works.routes.js | 9 ++-- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index 6f2b57f..ccc4609 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -9,15 +9,27 @@ import { getMentions } from '../utils/works'; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); + const [from, setFrom] = useState(0); const [filters] = useState({ doi: { value: null, matchMode: FilterMatchMode.CONTAINS }, }); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [search, setSearch] = useState(''); + const [size, setSize] = useState(20); const [timer, setTimer] = useState(); const [totalRecords, setTotalRecords] = useState(0); const [type, setType] = useState('software'); + const [lazyState, setlazyState] = useState({ + first: 0, + rows: 20, + page: 1, + sortField: null, + sortOrder: null, + filters: { + doi: { value: '', matchMode: 'contains' }, + }, + }); const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); @@ -27,13 +39,36 @@ export default function Mentions() { {rowData.doi} ); + const onPage = (event) => setFrom(event.first); + + const loadLazyData = async () => { + setLoading(true); + // imitate delay of a backend call + if ( + searchParams.get('search') + && searchParams.get('search')?.length > 0 + ) { + const data = await getMentions({ + from: searchParams.get('from'), + search: searchParams.get('search'), + size: searchParams.get('size'), + type: searchParams.get('type'), + }); + setMentions(data?.mentions ?? []); + setTotalRecords(data?.count ?? 0); + } + setLoading(false); + }; + useEffect(() => { if (timer) { clearTimeout(timer); } const timerTmp = setTimeout(() => { setSearchParams((params) => { + params.set('from', from); params.set('search', search); + params.set('size', size); params.set('type', type); return params; }); @@ -41,27 +76,30 @@ export default function Mentions() { setTimer(timerTmp); // The timer should not be tracked // eslint-disable-next-line react-hooks/exhaustive-deps - }, [search, type]); + }, [from, search, size, type]); useEffect(() => { - const getData = async () => { - setLoading(true); - if ( - searchParams.get('search') - && searchParams.get('search')?.length > 0 - ) { - const m = await getMentions({ - search: searchParams.get('search'), - type: searchParams.get('type'), - }); - setMentions(m); - } - setTotalRecords(mentions?.length ?? 0); - setLoading(false); - }; - getData(); + setlazyState({ + ...lazyState, + first: searchParams.get('from'), + // search: searchParams.get('search'), + rows: searchParams.get('size'), + // type: searchParams.get('type'), + filters: { + ...lazyState.filters, + doi: { + ...lazyState.filters.doi, + value: searchParams.get('search'), + }, + }, + }); + // setType(searchParams.get('type')); }, [searchParams]); + useEffect(() => { + loadLazyData(); + }, [lazyState]); + return ( diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index 7449260..61aadcb 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -224,7 +224,7 @@ router.route('/works').post(async (req, res) => { }); const getMentions = async ({ options }) => { - const { search, type } = options; + const { from, search, size, type } = options; let types = []; if (type === 'software') { types = ['software']; @@ -232,7 +232,8 @@ const getMentions = async ({ options }) => { types = ['dataset-implicit', 'dataset-name']; } const body = JSON.stringify({ - size: '50', + from, + size, query: { bool: { must: [ @@ -255,12 +256,14 @@ const getMentions = async ({ options }) => { const params = { body, method: 'POST', headers: { Authorization: process.env.ES_AUTH, 'content-type': 'application/json' } }; const response = await fetch(url, params); const data = await response.json(); + const count = data?.hits?.total?.value ?? 0; const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, id: mention._id, rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, })); - return mentions; + + return { count, mentions }; }; router.route('/mentions').post(async (req, res) => { From 361be2a0cba81760ff0acb8804e4be94646f8bc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 18:41:59 +0200 Subject: [PATCH 05/21] fix(mentions): Restore pagination and DOI search --- client/src/mentions/index.jsx | 64 +++++++++++++++++-------------- server/src/routes/works.routes.js | 10 +++-- 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index ccc4609..3bc39e3 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -1,5 +1,4 @@ import { Container, Fieldset, Radio, TextInput } from '@dataesr/dsfr-plus'; -import { FilterMatchMode } from 'primereact/api'; import { Column } from 'primereact/column'; import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; @@ -7,48 +6,49 @@ import { useSearchParams } from 'react-router-dom'; import { getMentions } from '../utils/works'; +const DEFAULT_ROWS = 50; + export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); + const [doi, setDoi] = useState(''); const [from, setFrom] = useState(0); - const [filters] = useState({ - doi: { value: null, matchMode: FilterMatchMode.CONTAINS }, - }); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [search, setSearch] = useState(''); - const [size, setSize] = useState(20); + const [rows, setRows] = useState(DEFAULT_ROWS); const [timer, setTimer] = useState(); const [totalRecords, setTotalRecords] = useState(0); const [type, setType] = useState('software'); const [lazyState, setlazyState] = useState({ + filters: { doi: { value: '', matchMode: 'contains' } }, first: 0, - rows: 20, page: 1, + rows: DEFAULT_ROWS, sortField: null, sortOrder: null, - filters: { - doi: { value: '', matchMode: 'contains' }, - }, }); - const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); + // Templates const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); - const sharedTemplate = (rowData) => (rowData.mention_context.shared ? 'True' : 'False'); - const doiTemplate = (rowData) => ( {rowData.doi} ); + const sharedTemplate = (rowData) => (rowData.mention_context.shared ? 'True' : 'False'); + const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); - const onPage = (event) => setFrom(event.first); + // Events + const onFilter = (event) => setDoi(event?.filters?.doi?.value ?? ''); + const onPage = (event) => { + setFrom(event.first); + setRows(event.rows); + }; + const onSort = (event) => setlazyState(event); const loadLazyData = async () => { setLoading(true); - // imitate delay of a backend call - if ( - searchParams.get('search') - && searchParams.get('search')?.length > 0 - ) { + if (searchParams.get('search') && searchParams.get('search')?.length > 0) { const data = await getMentions({ + doi: searchParams.get('doi'), from: searchParams.get('from'), search: searchParams.get('search'), size: searchParams.get('size'), @@ -60,40 +60,39 @@ export default function Mentions() { setLoading(false); }; + // Effects useEffect(() => { if (timer) { clearTimeout(timer); } const timerTmp = setTimeout(() => { setSearchParams((params) => { + params.set('doi', doi); params.set('from', from); params.set('search', search); - params.set('size', size); + params.set('size', rows); params.set('type', type); return params; }); - }, 500); + }, 800); setTimer(timerTmp); // The timer should not be tracked // eslint-disable-next-line react-hooks/exhaustive-deps - }, [from, search, size, type]); + }, [doi, from, search, rows, type]); useEffect(() => { setlazyState({ ...lazyState, - first: searchParams.get('from'), - // search: searchParams.get('search'), + first: parseInt(searchParams.get('from'), 10), rows: searchParams.get('size'), - // type: searchParams.get('type'), filters: { ...lazyState.filters, doi: { ...lazyState.filters.doi, - value: searchParams.get('search'), + value: searchParams.get('doi'), }, }, }); - // setType(searchParams.get('type')); }, [searchParams]); useEffect(() => { @@ -126,18 +125,25 @@ export default function Mentions() { /> { }); const getMentions = async ({ options }) => { - const { from, search, size, type } = options; + const { doi, from, search, size, type } = options; let types = []; if (type === 'software') { types = ['software']; } else if (type === 'datasets') { types = ['dataset-implicit', 'dataset-name']; } - const body = JSON.stringify({ + let body = { from, size, query: { @@ -251,7 +251,11 @@ const getMentions = async ({ options }) => { }, }, _source: ['context', 'dataset-name', 'doi', 'mention_context', 'rawForm', 'software-name', 'type'], - }); + }; + if (doi?.length > 0) { + body.query.bool.must.push({ term: { 'doi.keyword': doi } }); + } + body = JSON.stringify(body); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; const params = { body, method: 'POST', headers: { Authorization: process.env.ES_AUTH, 'content-type': 'application/json' } }; const response = await fetch(url, params); From a5ad23093c67a2d3dc7d63e58ce808ee07ef31d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 18:46:42 +0200 Subject: [PATCH 06/21] fix(mentions): Columns are not sortable anymore --- client/src/mentions/index.jsx | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index 3bc39e3..cb9338c 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -24,8 +24,6 @@ export default function Mentions() { first: 0, page: 1, rows: DEFAULT_ROWS, - sortField: null, - sortOrder: null, }); // Templates @@ -42,7 +40,6 @@ export default function Mentions() { setFrom(event.first); setRows(event.rows); }; - const onSort = (event) => setlazyState(event); const loadLazyData = async () => { setLoading(true); @@ -61,6 +58,8 @@ export default function Mentions() { }; // Effects + useEffect(() => setFrom(0), [type]); + useEffect(() => { if (timer) { clearTimeout(timer); @@ -134,7 +133,6 @@ export default function Mentions() { loading={loading} onFilter={onFilter} onPage={onPage} - onSort={onSort} paginator paginatorPosition="top bottom" paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" @@ -142,8 +140,6 @@ export default function Mentions() { rowsPerPageOptions={[20, 50, 100]} scrollable size="small" - sortField={lazyState.sortField} - sortOrder={lazyState.sortOrder} stripedRows style={{ fontSize: '14px', lineHeight: '13px' }} tableStyle={{ minWidth: '50rem' }} @@ -164,19 +160,16 @@ export default function Mentions() { body={usedTemplate} field="mention.mention_context.used" header="Used" - sortable /> From c1484012979a7d0270d38691dcec28d9b5cde8d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 19:07:44 +0200 Subject: [PATCH 07/21] feat(mentions): Add highlights on context --- client/src/mentions/index.jsx | 5 +++- server/src/routes/works.routes.js | 46 +++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index cb9338c..bdacdd4 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -27,6 +27,9 @@ export default function Mentions() { }); // Templates + const contextTemplate = (rowData) => ( + + ); const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); const doiTemplate = (rowData) => ( {rowData.doi} @@ -155,7 +158,7 @@ export default function Mentions() { /> - + { try { const options = req?.body ?? {}; if (!options?.affiliationStrings && !options?.rors) { - res - .status(400) - .json({ - message: 'You must provide at least one affiliation string or RoR.', - }); + res.status(400).json({ + message: 'You must provide at least one affiliation string or RoR.', + }); } else { const compressedResult = await getWorks({ options }); res.status(200).json(compressedResult); @@ -250,21 +248,49 @@ const getMentions = async ({ options }) => { ], }, }, - _source: ['context', 'dataset-name', 'doi', 'mention_context', 'rawForm', 'software-name', 'type'], + _source: [ + 'context', + 'dataset-name', + 'doi', + 'mention_context', + 'rawForm', + 'software-name', + 'type', + ], + highlight: { + number_of_fragments: 0, + fragment_size: 100, + require_field_match: 'true', + fields: [ + { + context: { pre_tags: [''], post_tags: [''] }, + }, + ], + }, }; if (doi?.length > 0) { body.query.bool.must.push({ term: { 'doi.keyword': doi } }); } body = JSON.stringify(body); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; - const params = { body, method: 'POST', headers: { Authorization: process.env.ES_AUTH, 'content-type': 'application/json' } }; + const params = { + body, + method: 'POST', + headers: { + Authorization: process.env.ES_AUTH, + 'content-type': 'application/json', + }, + }; const response = await fetch(url, params); const data = await response.json(); const count = data?.hits?.total?.value ?? 0; const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, id: mention._id, - rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, + rawForm: + mention._source?.['software-name']?.rawForm + ?? mention._source?.['dataset-name']?.rawForm, + context: mention.highlight.context, })); return { count, mentions }; @@ -276,7 +302,9 @@ router.route('/mentions').post(async (req, res) => { if (!options?.search) { res.status(400).json({ message: 'You must provide a search string.' }); } else if (!['datasets', 'software'].includes(options?.type)) { - res.status(400).json({ message: 'Type should be either "datasets" or "software".' }); + res + .status(400) + .json({ message: 'Type should be either "datasets" or "software".' }); } else { const mentions = await getMentions({ options }); res.status(200).json(mentions); From 2dd3d2e74bfb3f5b754c2d5e5a6e7066318e1aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Fri, 18 Oct 2024 19:27:10 +0200 Subject: [PATCH 08/21] feat(mentions): Add colors and icons, youpi ! --- client/src/mentions/index.jsx | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx index bdacdd4..c52ece7 100644 --- a/client/src/mentions/index.jsx +++ b/client/src/mentions/index.jsx @@ -30,12 +30,39 @@ export default function Mentions() { const contextTemplate = (rowData) => ( ); - const createdTemplate = (rowData) => (rowData.mention_context.created ? 'True' : 'False'); + const createdTemplate = (rowData) => ( + + ); const doiTemplate = (rowData) => ( {rowData.doi} ); - const sharedTemplate = (rowData) => (rowData.mention_context.shared ? 'True' : 'False'); - const usedTemplate = (rowData) => (rowData.mention_context.used ? 'True' : 'False'); + const sharedTemplate = (rowData) => ( + + ); + const usedTemplate = (rowData) => ( + + ); // Events const onFilter = (event) => setDoi(event?.filters?.doi?.value ?? ''); From ca07caa29998b0a0ec5cc897ecfed1f6dfdc6a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Mon, 21 Oct 2024 15:58:08 +0200 Subject: [PATCH 09/21] fix(style): Reduce margin width --- client/src/styles/index.scss | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/client/src/styles/index.scss b/client/src/styles/index.scss index 1dc4589..92b5a41 100644 --- a/client/src/styles/index.scss +++ b/client/src/styles/index.scss @@ -90,6 +90,12 @@ body { width: 200px; } +@media (min-width: 100em) { + .fr-container, .fr-container-sm, .fr-container-md, .fr-container-lg { + max-width: 100rem; + } +} + /* DataTable - PaginatorLeft */ @@ -112,6 +118,7 @@ body { content: ""; margin-right: 5px; } + .fr-toggle label[data-fr-unchecked-label][data-fr-checked-label]:before { margin: 0; } \ No newline at end of file From 0718ed2e8836096e304a1c1f4c1fbf6b616141a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Mon, 21 Oct 2024 16:59:31 +0200 Subject: [PATCH 10/21] feat(mentions): Separate software and datasets into tabs --- .../src/components/{File => file}/index.jsx | 0 client/src/mentions/index.jsx | 207 --------------- .../src/pages/actions/actionsAffiliations.jsx | 14 +- client/src/pages/mentions.jsx | 242 ++++++++++++++++++ client/src/router.jsx | 2 +- client/src/styles/index.scss | 1 + 6 files changed, 252 insertions(+), 214 deletions(-) rename client/src/components/{File => file}/index.jsx (100%) delete mode 100644 client/src/mentions/index.jsx create mode 100644 client/src/pages/mentions.jsx diff --git a/client/src/components/File/index.jsx b/client/src/components/file/index.jsx similarity index 100% rename from client/src/components/File/index.jsx rename to client/src/components/file/index.jsx diff --git a/client/src/mentions/index.jsx b/client/src/mentions/index.jsx deleted file mode 100644 index c52ece7..0000000 --- a/client/src/mentions/index.jsx +++ /dev/null @@ -1,207 +0,0 @@ -import { Container, Fieldset, Radio, TextInput } from '@dataesr/dsfr-plus'; -import { Column } from 'primereact/column'; -import { DataTable } from 'primereact/datatable'; -import { useEffect, useState } from 'react'; -import { useSearchParams } from 'react-router-dom'; - -import { getMentions } from '../utils/works'; - -const DEFAULT_ROWS = 50; - -export default function Mentions() { - const [searchParams, setSearchParams] = useSearchParams(); - const [doi, setDoi] = useState(''); - const [from, setFrom] = useState(0); - const [loading, setLoading] = useState(false); - const [mentions, setMentions] = useState([]); - const [search, setSearch] = useState(''); - const [rows, setRows] = useState(DEFAULT_ROWS); - const [timer, setTimer] = useState(); - const [totalRecords, setTotalRecords] = useState(0); - const [type, setType] = useState('software'); - const [lazyState, setlazyState] = useState({ - filters: { doi: { value: '', matchMode: 'contains' } }, - first: 0, - page: 1, - rows: DEFAULT_ROWS, - }); - - // Templates - const contextTemplate = (rowData) => ( - - ); - const createdTemplate = (rowData) => ( - - ); - const doiTemplate = (rowData) => ( - {rowData.doi} - ); - const sharedTemplate = (rowData) => ( - - ); - const usedTemplate = (rowData) => ( - - ); - - // Events - const onFilter = (event) => setDoi(event?.filters?.doi?.value ?? ''); - const onPage = (event) => { - setFrom(event.first); - setRows(event.rows); - }; - - const loadLazyData = async () => { - setLoading(true); - if (searchParams.get('search') && searchParams.get('search')?.length > 0) { - const data = await getMentions({ - doi: searchParams.get('doi'), - from: searchParams.get('from'), - search: searchParams.get('search'), - size: searchParams.get('size'), - type: searchParams.get('type'), - }); - setMentions(data?.mentions ?? []); - setTotalRecords(data?.count ?? 0); - } - setLoading(false); - }; - - // Effects - useEffect(() => setFrom(0), [type]); - - useEffect(() => { - if (timer) { - clearTimeout(timer); - } - const timerTmp = setTimeout(() => { - setSearchParams((params) => { - params.set('doi', doi); - params.set('from', from); - params.set('search', search); - params.set('size', rows); - params.set('type', type); - return params; - }); - }, 800); - setTimer(timerTmp); - // The timer should not be tracked - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [doi, from, search, rows, type]); - - useEffect(() => { - setlazyState({ - ...lazyState, - first: parseInt(searchParams.get('from'), 10), - rows: searchParams.get('size'), - filters: { - ...lazyState.filters, - doi: { - ...lazyState.filters.doi, - value: searchParams.get('doi'), - }, - }, - }); - }, [searchParams]); - - useEffect(() => { - loadLazyData(); - }, [lazyState]); - - return ( - - setSearch(e.target.value)} - value={search} - /> -
- setType('software')} - value="software" - /> - setType('datasets')} - value="datasets" - /> -
- - - - - - - - - - - ); -} diff --git a/client/src/pages/actions/actionsAffiliations.jsx b/client/src/pages/actions/actionsAffiliations.jsx index 48dd755..9d173d9 100644 --- a/client/src/pages/actions/actionsAffiliations.jsx +++ b/client/src/pages/actions/actionsAffiliations.jsx @@ -1,17 +1,19 @@ -import { useState } from 'react'; -import { useSearchParams } from 'react-router-dom'; -import PropTypes from 'prop-types'; import { Button, - Container, Row, Col, + Col, + Container, Modal, ModalContent, + Row, Title, } from '@dataesr/dsfr-plus'; -import useToast from '../../hooks/useToast'; +import PropTypes from 'prop-types'; +import { useState } from 'react'; +import { useSearchParams } from 'react-router-dom'; +import File from '../../components/file'; import { status } from '../../config'; +import useToast from '../../hooks/useToast'; import { export2json, importJson } from '../../utils/files'; -import File from '../../components/File'; export default function ActionsAffiliations({ allAffiliations, diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx new file mode 100644 index 0000000..7cb40be --- /dev/null +++ b/client/src/pages/mentions.jsx @@ -0,0 +1,242 @@ +import { Container, Tab, Tabs, TextInput } from '@dataesr/dsfr-plus'; +import { Column } from 'primereact/column'; +import { DataTable } from 'primereact/datatable'; +import { useEffect, useState } from 'react'; +import { useSearchParams } from 'react-router-dom'; + +import { getMentions } from '../utils/works'; + +const DEFAULT_ROWS = 50; + +export default function Mentions() { + const [searchParams, setSearchParams] = useSearchParams(); + const [currentTab, setCurrentTab] = useState(0); + const [doi, setDoi] = useState(''); + const [from, setFrom] = useState(0); + const [loading, setLoading] = useState(false); + const [mentions, setMentions] = useState([]); + const [rows, setRows] = useState(DEFAULT_ROWS); + const [search, setSearch] = useState(''); + const [timer, setTimer] = useState(); + const [totalRecords, setTotalRecords] = useState(0); + const [lazyState, setlazyState] = useState({ + filters: { doi: { value: '', matchMode: 'contains' } }, + first: 0, + page: 1, + rows: DEFAULT_ROWS, + }); + + // Templates + const contextTemplate = (rowData) => ( + + ); + const createdTemplate = (rowData) => ( + + ); + const doiTemplate = (rowData) => ( + {rowData.doi} + ); + const sharedTemplate = (rowData) => ( + + ); + const usedTemplate = (rowData) => ( + + ); + + // Events + const onFilter = (event) => setDoi(event?.filters?.doi?.value ?? ''); + const onPage = (event) => { + setFrom(event.first); + setRows(event.rows); + }; + + const loadLazyData = async () => { + setLoading(true); + if (searchParams.get('search') && searchParams.get('search')?.length > 0) { + const data = await getMentions({ + doi: searchParams.get('doi'), + from: searchParams.get('from'), + search: searchParams.get('search'), + size: searchParams.get('size'), + type: currentTab === 0 ? 'software' : 'datasets', + }); + setMentions(data?.mentions ?? []); + setTotalRecords(data?.count ?? 0); + } + setLoading(false); + }; + + // Effects + useEffect(() => setFrom(0), [currentTab]); + + useEffect(() => { + if (timer) { + clearTimeout(timer); + } + const timerTmp = setTimeout(() => { + setSearchParams((params) => { + params.set('doi', doi); + params.set('from', from); + params.set('search', search); + params.set('size', rows); + return params; + }); + }, 800); + setTimer(timerTmp); + // The timer should not be tracked + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [doi, from, search, rows]); + + useEffect(() => { + setlazyState({ + ...lazyState, + first: parseInt(searchParams.get('from'), 10), + rows: searchParams.get('size'), + filters: { + ...lazyState.filters, + doi: { + ...lazyState.filters.doi, + value: searchParams.get('doi'), + }, + }, + }); + }, [currentTab, searchParams]); + + useEffect(() => { + loadLazyData(); + }, [lazyState]); + + return ( + + setSearch(e.target.value)} + value={search} + /> + setCurrentTab(i)}> + + + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/client/src/router.jsx b/client/src/router.jsx index c001ea0..ebe1ffb 100644 --- a/client/src/router.jsx +++ b/client/src/router.jsx @@ -2,7 +2,7 @@ import { useState } from 'react'; import { Route, Routes } from 'react-router-dom'; import Layout from './layout'; -import Mentions from './mentions'; +import Mentions from './pages/mentions'; import Home from './pages'; export default function Router() { diff --git a/client/src/styles/index.scss b/client/src/styles/index.scss index 92b5a41..fe0f245 100644 --- a/client/src/styles/index.scss +++ b/client/src/styles/index.scss @@ -90,6 +90,7 @@ body { width: 200px; } +// Reduce margin width @media (min-width: 100em) { .fr-container, .fr-container-sm, .fr-container-md, .fr-container-lg { max-width: 100rem; From 2e5166c5eabd4f70210b46a06c01b68810bad74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Mon, 21 Oct 2024 18:34:14 +0200 Subject: [PATCH 11/21] feat(mentions): Search by author, affiliation or DOI --- client/src/pages/mentions.jsx | 209 +++++++++++++++++++----------- client/src/styles/index.scss | 18 ++- server/src/routes/works.routes.js | 36 +++-- 3 files changed, 173 insertions(+), 90 deletions(-) diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index 7cb40be..99e35fc 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -1,9 +1,18 @@ -import { Container, Tab, Tabs, TextInput } from '@dataesr/dsfr-plus'; +import { + Button, + Col, + Container, + Row, + Tab, + Tabs, + TextInput, +} from '@dataesr/dsfr-plus'; import { Column } from 'primereact/column'; import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; +import { authorsTemplate } from '../utils/templates'; import { getMentions } from '../utils/works'; const DEFAULT_ROWS = 50; @@ -11,22 +20,29 @@ const DEFAULT_ROWS = 50; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); const [currentTab, setCurrentTab] = useState(0); + const [affiliation, setAffiliation] = useState(''); + const [author, setAuthor] = useState(''); const [doi, setDoi] = useState(''); const [from, setFrom] = useState(0); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [rows, setRows] = useState(DEFAULT_ROWS); const [search, setSearch] = useState(''); - const [timer, setTimer] = useState(); const [totalRecords, setTotalRecords] = useState(0); const [lazyState, setlazyState] = useState({ - filters: { doi: { value: '', matchMode: 'contains' } }, first: 0, page: 1, rows: DEFAULT_ROWS, }); // Templates + const affiliationsTemplate = (rowData) => ( +
    + {rowData.affiliations.slice(0, 3).map((_affiliation) => ( +
  • {_affiliation}
    • + ))} +
+ ); const contextTemplate = (rowData) => ( ); @@ -62,88 +78,134 @@ export default function Mentions() { ); // Events - const onFilter = (event) => setDoi(event?.filters?.doi?.value ?? ''); + const loadLazyData = async () => { + setLoading(true); + const data = await getMentions({ + affiliation: searchParams.get('affiliation'), + author: searchParams.get('author'), + doi: searchParams.get('doi'), + from: searchParams.get('from'), + search: searchParams.get('search'), + size: searchParams.get('size'), + type: currentTab === 0 ? 'software' : 'datasets', + }); + setMentions(data?.mentions ?? []); + setTotalRecords(data?.count ?? 0); + setLoading(false); + }; const onPage = (event) => { setFrom(event.first); setRows(event.rows); }; - - const loadLazyData = async () => { - setLoading(true); - if (searchParams.get('search') && searchParams.get('search')?.length > 0) { - const data = await getMentions({ - doi: searchParams.get('doi'), - from: searchParams.get('from'), - search: searchParams.get('search'), - size: searchParams.get('size'), - type: currentTab === 0 ? 'software' : 'datasets', - }); - setMentions(data?.mentions ?? []); - setTotalRecords(data?.count ?? 0); - } - setLoading(false); + const onSubmit = () => { + setSearchParams((params) => { + params.set('affiliation', affiliation); + params.set('author', author); + params.set('doi', doi); + params.set('from', from); + params.set('search', search); + params.set('size', rows); + return params; + }); }; // Effects useEffect(() => setFrom(0), [currentTab]); - - useEffect(() => { - if (timer) { - clearTimeout(timer); - } - const timerTmp = setTimeout(() => { - setSearchParams((params) => { - params.set('doi', doi); - params.set('from', from); - params.set('search', search); - params.set('size', rows); - return params; - }); - }, 800); - setTimer(timerTmp); - // The timer should not be tracked - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [doi, from, search, rows]); - useEffect(() => { setlazyState({ ...lazyState, first: parseInt(searchParams.get('from'), 10), rows: searchParams.get('size'), - filters: { - ...lazyState.filters, - doi: { - ...lazyState.filters.doi, - value: searchParams.get('doi'), - }, - }, }); }, [currentTab, searchParams]); - useEffect(() => { loadLazyData(); }, [lazyState]); return ( - setSearch(e.target.value)} - value={search} - /> - setCurrentTab(i)}> + + + + +
Search
+
Example "Coq"
+ + + setSearch(e.target.value)} + value={search} + /> + + + + +
Affiliation
+
Example "Cern"
+ + + setAffiliation(e.target.value)} + value={affiliation} + /> + + + + +
Author
+
Example "Bruno Latour"
+ + + setAuthor(e.target.value)} + value={author} + /> + + + + +
DOI
+
+ Example "10.4000/proceedings.elpub.2019.20" +
+ + + setDoi(e.target.value)} + value={doi} + /> + + + + + + + + setCurrentTab(i)} + > - - + + + - - + + + diff --git a/client/src/styles/index.scss b/client/src/styles/index.scss index fe0f245..69d3a6d 100644 --- a/client/src/styles/index.scss +++ b/client/src/styles/index.scss @@ -122,4 +122,20 @@ body { .fr-toggle label[data-fr-unchecked-label][data-fr-checked-label]:before { margin: 0; -} \ No newline at end of file +} + +/* Pge Mentions */ +.mentions { + .label { + font-size: 1em; + font-weight: bold; + text-align: end; + } + + .hint { + color: grey; + font-size: .8em; + font-style: italic; + text-align: end; + } +} diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index c05fd48..d5cea69 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -222,7 +222,9 @@ router.route('/works').post(async (req, res) => { }); const getMentions = async ({ options }) => { - const { doi, from, search, size, type } = options; + const { + affiliation, author, doi, from, search, size, type, + } = options; let types = []; if (type === 'software') { types = ['software']; @@ -235,11 +237,6 @@ const getMentions = async ({ options }) => { query: { bool: { must: [ - { - term: { - context: search, - }, - }, { terms: { 'type.keyword': types, @@ -249,13 +246,13 @@ const getMentions = async ({ options }) => { }, }, _source: [ + 'authors', 'context', 'dataset-name', 'doi', 'mention_context', 'rawForm', 'software-name', - 'type', ], highlight: { number_of_fragments: 0, @@ -268,9 +265,20 @@ const getMentions = async ({ options }) => { ], }, }; + if (affiliation?.length > 0) { + body.query.bool.must.push({ + term: { 'authors.affiliations.name': affiliation }, + }); + } + if (author?.length > 0) { + body.query.bool.must.push({ term: { 'authors.last_name': author } }); + } if (doi?.length > 0) { body.query.bool.must.push({ term: { 'doi.keyword': doi } }); } + if (search?.length > 0) { + body.query.bool.must.push({ term: { context: search } }); + } body = JSON.stringify(body); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; const params = { @@ -286,11 +294,15 @@ const getMentions = async ({ options }) => { const count = data?.hits?.total?.value ?? 0; const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, + affiliations: + mention._source?.authors?.map((_author) => _author?.affiliations?.map((_affiliation) => _affiliation.name)).flat() ?? [], + authors: + mention._source?.authors?.map((_author) => _author.last_name) ?? [], + context: mention?.highlight?.context ?? mention._source.context, id: mention._id, rawForm: mention._source?.['software-name']?.rawForm ?? mention._source?.['dataset-name']?.rawForm, - context: mention.highlight.context, })); return { count, mentions }; @@ -299,15 +311,13 @@ const getMentions = async ({ options }) => { router.route('/mentions').post(async (req, res) => { try { const options = req?.body ?? {}; - if (!options?.search) { - res.status(400).json({ message: 'You must provide a search string.' }); - } else if (!['datasets', 'software'].includes(options?.type)) { + if (!['datasets', 'software'].includes(options?.type)) { res .status(400) .json({ message: 'Type should be either "datasets" or "software".' }); } else { - const mentions = await getMentions({ options }); - res.status(200).json(mentions); + const result = await getMentions({ options }); + res.status(200).json(result); } } catch (err) { console.error(err); From cca1b3c250d289825a1a2828e90ca4a4ae91288f Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 21 Oct 2024 19:49:39 +0200 Subject: [PATCH 12/21] mention colors + test request --- client/src/pages/mentions.jsx | 24 ++++++++++++++++++++---- server/src/routes/works.routes.js | 3 ++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index 99e35fc..ef34a22 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -53,6 +53,11 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} + style={{ color: + rowData.mention_context.created + ? '#8dc572' + : '#be6464', + }} /> ); const doiTemplate = (rowData) => ( @@ -65,6 +70,11 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} + style={{ color: + rowData.mention_context.shared + ? '#8dc572' + : '#be6464', + }} /> ); const usedTemplate = (rowData) => ( @@ -74,6 +84,11 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} + style={{ color: + rowData.mention_context.used + ? '#8dc572' + : '#be6464', + }} /> ); @@ -127,7 +142,7 @@ export default function Mentions() { - +
Search
Example "Coq"
@@ -141,7 +156,7 @@ export default function Mentions() { /> - +
Affiliation
Example "Cern"
@@ -156,7 +171,7 @@ export default function Mentions() { /> - +
Author
Example "Bruno Latour"
@@ -171,7 +186,7 @@ export default function Mentions() { /> - +
DOI
@@ -207,6 +222,7 @@ export default function Mentions() { lazy loading={loading} onPage={onPage} + onSort={onSort} sortField={lazyState.sortField} sortOrder={lazyState.sortOrder} paginator paginatorPosition="bottom" paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index d5cea69..28872d9 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -277,7 +277,8 @@ const getMentions = async ({ options }) => { body.query.bool.must.push({ term: { 'doi.keyword': doi } }); } if (search?.length > 0) { - body.query.bool.must.push({ term: { context: search } }); + // body.query.bool.should.push({ term: { context: search } }); + body.query.bool.must.push({ simple_query_string: { query: search } }); } body = JSON.stringify(body); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; From c93b2a55760f9910e7fe2b5dce1d2c302d1096f7 Mon Sep 17 00:00:00 2001 From: eric Date: Mon, 21 Oct 2024 20:02:01 +0200 Subject: [PATCH 13/21] bugfix --- client/src/pages/mentions.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index ef34a22..e086a86 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -222,7 +222,6 @@ export default function Mentions() { lazy loading={loading} onPage={onPage} - onSort={onSort} sortField={lazyState.sortField} sortOrder={lazyState.sortOrder} paginator paginatorPosition="bottom" paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" From a0001cab797e98df8194c9ba75bcaa413eb8244d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Tue, 22 Oct 2024 16:11:09 +0200 Subject: [PATCH 14/21] fix(mentions): Restore pagination --- client/src/pages/filters.jsx | 2 +- client/src/pages/mentions.jsx | 183 +++++++++--------------------- client/src/styles/index.scss | 32 +++--- client/src/utils/templates.jsx | 21 ++++ server/src/routes/works.routes.js | 25 ++-- 5 files changed, 97 insertions(+), 166 deletions(-) diff --git a/client/src/pages/filters.jsx b/client/src/pages/filters.jsx index 3b6e8f4..e4bcba1 100644 --- a/client/src/pages/filters.jsx +++ b/client/src/pages/filters.jsx @@ -62,7 +62,7 @@ export default function Filters({ useEffect(() => { const getData = async () => { if (searchParams.size === 0) { - // default values + // Set default params values setSearchParams({ affiliations: [], datasets: false, diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index e086a86..93a0d64 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -12,37 +12,26 @@ import { DataTable } from 'primereact/datatable'; import { useEffect, useState } from 'react'; import { useSearchParams } from 'react-router-dom'; -import { authorsTemplate } from '../utils/templates'; +import { affiliations2Template, authorsTemplate } from '../utils/templates'; import { getMentions } from '../utils/works'; -const DEFAULT_ROWS = 50; +const DEFAULT_SEARCH = ''; +const DEFAULT_SIZE = 50; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); - const [currentTab, setCurrentTab] = useState(0); - const [affiliation, setAffiliation] = useState(''); - const [author, setAuthor] = useState(''); - const [doi, setDoi] = useState(''); - const [from, setFrom] = useState(0); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); - const [rows, setRows] = useState(DEFAULT_ROWS); - const [search, setSearch] = useState(''); + const [search, setSearch] = useState(DEFAULT_SEARCH); const [totalRecords, setTotalRecords] = useState(0); - const [lazyState, setlazyState] = useState({ - first: 0, - page: 1, - rows: DEFAULT_ROWS, + const [urlSearchParams, setUrlSearchParams] = useState({ + from: 0, + search: DEFAULT_SEARCH, + size: DEFAULT_SIZE, + type: 'software', }); // Templates - const affiliationsTemplate = (rowData) => ( -
    - {rowData.affiliations.slice(0, 3).map((_affiliation) => ( -
  • {_affiliation}
    • - ))} -
- ); const contextTemplate = (rowData) => ( ); @@ -53,11 +42,7 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} - style={{ color: - rowData.mention_context.created - ? '#8dc572' - : '#be6464', - }} + style={{ color: rowData.mention_context.created ? '#8dc572' : '#be6464' }} /> ); const doiTemplate = (rowData) => ( @@ -70,11 +55,7 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} - style={{ color: - rowData.mention_context.shared - ? '#8dc572' - : '#be6464', - }} + style={{ color: rowData.mention_context.shared ? '#8dc572' : '#be6464' }} /> ); const usedTemplate = (rowData) => ( @@ -84,69 +65,60 @@ export default function Mentions() { ? 'fr-icon-check-line' : 'fr-icon-close-line' }`} - style={{ color: - rowData.mention_context.used - ? '#8dc572' - : '#be6464', - }} + style={{ color: rowData.mention_context.used ? '#8dc572' : '#be6464' }} /> ); // Events - const loadLazyData = async () => { - setLoading(true); - const data = await getMentions({ - affiliation: searchParams.get('affiliation'), - author: searchParams.get('author'), - doi: searchParams.get('doi'), - from: searchParams.get('from'), - search: searchParams.get('search'), - size: searchParams.get('size'), - type: currentTab === 0 ? 'software' : 'datasets', - }); - setMentions(data?.mentions ?? []); - setTotalRecords(data?.count ?? 0); - setLoading(false); - }; const onPage = (event) => { - setFrom(event.first); - setRows(event.rows); + searchParams.set('from', parseInt(event.first, 10)); + setSearchParams(searchParams); }; const onSubmit = () => { - setSearchParams((params) => { - params.set('affiliation', affiliation); - params.set('author', author); - params.set('doi', doi); - params.set('from', from); - params.set('search', search); - params.set('size', rows); - return params; - }); + searchParams.set('search', search); + setSearchParams(searchParams); }; // Effects - useEffect(() => setFrom(0), [currentTab]); useEffect(() => { - setlazyState({ - ...lazyState, - first: parseInt(searchParams.get('from'), 10), - rows: searchParams.get('size'), - }); - }, [currentTab, searchParams]); + console.log('searchParams changed'); + if (searchParams.size === 0) { + setSearchParams({ + from: 0, + search: DEFAULT_SEARCH, + size: DEFAULT_SIZE, + type: 'software', + }); + } else { + setUrlSearchParams({ + from: searchParams.get('from'), + search: searchParams.get('search'), + size: searchParams.get('size'), + type: searchParams.get('type'), + }); + } + }, [searchParams]); useEffect(() => { - loadLazyData(); - }, [lazyState]); + const getData = async () => { + setLoading(true); + const data = await getMentions(urlSearchParams); + setMentions(data?.mentions ?? []); + setTotalRecords(data?.count ?? 0); + setLoading(false); + }; + getData(); + }, [urlSearchParams]); return ( - +
Search
-
Example "Coq"
+
Example "Coq" or "Cern"
- + - - -
Affiliation
-
Example "Cern"
- - - setAffiliation(e.target.value)} - value={affiliation} - /> - - - - -
Author
-
Example "Bruno Latour"
- - - setAuthor(e.target.value)} - value={author} - /> - - - - -
DOI
-
- Example "10.4000/proceedings.elpub.2019.20" -
- - - setDoi(e.target.value)} - value={doi} - /> - - - + setCurrentTab(i)} + defaultActiveIndex={0} + onTabChange={(i) => setUrlSearchParams({ ...urlSearchParams, type: i === 0 ? 'software' : 'datasets' })} > @@ -265,14 +190,14 @@ export default function Mentions() { diff --git a/client/src/styles/index.scss b/client/src/styles/index.scss index 69d3a6d..016e08a 100644 --- a/client/src/styles/index.scss +++ b/client/src/styles/index.scss @@ -4,6 +4,10 @@ cursor: pointer; } +.list-none { + list-style: none; +} + .text-center { text-align: center; } @@ -20,10 +24,6 @@ vertical-align: middle; } -.list-none { - list-style: none; -} - /* General */ @@ -31,14 +31,6 @@ body { height: auto; } -.fr-tags-group > li { - line-height: 0rem; -} - -.text-right { - text-align: right; -} - .filters { background-color: white; border-bottom: 2px solid #000; @@ -47,8 +39,8 @@ body { .actions-menu { background-color: #eee; - border-bottom: 2px solid #000; border-bottom-right-radius: 5px; + border-bottom: 2px solid #000; border-top-right-radius: 5px; box-shadow: 0px 0px 10px 0px rgba(0, 0, 0, 0.3); left: 0; @@ -124,18 +116,20 @@ body { margin: 0; } + /* Pge Mentions */ -.mentions { - .label { - font-size: 1em; - font-weight: bold; - text-align: end; - } +.mentions { .hint { color: grey; font-size: .8em; font-style: italic; text-align: end; } + + .label { + font-size: 1em; + font-weight: bold; + text-align: end; + } } diff --git a/client/src/utils/templates.jsx b/client/src/utils/templates.jsx index dfb0fd7..f19a6c4 100644 --- a/client/src/utils/templates.jsx +++ b/client/src/utils/templates.jsx @@ -16,6 +16,26 @@ const affiliationsTemplate = (rowData) => ( ); +const affiliations2Template = (rowData) => { + let affiliationsHtml = `
    `; + affiliationsHtml += rowData.affiliations.slice(0, 3).map((affiliation, index) => `
  • ${affiliation}
    • `).join(''); + if (rowData.affiliations.length > 3) { + affiliationsHtml += `
  • others (${rowData.affiliations.length - 3})
    • `; + } + affiliationsHtml += '
'; + let affiliationsTooltip = '
    '; + affiliationsTooltip += rowData.affiliations.map((affiliation, index) => `
  • ${affiliation}
    • `).join(''); + affiliationsTooltip += '
'; + return ( + <> + + + + + + ); +}; + const statusesItemTemplate = (option) => (
{option.name} @@ -153,6 +173,7 @@ const hasCorrectionTemplate = (rowData) => (rowData?.hasCorrection export { affiliationsTemplate, + affiliations2Template, allIdsTemplate, authorsTemplate, correctionTemplate, diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index 28872d9..a2c248d 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -222,9 +222,7 @@ router.route('/works').post(async (req, res) => { }); const getMentions = async ({ options }) => { - const { - affiliation, author, doi, from, search, size, type, - } = options; + const { from, search, size, type } = options; let types = []; if (type === 'software') { types = ['software']; @@ -265,19 +263,7 @@ const getMentions = async ({ options }) => { ], }, }; - if (affiliation?.length > 0) { - body.query.bool.must.push({ - term: { 'authors.affiliations.name': affiliation }, - }); - } - if (author?.length > 0) { - body.query.bool.must.push({ term: { 'authors.last_name': author } }); - } - if (doi?.length > 0) { - body.query.bool.must.push({ term: { 'doi.keyword': doi } }); - } if (search?.length > 0) { - // body.query.bool.should.push({ term: { context: search } }); body.query.bool.must.push({ simple_query_string: { query: search } }); } body = JSON.stringify(body); @@ -295,8 +281,13 @@ const getMentions = async ({ options }) => { const count = data?.hits?.total?.value ?? 0; const mentions = (data?.hits?.hits ?? []).map((mention) => ({ ...mention._source, - affiliations: - mention._source?.authors?.map((_author) => _author?.affiliations?.map((_affiliation) => _affiliation.name)).flat() ?? [], + affiliations: [ + ...new Set( + mention._source?.authors + ?.map((_author) => _author?.affiliations?.map((_affiliation) => _affiliation.name)) + .flat().filter((item) => !!item) ?? [], + ), + ], authors: mention._source?.authors?.map((_author) => _author.last_name) ?? [], context: mention?.highlight?.context ?? mention._source.context, From fd870111c60f9d7c8dcad3b1a4be7995f285e56b Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 22 Oct 2024 16:42:05 +0200 Subject: [PATCH 15/21] logging --- server/src/routes/works.routes.js | 5 +++-- server/src/utils/s3.js | 12 ++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index 28872d9..d912bd0 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -53,7 +53,8 @@ const getWorks = async ({ options, resetCache = false }) => { const shasum = crypto.createHash('sha1'); shasum.update(JSON.stringify(options)); const searchId = shasum.digest('hex'); - const queryId = Math.floor(Math.random() * SEED_MAX); + const start = new Date(); + const queryId = start.toISOString().concat(' - ', Math.floor(Math.random() * SEED_MAX).toString()); let cache = false; if (USE_CACHE) { console.time( @@ -197,7 +198,7 @@ const getWorks = async ({ options, resetCache = false }) => { console.time( `7. Query ${queryId} | Save cache ${options.affiliationStrings}`, ); - await saveCache({ result, searchId }); + await saveCache({ result, searchId, queryId }); console.timeEnd( `7. Query ${queryId} | Save cache ${options.affiliationStrings}`, ); diff --git a/server/src/utils/s3.js b/server/src/utils/s3.js index cb2be59..d44ffec 100644 --- a/server/src/utils/s3.js +++ b/server/src/utils/s3.js @@ -35,7 +35,8 @@ const getCache = async ({ searchId }) => { return false; }; -const saveCache = async ({ result, searchId }) => { +const saveCache = async ({ result, searchId, queryId }) => { + console.log(queryId, 'start saving cache'); const fileName = getFileName(searchId); const remotePath = `${OS_CONTAINER}/${fileName}`; @@ -64,14 +65,21 @@ const saveCache = async ({ result, searchId }) => { }); const token = response?.headers?.get('x-subject-token'); if (token) { + const resultJson = JSON.stringify(result); + console.time( + `7b. Query ${queryId} | Uploading data to cloud`, + ); await fetch( `https://storage.gra.cloud.ovh.net/v1/AUTH_${OS_TENANT_ID}/${remotePath}`, { - body: JSON.stringify(result), + body: resultJson, headers: { 'Content-Type': 'application/json', 'X-Auth-Token': token, 'X-Delete-After': '604800' }, // 7 days method: 'PUT', }, ); + console.timeEnd( + `7b. Query ${queryId} | Uploading data to cloud`, + ); } }; From 28320792a49380a967a3c02d81f8010339f630a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Tue, 22 Oct 2024 17:23:00 +0200 Subject: [PATCH 16/21] feat(sort): Enable sort on doi, rawForm, used, created and shared --- client/src/pages/mentions.jsx | 53 +++++++++++++++++++++++-------- server/src/routes/works.routes.js | 33 +++++++++++++++++-- 2 files changed, 70 insertions(+), 16 deletions(-) diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index 93a0d64..8a6ecf9 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -15,8 +15,12 @@ import { useSearchParams } from 'react-router-dom'; import { affiliations2Template, authorsTemplate } from '../utils/templates'; import { getMentions } from '../utils/works'; +const DEFAULT_FROM = 0; const DEFAULT_SEARCH = ''; const DEFAULT_SIZE = 50; +const DEFAULT_SORTBY = ''; +const DEFAULT_SORTORDER = ''; +const DEFAULT_TYPE = 'software'; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); @@ -25,10 +29,12 @@ export default function Mentions() { const [search, setSearch] = useState(DEFAULT_SEARCH); const [totalRecords, setTotalRecords] = useState(0); const [urlSearchParams, setUrlSearchParams] = useState({ - from: 0, + from: DEFAULT_FROM, search: DEFAULT_SEARCH, size: DEFAULT_SIZE, - type: 'software', + sortBy: DEFAULT_SORTBY, + sortOrder: DEFAULT_SORTORDER, + type: DEFAULT_TYPE, }); // Templates @@ -74,6 +80,11 @@ export default function Mentions() { searchParams.set('from', parseInt(event.first, 10)); setSearchParams(searchParams); }; + const onSort = (event) => { + searchParams.set('sort-by', event.sortField); + searchParams.set('sort-order', event.sortOrder === 1 ? 'asc' : 'desc'); + setSearchParams(searchParams); + }; const onSubmit = () => { searchParams.set('search', search); setSearchParams(searchParams); @@ -81,26 +92,30 @@ export default function Mentions() { // Effects useEffect(() => { - console.log('searchParams changed'); + // Set default params values if (searchParams.size === 0) { setSearchParams({ - from: 0, + from: DEFAULT_FROM, search: DEFAULT_SEARCH, size: DEFAULT_SIZE, - type: 'software', + 'sort-by': DEFAULT_SORTBY, + 'sort-order': DEFAULT_SORTORDER, + type: DEFAULT_TYPE, }); } else { + setLoading(true); setUrlSearchParams({ from: searchParams.get('from'), search: searchParams.get('search'), size: searchParams.get('size'), + sortBy: searchParams.get('sort-by'), + sortOrder: searchParams.get('sort-order'), type: searchParams.get('type'), }); } - }, [searchParams]); + }, [searchParams, setSearchParams]); useEffect(() => { const getData = async () => { - setLoading(true); const data = await getMentions(urlSearchParams); setMentions(data?.mentions ?? []); setTotalRecords(data?.count ?? 0); @@ -121,8 +136,6 @@ export default function Mentions() { setSearch(e.target.value)} value={search} /> @@ -137,7 +150,10 @@ export default function Mentions() { setUrlSearchParams({ ...urlSearchParams, type: i === 0 ? 'software' : 'datasets' })} + onTabChange={(i) => setUrlSearchParams({ + ...urlSearchParams, + type: i === 0 ? 'software' : 'datasets', + })} > - - + + - - + + { }); const getMentions = async ({ options }) => { - const { from, search, size, type } = options; + const { + from, search, size, sortBy, sortOrder, type, + } = options; let types = []; if (type === 'software') { types = ['software']; @@ -267,6 +269,32 @@ const getMentions = async ({ options }) => { if (search?.length > 0) { body.query.bool.must.push({ simple_query_string: { query: search } }); } + if (sortBy && sortOrder) { + let sortFields = sortBy; + switch (sortBy) { + case 'doi': + sortFields = ['doi.keyword']; + break; + case 'rawForm': + sortFields = [ + 'dataset-name.rawForm.keyword', + 'software-name.rawForm.keyword', + ]; + break; + case 'mention.mention_context.used': + sortFields = ['mention_context.used']; + break; + case 'mention.mention_context.created': + sortFields = ['mention_context.created']; + break; + case 'mention.mention_context.shared': + sortFields = ['mention_context.shared']; + break; + default: + } + body.sort = []; + sortFields.map((sortField) => body.sort.push({ [sortField]: sortOrder })); + } body = JSON.stringify(body); const url = `${process.env.ES_URL}/${process.env.ES_INDEX_MENTIONS}/_search`; const params = { @@ -286,7 +314,8 @@ const getMentions = async ({ options }) => { ...new Set( mention._source?.authors ?.map((_author) => _author?.affiliations?.map((_affiliation) => _affiliation.name)) - .flat().filter((item) => !!item) ?? [], + .flat() + .filter((item) => !!item) ?? [], ), ], authors: From 8a94a88add8fd5de95919914e3ef1709af4cabec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anne=20L=27H=C3=B4te?= Date: Tue, 22 Oct 2024 18:42:46 +0200 Subject: [PATCH 17/21] feat(mentions): Display panel to chose caracterizations --- client/src/pages/mentions.jsx | 130 +++++++++++++++++++++++++++++++ client/src/pages/openalexTab.jsx | 2 +- 2 files changed, 131 insertions(+), 1 deletion(-) diff --git a/client/src/pages/mentions.jsx b/client/src/pages/mentions.jsx index 8a6ecf9..ccd5d8d 100644 --- a/client/src/pages/mentions.jsx +++ b/client/src/pages/mentions.jsx @@ -24,9 +24,15 @@ const DEFAULT_TYPE = 'software'; export default function Mentions() { const [searchParams, setSearchParams] = useSearchParams(); + const [correctionsUsed, setCorrectionsUsed] = useState(true); + const [correctionsCreated, setCorrectionsCreated] = useState(true); + const [correctionsShared, setCorrectionsShared] = useState(true); + const [fixedMenu, setFixedMenu] = useState(false); const [loading, setLoading] = useState(false); const [mentions, setMentions] = useState([]); const [search, setSearch] = useState(DEFAULT_SEARCH); + const [selectAll, setSelectAll] = useState(false); + const [selectedMentions, setSelectedMentions] = useState([]); const [totalRecords, setTotalRecords] = useState(0); const [urlSearchParams, setUrlSearchParams] = useState({ from: DEFAULT_FROM, @@ -80,6 +86,15 @@ export default function Mentions() { searchParams.set('from', parseInt(event.first, 10)); setSearchParams(searchParams); }; + const onSelectAllChange = (event) => { + if (event.checked) { + setSelectAll(true); + setSelectedMentions(mentions); + } else { + setSelectAll(false); + setSelectedMentions([]); + } + }; const onSort = (event) => { searchParams.set('sort-by', event.sortField); searchParams.set('sort-order', event.sortOrder === 1 ? 'asc' : 'desc'); @@ -126,6 +141,103 @@ export default function Mentions() { return ( +
+
+ {selectedMentions.length} + {`selected mention${selectedMentions.length === 1 ? '' : 's'}`} +
+ + + + +
+ +
+
@@ -137,6 +249,11 @@ export default function Mentions() { setSearch(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + onSubmit(); + } + }} value={search} /> @@ -163,6 +280,8 @@ export default function Mentions() { lazy loading={loading} onPage={onPage} + onSelectAllChange={onSelectAllChange} + onSelectionChange={(e) => setSelectedMentions(e.value)} onSort={onSort} paginator paginatorPosition="bottom" @@ -170,6 +289,8 @@ export default function Mentions() { rows={parseInt(urlSearchParams.size, 10)} rowsPerPageOptions={[20, 50, 100]} scrollable + selectAll={selectAll} + selection={selectedMentions} size="small" sortField={urlSearchParams.sortBy} sortOrder={urlSearchParams.sortOrder === 'asc' ? 1 : -1} @@ -179,6 +300,7 @@ export default function Mentions() { totalRecords={totalRecords} value={mentions} > + @@ -216,19 +338,27 @@ export default function Mentions() { lazy loading={loading} onPage={onPage} + onSelectAllChange={onSelectAllChange} + onSelectionChange={(e) => setSelectedMentions(e.value)} + onSort={onSort} paginator paginatorPosition="bottom" paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" rows={parseInt(urlSearchParams.size, 10)} rowsPerPageOptions={[20, 50, 100]} scrollable + selectAll={selectAll} + selection={selectedMentions} size="small" + sortField={urlSearchParams.sortBy} + sortOrder={urlSearchParams.sortOrder === 'asc' ? 1 : -1} stripedRows style={{ fontSize: '14px', lineHeight: '13px' }} tableStyle={{ minWidth: '50rem' }} totalRecords={totalRecords} value={mentions} > + diff --git a/client/src/pages/openalexTab.jsx b/client/src/pages/openalexTab.jsx index 2e27949..c145717 100644 --- a/client/src/pages/openalexTab.jsx +++ b/client/src/pages/openalexTab.jsx @@ -66,7 +66,7 @@ export default function OpenalexTab({ affiliation.key.replace('[ source: ', '').replace(' ]', ''), ); }); - // recompute corrections only when the array has changed + // Recompute corrections only when the array has changed if (filteredAffiliationsTmp.length !== filteredAffiliations.length) { const newCorrections = getCorrections(filteredAffiliationsTmp); setAllOpenalexCorrections(newCorrections); From d9d2cc88c35aebf0f05a7abd725da471c51e1d5d Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 22 Oct 2024 21:45:27 +0200 Subject: [PATCH 18/21] warning message when too many publications found --- client/src/pages/index.jsx | 5 +++-- client/src/utils/works.jsx | 19 ++++++++++++++++--- server/src/routes/works.routes.js | 12 ++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/client/src/pages/index.jsx b/client/src/pages/index.jsx index 4888008..ea502a6 100644 --- a/client/src/pages/index.jsx +++ b/client/src/pages/index.jsx @@ -10,6 +10,7 @@ import PublicationsTile from '../components/tiles/publications'; import { status } from '../config'; import { getWorks } from '../utils/works'; import Filters from './filters'; +import useToast from '../hooks/useToast'; import Datasets from './views/datasets'; import Openalex from './views/openalex'; import Publications from './views/publications'; @@ -24,6 +25,7 @@ export default function Home({ isSticky, setIsSticky }) { const [selectedAffiliations, setSelectedAffiliations] = useState([]); const [selectedDatasets, setSelectedDatasets] = useState([]); const [selectedPublications, setSelectedPublications] = useState([]); + const { toast } = useToast(); const setView = (_view) => { setSearchParams((params) => { @@ -34,7 +36,7 @@ export default function Home({ isSticky, setIsSticky }) { const { data, error, isFetched, isFetching, refetch } = useQuery({ queryKey: ['data', JSON.stringify(options)], - queryFn: () => getWorks(options), + queryFn: () => getWorks(options, toast), enabled: false, cacheTime: 60 * (60 * 1000), // 1h }); @@ -78,7 +80,6 @@ export default function Home({ isSticky, setIsSticky }) { .map((affiliation) => (affiliation.status = action)); setSelectedAffiliations([]); }; - return ( // TODO:do a cleaner way to display the spinner and views <> diff --git a/client/src/utils/works.jsx b/client/src/utils/works.jsx index bbf9c4e..88d6cf3 100644 --- a/client/src/utils/works.jsx +++ b/client/src/utils/works.jsx @@ -79,7 +79,7 @@ const getMentions = async (options) => { return mentions; }; -const getWorks = async (options) => { +const getWorks = async (options, toast) => { const response = await fetch(`${VITE_API}/works`, { body: JSON.stringify(options), headers: { 'Content-Type': 'application/json' }, @@ -89,11 +89,24 @@ const getWorks = async (options) => { if (!response.ok) { throw new Error('Oops... FOSM API request did not work for works !'); } - const { affiliations, datasets, publications } = await response.json(); + const { affiliations, datasets, publications, warnings } = await response.json(); const resAffiliations = await decompressAll(affiliations); datasets.results = await decompressAll(datasets.results); publications.results = await decompressAll(publications.results); - return { affiliations: resAffiliations, datasets, publications }; + let warningMessage = ''; + if (warnings?.isMaxFosmReached) { + warningMessage = warningMessage.concat(`More than ${warnings.maxFosmValue} publications found in French OSM, only the first ${warnings.maxFosmValue} were retrieved.\n`); + } + if (warnings?.isMaxOpenalexReached) { + warningMessage = warningMessage.concat(`More than ${warnings.maxOpenalexValue} publications found in OpenAlex, only the first ${warnings.maxOpenalexValue} were retrieved.\n`); + } + toast({ + description: warningMessage, + id: 'tooManyPublications', + title: 'Too Many publications found', + toastType: 'error', + }); + return { affiliations: resAffiliations, datasets, publications, warnings }; }; const normalizeName = (name) => name diff --git a/server/src/routes/works.routes.js b/server/src/routes/works.routes.js index 3b9ffe4..c0a1a3f 100644 --- a/server/src/routes/works.routes.js +++ b/server/src/routes/works.routes.js @@ -101,6 +101,17 @@ const getWorks = async ({ options, resetCache = false }) => { ); }); const responses = await Promise.all(queries); + const warnings = {}; + const MAX_FOSM = Number(process.env.ES_MAX_SIZE); + if (MAX_FOSM > 0 && responses.length > 0 && responses[0].length >= MAX_FOSM) { + warnings.isMaxFosmReached = true; + warnings.maxFosmValue = MAX_FOSM; + } + const MAX_OPENALEX = Number(process.env.OPENALEX_MAX_SIZE); + if (MAX_OPENALEX > 0 && responses.length > 1 && responses[1].length >= MAX_OPENALEX) { + warnings.isMaxOpenalexReached = true; + warnings.maxOpenalexValue = MAX_OPENALEX; + } console.timeEnd( `1. Query ${queryId} | Requests ${options.affiliationStrings}`, ); @@ -191,6 +202,7 @@ const getWorks = async ({ options, resetCache = false }) => { years: publicationsYears, }, extractionDate: Date.now(), + warnings, }; console.timeEnd( `6. Query ${queryId} | Serialization ${options.affiliationStrings}`, From a0bf98bc53417cb018b73998e11749e32e6189ce Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 22 Oct 2024 22:06:32 +0200 Subject: [PATCH 19/21] fix --- client/src/utils/works.jsx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/client/src/utils/works.jsx b/client/src/utils/works.jsx index 88d6cf3..e0a1748 100644 --- a/client/src/utils/works.jsx +++ b/client/src/utils/works.jsx @@ -100,12 +100,14 @@ const getWorks = async (options, toast) => { if (warnings?.isMaxOpenalexReached) { warningMessage = warningMessage.concat(`More than ${warnings.maxOpenalexValue} publications found in OpenAlex, only the first ${warnings.maxOpenalexValue} were retrieved.\n`); } - toast({ - description: warningMessage, - id: 'tooManyPublications', - title: 'Too Many publications found', - toastType: 'error', - }); + if (warningMessage) { + toast({ + description: warningMessage, + id: 'tooManyPublications', + title: 'Too Many publications found', + toastType: 'error', + }); + } return { affiliations: resAffiliations, datasets, publications, warnings }; }; From 6a2700f683c5d6cefcf1dc3a500ef19b4ff66aa5 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 22 Oct 2024 22:19:41 +0200 Subject: [PATCH 20/21] check limits in logs --- server/src/utils/works.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/utils/works.js b/server/src/utils/works.js index 58f8dc0..312728c 100644 --- a/server/src/utils/works.js +++ b/server/src/utils/works.js @@ -1,5 +1,8 @@ import { cleanId, getAuthorOrcid, intersectArrays, removeDiacritics } from './utils'; +const MAX_FOSM = Number(process.env.ES_MAX_SIZE); +const MAX_OPENALEX = Number(process.env.OPENALEX_MAX_SIZE); + const datasetsType = ['dataset', 'physicalobject', 'collection', 'audiovisual', 'sound', 'software', 'computationalnotebook', 'film', 'image']; @@ -205,6 +208,7 @@ const formatFosmResult = (result, options) => { }; const getFosmWorksByYear = async ({ remainingTries = 3, results = [], options, pit, searchAfter }) => { + console.log('getFosmWorksByYear', `MAX_FOSM = ${MAX_FOSM}`); if (!pit) { const response = await fetch( `${process.env.ES_URL}/${process.env.ES_INDEX}/_pit?keep_alive=${process.env.ES_PIT_KEEP_ALIVE}`, @@ -234,7 +238,7 @@ const getFosmWorksByYear = async ({ remainingTries = 3, results = [], options, p const hits = response?.hits?.hits ?? []; // eslint-disable-next-line no-param-reassign results = results.concat(hits.map((result) => formatFosmResult(result, options))).filter((r) => r.levelCertainty !== '3.low'); - if (hits.length > 0 && (Number(process.env.ES_MAX_SIZE) === 0 || results.length < Number(process.env.ES_MAX_SIZE))) { + if (hits.length > 0 && (MAX_FOSM === 0 || results.length < MAX_FOSM)) { // eslint-disable-next-line no-param-reassign searchAfter = hits.at('-1').sort; return getFosmWorksByYear({ results, options, pit, searchAfter }); @@ -334,6 +338,7 @@ const getOpenAlexAffiliation = (author) => { }; const getOpenAlexPublicationsByYear = (options, cursor = '*', previousResponse = [], remainingTries = 3) => { + console.log('getOpenAlexPublicationsByYear', `MAX_OPENALEX = ${MAX_OPENALEX}`); let url = `https://api.openalex.org/works?per_page=${process.env.OPENALEX_PER_PAGE}`; url += '&filter=is_paratext:false'; url += `,publication_year:${Number(options.year)}-${Number(options?.year)}`; @@ -392,7 +397,7 @@ const getOpenAlexPublicationsByYear = (options, cursor = '*', previousResponse = return answer; })); const nextCursor = response?.meta?.next_cursor; - if (nextCursor && hits.length > 0 && (Number(process.env.OPENALEX_MAX_SIZE) === 0 || results.length < Number(process.env.OPENALEX_MAX_SIZE))) { + if (nextCursor && hits.length > 0 && (MAX_OPENALEX === 0 || results.length < MAX_OPENALEX)) { return getOpenAlexPublicationsByYear(options, nextCursor, results); } return results; From 3d4fe2ce37819df5cc8abfdc478401f950c77df9 Mon Sep 17 00:00:00 2001 From: eric Date: Tue, 22 Oct 2024 22:31:56 +0200 Subject: [PATCH 21/21] more logs --- server/src/utils/works.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/utils/works.js b/server/src/utils/works.js index 312728c..b5fcc5c 100644 --- a/server/src/utils/works.js +++ b/server/src/utils/works.js @@ -338,7 +338,7 @@ const getOpenAlexAffiliation = (author) => { }; const getOpenAlexPublicationsByYear = (options, cursor = '*', previousResponse = [], remainingTries = 3) => { - console.log('getOpenAlexPublicationsByYear', `MAX_OPENALEX = ${MAX_OPENALEX}`); + console.log('getOpenAlexPublicationsByYear', `MAX_OPENALEX = ${MAX_OPENALEX}, currentResponseLength = ${previousResponse.length}`); let url = `https://api.openalex.org/works?per_page=${process.env.OPENALEX_PER_PAGE}`; url += '&filter=is_paratext:false'; url += `,publication_year:${Number(options.year)}-${Number(options?.year)}`; @@ -365,6 +365,7 @@ const getOpenAlexPublicationsByYear = (options, cursor = '*', previousResponse = .then((response) => { if (response.ok) return response.json(); if (response.status === 429) { + console.log('Error 429', 'Getting error 429 from OpenAlex'); return new Promise((resolve) => setTimeout(resolve, Math.round(Math.random() * 1000))).then(() => getOpenAlexPublicationsByYear(options, cursor, previousResponse)); } console.error(`Error while fetching ${url} :`);