diff --git a/invenio_vocabularies/administration/__init__.py b/invenio_vocabularies/administration/__init__.py new file mode 100644 index 00000000..f27a100e --- /dev/null +++ b/invenio_vocabularies/administration/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# Copyright (C) 2024 Uni Münster. +# +# Invenio-Vocabularies is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Invenio administration views module for vocabularies.""" diff --git a/invenio_vocabularies/administration/views/__init__.py b/invenio_vocabularies/administration/views/__init__.py new file mode 100644 index 00000000..b041d1cb --- /dev/null +++ b/invenio_vocabularies/administration/views/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020-2021 CERN. +# Copyright (C) 2024 Uni Münster. +# +# Invenio-Vocabularies is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Invenio administration views module for Vocabularies.""" diff --git a/invenio_vocabularies/administration/views/vocabularies.py b/invenio_vocabularies/administration/views/vocabularies.py new file mode 100644 index 00000000..db8a2fbd --- /dev/null +++ b/invenio_vocabularies/administration/views/vocabularies.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020-2021 CERN. +# Copyright (C) 2024 Uni Münster. +# +# Invenio-Vocabularies is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Vocabularies admin interface.""" +from flask import current_app +from invenio_administration.views.base import ( + AdminResourceEditView, + AdminResourceListView, +) +from invenio_i18n import lazy_gettext as _ + + +class VocabulariesListView(AdminResourceListView): + """Configuration for vocabularies list view.""" + + api_endpoint = "/vocabularies/" + name = "vocabulary-types" + resource_config = "vocabulary_admin_resource" + search_request_headers = {"Accept": "application/json"} + title = "Vocabulary Types" + category = "Site management" + + pid_path = "id" + icon = "exchange" + template = "invenio_administration/search.html" + + display_search = True + display_delete = False + display_create = False + display_edit = False + + item_field_list = { + "id": {"text": "Name", "order": 1}, + "count": {"text": "Number of entries", "order": 2}, + } + + search_config_name = "VOCABULARIES_TYPES_SEARCH" + search_facets_config_name = "VOCABULARIES_TYPES_FACETS" + search_sort_config_name = "VOCABULARIES_TYPES_SORT_OPTIONS" + + # resource_name = "vocabulary_admin_resource" + + +class VocabularyTypesDetailsView(AdminResourceListView): + """Configuration for specific vocabularies list view after initial list view.""" + + def get_api_endpoint(self, vocab_type=None): + """overwrite get_api_endpoint to accept pid_value""" + + if vocab_type in current_app.config.get( + "VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [] + ): + return f"/api/{vocab_type}" + else: + return f"/api/vocabularies/{vocab_type}" + + # def get_context(self, **kwargs): + # """Create details view context.""" + # search_conf = self.init_search_config(**kwargs) + # schema = self.get_service_schema() + # serialized_schema = self._schema_to_json(schema) + # pid_value = kwargs.get("pid_value", "") + # return { + # "search_config": search_conf, + # "title": f"{pid_value} vocabulary items", + # "name": self.name, + # "resource_schema": serialized_schema, + # "fields": self.item_field_list, + # "display_search": self.display_search, + # "display_create": self.display_create, + # "display_edit": self.display_edit, + # "display_delete": self.display_delete, + # "display_read": self.display_read, + # "actions": self.serialize_actions(), + # "pid_path": self.pid_path, + # "pid_value": pid_value, + # "create_ui_endpoint": self.get_create_view_endpoint(), + # "list_ui_endpoint": self.get_list_view_endpoint(), + # "resource_name": ( + # self.resource_name if self.resource_name else self.pid_path + # ), + # } + + name = "vocabularies-details" + url = "/vocabulary-types/" + + # api_endpoint = "/vocabularies/" + + # INFO name of the resource's list view name, enables navigation between detail view and list view. + list_view_name = "vocabulary-types" + + resource_config = "vocabulary_admin_resource" + search_request_headers = {"Accept": "application/json"} + pid_path = "id" + + # INFO only if disabled() (as a function) its not in the sidebar, see https://github.com/inveniosoftware/invenio-administration/blob/main/invenio_administration/menu/menu.py#L54 + disabled = lambda _: True + + template = "invenio_administration/search.html" + + display_delete = False + display_create = False + display_edit = True + display_search = False + + item_field_list = { + "id": {"text": "ID", "order": 1}, + "created": {"text": "Created", "order": 2}, + } + + search_config_name = "VOCABULARIES_TYPES_ITEMS_SEARCH" + search_facets_config_name = "VOCABULARIES_TYPES_ITEMS_FACETS" + search_sort_config_name = "VOCABULARIES_TYPES_ITEMS_SORT_OPTIONS" + + +class VocabularyTypesDetailsEditView(AdminResourceEditView): + """Configuration for vocabulary item edit view.""" + + # Edit view for vocabulary items from a specific vocabulary type + def get_api_endpoint(vocab_type=None, pid=None): + """overwrite get_api_endpoint to accept pid_value""" + if vocab_type in current_app.config.get( + "VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [] + ): + return f"/api/{vocab_type}/{pid}" + else: + return f"/api/vocabulary-types/{vocab_type}/{pid}" + + name = "vocabularies_details_edit" + url = "/vocabulary-types///edit" + resource_config = "vocabulary_admin_resource" + pid_path = "id" + title = "Edit vocabulary item" + + list_view_name = "vocabularies_details" + + # form_fields = { + # "ID": { + # "order": 1, + # "text": "Set ID", + # "description": _("Some ID."), + # }, + # "created": {"order": 2}, + # "updated": {"order": 3}, + # } diff --git a/invenio_vocabularies/config.py b/invenio_vocabularies/config.py index 789963a5..f615919a 100644 --- a/invenio_vocabularies/config.py +++ b/invenio_vocabularies/config.py @@ -25,8 +25,8 @@ ) from .datastreams.transformers import XMLTransformer from .datastreams.writers import ServiceWriter, YamlWriter -from .resources.resource import VocabulariesResourceConfig -from .services.service import VocabulariesServiceConfig +from .resources import VocabulariesResourceConfig +from .services.config import VocabulariesServiceConfig VOCABULARIES_RESOURCE_CONFIG = VocabulariesResourceConfig """Configure the resource.""" @@ -103,6 +103,15 @@ } """Names allowed identifier schemes.""" +# configure CUSTOM_VOCABULARY_TYPES to differentiate output. Is used in VocabulariesServiceConfig +VOCABULARIES_CUSTOM_VOCABULARY_TYPES = [ + "names", + "affiliations", + "awards", + "funders", + "subjects", +] + VOCABULARIES_DATASTREAM_READERS = { "csv": CSVReader, "json": JsonReader, @@ -126,3 +135,38 @@ "yaml": YamlWriter, } """Data Streams writers.""" + +VOCABULARIES_TYPES_SORT_OPTIONS = { + "name": dict( + title=_("Name"), + fields=["id"], + ), + "count": dict( + title=_("Number of entries"), + fields=["count"], + ), +} +VOCABULARIES_TYPES_SEARCH = { + "facets": [], + "sort": ["name", "count"], +} + + +VOCABULARIES_TYPES_ITEMS_SORT_OPTIONS = { + "name": dict( + title=_("Name"), + fields=["id"], + ), + "created": dict( + title=_("Created"), + fields=["created"], + ), +} +VOCABULARIES_TYPES_ITEMS_SEARCH = { + "facets": [], + "sort": ["name", "created"], +} +"""Definitions of available Vocabulary types sort options. """ + + +"""Vocabulary type search configuration.""" diff --git a/invenio_vocabularies/ext.py b/invenio_vocabularies/ext.py index 4e7ab481..3792f1bc 100644 --- a/invenio_vocabularies/ext.py +++ b/invenio_vocabularies/ext.py @@ -40,8 +40,16 @@ SubjectsService, SubjectsServiceConfig, ) -from .resources.resource import VocabulariesResource -from .services.service import VocabulariesService +from .resources import ( + VocabulariesAdminResource, + VocabulariesResource, + VocabulariesResourceConfig, + VocabularyTypeResourceConfig, +) +from .services.config import VocabularyTypesServiceConfig +from .services.service import VocabulariesService, VocabularyTypeService + +# from .contrib.information import InformationResource, InformationResourceConfig class InvenioVocabularies(object): @@ -76,6 +84,7 @@ class ServiceConfigs: funders = FundersServiceConfig names = NamesServiceConfig subjects = SubjectsServiceConfig + vocabulary_types = VocabularyTypesServiceConfig return ServiceConfigs @@ -93,9 +102,12 @@ def init_services(self, app): self.funders_service = FundersService(config=service_configs.funders) self.names_service = NamesService(config=service_configs.names) self.subjects_service = SubjectsService(config=service_configs.subjects) - self.service = VocabulariesService( + self.vocabularies_service = VocabulariesService( config=app.config["VOCABULARIES_SERVICE_CONFIG"], ) + self.vocabulary_types_service = VocabularyTypeService( + config=service_configs.vocabulary_types + ) def init_resource(self, app): """Initialize vocabulary resources.""" @@ -121,9 +133,13 @@ def init_resource(self, app): config=SubjectsResourceConfig, ) self.resource = VocabulariesResource( - service=self.service, + service=self.vocabularies_service, config=app.config["VOCABULARIES_RESOURCE_CONFIG"], ) + self.vocabulary_admin_resource = VocabulariesAdminResource( + service=self.vocabulary_types_service, + config=VocabularyTypeResourceConfig, + ) def finalize_app(app): @@ -153,7 +169,8 @@ def init(app): sregistry.register(ext.funders_service, service_id="funders") sregistry.register(ext.names_service, service_id="names") sregistry.register(ext.subjects_service, service_id="subjects") - sregistry.register(ext.service, service_id="vocabularies") + sregistry.register(ext.vocabularies_service, service_id="vocabularies") + sregistry.register(ext.vocabulary_types_service, service_id="vocabulary-types") # Register indexers iregistry = app.extensions["invenio-indexer"].registry iregistry.register(ext.affiliations_service.indexer, indexer_id="affiliations") @@ -161,4 +178,4 @@ def init(app): iregistry.register(ext.funders_service.indexer, indexer_id="funders") iregistry.register(ext.names_service.indexer, indexer_id="names") iregistry.register(ext.subjects_service.indexer, indexer_id="subjects") - iregistry.register(ext.service.indexer, indexer_id="vocabularies") + iregistry.register(ext.vocabularies_service.indexer, indexer_id="vocabularies") diff --git a/invenio_vocabularies/proxies.py b/invenio_vocabularies/proxies.py index f6b9704a..434db1ec 100644 --- a/invenio_vocabularies/proxies.py +++ b/invenio_vocabularies/proxies.py @@ -19,7 +19,7 @@ def _ext_proxy(attr): ) -current_service = _ext_proxy("service") +current_service = _ext_proxy("vocabularies_service") """Proxy to the instantiated vocabulary service.""" diff --git a/invenio_vocabularies/resources/__init__.py b/invenio_vocabularies/resources/__init__.py index 9c379e62..486e905e 100644 --- a/invenio_vocabularies/resources/__init__.py +++ b/invenio_vocabularies/resources/__init__.py @@ -9,7 +9,14 @@ """Resources module.""" from invenio_vocabularies.resources.schema import L10NString, VocabularyL10Schema +from .config import VocabulariesResourceConfig, VocabularyTypeResourceConfig +from .resource import VocabulariesAdminResource, VocabulariesResource + __all__ = ( "VocabularyL10Schema", "L10NString", + "VocabulariesResourceConfig", + "VocabularyTypeResourceConfig", + "VocabulariesAdminResource", + "VocabulariesResource", ) diff --git a/invenio_vocabularies/resources/config.py b/invenio_vocabularies/resources/config.py new file mode 100644 index 00000000..2b1f3080 --- /dev/null +++ b/invenio_vocabularies/resources/config.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# Copyright (C) 2024 University of Münster. +# +# Invenio-Vocabularies is free software; you can redistringibute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""Resources config.""" + +import marshmallow as ma +from flask_resources import ( + BaseListSchema, + HTTPJSONException, + JSONSerializer, + MarshmallowSerializer, + ResourceConfig, + ResponseHandler, + create_error_handler, +) +from invenio_records_resources.resources import ( + RecordResource, + RecordResourceConfig, + SearchRequestArgsSchema, +) +from invenio_records_resources.resources.errors import ErrorHandlersMixin +from invenio_records_resources.resources.records.args import SearchRequestArgsSchema +from invenio_records_resources.resources.records.headers import etag_headers +from invenio_records_resources.services.base.config import ConfiguratorMixin + +from .serializer import VocabularyL10NItemSchema + + +class VocabularySearchRequestArgsSchema(SearchRequestArgsSchema): + """Vocabularies search request parameters.""" + + tags = ma.fields.Str() + active = ma.fields.Boolean() + status = ma.fields.Boolean() + + +class VocabulariesResourceConfig(RecordResourceConfig): + """Vocabulary resource configuration.""" + + blueprint_name = "vocabularies" + url_prefix = "/vocabularies" + routes = { + "list": "/", + "item": "//", + "tasks": "/tasks", + } + + request_view_args = { + "pid_value": ma.fields.Str(), + "type": ma.fields.Str(required=True), + } + + request_search_args = VocabularySearchRequestArgsSchema + + response_handlers = { + "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers), + "application/vnd.inveniordm.v1+json": ResponseHandler( + MarshmallowSerializer( + format_serializer_cls=JSONSerializer, + object_schema_cls=VocabularyL10NItemSchema, + list_schema_cls=BaseListSchema, + ), + headers=etag_headers, + ), + } + + +class VocabularyTypeResourceConfig(ResourceConfig, ConfiguratorMixin): + """Vocabulary list resource config.""" + + # /vocabulary/vocabulary_id + # Blueprint configuration + blueprint_name = "vocabulary_list" + url_prefix = "/vocabularies" + + routes = { + "all": "/", + "list": "/vocabularies/", + "item": "/vocabularies//", + } + + # Request parsing + request_read_args = {} + request_view_args = { + "pid_value": ma.fields.String, + "type": ma.fields.String, + } + request_search_args = VocabularySearchRequestArgsSchema + + error_handlers = { + **ErrorHandlersMixin.error_handlers, + # TODO: Add custom error handlers here + } + response_handlers = { + "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers), + "application/vnd.inveniordm.v1+json": ResponseHandler( + MarshmallowSerializer( + format_serializer_cls=JSONSerializer, + object_schema_cls=VocabularyL10NItemSchema, + list_schema_cls=BaseListSchema, + ), + headers=etag_headers, + ), + } diff --git a/invenio_vocabularies/resources/resource.py b/invenio_vocabularies/resources/resource.py index ee10a685..92f85f64 100644 --- a/invenio_vocabularies/resources/resource.py +++ b/invenio_vocabularies/resources/resource.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # Copyright (C) 2020-2021 CERN. +# Copyright (C) 2024 Uni Münster. # # Invenio-Vocabularies is free software; you can redistribute it and/or # modify it under the terms of the MIT License; see LICENSE file for more @@ -8,6 +9,8 @@ """Vocabulary resource.""" +import json + import marshmallow as ma from flask import g from flask_resources import ( @@ -18,6 +21,7 @@ resource_requestctx, response_handler, ) +from invenio_access.permissions import system_identity from invenio_records_resources.resources import ( RecordResource, RecordResourceConfig, @@ -34,58 +38,28 @@ from invenio_records_resources.resources.records.utils import search_preference from marshmallow import fields -from .serializer import VocabularyL10NItemSchema - - -# -# Request args -# -class VocabularySearchRequestArgsSchema(SearchRequestArgsSchema): - """Add parameter to parse tags.""" - - tags = fields.Str() - - -# -# Resource config -# -class VocabulariesResourceConfig(RecordResourceConfig): - """Vocabulary resource configuration.""" - - blueprint_name = "vocabularies" - url_prefix = "/vocabularies" - routes = {"list": "/", "item": "//", "tasks": "/tasks"} - - request_view_args = { - "pid_value": ma.fields.Str(), - "type": ma.fields.Str(required=True), - } +from invenio_vocabularies.proxies import current_service +from invenio_vocabularies.services.service import VocabularyTypeService - request_search_args = VocabularySearchRequestArgsSchema +from .serializer import VocabularyL10NItemSchema - response_handlers = { - "application/json": ResponseHandler(JSONSerializer(), headers=etag_headers), - "application/vnd.inveniordm.v1+json": ResponseHandler( - MarshmallowSerializer( - format_serializer_cls=JSONSerializer, - object_schema_cls=VocabularyL10NItemSchema, - list_schema_cls=BaseListSchema, - ), - headers=etag_headers, - ), - } +# from invenio_vocabularies.proxies import current_service # # Resource # class VocabulariesResource(RecordResource): - """Resource for generic vocabularies.""" + """Resource for generic vocabularies. + + Provide the API /api/vocabularies/ + """ def create_url_rules(self): """Create the URL rules for the record resource.""" routes = self.config.routes rules = super().create_url_rules() + rules.append( route("POST", routes["tasks"], self.launch), ) @@ -164,3 +138,37 @@ def launch(self): """Create a task.""" self.service.launch(g.identity, resource_requestctx.data or {}) return "", 202 + + +class VocabulariesAdminResource(RecordResource): + """Resource for vocabularies admin interface.""" + + def create_url_rules(self): + """Create the URL rules for the record resource.""" + routes = self.config.routes + rules = super().create_url_rules() + + rules.append( + route("GET", routes["all"], self.get_all_vocabulary_types), + ) + return rules + + @request_search_args + @response_handler(many=True) + def get_all_vocabulary_types(self): + """Return information about _all_ vocabularies.""" + identity = g.identity + hits = self.service.search(identity, params=resource_requestctx.args) + + return hits.to_dict(), 200 + + @request_view_args + @response_handler() + def read(self): + """Read an item.""" + pid_value = ( + resource_requestctx.view_args["type"], + resource_requestctx.view_args["pid_value"], + ) + item = self.service.read(g.identity, pid_value) + return item.to_dict(), 200 diff --git a/invenio_vocabularies/services/__init__.py b/invenio_vocabularies/services/__init__.py index 4fa45609..da6a91ea 100644 --- a/invenio_vocabularies/services/__init__.py +++ b/invenio_vocabularies/services/__init__.py @@ -8,9 +8,12 @@ """Services module.""" -from .service import VocabulariesService, VocabulariesServiceConfig +from .config import VocabulariesServiceConfig, VocabularyTypesServiceConfig +from .service import VocabulariesService, VocabularyTypeService __all__ = ( "VocabulariesService", + "VocabularyTypeService", "VocabulariesServiceConfig", + "VocabularyTypesServiceConfig", ) diff --git a/invenio_vocabularies/services/config.py b/invenio_vocabularies/services/config.py new file mode 100644 index 00000000..7a75f4c9 --- /dev/null +++ b/invenio_vocabularies/services/config.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Vocabularies is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Vocabulary services configs.""" + +from flask import current_app +from invenio_i18n import lazy_gettext as _ +from invenio_records_resources.services import ( + Link, + LinksTemplate, + RecordService, + RecordServiceConfig, + SearchOptions, + pagination_links, +) +from invenio_records_resources.services.base import ( + ConditionalLink, + Service, + ServiceListResult, +) +from invenio_records_resources.services.records.components import DataComponent +from invenio_records_resources.services.records.params import ( + FilterParam, + SuggestQueryParser, +) +from sqlalchemy import asc, desc + +from ..records.api import Vocabulary +from . import results +from .components import PIDComponent, VocabularyTypeComponent +from .permissions import PermissionPolicy +from .schema import TaskSchema, VocabularySchema + + +def is_custom_vocabulary_type(vocabulary_type, context): + """Check if the vocabulary type is a custom vocabulary type.""" + return vocabulary_type["id"] in current_app.config.get( + "VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [] + ) + + +class VocabularySearchOptions(SearchOptions): + """Search options for vocabularies.""" + + params_interpreters_cls = [ + FilterParam.factory(param="tags", field="tags"), + ] + SearchOptions.params_interpreters_cls + + suggest_parser_cls = SuggestQueryParser.factory( + fields=[ + "id.text^100", + "id.text._2gram", + "id.text._3gram", + "title.en^5", + "title.en._2gram", + "title.en._3gram", + ], + ) + + sort_default = "bestmatch" + + sort_default_no_query = "title" + + sort_options = { + "bestmatch": dict( + title=_("Best match"), + fields=["_score"], # ES defaults to desc on `_score` field + ), + "title": dict( + title=_("Title"), + fields=["title_sort"], + ), + "newest": dict( + title=_("Newest"), + fields=["-created"], + ), + "oldest": dict( + title=_("Oldest"), + fields=["created"], + ), + } + + +class VocabularyTypeSearchOptions(SearchOptions): + """Search options for vocabulary types.""" + + # TODO: Is this still necessary here? + params_interpreters_cls = [ + FilterParam.factory(param="tags", field="tags"), + ] + SearchOptions.params_interpreters_cls + + # TODO: Is this still necessary here? + suggest_parser_cls = SuggestQueryParser.factory( + fields=[ + "id.text^100", + "id.text._2gram", + "id.text._3gram", + ], + ) + + sort_options = { + "id": dict( + title=_("ID"), + fields=["id"], + ), + "created": dict( + title=_("Created"), + fields=["created"], + ), + } + + sort_default = "id" + + sort_default_no_query = "id" + + # TODO: Check if these options are actually necessary + sort_direction_options = { + "asc": dict(title=_("Ascending"), fn=asc), + "desc": dict(title=_("Descending"), fn=desc), + } + + sort_direction_default = "asc" + + +class VocabulariesServiceConfig(RecordServiceConfig): + """Vocabulary service configuration.""" + + service_id = "vocabularies" + indexer_queue_name = "vocabularies" + permission_policy_cls = PermissionPolicy + record_cls = Vocabulary + schema = VocabularySchema + task_schema = TaskSchema + + search = VocabularySearchOptions + + components = [ + # Order of components are important! + VocabularyTypeComponent, + DataComponent, + PIDComponent, + ] + + links_item = { + "self": Link( + "{+api}/vocabularies/{type}/{id}", + vars=lambda record, vars: vars.update( + { + "id": record.pid.pid_value, + "type": record.type.id, + } + ), + ), + } + + links_search = pagination_links("{+api}/vocabularies/{type}{?args*}") + + +class VocabularyTypesServiceConfig(RecordServiceConfig): + """Vocabulary types service configuration.""" + + service_id = "vocabulary_types" + permission_policy_cls = PermissionPolicy + record_cls = Vocabulary # TODO: Is this correct? + schema = VocabularySchema + task_schema = TaskSchema + vocabularies_listing_resultlist_cls = results.VocabularyMetadataList + + vocabularies_listing_item = { + "self": ConditionalLink( + cond=is_custom_vocabulary_type, + if_=Link( + "{+api}/{id}", + vars=lambda vocab_type, vars: vars.update({"id": vocab_type["id"]}), + ), + else_=Link( + "{+api}/vocabularies/{id}", + vars=lambda vocab_type, vars: vars.update({"id": vocab_type["id"]}), + ), + ) + } + + search = VocabularyTypeSearchOptions + + components = [ + # Order of components are important! + VocabularyTypeComponent, + DataComponent, + PIDComponent, + ] + + links_item = { + "self": Link( + "{+api}/vocabularies/{type}/{id}", + vars=lambda record, vars: vars.update( + { + "id": record.pid.pid_value, + "type": record.type.id, + } + ), + ), + } + + links_search = pagination_links("{+api}/vocabularies/{type}{?args*}") diff --git a/invenio_vocabularies/services/permissions.py b/invenio_vocabularies/services/permissions.py index 6a8c7d57..2ca6d302 100644 --- a/invenio_vocabularies/services/permissions.py +++ b/invenio_vocabularies/services/permissions.py @@ -21,3 +21,5 @@ class PermissionPolicy(RecordPermissionPolicy): can_update = [SystemProcess()] can_delete = [SystemProcess()] can_manage = [SystemProcess()] + # this permission is needed for the /api/vocabularies/ endpoint + can_list_vocabularies = [SystemProcess(), AnyUser()] diff --git a/invenio_vocabularies/services/results.py b/invenio_vocabularies/services/results.py new file mode 100644 index 00000000..44f644f8 --- /dev/null +++ b/invenio_vocabularies/services/results.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Vocabularies is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Vocabulary results.""" + +from flask import current_app +from invenio_records_resources.proxies import current_service_registry +from invenio_records_resources.services import RecordServiceConfig +from invenio_records_resources.services.records.results import RecordList +from invenio_search import current_search_client + +from ..proxies import current_service + + +class VocabularyMetadataList(RecordList): + """Ensures that vocabulary metadata is returned in the proper format.""" + + @property + def total(self): + """Get total number of hits.""" + return self._results.total + + @property + def custom_vocabulary_names(self): + """Checks whether vocabulary is a custom vocabulary.""" + return current_app.config.get("VOCABULARIES_CUSTOM_VOCABULARY_TYPES", []) + + def to_dict(self): + """Formats result to a dict of hits.""" + # hits = list(self._results) + + config_vocab_types = current_app.config.get( + "INVENIO_VOCABULARY_TYPE_METADATA", {} + ) + + count_terms_agg = {} + generic_stats = self._generic_vocabulary_statistics() + custom_stats = self._custom_vocabulary_statistics() + + for k in generic_stats.keys() | custom_stats.keys(): + count_terms_agg[k] = generic_stats.get(k, 0) + custom_stats.get(k, 0) + + hits = self._results.items + + # Extend database data with configuration & aggregation data. + results = [] + for db_vocab_type in hits: + result = { + "id": db_vocab_type.id, + "pid_type": db_vocab_type.pid_type, + "count": count_terms_agg.get(db_vocab_type.id, 0), + "is_custom_vocabulary": db_vocab_type.id + in self.custom_vocabulary_names, + } + + if db_vocab_type.id in config_vocab_types: + for k, v in config_vocab_types[db_vocab_type.id].items(): + result[k] = v + + results.append(result) + + for hit in results: + if self._links_item_tpl: + hit["links"] = self._links_item_tpl.expand(self._identity, hit) + + res = { + "hits": { + "hits": results, + "total": self.total, + } + } + + if self._params: + if self._links_tpl: + res["links"] = self._links_tpl.expand(self._identity, self.pagination) + + return res + + def _custom_vocabulary_statistics(self): + # query database for count of terms in custom vocabularies + returndict = {} + for vocab_type in self.custom_vocabulary_names: + custom_service = current_service_registry.get(vocab_type) + record_cls = custom_service.config.record_cls + returndict[vocab_type] = record_cls.model_cls.query.count() + + return returndict + + def _generic_vocabulary_statistics(self): + # Opensearch query for generic vocabularies + config: RecordServiceConfig = ( + current_service.config + ) # TODO: Where to get the config from here? current_service is None + search_opts = config.search + + search = search_opts.search_cls( + using=current_search_client, + index=config.record_cls.index.search_alias, + ) + + search.aggs.bucket("vocabularies", {"terms": {"field": "type.id", "size": 100}}) + + search_result = search.execute() + buckets = search_result.aggs.to_dict()["vocabularies"]["buckets"] + + return {bucket["key"]: bucket["doc_count"] for bucket in buckets} diff --git a/invenio_vocabularies/services/service.py b/invenio_vocabularies/services/service.py index 817d4989..11825f6a 100644 --- a/invenio_vocabularies/services/service.py +++ b/invenio_vocabularies/services/service.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2020-2021 CERN. +# Copyright (C) 2024 CERN. # Copyright (C) 2021 Northwestern University. # # Invenio-Vocabularies is free software; you can redistribute it and/or @@ -8,10 +8,10 @@ # details. """Vocabulary service.""" +from functools import partial +import sqlalchemy as sa from invenio_cache import current_cache -from invenio_db import db -from invenio_i18n import lazy_gettext as _ from invenio_records_resources.services import ( Link, LinksTemplate, @@ -20,97 +20,268 @@ SearchOptions, pagination_links, ) -from invenio_records_resources.services.records.components import DataComponent -from invenio_records_resources.services.records.params import ( - FilterParam, - SuggestQueryParser, +from invenio_records_resources.services.base.results import ( + ServiceItemResult, + ServiceListResult, ) +from invenio_records_resources.services.base.utils import map_search_params from invenio_records_resources.services.records.schema import ServiceSchemaWrapper from invenio_records_resources.services.uow import unit_of_work +from invenio_search import current_search_client from invenio_search.engine import dsl -from ..records.api import Vocabulary from ..records.models import VocabularyType -from .components import PIDComponent, VocabularyTypeComponent -from .permissions import PermissionPolicy -from .schema import TaskSchema, VocabularySchema from .tasks import process_datastream -class VocabularySearchOptions(SearchOptions): - """Search options.""" +def is_custom_vocabulary_type(vocabulary_type, context): + """Check if the vocabulary type is a custom vocabulary type.""" + return vocabulary_type["id"] in current_app.config.get( + "VOCABULARIES_CUSTOM_VOCABULARY_TYPES", [] + ) - params_interpreters_cls = [ - FilterParam.factory(param="tags", field="tags"), - ] + SearchOptions.params_interpreters_cls - suggest_parser_cls = SuggestQueryParser.factory( - fields=[ - "id.text^100", - "id.text._2gram", - "id.text._3gram", - "title.en^5", - "title.en._2gram", - "title.en._3gram", - ], - ) +class VocabularyMetadataList(ServiceListResult): + """Ensures that vocabulary metadata is returned in the proper format.""" - sort_default = "bestmatch" - - sort_default_no_query = "title" - - sort_options = { - "bestmatch": dict( - title=_("Best match"), - fields=["_score"], # ES defaults to desc on `_score` field - ), - "title": dict( - title=_("Title"), - fields=["title_sort"], - ), - "newest": dict( - title=_("Newest"), - fields=["-created"], - ), - "oldest": dict( - title=_("Oldest"), - fields=["created"], - ), - } - - -class VocabulariesServiceConfig(RecordServiceConfig): - """Vocabulary service configuration.""" - - service_id = "vocabularies" - indexer_queue_name = "vocabularies" - permission_policy_cls = PermissionPolicy - record_cls = Vocabulary - schema = VocabularySchema - task_schema = TaskSchema - - search = VocabularySearchOptions - - components = [ - # Order of components are important! - VocabularyTypeComponent, - DataComponent, - PIDComponent, - ] - - links_item = { - "self": Link( - "{+api}/vocabularies/{type}/{id}", - vars=lambda record, vars: vars.update( - { - "id": record.pid.pid_value, - "type": record.type.id, - } - ), - ), - } + def __init__( + self, + service, + identity, + results, + params=None, + links_tpl=None, + links_item_tpl=None, + ): + """Constructor. + + :params service: a service instance + :params identity: an identity that performed the service request + :params results: the search results + """ + self._identity = identity + self._results = results + self._params = params + self._service = service + self._links_tpl = links_tpl + self._links_item_tpl = links_item_tpl + + @property + def total(self): + """Get total number of hits.""" + return len(list(self._results)) + + @property + def pagination(self): + """Create a pagination object.""" + return Pagination( + self._params["size"], + self._params["page"], + self.total, + ) + + def to_dict(self): + """Formats result to a dict of hits.""" + hits = list(self._results) + + for hit in hits: + if self._links_item_tpl: + hit["links"] = self._links_item_tpl.expand(self._identity, hit) + + res = { + "hits": { + "hits": hits, + "total": self.total, + } + } + + if self._params: + if self._links_tpl: + res["links"] = self._links_tpl.expand(self._identity, self.pagination) + + return res + + +class VocabularyTypeService(RecordService): + """Vocabulary type service.""" + + @property + def schema(self): + """Returns the data schema instance.""" + return ServiceSchemaWrapper(self, schema=self.config.schema) - links_search = pagination_links("{+api}/vocabularies/{type}{?args*}") + @property + def links_item_tpl(self): + """Item links template.""" + return LinksTemplate( + self.config.vocabularies_listing_item, + ) + + @property + def custom_vocabulary_names(self): + """Checks whether vocabulary is a custom vocabulary.""" + return current_app.config.get("VOCABULARIES_CUSTOM_VOCABULARY_TYPES", []) + + def search(self, identity, params=None): + """Search for vocabulary types entries.""" + self.require_permission(identity, "list_vocabularies") + + vocabulary_types = VocabularyType.query.all() + + search_params = map_search_params(self.config.search, params) + + query_param = search_params["q"] + + if query_param: + vocabulary_types = [ + voc_type + for voc_type in vocabulary_types + if (query_param in voc_type.id.lower()) + ] + + sort_direction = search_params["sort_direction"] + vocabulary_types = sort_direction(vocabulary_types) + + config_vocab_types = current_app.config.get( + "INVENIO_VOCABULARY_TYPE_METADATA", {} + ) + + count_terms_agg = {} + generic_stats = self._generic_vocabulary_statistics() + custom_stats = self._custom_vocabulary_statistics() + + for k in generic_stats.keys() | custom_stats.keys(): + count_terms_agg[k] = generic_stats.get(k, 0) + custom_stats.get(k, 0) + + # Extend database data with configuration & aggregation data. + results = [] + for db_vocab_type in vocabulary_types: + result = { + "id": db_vocab_type.id, + "pid_type": db_vocab_type.pid_type, + "count": count_terms_agg.get(db_vocab_type.id, 0), + "is_custom_vocabulary": db_vocab_type.id + in self.custom_vocabulary_names, + } + + if db_vocab_type.id in config_vocab_types: + for k, v in config_vocab_types[db_vocab_type.id].items(): + result[k] = v + + results.append(result) + + return self.config.vocabularies_listing_resultlist_cls( + self, + identity, + results, + params, + links_tpl=LinksTemplate(self.config.links_search, context={"args": params}), + links_item_tpl=self.links_item_tpl, + ) + + @unit_of_work() + def update(self, identity, id_, data, uow=None): + """Update an OAI set.""" + self.require_permission(identity, "update") + + vocabulary_item, errors = self._get_one(id=id_) + # if oai_set.system_created: + # raise OAIPMHSetNotEditable(oai_set.id) + + valid_data, errors = self.schema.load( + data, + context={"identity": identity}, + raise_errors=True, + ) + + for key, value in valid_data.items(): + setattr(vocabulary_item, key, value) + # uow.register(OAISetCommitOp(oai_set)) + + return self.result_item( + service=self, + identity=identity, + item=vocabulary_item, + links_tpl=self.links_item_tpl, + ) + + def _custom_vocabulary_statistics(self): + # query database for count of terms in custom vocabularies + returndict = {} + for vocab_type in self.custom_vocabulary_names: + custom_service = current_service_registry.get(vocab_type) + record_cls = custom_service.config.record_cls + returndict[vocab_type] = record_cls.model_cls.query.count() + + return returndict + + def _generic_vocabulary_statistics(self): + # Opensearch query for generic vocabularies + config: RecordServiceConfig = current_service.config + search_opts = config.search + + search = search_opts.search_cls( + using=current_search_client, + index=config.record_cls.index.search_alias, + ) + + search.aggs.bucket("vocabularies", {"terms": {"field": "type.id", "size": 100}}) + + search_result = search.execute() + buckets = search_result.aggs.to_dict()["vocabularies"]["buckets"] + + return {bucket["key"]: bucket["doc_count"] for bucket in buckets} + + +class VocabularyTypeService(RecordService): + """Vocabulary type service.""" + + @property + def schema(self): + """Returns the data schema instance.""" + return ServiceSchemaWrapper(self, schema=self.config.schema) + + @property + def links_item_tpl(self): + """Item links template.""" + return LinksTemplate( + self.config.vocabularies_listing_item, + ) + + def search(self, identity, params=None): + """Search for vocabulary types entries.""" + self.require_permission(identity, "list_vocabularies") + + search_params = map_search_params(self.config.search, params) + + query_param = search_params["q"] + + filters = [] + if query_param: + filters.extend([VocabularyType.id.ilike(f"%{query_param}%")]) + + vocabulary_types = ( + VocabularyType.query.filter(sa.or_(*filters)) + .order_by( + search_params["sort_direction"]( + sa.text(",".join(search_params["sort"])) + ) + ) + .paginate( + page=search_params["page"], + per_page=search_params["size"], + error_out=False, + ) + ) + + return self.config.vocabularies_listing_resultlist_cls( + self, + identity, + vocabulary_types, + search_params, + links_tpl=LinksTemplate(self.config.links_search, context={"args": params}), + links_item_tpl=self.links_item_tpl, + ) class VocabulariesService(RecordService): @@ -145,7 +316,7 @@ def search( params, search_preference, extra_filter=dsl.Q("term", type__id=vocabulary_type.id), - **kwargs + **kwargs, ).execute() return self.result_list( diff --git a/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabularies-list.html b/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabularies-list.html new file mode 100644 index 00000000..8b6d2e65 --- /dev/null +++ b/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabularies-list.html @@ -0,0 +1,12 @@ +{# + Copyright (C) 2024 Uni Münster. + + Invenio App RDM is free software; you can redistribute it and/or modify it + under the terms of the MIT License; see LICENSE file for more details. + #} + {% extends "invenio_administration/search.html" %} + + {% block javascript %} + {{ super() }} + {{ webpack['invenio-vocabularies-search.js'] }} +{% endblock %} \ No newline at end of file diff --git a/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabulary-details.html b/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabulary-details.html new file mode 100644 index 00000000..4dcdeac7 --- /dev/null +++ b/invenio_vocabularies/templates/semantic-ui/invenio_vocabularies/vocabulary-details.html @@ -0,0 +1,71 @@ +{# + Copyright (C) 2024 CERN. + + Invenio App RDM is free software; you can redistribute it and/or modify it + under the terms of the MIT License; see LICENSE file for more details. + #} + + {%- from "invenio_administration/macros.html" import go_back %} + + {% extends "invenio_administration/search.html" %} + + {% block admin_main_column %} +
+ +
+ {{ go_back() }} +
+
+

{{ title or name }}

+
+
+ + {{ _("Settings") }} + + + {{ _("Schedule") }} + + + {{ _("Run now") }} + +
+
+ {% block admin_page_content %} + + + {%- block search_app %} +
+
+ {%- endblock search_app %} + {% endblock admin_page_content %} +
+
+ {% endblock %} + + {% block javascript %} + {{ super() }} + {{ webpack['invenio-jobs-details.js'] }} + {% endblock %} \ No newline at end of file diff --git a/invenio_vocabularies/views.py b/invenio_vocabularies/views.py index 4702c9e9..c5877ac0 100644 --- a/invenio_vocabularies/views.py +++ b/invenio_vocabularies/views.py @@ -44,3 +44,10 @@ def create_names_blueprint_from_app(app): def create_subjects_blueprint_from_app(app): """Create app blueprint.""" return app.extensions["invenio-vocabularies"].subjects_resource.as_blueprint() + + +def create_list_blueprint_from_app(app): + """Create app blueprint.""" + return app.extensions[ + "invenio-vocabularies" + ].vocabulary_admin_resource.as_blueprint() diff --git a/setup.cfg b/setup.cfg index a194808a..b136e4ba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ zip_safe = False install_requires = invenio-i18n>=2.0.0,<3.0.0 invenio-records-resources>=6.0.0,<7.0.0 + invenio-administration>=2.0.0,<3.0.0 lxml>=4.5.0 PyYAML>=5.4.1 oaipmh-scythe @ git+https://github.com/ulbmuenster/invenio-oaipmh-scythe.git @@ -55,6 +56,10 @@ sqlite = [options.entry_points] flask.commands = vocabularies = invenio_vocabularies.cli:vocabularies +invenio_administration.views = + vocabularies_list = invenio_vocabularies.administration.views.vocabularies:VocabulariesListView + vocabulary_types_details = invenio_vocabularies.administration.views.vocabularies:VocabularyTypesDetailsView + vocabulary_types_details_edit = invenio_vocabularies.administration.views.vocabularies:VocabularyTypesDetailsEditView invenio_base.apps = invenio_vocabularies = invenio_vocabularies:InvenioVocabularies invenio_base.api_apps = @@ -69,6 +74,7 @@ invenio_base.api_blueprints = invenio_vocabularies_names = invenio_vocabularies.views:create_names_blueprint_from_app invenio_vocabularies_subjects = invenio_vocabularies.views:create_subjects_blueprint_from_app invenio_vocabularies_ext = invenio_vocabularies.views:blueprint + invenio_vocabularies_list = invenio_vocabularies.views:create_list_blueprint_from_app invenio_base.api_finalize_app = invenio_vocabularies = invenio_vocabularies.ext:api_finalize_app invenio_base.finalize_app = @@ -134,7 +140,7 @@ input-file = invenio_vocabularies/translations/messages.pot output-dir = invenio_vocabularies/translations/ [isort] -profile=black +profile = black [check-manifest] ignore =