From 04480c0d4f98de3595f0eaa351e6ac61f37d51ba Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Mon, 9 Dec 2024 16:42:53 +0100 Subject: [PATCH 01/15] refactor pagination --- openatlas/api/endpoints/endpoint.py | 79 +++++++++++++++++------------ 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 41bbfd58a..41ba8ceec 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -30,6 +30,38 @@ def __init__( parser: dict[str, Any]) -> None: self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) + self.pagination = None + + def get_pagination(self) -> None: + total = [e.id for e in self.entities] + count = len(total) + self.parser.limit = self.parser.limit or count + # List of start ids for the index/pages + e_list = [] + if total: + e_list = list(itertools.islice(total, 0, None, self.parser.limit)) + # Creating index + index = \ + [{'page': i + 1, 'startId': id_} for i, id_ in enumerate(e_list)] + if self.parser.page: + self.parser.first = self.parser.get_by_page(index) + # Get which entity should be displayed (first or last) + if self.parser.last or self.parser.first: + total = self.parser.get_start_entity(total) + # Finding position in list of first entity + entity_list_index = 0 + for index_, entity in enumerate(self.entities): + if entity.id == total[0]: + entity_list_index = index_ + break + + self.pagination = { + 'count': count, 'index': index, 'entity_index': entity_list_index} + + def reduce_entities_list(self) -> None: + start_index = self.pagination['entity_index'] + end_index = start_index + int(self.parser.limit) + self.entities = self.entities[start_index:end_index] def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.type_id: @@ -37,13 +69,23 @@ def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.search: self.entities = [ e for e in self.entities if self.parser.search_filter(e)] + + self.remove_duplicate_entities() + self.sort_entities() + self.get_pagination() + self.reduce_entities_list() + if self.parser.export == 'csv': return self.export_entities_csv() if self.parser.export == 'csvNetwork': return self.export_csv_for_network_analysis() - self.remove_duplicate_entities() - self.sorting() - result = self.get_json_output() + result = { + "results": self.get_entities_formatted() if self.entities else [], + "pagination": { + 'entitiesPerPage': int(self.parser.limit), + 'entities': self.pagination['count'], + 'index': self.pagination['index'], + 'totalPages': len(self.pagination['index'])}} if self.parser.format in app.config['RDF_FORMATS']: # pragma: no cover return Response( self.parser.rdf_output(result['results']), @@ -145,7 +187,7 @@ def link_parser_check(self, inverse: bool = False) -> list[Link]: inverse=inverse) return links - def sorting(self) -> None: + def sort_entities(self) -> None: if 'latest' in request.path: return @@ -160,36 +202,7 @@ def remove_duplicate_entities(self) -> None: self.entities = \ [e for e in self.entities if not (e.id in seen or seen_add(e.id))] - def get_json_output(self) -> dict[str, Any]: - total = [e.id for e in self.entities] - count = len(total) - if self.parser.limit == 0: - self.parser.limit = count - e_list = [] - if total: - e_list = list(itertools.islice(total, 0, None, self.parser.limit)) - index = \ - [{'page': num + 1, 'startId': i} for num, i in enumerate(e_list)] - if index: - self.parser.first = self.parser.get_by_page(index) \ - if self.parser.page else self.parser.first - total = self.parser.get_start_entity(total) \ - if self.parser.last or self.parser.first else total - j = [i for i, x in enumerate(self.entities) if x.id == total[0]] - formatted_entities = [] - if self.entities: - self.entities = [e for idx, e in enumerate(self.entities[j[0]:])] - formatted_entities = self.get_entities_formatted() - return { - "results": formatted_entities, - "pagination": { - 'entitiesPerPage': int(self.parser.limit), - 'entities': count, - 'index': index, - 'totalPages': len(index)}} - def get_entities_formatted(self) -> list[dict[str, Any]]: - self.entities = self.entities[:int(self.parser.limit)] if self.parser.format == 'geojson': return [self.get_geojson()] if self.parser.format == 'geojson-v2': From 5fbaea13529cf0416438bc05ccd16689fff38539 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Mon, 9 Dec 2024 16:52:09 +0100 Subject: [PATCH 02/15] refactor pagination --- openatlas/api/endpoints/endpoint.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 41ba8ceec..3d2bf9dd8 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -17,7 +17,8 @@ from openatlas.api.resources.resolve_endpoints import ( download, parse_loud_context) from openatlas.api.resources.templates import ( - geojson_collection_template, linked_places_template, loud_template) + geojson_collection_template, linked_places_template, loud_template, + ) from openatlas.api.resources.util import ( get_linked_entities_api, get_location_link, remove_duplicate_entities) from openatlas.models.entity import Entity, Link @@ -73,6 +74,8 @@ def resolve_entities(self) -> Response | dict[str, Any]: self.remove_duplicate_entities() self.sort_entities() self.get_pagination() + if self.parser.count == 'true': + return jsonify(self.pagination['count']) self.reduce_entities_list() if self.parser.export == 'csv': @@ -90,8 +93,7 @@ def resolve_entities(self) -> Response | dict[str, Any]: return Response( self.parser.rdf_output(result['results']), mimetype=app.config['RDF_FORMATS'][self.parser.format]) - if self.parser.count == 'true': - return jsonify(result['pagination']['entities']) + if self.parser.download == 'true': return download(result, self.parser.get_entities_template()) return marshal(result, self.parser.get_entities_template()) From 038d924c7ec47c94a2da10bc3fffe4927f6d4fd4 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Mon, 9 Dec 2024 17:03:22 +0100 Subject: [PATCH 03/15] small changes --- openatlas/api/endpoints/endpoint.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 3d2bf9dd8..41764fb29 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -17,8 +17,7 @@ from openatlas.api.resources.resolve_endpoints import ( download, parse_loud_context) from openatlas.api.resources.templates import ( - geojson_collection_template, linked_places_template, loud_template, - ) + geojson_collection_template, linked_places_template, loud_template) from openatlas.api.resources.util import ( get_linked_entities_api, get_location_link, remove_duplicate_entities) from openatlas.models.entity import Entity, Link @@ -70,14 +69,12 @@ def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.search: self.entities = [ e for e in self.entities if self.parser.search_filter(e)] - self.remove_duplicate_entities() + if self.parser.count == 'true': + return jsonify(len(self.entities)) self.sort_entities() self.get_pagination() - if self.parser.count == 'true': - return jsonify(self.pagination['count']) self.reduce_entities_list() - if self.parser.export == 'csv': return self.export_entities_csv() if self.parser.export == 'csvNetwork': @@ -93,7 +90,6 @@ def resolve_entities(self) -> Response | dict[str, Any]: return Response( self.parser.rdf_output(result['results']), mimetype=app.config['RDF_FORMATS'][self.parser.format]) - if self.parser.download == 'true': return download(result, self.parser.get_entities_template()) return marshal(result, self.parser.get_entities_template()) From 6d0e29d346879820b2ce7bacce89739b14163879 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 10:59:44 +0100 Subject: [PATCH 04/15] refactored the format entities function --- openatlas/api/endpoints/endpoint.py | 59 ++++++++++++++++++----------- openatlas/api/resources/parser.py | 1 + 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 41764fb29..006abbf4e 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -31,6 +31,19 @@ def __init__( self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) self.pagination = None + self.entities_with_links: dict[int, dict[str, Any]] = {} + + + def get_links_for_entities(self) -> None: + for entity in self.entities: + self.entities_with_links[entity.id] = { + 'entity': entity, + 'links': [], + 'links_inverse': []} + for link_ in self.link_parser_check(): + self.entities_with_links[link_.domain.id]['links'].append(link_) + for link_ in self.link_parser_check(inverse=True): + self.entities_with_links[link_.range.id]['links_inverse'].append(link_) def get_pagination(self) -> None: total = [e.id for e in self.entities] @@ -75,6 +88,8 @@ def resolve_entities(self) -> Response | dict[str, Any]: self.sort_entities() self.get_pagination() self.reduce_entities_list() + if self.entities: + self.get_links_for_entities() if self.parser.export == 'csv': return self.export_entities_csv() if self.parser.export == 'csvNetwork': @@ -201,28 +216,28 @@ def remove_duplicate_entities(self) -> None: [e for e in self.entities if not (e.id in seen or seen_add(e.id))] def get_entities_formatted(self) -> list[dict[str, Any]]: - if self.parser.format == 'geojson': - return [self.get_geojson()] - if self.parser.format == 'geojson-v2': - return [self.get_geojson_v2()] - entities_dict: dict[int, dict[str, Any]] = {} - for entity in self.entities: - entities_dict[entity.id] = { - 'entity': entity, - 'links': [], - 'links_inverse': []} - for link_ in self.link_parser_check(): - entities_dict[link_.domain.id]['links'].append(link_) - for link_ in self.link_parser_check(inverse=True): - entities_dict[link_.range.id]['links_inverse'].append(link_) - if self.parser.format == 'loud' \ - or self.parser.format in app.config['RDF_FORMATS']: - return [ - get_loud_entities(item, parse_loud_context()) - for item in entities_dict.values()] - return [ - self.parser.get_linked_places_entity(item) - for item in entities_dict.values()] + match self.parser.format: + case 'geojson': + entities= [self.get_geojson()] + case 'geojson-v2': + entities= [self.get_geojson_v2()] + case 'loud': + parsed_context = parse_loud_context() + entities = [ + get_loud_entities(item, parsed_context) + for item in self.entities_with_links.values()] + case 'lp' | 'lpx': + entities= [ + self.parser.get_linked_places_entity(item) + for item in self.entities_with_links.values()] + case _ if self.parser.format in app.config['RDF_FORMATS']: + parsed_context = parse_loud_context() + entities = [ + get_loud_entities(item, parsed_context) + for item in self.entities_with_links.values()] + case _: + entities = [] + return entities def get_geojson(self) -> dict[str, Any]: out = [] diff --git a/openatlas/api/resources/parser.py b/openatlas/api/resources/parser.py index bc5d461a4..079904936 100644 --- a/openatlas/api/resources/parser.py +++ b/openatlas/api/resources/parser.py @@ -106,6 +106,7 @@ type=str, help='{error_msg}', case_sensitive=False, + default='lp', choices=frozenset(app.config['API_FORMATS']), location='args') entity_.add_argument( From 0a6ee16916d8958fe3f71dbb3ad1fd354a29e14e Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 12:37:12 +0100 Subject: [PATCH 05/15] merged entity resolved functions --- openatlas/api/endpoints/endpoint.py | 138 ++++++++++++++-------------- openatlas/api/endpoints/entities.py | 3 +- openatlas/api/endpoints/parser.py | 9 -- openatlas/api/formats/csv.py | 3 +- tests/test_api.py | 27 +++--- 5 files changed, 85 insertions(+), 95 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 006abbf4e..fd6164ed8 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -13,11 +13,11 @@ from openatlas.api.formats.csv import ( build_dataframe, build_link_dataframe) from openatlas.api.formats.loud import get_loud_entities -from openatlas.api.resources.api_entity import ApiEntity from openatlas.api.resources.resolve_endpoints import ( download, parse_loud_context) from openatlas.api.resources.templates import ( - geojson_collection_template, linked_places_template, loud_template) + geojson_collection_template, geojson_pagination, linked_place_pagination, + linked_places_template, loud_pagination, loud_template) from openatlas.api.resources.util import ( get_linked_entities_api, get_location_link, remove_duplicate_entities) from openatlas.models.entity import Entity, Link @@ -27,12 +27,14 @@ class Endpoint: def __init__( self, entities: Entity | list[Entity], - parser: dict[str, Any]) -> None: + parser: dict[str, Any], + single: bool = False) -> None: self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) self.pagination = None + self.single = single self.entities_with_links: dict[int, dict[str, Any]] = {} - + self.formated_entities = [] def get_links_for_entities(self) -> None: for entity in self.entities: @@ -43,7 +45,8 @@ def get_links_for_entities(self) -> None: for link_ in self.link_parser_check(): self.entities_with_links[link_.domain.id]['links'].append(link_) for link_ in self.link_parser_check(inverse=True): - self.entities_with_links[link_.range.id]['links_inverse'].append(link_) + self.entities_with_links[ + link_.range.id]['links_inverse'].append(link_) def get_pagination(self) -> None: total = [e.id for e in self.entities] @@ -67,83 +70,58 @@ def get_pagination(self) -> None: if entity.id == total[0]: entity_list_index = index_ break - self.pagination = { 'count': count, 'index': index, 'entity_index': entity_list_index} - def reduce_entities_list(self) -> None: + def reduce_entities_to_limit(self) -> None: start_index = self.pagination['entity_index'] end_index = start_index + int(self.parser.limit) self.entities = self.entities[start_index:end_index] def resolve_entities(self) -> Response | dict[str, Any]: - if self.parser.type_id: - self.entities = self.filter_by_type() - if self.parser.search: - self.entities = [ - e for e in self.entities if self.parser.search_filter(e)] - self.remove_duplicate_entities() - if self.parser.count == 'true': - return jsonify(len(self.entities)) - self.sort_entities() - self.get_pagination() - self.reduce_entities_list() + if not self.single: + if self.parser.type_id: + self.entities = self.filter_by_type() + if self.parser.search: + self.entities = [ + e for e in self.entities if self.parser.search_filter(e)] + self.remove_duplicate_entities() + if self.parser.count == 'true': + return jsonify(len(self.entities)) + self.sort_entities() + self.get_pagination() + self.reduce_entities_to_limit() + if self.parser.export == 'csvNetwork': + return self.export_csv_for_network_analysis() if self.entities: self.get_links_for_entities() if self.parser.export == 'csv': return self.export_entities_csv() - if self.parser.export == 'csvNetwork': - return self.export_csv_for_network_analysis() - result = { - "results": self.get_entities_formatted() if self.entities else [], - "pagination": { - 'entitiesPerPage': int(self.parser.limit), - 'entities': self.pagination['count'], - 'index': self.pagination['index'], - 'totalPages': len(self.pagination['index'])}} + if self.entities: + self.get_entities_formatted() + if self.parser.format in app.config['RDF_FORMATS']: # pragma: no cover return Response( - self.parser.rdf_output(result['results']), + self.parser.rdf_output(self.formated_entities), mimetype=app.config['RDF_FORMATS'][self.parser.format]) - if self.parser.download == 'true': - return download(result, self.parser.get_entities_template()) - return marshal(result, self.parser.get_entities_template()) - def resolve_entity(self) -> Response | dict[str, Any] | tuple[Any, int]: - if self.parser.export == 'csv': - return self.export_entities_csv() - if self.parser.export == 'csvNetwork': - return self.export_csv_for_network_analysis() - result = self.get_entity_formatted() - if (self.parser.format - in app.config['RDF_FORMATS']): # pragma: no cover - return Response( - self.parser.rdf_output(result), - mimetype=app.config['RDF_FORMATS'][self.parser.format]) - template = linked_places_template(self.parser) - if self.parser.format in ['geojson', 'geojson-v2']: - template = geojson_collection_template() - if self.parser.format == 'loud': - template = loud_template(result) - if self.parser.download: - return download(result, template) - return marshal(result, template), 200 + result = self.get_json_output() + if self.parser.download == 'true': + return download(result, self.get_entities_template(result)) + return marshal(result, self.get_entities_template(result)) - def get_entity_formatted(self) -> dict[str, Any]: - if self.parser.format == 'geojson': - return self.get_geojson() - if self.parser.format == 'geojson-v2': - return self.get_geojson_v2() - entity = self.entities[0] - entity_dict = { - 'entity': entity, - 'links': ApiEntity.get_links_of_entities(entity.id), - 'links_inverse': ApiEntity.get_links_of_entities( - entity.id, inverse=True)} - if self.parser.format == 'loud' \ - or self.parser.format in app.config['RDF_FORMATS']: - return get_loud_entities(entity_dict, parse_loud_context()) - return self.parser.get_linked_places_entity(entity_dict) + def get_json_output(self) -> dict[str, Any]: + if not self.single: + result = { + "results": self.formated_entities, + "pagination": { + 'entitiesPerPage': int(self.parser.limit), + 'entities': self.pagination['count'], + 'index': self.pagination['index'], + 'totalPages': len(self.pagination['index'])}} + else: + result = dict(self.formated_entities[0]) + return result def filter_by_type(self) -> list[Entity]: result = [] @@ -154,7 +132,9 @@ def filter_by_type(self) -> list[Entity]: return result def export_entities_csv(self) -> Response: - frames = [build_dataframe(e, relations=True) for e in self.entities] + frames = [ + build_dataframe(e, relations=True) + for e in self.entities_with_links.values()] return Response( pd.DataFrame(data=frames).to_csv(), mimetype='text/csv', @@ -215,19 +195,19 @@ def remove_duplicate_entities(self) -> None: self.entities = \ [e for e in self.entities if not (e.id in seen or seen_add(e.id))] - def get_entities_formatted(self) -> list[dict[str, Any]]: + def get_entities_formatted(self) -> None: match self.parser.format: case 'geojson': - entities= [self.get_geojson()] + entities = [self.get_geojson()] case 'geojson-v2': - entities= [self.get_geojson_v2()] + entities = [self.get_geojson_v2()] case 'loud': parsed_context = parse_loud_context() entities = [ get_loud_entities(item, parsed_context) for item in self.entities_with_links.values()] case 'lp' | 'lpx': - entities= [ + entities = [ self.parser.get_linked_places_entity(item) for item in self.entities_with_links.values()] case _ if self.parser.format in app.config['RDF_FORMATS']: @@ -237,7 +217,7 @@ def get_entities_formatted(self) -> list[dict[str, Any]]: for item in self.entities_with_links.values()] case _: entities = [] - return entities + self.formated_entities = entities def get_geojson(self) -> dict[str, Any]: out = [] @@ -273,3 +253,19 @@ def get_geojson_v2(self) -> dict[str, Any]: [l_.range.id for l_ in entity_links]): out.append(self.parser.get_geojson_dict(entity, geom)) return {'type': 'FeatureCollection', 'features': out} + + def get_entities_template(self, result: dict[str, Any]) -> dict[str, Any]: + match self.parser.format: + case 'geojson' | 'geojson-v2': + template = geojson_collection_template() + if not self.single: + template = geojson_pagination() + case 'loud': + template = loud_template(result) + if not self.single: + template = loud_pagination() + case 'lp' | 'lpx' | _: + template = linked_places_template(self.parser) + if not self.single: + template = linked_place_pagination(self.parser) + return template diff --git a/openatlas/api/endpoints/entities.py b/openatlas/api/endpoints/entities.py index 666cb417d..11adb5121 100644 --- a/openatlas/api/endpoints/entities.py +++ b/openatlas/api/endpoints/entities.py @@ -61,7 +61,8 @@ class GetEntity(Resource): def get(id_: int) -> tuple[Resource, int] | Response | dict[str, Any]: return Endpoint( ApiEntity.get_by_id(id_, types=True, aliases=True), - entity_.parse_args()).resolve_entity() + entity_.parse_args(), + True).resolve_entities() class GetLatest(Resource): diff --git a/openatlas/api/endpoints/parser.py b/openatlas/api/endpoints/parser.py index dc425b7aa..453828961 100644 --- a/openatlas/api/endpoints/parser.py +++ b/openatlas/api/endpoints/parser.py @@ -21,8 +21,6 @@ from openatlas.api.resources.search import get_search_values, search_entity from openatlas.api.resources.search_validation import ( check_if_date_search, validate_search_parameters) -from openatlas.api.resources.templates import ( - geojson_pagination, linked_place_pagination, loud_pagination) from openatlas.api.resources.util import ( flatten_list_and_remove_duplicates, get_geometric_collection, get_geoms_dict, get_location_link, get_reference_systems, @@ -289,13 +287,6 @@ def rdf_output( graph = Graph().parse(data=json.dumps(data), format='json-ld') return graph.serialize(format=self.format, encoding='utf-8') - def get_entities_template(self) -> dict[str, Any]: - if self.format in ['geojson', 'geojson-v2']: - return geojson_pagination() - if self.format == 'loud': - return loud_pagination() - return linked_place_pagination(self) - def is_valid_url(self) -> None: if self.url and isinstance( validators.url(self.url), diff --git a/openatlas/api/formats/csv.py b/openatlas/api/formats/csv.py index d9930e47a..69872a816 100644 --- a/openatlas/api/formats/csv.py +++ b/openatlas/api/formats/csv.py @@ -12,8 +12,9 @@ def build_dataframe( - entity: Entity, + entity_dict: dict[str, Any], relations: bool = False) -> dict[str, Any]: + entity = entity_dict['entity'] geom = get_csv_geom_entry(entity) data = { 'id': str(entity.id), diff --git a/tests/test_api.py b/tests/test_api.py index 6b657f154..a34fb186f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -263,20 +263,21 @@ def test_api(self) -> None: export='csv'))]: assert b'Shire' in rv.data assert 'text/csv' in rv.headers.get('Content-Type') + # + # for rv in [ + # c.get( + # url_for('api_04.entity', id_=place.id, export='csvNetwork')), + # c.get( + # url_for( + # 'api_04.query', + # entities=location.id, + # cidoc_classes='E18', + # view_classes='artifact', + # system_classes='person', + # export='csvNetwork'))]: + # assert b'Shire' in rv.data + # assert 'application/zip' in rv.headers.get('Content-Type') - for rv in [ - c.get( - url_for('api_04.entity', id_=place.id, export='csvNetwork')), - c.get( - url_for( - 'api_04.query', - entities=location.id, - cidoc_classes='E18', - view_classes='artifact', - system_classes='person', - export='csvNetwork'))]: - assert b'Shire' in rv.data - assert 'application/zip' in rv.headers.get('Content-Type') rv = c.get( url_for( 'api_04.linked_entities_by_properties_recursive', From d57f4d508e7e3de5c13e196ba1b5e2b6114af307 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 13:20:23 +0100 Subject: [PATCH 06/15] refactored csv --- openatlas/api/endpoints/endpoint.py | 26 ++++++++++++++------------ openatlas/api/endpoints/entities.py | 2 +- openatlas/api/formats/csv.py | 18 +++++++++--------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index fd6164ed8..50efe226d 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -28,11 +28,11 @@ def __init__( self, entities: Entity | list[Entity], parser: dict[str, Any], - single: bool = False) -> None: + single_entity: bool = False) -> None: self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) self.pagination = None - self.single = single + self.single_entity = single_entity self.entities_with_links: dict[int, dict[str, Any]] = {} self.formated_entities = [] @@ -79,7 +79,7 @@ def reduce_entities_to_limit(self) -> None: self.entities = self.entities[start_index:end_index] def resolve_entities(self) -> Response | dict[str, Any]: - if not self.single: + if not self.single_entity: if self.parser.type_id: self.entities = self.filter_by_type() if self.parser.search: @@ -111,7 +111,7 @@ def resolve_entities(self) -> Response | dict[str, Any]: return marshal(result, self.get_entities_template(result)) def get_json_output(self) -> dict[str, Any]: - if not self.single: + if not self.single_entity: result = { "results": self.formated_entities, "pagination": { @@ -145,8 +145,10 @@ def export_csv_for_network_analysis(self) -> Response: with zipfile.ZipFile(archive, 'w') as zipped_file: for key, frame in self.get_entities_grouped_by_class().items(): with zipped_file.open(f'{key}.csv', 'w') as file: - file.write(bytes( - pd.DataFrame(data=frame).to_csv(), encoding='utf8')) + file.write( + bytes( + pd.DataFrame(data=frame).to_csv(), + encoding='utf8')) with zipped_file.open('links.csv', 'w') as file: link_frame = [ build_link_dataframe(link_) for link_ in @@ -196,6 +198,7 @@ def remove_duplicate_entities(self) -> None: [e for e in self.entities if not (e.id in seen or seen_add(e.id))] def get_entities_formatted(self) -> None: + entities = [] match self.parser.format: case 'geojson': entities = [self.get_geojson()] @@ -210,13 +213,12 @@ def get_entities_formatted(self) -> None: entities = [ self.parser.get_linked_places_entity(item) for item in self.entities_with_links.values()] - case _ if self.parser.format in app.config['RDF_FORMATS']: + case _ if self.parser.format \ + in app.config['RDF_FORMATS']: # pragma: no cover parsed_context = parse_loud_context() entities = [ get_loud_entities(item, parsed_context) for item in self.entities_with_links.values()] - case _: - entities = [] self.formated_entities = entities def get_geojson(self) -> dict[str, Any]: @@ -258,14 +260,14 @@ def get_entities_template(self, result: dict[str, Any]) -> dict[str, Any]: match self.parser.format: case 'geojson' | 'geojson-v2': template = geojson_collection_template() - if not self.single: + if not self.single_entity: template = geojson_pagination() case 'loud': template = loud_template(result) - if not self.single: + if not self.single_entity: template = loud_pagination() case 'lp' | 'lpx' | _: template = linked_places_template(self.parser) - if not self.single: + if not self.single_entity: template = linked_place_pagination(self.parser) return template diff --git a/openatlas/api/endpoints/entities.py b/openatlas/api/endpoints/entities.py index 11adb5121..40a27b624 100644 --- a/openatlas/api/endpoints/entities.py +++ b/openatlas/api/endpoints/entities.py @@ -62,7 +62,7 @@ def get(id_: int) -> tuple[Resource, int] | Response | dict[str, Any]: return Endpoint( ApiEntity.get_by_id(id_, types=True, aliases=True), entity_.parse_args(), - True).resolve_entities() + single_entity=True).resolve_entities() class GetLatest(Resource): diff --git a/openatlas/api/formats/csv.py b/openatlas/api/formats/csv.py index 69872a816..bb78c1d4a 100644 --- a/openatlas/api/formats/csv.py +++ b/openatlas/api/formats/csv.py @@ -31,9 +31,9 @@ def build_dataframe( 'geom_type': geom['type'], 'coordinates': geom['coordinates']} if relations: - for key, value in get_csv_links(entity).items(): + for key, value in get_csv_links(entity_dict).items(): data[key] = ' | '.join(list(map(str, value))) - for key, value in get_csv_types(entity).items(): + for key, value in get_csv_types(entity_dict).items(): data[key] = ' | '.join(list(map(str, value))) return data @@ -54,28 +54,28 @@ def build_link_dataframe(link: Link) -> dict[str, Any]: 'end_comment': link.end_comment} -def get_csv_types(entity: Entity) -> dict[Any, list[Any]]: +def get_csv_types(entity_dict: dict[str, Any]) -> dict[Any, list[Any]]: types: dict[str, Any] = defaultdict(list) - for type_ in entity.types: + for type_ in entity_dict['entity'].types: hierarchy = [g.types[root].name for root in type_.root] value = '' - for link in Entity.get_links_of_entities(entity.id): + for link in entity_dict['links']: if link.range.id == type_.id and link.description: value += link.description if link.range.id == type_.id and type_.description: value += f' {type_.description}' key = ' > '.join(map(str, hierarchy)) - types[key].append(f"{type_.name}: {value or ''}") + types[key].append(type_.name + (f": {value}" if value else '')) return types -def get_csv_links(entity: Entity) -> dict[str, Any]: +def get_csv_links(entity_dict: dict[str, Any]) -> dict[str, Any]: links: dict[str, Any] = defaultdict(list) - for link in Entity.get_links_of_entities(entity.id): + for link in entity_dict['links']: key = f"{link.property.i18n['en'].replace(' ', '_')}_" \ f"{link.range.class_.name}" links[key].append(link.range.name) - for link in Entity.get_links_of_entities(entity.id, inverse=True): + for link in entity_dict['links_inverse']: key = f"{link.property.i18n['en'].replace(' ', '_')}_" \ f"{link.range.class_.name}" if link.property.i18n_inverse['en']: From f2a035353d454f48d900b1a72f3a67a9b5bd23c4 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 15:13:36 +0100 Subject: [PATCH 07/15] fixed csv network --- openatlas/api/endpoints/endpoint.py | 72 ++++++++++++++++------------- openatlas/api/endpoints/entities.py | 3 +- openatlas/api/formats/csv.py | 19 ++++++++ tests/test_api.py | 28 +++++------ 4 files changed, 74 insertions(+), 48 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 50efe226d..3c2f6f4f1 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -11,7 +11,7 @@ from openatlas import app from openatlas.api.endpoints.parser import Parser from openatlas.api.formats.csv import ( - build_dataframe, build_link_dataframe) + build_dataframe, build_dataframe_network, build_link_dataframe) from openatlas.api.formats.loud import get_loud_entities from openatlas.api.resources.resolve_endpoints import ( download, parse_loud_context) @@ -19,7 +19,7 @@ geojson_collection_template, geojson_pagination, linked_place_pagination, linked_places_template, loud_pagination, loud_template) from openatlas.api.resources.util import ( - get_linked_entities_api, get_location_link, remove_duplicate_entities) + get_linked_entities_api, get_location_link) from openatlas.models.entity import Entity, Link @@ -28,11 +28,11 @@ def __init__( self, entities: Entity | list[Entity], parser: dict[str, Any], - single_entity: bool = False) -> None: + single: bool = False) -> None: self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) self.pagination = None - self.single_entity = single_entity + self.single = single self.entities_with_links: dict[int, dict[str, Any]] = {} self.formated_entities = [] @@ -79,24 +79,23 @@ def reduce_entities_to_limit(self) -> None: self.entities = self.entities[start_index:end_index] def resolve_entities(self) -> Response | dict[str, Any]: - if not self.single_entity: - if self.parser.type_id: - self.entities = self.filter_by_type() - if self.parser.search: - self.entities = [ - e for e in self.entities if self.parser.search_filter(e)] - self.remove_duplicate_entities() - if self.parser.count == 'true': - return jsonify(len(self.entities)) - self.sort_entities() - self.get_pagination() - self.reduce_entities_to_limit() - if self.parser.export == 'csvNetwork': - return self.export_csv_for_network_analysis() + if self.parser.type_id: + self.entities = self.filter_by_type() + if self.parser.search: + self.entities = [ + e for e in self.entities if self.parser.search_filter(e)] + self.remove_duplicate_entities() + if self.parser.count == 'true': + return jsonify(len(self.entities)) + self.sort_entities() + self.get_pagination() + self.reduce_entities_to_limit() if self.entities: self.get_links_for_entities() if self.parser.export == 'csv': return self.export_entities_csv() + if self.parser.export == 'csvNetwork': + return self.export_csv_for_network_analysis() if self.entities: self.get_entities_formatted() @@ -111,7 +110,7 @@ def resolve_entities(self) -> Response | dict[str, Any]: return marshal(result, self.get_entities_template(result)) def get_json_output(self) -> dict[str, Any]: - if not self.single_entity: + if not self.single: result = { "results": self.formated_entities, "pagination": { @@ -141,6 +140,19 @@ def export_entities_csv(self) -> Response: headers={'Content-Disposition': 'attachment;filename=result.csv'}) def export_csv_for_network_analysis(self) -> Response: + entities = [] + links = [] + for items in self.entities_with_links.values(): + entities.append(items['entity']) + for link_ in items['links']: + entities.append(link_.range) + links.append(link_) + for link_inverse in items['links']: + entities.append(link_inverse.domain) + links.append(link_inverse) + self.entities = entities + self.remove_duplicate_entities() + self.get_links_for_entities() archive = BytesIO() with zipfile.ZipFile(archive, 'w') as zipped_file: for key, frame in self.get_entities_grouped_by_class().items(): @@ -150,11 +162,9 @@ def export_csv_for_network_analysis(self) -> Response: pd.DataFrame(data=frame).to_csv(), encoding='utf8')) with zipped_file.open('links.csv', 'w') as file: - link_frame = [ - build_link_dataframe(link_) for link_ in - (self.link_parser_check() - + self.link_parser_check(inverse=True))] - file.write(bytes( + link_frame = [build_link_dataframe(link_) for link_ in links] + file.write( + bytes( pd.DataFrame(data=link_frame).to_csv(), encoding='utf8')) return Response( archive.getvalue(), @@ -162,14 +172,12 @@ def export_csv_for_network_analysis(self) -> Response: headers={'Content-Disposition': 'attachment;filename=oa_csv.zip'}) def get_entities_grouped_by_class(self) -> dict[str, Any]: - self.entities += get_linked_entities_api([e.id for e in self.entities]) - entities = remove_duplicate_entities(self.entities) grouped_entities = {} for class_, entities_ in groupby( - sorted(entities, key=lambda entity: entity.class_.name), + sorted(self.entities, key=lambda entity: entity.class_.name), key=lambda entity: entity.class_.name): - grouped_entities[class_] = \ - [build_dataframe(entity) for entity in entities_] + grouped_entities[class_] = \ + [build_dataframe_network(entity) for entity in entities_] return grouped_entities def link_parser_check(self, inverse: bool = False) -> list[Link]: @@ -260,14 +268,14 @@ def get_entities_template(self, result: dict[str, Any]) -> dict[str, Any]: match self.parser.format: case 'geojson' | 'geojson-v2': template = geojson_collection_template() - if not self.single_entity: + if not self.single: template = geojson_pagination() case 'loud': template = loud_template(result) - if not self.single_entity: + if not self.single: template = loud_pagination() case 'lp' | 'lpx' | _: template = linked_places_template(self.parser) - if not self.single_entity: + if not self.single: template = linked_place_pagination(self.parser) return template diff --git a/openatlas/api/endpoints/entities.py b/openatlas/api/endpoints/entities.py index 40a27b624..c723d6ba3 100644 --- a/openatlas/api/endpoints/entities.py +++ b/openatlas/api/endpoints/entities.py @@ -20,7 +20,6 @@ def get(class_: str) -> tuple[Resource, int] | Response | dict[str, Any]: ApiEntity.get_by_cidoc_classes([class_]), entity_.parse_args()).resolve_entities() - class GetBySystemClass(Resource): @staticmethod def get(class_: str) -> tuple[Resource, int] | Response | dict[str, Any]: @@ -62,7 +61,7 @@ def get(id_: int) -> tuple[Resource, int] | Response | dict[str, Any]: return Endpoint( ApiEntity.get_by_id(id_, types=True, aliases=True), entity_.parse_args(), - single_entity=True).resolve_entities() + single=True).resolve_entities() class GetLatest(Resource): diff --git a/openatlas/api/formats/csv.py b/openatlas/api/formats/csv.py index bb78c1d4a..d7dc0260e 100644 --- a/openatlas/api/formats/csv.py +++ b/openatlas/api/formats/csv.py @@ -38,6 +38,25 @@ def build_dataframe( return data +def build_dataframe_network(entity: Entity) -> dict[str, Any]: + geom = get_csv_geom_entry(entity) + data = { + 'id': str(entity.id), + 'name': entity.name, + 'description': entity.description, + 'begin_from': entity.begin_from, + 'begin_to': entity.begin_to, + 'begin_comment': entity.begin_comment, + 'end_from': entity.end_from, + 'end_to': entity.end_to, + 'end_comment': entity.end_comment, + 'cidoc_class': entity.cidoc_class.name, + 'system_class': entity.class_.name, + 'geom_type': geom['type'], + 'coordinates': geom['coordinates']} + return data + + def build_link_dataframe(link: Link) -> dict[str, Any]: return { 'id': link.id, diff --git a/tests/test_api.py b/tests/test_api.py index a34fb186f..b92204e7e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -263,20 +263,20 @@ def test_api(self) -> None: export='csv'))]: assert b'Shire' in rv.data assert 'text/csv' in rv.headers.get('Content-Type') - # - # for rv in [ - # c.get( - # url_for('api_04.entity', id_=place.id, export='csvNetwork')), - # c.get( - # url_for( - # 'api_04.query', - # entities=location.id, - # cidoc_classes='E18', - # view_classes='artifact', - # system_classes='person', - # export='csvNetwork'))]: - # assert b'Shire' in rv.data - # assert 'application/zip' in rv.headers.get('Content-Type') + + for rv in [ + c.get( + url_for('api_04.entity', id_=place.id, export='csvNetwork')), + c.get( + url_for( + 'api_04.query', + entities=location.id, + cidoc_classes='E18', + view_classes='artifact', + system_classes='person', + export='csvNetwork'))]: + assert b'Shire' in rv.data + assert 'application/zip' in rv.headers.get('Content-Type') rv = c.get( url_for( From 31af09f010bc3a0c0f263e8747bd453fb3d55740 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 15:42:34 +0100 Subject: [PATCH 08/15] wip --- openatlas/api/endpoints/endpoint.py | 29 +++++++++++++++++------- openatlas/api/formats/csv.py | 35 ++++++++--------------------- openatlas/models/entity.py | 7 ++++++ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 3c2f6f4f1..b1174e8ba 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -11,15 +11,14 @@ from openatlas import app from openatlas.api.endpoints.parser import Parser from openatlas.api.formats.csv import ( - build_dataframe, build_dataframe_network, build_link_dataframe) + build_dataframe_with_relations, build_dataframe, build_link_dataframe) from openatlas.api.formats.loud import get_loud_entities from openatlas.api.resources.resolve_endpoints import ( download, parse_loud_context) from openatlas.api.resources.templates import ( geojson_collection_template, geojson_pagination, linked_place_pagination, linked_places_template, loud_pagination, loud_template) -from openatlas.api.resources.util import ( - get_linked_entities_api, get_location_link) +from openatlas.api.resources.util import get_location_link from openatlas.models.entity import Entity, Link @@ -84,7 +83,15 @@ def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.search: self.entities = [ e for e in self.entities if self.parser.search_filter(e)] - self.remove_duplicate_entities() + before = len(self.entities) + # self.remove_duplicate_entities() + self.entities = set(self.entities) + after = len(self.entities) + if before != after: + print(before) + print(after) + self.entities = list(self.entities) + if self.parser.count == 'true': return jsonify(len(self.entities)) self.sort_entities() @@ -132,7 +139,7 @@ def filter_by_type(self) -> list[Entity]: def export_entities_csv(self) -> Response: frames = [ - build_dataframe(e, relations=True) + build_dataframe_with_relations(e) for e in self.entities_with_links.values()] return Response( pd.DataFrame(data=frames).to_csv(), @@ -177,7 +184,7 @@ def get_entities_grouped_by_class(self) -> dict[str, Any]: sorted(self.entities, key=lambda entity: entity.class_.name), key=lambda entity: entity.class_.name): grouped_entities[class_] = \ - [build_dataframe_network(entity) for entity in entities_] + [build_dataframe(entity) for entity in entities_] return grouped_entities def link_parser_check(self, inverse: bool = False) -> list[Link]: @@ -202,8 +209,14 @@ def sort_entities(self) -> None: def remove_duplicate_entities(self) -> None: seen: set[int] = set() seen_add = seen.add # Faster than always call seen.add() - self.entities = \ - [e for e in self.entities if not (e.id in seen or seen_add(e.id))] + entities = [] + for e in self.entities: + if e.id not in seen: + seen_add(e.id) + entities.append(e) + self.entities = entities + # self.entities = \ + # [e for e in self.entities if not (e.id in seen or seen_add(e.id))] def get_entities_formatted(self) -> None: entities = [] diff --git a/openatlas/api/formats/csv.py b/openatlas/api/formats/csv.py index d7dc0260e..1798981a1 100644 --- a/openatlas/api/formats/csv.py +++ b/openatlas/api/formats/csv.py @@ -11,36 +11,20 @@ from openatlas.models.gis import Gis -def build_dataframe( - entity_dict: dict[str, Any], - relations: bool = False) -> dict[str, Any]: +def build_dataframe_with_relations( + entity_dict: dict[str, Any]) -> dict[str, Any]: entity = entity_dict['entity'] - geom = get_csv_geom_entry(entity) - data = { - 'id': str(entity.id), - 'name': entity.name, - 'description': entity.description, - 'begin_from': entity.begin_from, - 'begin_to': entity.begin_to, - 'begin_comment': entity.begin_comment, - 'end_from': entity.end_from, - 'end_to': entity.end_to, - 'end_comment': entity.end_comment, - 'cidoc_class': entity.cidoc_class.name, - 'system_class': entity.class_.name, - 'geom_type': geom['type'], - 'coordinates': geom['coordinates']} - if relations: - for key, value in get_csv_links(entity_dict).items(): - data[key] = ' | '.join(list(map(str, value))) - for key, value in get_csv_types(entity_dict).items(): - data[key] = ' | '.join(list(map(str, value))) + data = build_dataframe(entity) + for key, value in get_csv_links(entity_dict).items(): + data[key] = ' | '.join(list(map(str, value))) + for key, value in get_csv_types(entity_dict).items(): + data[key] = ' | '.join(list(map(str, value))) return data -def build_dataframe_network(entity: Entity) -> dict[str, Any]: +def build_dataframe(entity: Entity) -> dict[str, Any]: geom = get_csv_geom_entry(entity) - data = { + return { 'id': str(entity.id), 'name': entity.name, 'description': entity.description, @@ -54,7 +38,6 @@ def build_dataframe_network(entity: Entity) -> dict[str, Any]: 'system_class': entity.class_.name, 'geom_type': geom['type'], 'coordinates': geom['coordinates']} - return data def build_link_dataframe(link: Link) -> dict[str, Any]: diff --git a/openatlas/models/entity.py b/openatlas/models/entity.py index 64d923b31..a5eb31371 100644 --- a/openatlas/models/entity.py +++ b/openatlas/models/entity.py @@ -82,6 +82,12 @@ def __init__(self, data: dict[str, Any]) -> None: self.creator = g.file_info[self.id]['creator'] self.license_holder = g.file_info[self.id]['license_holder'] + def __eq__(self, other): + return self.id == other.id + + def __hash__(self): + return hash(('id', self.id)) + def get_linked_entity( self, code: str, @@ -586,6 +592,7 @@ def __init__( self.last = format_date_part(self.end_to, 'year') \ if self.end_to else self.last + def update(self) -> None: db_link.update({ 'id': self.id, From 3037380a8dec6dd108bbf90d982a95331fa514cf Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 16:25:49 +0100 Subject: [PATCH 09/15] refactor --- openatlas/api/endpoints/endpoint.py | 35 ++++++++++------------------- openatlas/models/entity.py | 6 ----- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index b1174e8ba..7172c95c0 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -30,12 +30,14 @@ def __init__( single: bool = False) -> None: self.entities = entities if isinstance(entities, list) else [entities] self.parser = Parser(parser) - self.pagination = None + self.pagination: dict[str, Any] = {} self.single = single self.entities_with_links: dict[int, dict[str, Any]] = {} - self.formated_entities = [] + self.formated_entities: list[dict[str, Any]] = [] def get_links_for_entities(self) -> None: + if not self.entities: + return for entity in self.entities: self.entities_with_links[entity.id] = { 'entity': entity, @@ -83,28 +85,19 @@ def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.search: self.entities = [ e for e in self.entities if self.parser.search_filter(e)] - before = len(self.entities) - # self.remove_duplicate_entities() - self.entities = set(self.entities) - after = len(self.entities) - if before != after: - print(before) - print(after) - self.entities = list(self.entities) - + self.remove_duplicate_entities() if self.parser.count == 'true': return jsonify(len(self.entities)) + self.sort_entities() self.get_pagination() self.reduce_entities_to_limit() - if self.entities: - self.get_links_for_entities() + self.get_links_for_entities() if self.parser.export == 'csv': return self.export_entities_csv() if self.parser.export == 'csvNetwork': return self.export_csv_for_network_analysis() - if self.entities: - self.get_entities_formatted() + self.get_entities_formatted() if self.parser.format in app.config['RDF_FORMATS']: # pragma: no cover return Response( @@ -209,16 +202,12 @@ def sort_entities(self) -> None: def remove_duplicate_entities(self) -> None: seen: set[int] = set() seen_add = seen.add # Faster than always call seen.add() - entities = [] - for e in self.entities: - if e.id not in seen: - seen_add(e.id) - entities.append(e) - self.entities = entities - # self.entities = \ - # [e for e in self.entities if not (e.id in seen or seen_add(e.id))] + self.entities = \ + [e for e in self.entities if not (e.id in seen or seen_add(e.id))] def get_entities_formatted(self) -> None: + if not self.entities: + return entities = [] match self.parser.format: case 'geojson': diff --git a/openatlas/models/entity.py b/openatlas/models/entity.py index a5eb31371..2075a97cf 100644 --- a/openatlas/models/entity.py +++ b/openatlas/models/entity.py @@ -82,12 +82,6 @@ def __init__(self, data: dict[str, Any]) -> None: self.creator = g.file_info[self.id]['creator'] self.license_holder = g.file_info[self.id]['license_holder'] - def __eq__(self, other): - return self.id == other.id - - def __hash__(self): - return hash(('id', self.id)) - def get_linked_entity( self, code: str, From 85091be89d327e9e68552f6488272f766d1a7a79 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Tue, 10 Dec 2024 17:11:12 +0100 Subject: [PATCH 10/15] small refactor --- openatlas/api/endpoints/endpoint.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openatlas/api/endpoints/endpoint.py b/openatlas/api/endpoints/endpoint.py index 7172c95c0..c925b7ebc 100644 --- a/openatlas/api/endpoints/endpoint.py +++ b/openatlas/api/endpoints/endpoint.py @@ -85,7 +85,7 @@ def resolve_entities(self) -> Response | dict[str, Any]: if self.parser.search: self.entities = [ e for e in self.entities if self.parser.search_filter(e)] - self.remove_duplicate_entities() + self.remove_duplicates() if self.parser.count == 'true': return jsonify(len(self.entities)) @@ -151,7 +151,7 @@ def export_csv_for_network_analysis(self) -> Response: entities.append(link_inverse.domain) links.append(link_inverse) self.entities = entities - self.remove_duplicate_entities() + self.remove_duplicates() self.get_links_for_entities() archive = BytesIO() with zipfile.ZipFile(archive, 'w') as zipped_file: @@ -176,7 +176,7 @@ def get_entities_grouped_by_class(self) -> dict[str, Any]: for class_, entities_ in groupby( sorted(self.entities, key=lambda entity: entity.class_.name), key=lambda entity: entity.class_.name): - grouped_entities[class_] = \ + grouped_entities[class_] = \ [build_dataframe(entity) for entity in entities_] return grouped_entities @@ -199,11 +199,11 @@ def sort_entities(self) -> None: key=self.parser.get_key, reverse=bool(self.parser.sort == 'desc')) - def remove_duplicate_entities(self) -> None: - seen: set[int] = set() - seen_add = seen.add # Faster than always call seen.add() + def remove_duplicates(self) -> None: + exists: set[int] = set() + add_ = exists.add # Faster than always call exists.add() self.entities = \ - [e for e in self.entities if not (e.id in seen or seen_add(e.id))] + [e for e in self.entities if not (e.id in exists or add_(e.id))] def get_entities_formatted(self) -> None: if not self.entities: From 8bcbe61d972b7a3b3829892a656b68cdabb85622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Katharina=20W=C3=BCnsche?= Date: Fri, 10 Jan 2025 17:13:38 +0100 Subject: [PATCH 11/15] Make count from tabs more visible (#2417) --- openatlas/static/css/style.css | 9 +++++++++ openatlas/templates/tabs.html | 17 +++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/openatlas/static/css/style.css b/openatlas/static/css/style.css index 5e3578e70..b897ab2eb 100644 --- a/openatlas/static/css/style.css +++ b/openatlas/static/css/style.css @@ -257,3 +257,12 @@ img.schema_image { min-width: 12.25rem; } +.active > span > .tab-counter.badge { + color: var(--bs-primary); + background-color: white; +} + +:not(.active) > span > .tab-counter.badge { + background-color: var(--bs-primary); + color: white; +} diff --git a/openatlas/templates/tabs.html b/openatlas/templates/tabs.html index cc6ee01e1..1da3687e9 100644 --- a/openatlas/templates/tabs.html +++ b/openatlas/templates/tabs.html @@ -14,14 +14,15 @@ aria-selected="{% if active %}true{% else %}false{% endif %}" href="#tab-{{ tab.name|replace('_', '-') }}"> - {{ _(tab.name|replace('_', ' '))|uc_first }} - {% if tab.table.rows|length %} - {{ '{0:,}'.format(tab.table.rows|length) }} - {% endif %} + data-bs-toggle="tooltip" + title="{{ tab.tooltip }}" + data-bs-placement="top" + class="d-flex gap-1"> + {{ _(tab.name|replace('_', ' '))|uc_first }} + {% if tab.table.rows|length %} + {{ + '{0:,}'.format(tab.table.rows|length) }} + {% endif %} From 45227ffb29d986169a7d853f9ef075fa969f2b22 Mon Sep 17 00:00:00 2001 From: Alexander Watzinger Date: Sat, 11 Jan 2025 17:25:14 +0100 Subject: [PATCH 12/15] Fixed presentation site link being full page width --- openatlas/templates/entity/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openatlas/templates/entity/view.html b/openatlas/templates/entity/view.html index 52835848f..0d5237187 100644 --- a/openatlas/templates/entity/view.html +++ b/openatlas/templates/entity/view.html @@ -10,7 +10,7 @@

{{ entity.name }}

{% if frontend_link %} - {{ frontend_link|safe }} + {{ frontend_link|safe }} {% endif %}
{{ entity|profile_image|safe }} From 8934bc687d872c6acb1e12ccf93dda33cb53671a Mon Sep 17 00:00:00 2001 From: Alexander Watzinger Date: Sat, 11 Jan 2025 17:29:37 +0100 Subject: [PATCH 13/15] Tests, update changelog --- openatlas/views/changelog.py | 7 ++++++- tests/test_admin.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openatlas/views/changelog.py b/openatlas/views/changelog.py index 0d9362a94..f3fef7061 100644 --- a/openatlas/views/changelog.py +++ b/openatlas/views/changelog.py @@ -15,7 +15,12 @@ def index_changelog() -> str: # pylint: disable=too-many-lines versions = { - '8.10.0': ['TBA', {}], + '8.10.0': ['TBA', { + 'feature': { + '2417': 'Make count from tabs more visible', + '2415': 'Manual: How to report an issue on redmine', + '2444': 'Refactor and minor improvements'} + }], '8.9.0': ['2025-01-01', { 'feature': { '2079': 'Text annotation', diff --git a/tests/test_admin.py b/tests/test_admin.py index 0cf99757a..2af512dc4 100644 --- a/tests/test_admin.py +++ b/tests/test_admin.py @@ -96,7 +96,7 @@ def test_admin(self) -> None: source.link('P2', g.types[source_type.subs[1]]) rv = c.get(url_for('check_dates')) - assert b'' in rv.data + assert b'tab-counter' in rv.data rv = c.get(url_for('check_link_duplicates')) assert b'Event Horizon' in rv.data From c496b38d11dba73178eaaed89a3a1a0e39ec9f29 Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Mon, 13 Jan 2025 16:06:32 +0100 Subject: [PATCH 14/15] pylint --- openatlas/api/formats/csv.py | 4 +- openatlas/api/formats/loud.py | 82 ++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/openatlas/api/formats/csv.py b/openatlas/api/formats/csv.py index 1798981a1..b3a0b5357 100644 --- a/openatlas/api/formats/csv.py +++ b/openatlas/api/formats/csv.py @@ -127,12 +127,12 @@ def export_database_csv(tables: dict[str, Any], filename: str) -> Response: f'{system_class}.csv', 'w') as file: file.write(bytes( pd.DataFrame(data=frame).to_csv(), - encoding='utf8')) + encoding='utf-8')) else: with zipped_file.open(f'{name}.csv', 'w') as file: file.write(bytes( pd.DataFrame(data=entries).to_csv(), - encoding='utf8')) + encoding='utf-8')) return Response( archive.getvalue(), mimetype='application/zip', diff --git a/openatlas/api/formats/loud.py b/openatlas/api/formats/loud.py index 24a799baa..8bac1b2a0 100644 --- a/openatlas/api/formats/loud.py +++ b/openatlas/api/formats/loud.py @@ -8,7 +8,7 @@ from openatlas.api.resources.util import ( remove_spaces_dashes, date_to_str, get_crm_relation, get_crm_code) from openatlas.display.util import get_file_path -from openatlas.models.entity import Entity +from openatlas.models.entity import Entity, Link from openatlas.models.gis import Gis from openatlas.models.type import Type @@ -45,10 +45,8 @@ def get_domain_links() -> dict[str, Any]: for link_ in data['links']: if link_.property.code in ['OA7', 'OA8', 'OA9']: continue - if link_.property.code == 'P127': - property_name = 'broader' - else: - property_name = loud[get_crm_relation(link_).replace(' ', '_')] + property_name = get_loud_property_name(loud, link_) + if link_.property.code == 'P53': for geom in Gis.get_wkt_by_id(link_.range.id): base_property = get_range_links() | geom @@ -61,11 +59,7 @@ def get_domain_links() -> dict[str, Any]: for link_ in data['links_inverse']: if link_.property.code in ['OA7', 'OA8', 'OA9']: continue - if link_.property.code == 'P127': - property_name = 'broader' - else: - property_name = \ - loud[get_crm_relation(link_, True).replace(' ', '_')] + property_name = get_loud_property_name(loud, link_, inverse=True) if link_.property.code == 'P53': for geom in Gis.get_wkt_by_id(link_.range.id): @@ -77,37 +71,53 @@ def get_domain_links() -> dict[str, Any]: if link_.domain.class_.name == 'file' and g.files.get(link_.domain.id): image_links.append(link_) + if image_links: - profile_image = Entity.get_profile_image_id(data['entity']) - representation: dict[str, Any] = { - 'type': 'VisualItem', - 'digitally_shown_by': []} - for link_ in image_links: - id_ = link_.domain.id - mime_type, _ = mimetypes.guess_type(g.files[id_]) - if not mime_type: - continue # pragma: no cover - file_ = get_file_path(id_) - image = { - 'id': url_for('api.entity', id_=id_, _external=True), - '_label': link_.domain.name, - 'type': 'DigitalObject', - 'format': mime_type, - 'access_point': [{ - 'id': url_for( - 'api.display', - filename=file_.stem if file_ else '', - _external=True), - 'type': 'DigitalObject', - '_label': 'ProfileImage' if id_ == profile_image else ''}]} - if type_ := get_standard_type_loud(link_.domain.types): - image['classified_as'] = get_type_property(type_) - representation['digitally_shown_by'].append(image) - properties_set['representation'].append(representation) + properties_set['representation'].append( + get_loud_images(data['entity'], image_links)) return {'@context': app.config['API_CONTEXT']['LOUD']} | \ base_entity_dict() | properties_set +def get_loud_property_name( + loud: dict[str, str], + link_: Link, + inverse: bool = False) -> str: + name = 'broader' + if not link_.property.code == 'P127': + name = loud[get_crm_relation(link_, inverse).replace(' ', '_')] + return name + + +def get_loud_images(entity: Entity, image_links: list[Link]) -> dict[str, Any]: + profile_image = Entity.get_profile_image_id(entity) + representation: dict[str, Any] = { + 'type': 'VisualItem', + 'digitally_shown_by': []} + for link_ in image_links: + id_ = link_.domain.id + mime_type, _ = mimetypes.guess_type(g.files[id_]) + if not mime_type: + continue # pragma: no cover + file_ = get_file_path(id_) + image = { + 'id': url_for('api.entity', id_=id_, _external=True), + '_label': link_.domain.name, + 'type': 'DigitalObject', + 'format': mime_type, + 'access_point': [{ + 'id': url_for( + 'api.display', + filename=file_.stem if file_ else '', + _external=True), + 'type': 'DigitalObject', + '_label': 'ProfileImage' if id_ == profile_image else ''}]} + if type_ := get_standard_type_loud(link_.domain.types): + image['classified_as'] = get_type_property(type_) + representation['digitally_shown_by'].append(image) + return representation + + def get_loud_timespan(entity: Entity) -> dict[str, Any]: return { 'type': 'TimeSpan', From 6f4cd61a728569c033d23cc6c4fc93ec7f7ef85b Mon Sep 17 00:00:00 2001 From: Bernhard Koschicek-Krombholz Date: Mon, 13 Jan 2025 17:12:06 +0100 Subject: [PATCH 15/15] added fix for k8s deployment --- .github/workflows/starter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/starter.yaml b/.github/workflows/starter.yaml index 1c1bd5f72..4594cfe0c 100644 --- a/.github/workflows/starter.yaml +++ b/.github/workflows/starter.yaml @@ -40,7 +40,7 @@ jobs: else echo "environment=review/${{ github.ref_name }}" echo "environment=review/${{ github.ref_name }}" >> $GITHUB_OUTPUT - echo "environment_short=$(echo -n ${{ github.ref_name }} | sed s/feature_// | tr '_' '-' | tr '[:upper:]' '[:lower:]' )" >> $GITHUB_OUTPUT + echo "environment_short=$(echo -n ${{ github.ref_name }} | sed 's/feat\(ure\)\{0,1\}[_/]//' | tr '_' '-' | tr '[:upper:]' '[:lower:]' | cut -c -63 )" >> $GITHUB_OUTPUT fi build_openatlas: needs: [setup_workflow_env]