From 61b98b024809b476a0fb1f9fa2cd36e3513707ff Mon Sep 17 00:00:00 2001 From: csae8092 Date: Thu, 19 Dec 2024 12:21:27 +0100 Subject: [PATCH 1/2] closes #282 emt persons [skip ci] --- notebooks/issue__282_emt.ipynb | 93 ++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 notebooks/issue__282_emt.ipynb diff --git a/notebooks/issue__282_emt.ipynb b/notebooks/issue__282_emt.ipynb new file mode 100644 index 0000000..5177429 --- /dev/null +++ b/notebooks/issue__282_emt.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "a7cbcfd0-b9f4-4897-9cf5-a8cec08683da", + "metadata": {}, + "outputs": [], + "source": [ + "# run against production 2024-12-19\n", + "from acdh_tei_pyutils.tei import TeiReader\n", + "from acdh_tei_pyutils.utils import get_xmlid\n", + "from acdh_xml_pyutils.xml import NSMAP\n", + "from normdata.utils import import_from_normdata\n", + "from tqdm import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5babd6f6-4da8-4ad5-9ea3-955d77c768bd", + "metadata": {}, + "outputs": [], + "source": [ + "doc = TeiReader(\"https://emt.acdh-dev.oeaw.ac.at/listperson.xml\")\n", + "domain = \"kaiserin-eleonora\"\n", + "col, _ = Collection.objects.get_or_create(name=domain)\n", + "col" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c769236-3a4c-4d50-bf26-13735f073fdd", + "metadata": {}, + "outputs": [], + "source": [ + "broken_gnd = []\n", + "for x in tqdm(doc.any_xpath(\".//tei:person[@xml:id and ./tei:idno[@type='GND']]\")):\n", + " gnd_uri = x.xpath(\"./tei:idno[@type='GND']/text()\", namespaces=NSMAP)[0]\n", + " xmlid = get_xmlid(x)\n", + " uri = \"https://kaiserin-eleonora.oeaw.ac.at/{}.html\".format(xmlid)\n", + " entity = import_from_normdata(gnd_uri, \"person\")\n", + " if entity:\n", + " pmb_uri, _ = Uri.objects.get_or_create(uri=uri, domain=domain)\n", + " pmb_uri.entity = entity\n", + " pmb_uri.save()\n", + " entity.collection.add(col)\n", + " else:\n", + " broken_gnd.append(gnd_uri)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4fb4dab6-7057-44d7-b939-3ff96473c442", + "metadata": {}, + "outputs": [], + "source": [ + "print(broken_gnd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dfd3fe3-21d3-4b8c-aee1-0e375cb5c607", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Django Shell-Plus", + "language": "python", + "name": "django_extensions" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 631b69f376a2e90308056dcae5737356e8fbb77b Mon Sep 17 00:00:00 2001 From: Peter Andorfer Date: Thu, 2 Jan 2025 15:16:51 +0100 Subject: [PATCH 2/2] 3d calendar (#283) * wip cool 3d calender * wip cool 3d calendar * better labels * looks good so far * minor * linting --- network/static/network/calendar-to-coords.js | 70 +++++++++++++ network/static/network/calender.js | 102 ++++++++++++++++++ network/static/network/map.js | 105 +++++++++++++++++++ network/templates/network/calender.html | 26 +++++ network/templates/network/list_view.html | 1 + network/templates/network/map.html | 98 +---------------- network/tests.py | 25 +++++ network/urls.py | 4 + network/utils.py | 53 ++++++++++ network/views.py | 51 ++++++++- pmb/settings.py | 5 - 11 files changed, 437 insertions(+), 103 deletions(-) create mode 100644 network/static/network/calendar-to-coords.js create mode 100644 network/static/network/calender.js create mode 100644 network/static/network/map.js create mode 100644 network/templates/network/calender.html diff --git a/network/static/network/calendar-to-coords.js b/network/static/network/calendar-to-coords.js new file mode 100644 index 0000000..9ea2eff --- /dev/null +++ b/network/static/network/calendar-to-coords.js @@ -0,0 +1,70 @@ +function dateToCoordinates(isoDate, minYear = 1500, maxYear = 1800) { + // Parse the ISO date + const date = new Date(isoDate); + if (isNaN(date)) { + throw new Error("Invalid date format"); + } + + // Extract year, month, and day + const year = date.getUTCFullYear(); + const month = date.getUTCMonth() + 1; // Months are 0-based + const day = date.getUTCDate(); + + // Map year to latitude (-90 to 90) with larger gaps between years + const yearFactor = (year - minYear) / (maxYear - minYear); + const latitude = yearFactor * 180 - 90; + + // Map month and day to longitude (-180 to 180) + const maxMonth = 12; + const maxDay = 31; // Approximation for simplicity + const monthFactor = (month - 1) / (maxMonth - 1); + const dayFactor = (day - 1) / (maxDay - 1); + const longitude = ((monthFactor + dayFactor) / 2) * 360 - 180; + + return { lat: latitude, lng: longitude }; +} + +function scaleToRange( + x, + originalMin = 1, + originalMax = 10, + targetMin = 0, + targetMax = 250 +) { + return ( + Math.round( + ((x - originalMin) * (targetMax - targetMin)) / + (originalMax - originalMin) + ) + targetMin + ); +} + +function mapValueToColor(value) { + if (value < 1 || value > 10) { + return [0, 0, 0]; + } + + // Normalize the value to a range of 0 to 1 + const normalized = (value - 1) / 9; + + // Define the gradient colors (red -> yellow -> green) + const startColor = [255, 0, 0]; // Red + const midColor = [255, 255, 0]; // Yellow + const endColor = [0, 255, 0]; // Green + + let color; + if (normalized <= 0.5) { + // Interpolate between startColor and midColor + const t = normalized * 2; // Scale to [0, 1] + color = startColor.map((start, i) => + Math.round(start + t * (midColor[i] - start)) + ); + } else { + // Interpolate between midColor and endColor + const t = (normalized - 0.5) * 2; // Scale to [0, 1] + color = midColor.map((mid, i) => Math.round(mid + t * (endColor[i] - mid))); + } + + // Convert to RGB format + return [color[0], color[1], color[2]]; +} diff --git a/network/static/network/calender.js b/network/static/network/calender.js new file mode 100644 index 0000000..acba6b8 --- /dev/null +++ b/network/static/network/calender.js @@ -0,0 +1,102 @@ +const url = document.getElementById("url").textContent; +console.log("fetching data"); +fetch(url) + .then((response) => { + if (!response.ok) { + throw new Error("Calender data response was not ok"); + } + return response.json(); + }) + .then((data) => { + const legendDiv = document.getElementById("legend"); + const dl = document.createElement("dl"); // Create the
element + + data.metadata.query_params.forEach((param) => { + for (const [key, value] of Object.entries(param)) { + const dt = document.createElement("dt"); // Create the
element + dt.textContent = key; + const dd = document.createElement("dd"); // Create the
element + dd.textContent = value; + + dl.appendChild(dt); + dl.appendChild(dd); + } + }); + + // Ensure each event has a label property + const validEvents = data.events + .filter((event) => event.latitude && event.longitude) + .map((event) => ({ + ...event, + label: event.label || "Unknown Event", // Default to 'Unknown Event' if label is missing + })); + + console.log(validEvents); + const deckgl = new deck.DeckGL({ + container: "map", + initialViewState: { + altitude: 1.5, + height: 700, + longitude: 80, + latitude: 35, + zoom: 2, + pitch: 60, + // bearing: -1.7 + }, + controller: true, + // onViewStateChange: ({ viewState }) => { + // console.log("Current view state:", viewState); + // }, + layers: [ + new deck.HexagonLayer({ + data: validEvents, + getPosition: (d) => [d.longitude, d.latitude], + radius: 50000, + elevationScale: 4000, + elevationRange: [0, 50], + extruded: true, + pickable: true, + onHover: ({ object, x, y }) => { + const tooltip = document.getElementById("tooltip"); + if (object) { + const eventLabels = object.points.map((p) => p.source.label); + const curDate = object.points[0].source.date; + const listItems = eventLabels + .map((label) => `
  • ${label}
    • `) + .join(""); + tooltip.style.display = "block"; + tooltip.style.left = `${x}px`; + tooltip.style.top = `${y}px`; + tooltip.innerHTML = `${curDate}
      ${listItems}
    `; + } else { + tooltip.style.display = "none"; + } + }, + }), + ], + }); + + legendDiv.appendChild(dl); + }) + .catch((error) => { + console.error("Something went wrong:", error); + }); + +// Add this CSS for the tooltip +const style = document.createElement("style"); +style.innerHTML = ` + #tooltip { + position: absolute; + background: white; + padding: 5px; + border: 1px solid black; + display: none; + pointer-events: none; + } +`; +document.head.appendChild(style); + +// Add this HTML for the tooltip +const tooltip = document.createElement("div"); +tooltip.id = "tooltip"; +document.body.appendChild(tooltip); diff --git a/network/static/network/map.js b/network/static/network/map.js new file mode 100644 index 0000000..595ac9d --- /dev/null +++ b/network/static/network/map.js @@ -0,0 +1,105 @@ +const url = document.getElementById("url").textContent; +console.log("fetching data"); +fetch(url) + .then((response) => { + if (!response.ok) { + throw new Error("Geojson response was not ok"); + } + return response.json(); + }) + .then((data) => { + var map = L.map("map"); + const legendDiv = document.getElementById("legend"); + const dl = document.createElement("dl"); // Create the
    element + + data.metadata.query_params.forEach((param) => { + for (const [key, value] of Object.entries(param)) { + const dt = document.createElement("dt"); // Create the
    element + dt.textContent = key; + const dd = document.createElement("dd"); // Create the
    element + dd.textContent = value; + + dl.appendChild(dt); // Append
    to
    + dl.appendChild(dd); // Append
    to
    + } + }); + + legendDiv.appendChild(dl); + var OSMBaseLayer = L.tileLayer( + "https://tile.openstreetmap.org/{z}/{x}/{y}.png", + { + maxZoom: 19, + attribution: + '© OpenStreetMap', + } + ).addTo(map); + + var CartoDB_PositronNoLabels = L.tileLayer( + "https://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}{r}.png", + { + attribution: + '© OpenStreetMap contributors © CARTO', + subdomains: "abcd", + maxZoom: 20, + } + ); + + var CartoDB_DarkMatterNoLabels = L.tileLayer( + "https://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}{r}.png", + { + attribution: + '© OpenStreetMap contributors © CARTO', + subdomains: "abcd", + maxZoom: 20, + } + ); + + const markers = L.markerClusterGroup(); + const geojsonLayer = L.geoJSON(data, { + onEachFeature: function (feature, layer) { + layer.bindPopup(feature.properties.label); + }, + pointToLayer: function (feature, latlng) { + return L.marker(latlng); + }, + }); + markers.addTo(map); + geojsonLayer.eachLayer((layer) => markers.addLayer(layer)); + + var heatData = []; + L.geoJSON(data, { + onEachFeature: function (feature, layer) { + if (feature.geometry.type === "Point") { + var lat = feature.geometry.coordinates[1]; + var lng = feature.geometry.coordinates[0]; + heatData.push([lat, lng]); + } + }, + }); + + // Create the heatmap layer + var heatmapLayer = L.heatLayer(heatData, { + radius: 25, + blur: 10, + maxZoom: 17, + max: 0.7, + gradient: { 0: "white", 0.5: "lime", 1: "red" }, + }); + + var baseMaps = { + "Base Layer": OSMBaseLayer, + "CartoDB hell": CartoDB_PositronNoLabels, + "CartoDB dunkel": CartoDB_DarkMatterNoLabels, + }; + + const overlayMaps = { + "Marker Cluster": markers, + Heatmap: heatmapLayer, + }; + + L.control.layers(baseMaps, overlayMaps, { collapsed: false }).addTo(map); + map.fitBounds(geojsonLayer.getBounds()); + }) + .catch((error) => { + console.error("Something went wrong:", error); + }); diff --git a/network/templates/network/calender.html b/network/templates/network/calender.html new file mode 100644 index 0000000..77b6d5c --- /dev/null +++ b/network/templates/network/calender.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} +{% load static %} +{% block title %}Map{% endblock %} +{% block scriptHeader %} +{% endblock %} +{% block content %} + + +
    +

    Kalender

    +
    +
    +
    +

    gewählte Filterparameter

    +
    +
    +
    + + + +{% endblock %} \ No newline at end of file diff --git a/network/templates/network/list_view.html b/network/templates/network/list_view.html index f263f26..7787346 100644 --- a/network/templates/network/list_view.html +++ b/network/templates/network/list_view.html @@ -16,6 +16,7 @@

    CSV JSON Als Netzwerk + Als Kalender GeoJson Karte diff --git a/network/templates/network/map.html b/network/templates/network/map.html index f45d09e..859378b 100644 --- a/network/templates/network/map.html +++ b/network/templates/network/map.html @@ -22,101 +22,5 @@

    gewählte Filterparameter

    - + {% endblock %} \ No newline at end of file diff --git a/network/tests.py b/network/tests.py index 6bfa4ef..25426ab 100644 --- a/network/tests.py +++ b/network/tests.py @@ -24,6 +24,26 @@ def test_01_network(self): response = client.get(url) self.assertEqual(response.status_code, 200) + def test_01a_network_data(self): + url = reverse("network:data") + response = client.get(url) + self.assertEqual(response.status_code, 200) + + def test_01b_network_data(self): + url = f'{reverse("network:data")}?format=hansi' + response = client.get(url) + self.assertEqual(response.status_code, 200) + + def test_01c_network_data(self): + url = f'{reverse("network:data")}?format=json' + response = client.get(url) + self.assertEqual(response.status_code, 200) + + def test_01d_network_data(self): + url = f'{reverse("network:data")}?format=cosmograph' + response = client.get(url) + self.assertEqual(response.status_code, 200) + def test_02_edge_list_view(self): url = reverse("network:edges_browse") response = client.get(url) @@ -38,3 +58,8 @@ def test_04_geojson_view(self): url = reverse("network:geojson") response = client.get(url) self.assertEqual(response.status_code, 200) + + def test_05_calendar_data(self): + url = reverse("network:calender_data") + response = client.get(url) + self.assertEqual(response.status_code, 200) diff --git a/network/urls.py b/network/urls.py index 24d3ac1..f0eceb8 100644 --- a/network/urls.py +++ b/network/urls.py @@ -3,7 +3,9 @@ EdgeListViews, network_data, NetworkView, + CalenderView, edges_as_geojson, + edges_as_calender, MapView, ) @@ -14,5 +16,7 @@ path("network-data/", network_data, name="data"), path("network/", NetworkView.as_view(), name="network"), path("geojson-data/", edges_as_geojson, name="geojson"), + path("calender-data/", edges_as_calender, name="calender_data"), + path("calender/", CalenderView.as_view(), name="calender"), path("map/", MapView.as_view(), name="map"), ] diff --git a/network/utils.py b/network/utils.py index a2aa805..82505b2 100644 --- a/network/utils.py +++ b/network/utils.py @@ -1,4 +1,5 @@ import pandas as pd +from datetime import datetime, date def df_to_geojson_vect( @@ -35,3 +36,55 @@ def get_coords(row): return (row["target_lat"], row["target_lng"]) else: return row["source_lat"], row["source_lng"] + + +def iso_to_lat_long(iso_date, start_date="1700-01-01", end_date="1990-12-31"): + """ + Maps an ISO date string or datetime.date to latitude and longitude, ensuring + earlier dates are more south (latitude) and earlier days within a year are more west (longitude). + + Args: + iso_date (str | datetime.date): An ISO-formatted date string (e.g., "2023-01-01") + or a datetime.date object. + start_date (str): Start of the date range in ISO format (default: "1900-01-01"). + end_date (str): End of the date range in ISO format (default: "2100-12-31"). + + Returns: + tuple: A tuple containing latitude and longitude (both as floats). + """ + try: + # Ensure iso_date is a datetime.date object + if isinstance(iso_date, str): + date_obj = datetime.strptime(iso_date, "%Y-%m-%d").date() + elif isinstance(iso_date, date): + date_obj = iso_date + else: + raise ValueError("Invalid input type. Must be a string or datetime.date.") + + # Convert start_date and end_date to datetime.date objects + start_date_obj = datetime.strptime(start_date, "%Y-%m-%d").date() + end_date_obj = datetime.strptime(end_date, "%Y-%m-%d").date() + + # Ensure date_obj is within range + if not (start_date_obj <= date_obj <= end_date_obj): + raise ValueError("Date is out of the specified range.") + + # Latitude: Based on the position of the date within the range (0–90) + total_days = (end_date_obj - start_date_obj).days + date_position = (date_obj - start_date_obj).days / total_days + lat = 90 * date_position + + # Longitude: Inverted position of the day within the year (0–180) + day_of_year = date_obj.timetuple().tm_yday + days_in_year = ( + date(datetime(date_obj.year, 12, 31).year, 12, 31) + - date(datetime(date_obj.year, 1, 1).year, 1, 1) + ).days + 1 + day_position = ( + day_of_year - 1 + ) / days_in_year # Normalize day position in the year + lon = 180 * day_position + + return lat, lon + except Exception as e: + raise ValueError(f"Invalid input: {iso_date}") from e diff --git a/network/views.py b/network/views.py index bb3c77b..65c2cd8 100644 --- a/network/views.py +++ b/network/views.py @@ -14,7 +14,7 @@ from network.forms import EdgeFilterFormHelper from network.models import Edge from network.tables import EdgeTable -from network.utils import get_coords, df_to_geojson_vect +from network.utils import get_coords, df_to_geojson_vect, iso_to_lat_long class NetworkView(TemplateView): @@ -42,6 +42,10 @@ class MapView(TemplateView): template_name = "network/map.html" +class CalenderView(TemplateView): + template_name = "network/calender.html" + + class EdgeListViews(GenericListView): model = Edge filter_class = EdgeListFilter @@ -57,6 +61,51 @@ class EdgeListViews(GenericListView): template_name = "network/list_view.html" +def edges_as_calender(request): + query_params = request.GET + queryset = ( + Edge.objects.filter() + .exclude(start_date__isnull=True) + .exclude(start_date__lte="0001-01-01") + ) + values_list = [x.name for x in Edge._meta.get_fields()] + qs = EdgeListFilter(request.GET, queryset=queryset).qs + items = list(qs.values_list(*values_list)) + df = pd.DataFrame(list(items), columns=values_list) + start_date = str(df["start_date"].min()) + end_date = str(df["start_date"].max()) + df["latitude"], df["longitude"] = zip( + *df["start_date"].map( + lambda date: iso_to_lat_long(date, start_date=start_date, end_date=end_date) + ) + ) + df["label"] = df[["source_label", "edge_label", "target_label"]].agg( + " ".join, axis=1 + ) + df = df.sort_values(by="start_date") + items = df.apply( + lambda row: { + "date": str(row["start_date"]), + "label": row["label"], + "edge_label": row["edge_label"], + "kind": row["edge_kind"], + "latitude": row["latitude"], + "longitude": row["longitude"], + "id": row["edge_id"], + }, + axis=1, + ).tolist() + data = {} + data = {"metadata": {}, "events": items} + data["metadata"] = {"number of objects": len(items)} + data["metadata"]["query_params"] = [ + {key: value} for key, value in query_params.items() + ] + data["metadata"]["start_date"] = str(df["start_date"].min()) + data["metadata"]["end_date"] = str(df["start_date"].max()) + return JsonResponse(data=data, json_dumps_params={"ensure_ascii": False}) + + def edges_as_geojson(request): query_params = request.GET values_list = [x.name for x in Edge._meta.get_fields()] diff --git a/pmb/settings.py b/pmb/settings.py index ba3ee85..be89df6 100644 --- a/pmb/settings.py +++ b/pmb/settings.py @@ -160,11 +160,6 @@ USE_THOUSAND_SEPARATOR = False -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/3.2/howto/static-files/ - -STATIC_URL = "/static/" - # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field