Skip to content

Commit

Permalink
Merge pull request #98 from arthur-schnitzler/main
Browse files Browse the repository at this point in the history
code updates
  • Loading branch information
csae8092 authored Oct 29, 2024
2 parents 972e6d8 + b7c3e46 commit 7f3d5ad
Show file tree
Hide file tree
Showing 11 changed files with 343 additions and 19 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -191,3 +191,6 @@ featureCodes_en.tsv
listperson.xml
schubert_pmb.csv
brahms_pmb.csv
hanslick-werke-wikidata.csv
Untitled1.ipynb
event-types.csv
2 changes: 1 addition & 1 deletion apis_core/apis_entities/detail_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def get(self, request, *args, **kwargs):
"".join([x.title() for x in match]),
tb_object_open,
link_to_relations,
rel_type
rel_type,
)
)
object_lod = Uri.objects.filter(entity=instance)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ <h3 class="text-center mt-5">Beziehungen</h3>
{% block relations %}
{% for obj in right_card %}
{% if obj.1.data|length > 0 %}
<h4>{{ obj.0 }} <a class="apis-{{ obj.5 }}" title="Gehe zu {{ object }}–{{ obj.0 }} Relationen" href="{{ obj.4 }}"><i class="bi bi-box-arrow-up-right" aria-hidden="true"><span class="visually-hidden">Gehe zu {{ object }} - {{ obj.5 }} Relationen</span></i></a></h4>
<h4>{{ obj.0 }} {% if obj.5 %}<a class="apis-{{ obj.5 }}" title="Gehe zu {{ object }}–{{ obj.0 }} Relationen" href="{{ obj.4 }}"><i class="bi bi-box-arrow-up-right" aria-hidden="true"><span class="visually-hidden">Gehe zu {{ object }} - {{ obj.5 }} Relationen</span></i></a>{% endif %}</h4>

<div id="tab_{{ obj.2 }}" class="mb-5">
{% render_table obj.1 %}
Expand Down
22 changes: 22 additions & 0 deletions apis_core/apis_entities/tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from normdata.utils import (
get_or_create_person_from_wikidata,
get_or_create_place_from_wikidata,
get_or_create_work_from_wikidata,
import_from_normdata,
)

Expand Down Expand Up @@ -253,6 +254,27 @@ def test_013_import_normdata_no_wikidata(self):
response = client.post(url, payload, follow=True)
self.assertEqual(response.status_code, 200)

def test_013a_import_normdata_for_work(self):
client.login(**USER)
payload = {
"normdata_url": "https://www.wikidata.org/wiki/Q105745657",
"entity_type": "work",
}
url = reverse(
"normdata:import_from_normdata",
)
response = client.post(url, payload, follow=True)
self.assertEqual(response.status_code, 200)

def test_013b_wikidata_work_exist(self):
entity = get_or_create_work_from_wikidata(
"https://www.wikidata.org/wiki/Q105745657"
)
ic(entity)
for x in entity.uri_set.all():
entity = get_or_create_work_from_wikidata(x.uri)
self.assertTrue(entity)

def test_014_wikidata_place_exist(self):
entity = get_or_create_place_from_wikidata(
"http://www.wikidata.org/entity/Q1741"
Expand Down
8 changes: 6 additions & 2 deletions apis_core/apis_relations/forms2.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,15 @@ def get_html_table(self, entity_type, request, site_instance, form_match):
if form_match.group(1) == form_match.group(2):
dic_a = {"related_" + entity_type.lower() + "a": site_instance}
dic_b = {"related_" + entity_type.lower() + "b": site_instance}
objects = self.relation_form.objects.filter(Q(**dic_a) | Q(**dic_b)).order_by("-updated")[:10]
objects = self.relation_form.objects.filter(
Q(**dic_a) | Q(**dic_b)
).order_by("-updated")[:10]
table_html = table(data=objects, prefix=prefix)
else:
tab_query = {"related_" + entity_type.lower(): site_instance}
ttab = self.relation_form.objects.filter(**tab_query).order_by("-updated")[:10]
ttab = self.relation_form.objects.filter(**tab_query).order_by("-updated")[
:10
]
table_html = table(data=ttab, prefix=prefix)
return table_html

Expand Down
108 changes: 108 additions & 0 deletions issue__230_hanslicks_werke.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "16556ed2-f178-4edd-9c4c-5639fbdcc851",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import requests\n",
"from acdh_id_reconciler import gnd_to_wikidata\n",
"from tqdm import tqdm"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6dcbb6ef-dfa6-4ff9-b256-c3a4ce35a5bd",
"metadata": {},
"outputs": [],
"source": [
"orig_data = requests.get(\"https://raw.githubusercontent.com/Hanslick-Online/hsl-entities/refs/heads/main/json_dumps/Werke.json\").json()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "95db218e-6b72-4451-aa80-4b66af239d31",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"data = []\n",
"for key, value in tqdm(orig_data.items(), total=len(orig_data)):\n",
" gnd = value[\"gnd\"]\n",
" if gnd:\n",
" try:\n",
" wikidata = gnd_to_wikidata(gnd)\n",
" wikidata[\"id\"] = key\n",
" data.append(wikidata)\n",
" except IndexError:\n",
" continue\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "67be47c8-5a4f-4d94-98b2-61d2cd6364bd",
"metadata": {},
"outputs": [],
"source": [
"data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4be7625f-010e-4730-b4f7-0cce11d86b80",
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fae7f32c-405a-4dc9-ab5d-c6d0737e3620",
"metadata": {},
"outputs": [],
"source": [
"df.to_csv(\"hanslick-werke-wikidata.csv\", index=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "72f574ac-259b-4d7d-84d9-33da01a47ef1",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Django Shell-Plus",
"language": "python",
"name": "django_extensions"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
1 change: 1 addition & 0 deletions normdata/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,6 @@ class NormDataImportForm(forms.Form):
("person", "Person"),
("place", "Ort"),
("institution", "Institution"),
("work", "Werk"),
),
)
114 changes: 110 additions & 4 deletions normdata/utils.py
Original file line number Diff line number Diff line change
@@ -1,25 +1,32 @@
from acdh_geonames_utils.gn_client import gn_as_object
from acdh_id_reconciler import geonames_to_wikidata, gnd_to_wikidata
from acdh_wikidata_pyutils import WikiDataPerson, WikiDataPlace, WikiDataOrg
from acdh_wikidata_pyutils import (
WikiDataPerson,
WikiDataPlace,
WikiDataOrg,
WikiDataEntity,
)
from AcdhArcheAssets.uri_norm_rules import get_normalized_uri
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.db.utils import IntegrityError
from icecream import ic
from pylobid.pylobid import PyLobidPerson, PyLobidPlace, PyLobidOrg
from pylobid.pylobid import PyLobidPerson, PyLobidPlace, PyLobidOrg, PyLobidWork

from apis_core.apis_entities.models import Person, Place, Institution
from apis_core.apis_entities.models import Person, Place, Institution, Work
from apis_core.apis_metainfo.models import Uri
from apis_core.apis_relations.models import PersonPlace, InstitutionPlace
from apis_core.apis_relations.models import PersonPlace, InstitutionPlace, PersonWork
from apis_core.apis_vocabularies.models import (
PersonPlaceRelation,
InstitutionPlaceRelation,
PersonWorkRelation,
)

DOMAIN_MAPPING = settings.DOMAIN_MAPPING
BIRTH_REL = getattr(settings, "BIRTH_REL")
DEATH_REL = getattr(settings, "DEATH_REL")
LOCATED_REL = getattr(settings, "ORG_LOCATED_IN")
CREATED_REL = getattr(settings, "AUTHOR_RELS")


def get_uri_domain(uri):
Expand Down Expand Up @@ -133,6 +140,7 @@ def get_or_create_place_from_wikidata(uri):


def get_or_create_person_from_gnd(uri):
uri = get_normalized_uri(uri)
try:
entity = Uri.objects.get(uri=uri).entity
entity = Person.objects.get(id=entity.id)
Expand Down Expand Up @@ -162,6 +170,7 @@ def get_or_create_person_from_gnd(uri):


def get_or_create_org_from_gnd(uri):
uri = get_normalized_uri(uri.strip())
try:
entity = Uri.objects.get(uri=uri).entity
entity = Institution.objects.get(id=entity.id)
Expand All @@ -182,6 +191,7 @@ def get_or_create_org_from_gnd(uri):


def get_or_create_person_from_wikidata(uri):
uri = get_normalized_uri(uri)
try:
entity = Uri.objects.get(uri=uri).entity
entity = Person.objects.get(id=entity.id)
Expand Down Expand Up @@ -240,6 +250,93 @@ def get_or_create_person_from_wikidata(uri):
return entity


def get_or_create_work_from_gnd(uri):
try:
entity = Uri.objects.get(uri=uri).entity
entity = Work.objects.get(id=entity.id)
return entity
except ObjectDoesNotExist:
fetched_item = PyLobidWork(uri)
try:
start_date_written, end_date_written = fetched_item.date_of_creation.split(
"-"
)
except ValueError:
start_date_written, end_date_written = fetched_item.date_of_creation, ""
apis_entity = {
"name": fetched_item.pref_name,
"start_date_written": start_date_written,
"end_date_written": end_date_written,
}
entity = Work.objects.create(**apis_entity)
Uri.objects.create(
uri=uri,
domain="gnd",
entity=entity,
)
try:
start_date_written, end_date_written = (
fetched_item.date_of_production.split("-")
)
except ValueError:
pass
for x in fetched_item.creators:
if x["role"] in ["firstAuthor", "author", "firstComposer"]:
print(x)
try:
wikidata_url = gnd_to_wikidata(x["id"])["wikidata"]
print(wikidata_url)
creator = get_or_create_person_from_wikidata(wikidata_url)
except IndexError:
creator = get_or_create_person_from_gnd(x["id"])
try:
relation_type = PersonWorkRelation.objects.get(id=CREATED_REL[0])
except ObjectDoesNotExist:
relation_type, _ = PersonWorkRelation.objects.get_or_create(
name="hat geschaffen"
)
rel, _ = PersonWork.objects.get_or_create(
related_person=creator,
related_work=entity,
relation_type=relation_type,
start_date_written=start_date_written,
end_date_written=end_date_written,
)
print(rel)
return entity


def get_or_create_work_from_wikidata(uri):
try:
entity = Uri.objects.get(uri=uri).entity
entity = Work.objects.get(id=entity.id)
return entity
except ObjectDoesNotExist:
wd_entity = WikiDataEntity(uri)
if wd_entity.gnd_uri:
try:
entity = Uri.objects.get(uri=wd_entity.gnd_uri).entity
entity = Work.objects.get(id=entity.id)
return entity
except ObjectDoesNotExist:
entity = get_or_create_work_from_gnd(wd_entity.gnd_uri)
Uri.objects.create(
uri=get_normalized_uri(uri),
domain="wikidata",
entity=entity,
)
return entity
else:
apis_entity = wd_entity.get_apis_entity()
entity = Work.objects.create(**apis_entity)
Uri.objects.create(
uri=get_normalized_uri(uri),
domain="wikidata",
entity=entity,
)
return entity


def get_or_create_org_from_wikidata(uri):
try:
entity = Uri.objects.get(uri=uri).entity
Expand Down Expand Up @@ -299,6 +396,13 @@ def import_from_normdata(raw_url, entity_type):
try:
wikidata_url = gnd_to_wikidata(normalized_url)["wikidata"]
except (IndexError, KeyError):
if entity_type == "work":
try:
entity = get_or_create_work_from_gnd(normalized_url)
return entity
except Exception as e:
ic(e)
wikidata_url = False
if entity_type == "place":
try:
entity = get_or_create_place_from_gnd(normalized_url)
Expand Down Expand Up @@ -344,6 +448,8 @@ def import_from_normdata(raw_url, entity_type):
entity = get_or_create_place_from_wikidata(wikidata_url)
elif entity_type == "person":
entity = get_or_create_person_from_wikidata(wikidata_url)
elif entity_type == "work":
entity = get_or_create_work_from_wikidata(wikidata_url)
else:
entity = get_or_create_org_from_wikidata(wikidata_url)
else:
Expand Down
Loading

0 comments on commit 7f3d5ad

Please sign in to comment.