feat: consolidate author remote_ids and wikidata identifiers

openlibrary/core/models.py

@@ -31,7 +31,7 @@
 from openlibrary.core.observations import Observations
 from openlibrary.core.ratings import Ratings
 from openlibrary.core.vendors import get_amazon_metadata
-from openlibrary.core.wikidata import WikidataEntity, get_wikidata_entity
+from openlibrary.core.wikidata import REMOTE_IDS, WikidataEntity, get_wikidata_entity
 from openlibrary.utils import extract_numeric_id_from_olid
 from openlibrary.utils.isbn import canonical, isbn_13_to_isbn_10, to_isbn_13
 
@@ -807,6 +807,27 @@ def get_edition_count(self):
     def get_lists(self, limit=50, offset=0, sort=True):
         return self._get_lists(limit=limit, offset=offset, sort=sort)
 
+    def merge_remote_ids(
+        self, incoming_ids: dict[str, str]
+    ) -> tuple[dict[str, str], int]:
+        output = {**self.remote_ids}
+        if len(incoming_ids.items()) == 0:
+            return output, -1
+        matches = 0
+        conflicts = 0
+        for identifier in REMOTE_IDS:
+            if identifier in output and identifier in incoming_ids:
+                if output[identifier] != incoming_ids[identifier]:
+                    conflicts = conflicts + 1
+                else:
+                    output[identifier] = incoming_ids[identifier]
+                    matches = matches + 1
+        if conflicts > matches:
+            # This means that the identifiers we already have for this author have too many conflicts with whichever identifiers we're trying to merge into it.
+            # TODO: Raise this to librarians, somehow.
+            return self.remote_ids, -1
+        return output, matches
+
 
 class User(Thing):
     DEFAULT_PREFERENCES = {

openlibrary/core/wikidata.py

@@ -7,13 +7,17 @@
 
 import json
 import logging
+import re
 from dataclasses import dataclass
 from datetime import datetime
+from typing import Any
 
 import requests
+import web
 
 from openlibrary.core import db
 from openlibrary.core.helpers import days_since
+from openlibrary.utils import extract_numeric_id_from_olid
 
 logger = logging.getLogger("core.wikidata")
 
@@ -30,6 +34,25 @@
     }
 ]
 
+# The keys in this dict need to match their corresponding names in openlibrary/plugins/openlibrary/config/author/identifiers.yml
+# Ordered by what I assume is most (viaf) to least (amazon/youtube) reliable for author matching
+REMOTE_IDS = {
+    "viaf": "P214",
+    "isni": "P213",
+    "lc_naf": "P244",
+    "opac_sbn": "P396",
+    "project_gutenberg": "P1938",
+    "librivox": "P1899",
+    "bookbrainz": "P2607",
+    "musicbrainz": "P434",
+    "librarything": "P7400",
+    "goodreads": "P2963",
+    "storygraph": "P12430",
+    "imdb": "P345",
+    "amazon": "P4862",
+    "youtube": "P2397",
+}
+
 
 @dataclass
 class WikidataEntity:
@@ -98,6 +121,33 @@ def get_external_profiles(self, language: str = 'en') -> list[dict]:
             )
         return profiles
 
+    def get_remote_ids(self) -> dict[str, Any]:
+        """
+        Get remote IDs like viaf, isni, etc.
+
+        Returns:
+            Dict containing identifier names as keys and lists of corresponding values
+        """
+        remote_ids = {}
+
+        for service, id in REMOTE_IDS.items():
+            values = self._get_statement_values(id)
+            remote_ids[service] = values
+        return remote_ids
+
+    def get_openlibrary_id(self) -> str | None:
+        """
+        Get open library ID of WD item. Mostly used to validate connection between WD item and OL thing
+
+        Returns:
+            OL ID, if it exists
+        """
+        res = self._get_statement_values("P648")
+        if len(res) > 0:
+            return min(res, key=extract_numeric_id_from_olid)
+
+        return None
+
     def _get_wiki_profiles(self, language: str) -> list[dict]:
         """
         Get formatted Wikipedia and Wikidata profile data for rendering.
@@ -161,6 +211,38 @@ def _get_statement_values(self, property_id: str) -> list[str]:
             if "value" in statement and "content" in statement["value"]
         ]
 
+    # can this move into author def'n?
+    def consolidate_remote_author_ids(self) -> None:
+        output = {"wikidata": self.id}
+        ol_id = self.get_openlibrary_id()
+        if ol_id is None or not re.fullmatch(r"^OL\d+A$", ol_id):
+            return
+
+        key = "/authors/" + ol_id
+        q = {"type": "/type/author", "key~": key}
+        reply = list(web.ctx.site.things(q))
+        authors = [web.ctx.site.get(k) for k in reply]
+        if len(authors) != 1:
+            # There should never, ever be len(authors) > 1, because that would imply two OL author entities have the same OL ID.
+            return
+        author = authors[0]
+        if author.wikidata() is not None and author.wikidata().id != self.id:
+            # TODO: Flag this to librarians. This means the OL entity identified by the Wikidata JSON has a different Wikidata ID than the JSON expects.
+            return
+        wd_remote_ids: dict[str, str] = {
+            key: value[0] for key, value in self.get_remote_ids().items() if value != []
+        }
+
+        # Verify that the author's IDs are not significantly different.
+        output, matches = author.merge_remote_ids(wd_remote_ids)
+
+        # save if there are new identifiers to save
+        if matches >= 0:
+            author.remote_ids = output
+            web.ctx.site.save(
+                query={**author, "key": key, "type": {"key": "/type/author"}}
+            )
+
 
 def _cache_expired(entity: WikidataEntity) -> bool:
     return days_since(entity._updated) > WIKIDATA_CACHE_TTL_DAYS
@@ -228,6 +310,11 @@ def _add_to_cache(entity: WikidataEntity) -> None:
     # TODO: after we upgrade to postgres 9.5+ we should use upsert here
     oldb = db.get_db()
     json_data = entity.to_wikidata_api_json_format()
+    ol_id = entity.get_openlibrary_id()
+
+    # here, we should write WD data to author remote ids
+    if ol_id is not None and re.fullmatch(r"^OL\d+A$", ol_id):
+        entity.consolidate_remote_author_ids()
 
     if _get_from_cache(entity.id):
         return oldb.update(

scripts/backfill_author_identifiers.py

@@ -0,0 +1,40 @@
+"""
+Copies all author identifiers from the author's stored Wikidata info into their remote_ids.
+
+To Run:
+
+PYTHONPATH=. python ./scripts/populate_author_identifiers.py /olsystem/etc/openlibrary.yml
+
+(If testing locally, run inside `docker compose exec web bash` and use ./conf/openlibrary.yml)
+"""
+
+#!/usr/bin/env python
+import web
+
+import infogami
+from openlibrary.config import load_config
+from openlibrary.core import db
+from openlibrary.core.wikidata import get_wikidata_entity
+from scripts.solr_builder.solr_builder.fn_to_cli import FnToCLI
+
+
+def main(ol_config: str):
+    """
+    :param str ol_config: Path to openlibrary.yml file
+    """
+    load_config(ol_config)
+    infogami._setup()
+
+    # how i fix this lol there's no IP when running from within docker
+    web.ctx.ip = '127.0.0.1'
+
+    for row in db.query("select id from wikidata"):
+        e = get_wikidata_entity(row.id)
+        if e is not None:
+            e.consolidate_remote_author_ids()
+
+
+# Get wikidata for authors who dont have it yet?
+
+if __name__ == "__main__":
+    FnToCLI(main).run()