From 7b1a911a9fa22980c086d4254a31c36aeb90ae90 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Wed, 3 Jul 2024 16:28:10 -0500 Subject: [PATCH 1/2] Don't update a full taxon record with a partial one --- pyinaturalist_convert/_models.py | 10 +++++++--- test/test_db.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/pyinaturalist_convert/_models.py b/pyinaturalist_convert/_models.py index 92b1688..ca2bdbb 100644 --- a/pyinaturalist_convert/_models.py +++ b/pyinaturalist_convert/_models.py @@ -229,14 +229,18 @@ def to_model(self) -> Taxon: ) def update(self, taxon: Taxon): - """Update an existing record""" + """Merge new values into an existing record""" + # Don't update a full taxon record with a partial one + if self.partial is False and taxon._partial is True: + return + new_taxon = self.from_model(taxon) for col in [c.name for c in inspect(self).mapper.columns]: - if not (new_val := getattr(new_taxon, col)): + if (new_val := getattr(new_taxon, col)) is None: continue if col not in PRECOMPUTED_COLUMNS: setattr(self, col, new_val) - # Precomputed columns: Don't overwrite non-null values + # Precomputed columns: Only overwrite null values elif getattr(self, col) in [None, 0]: setattr(self, col, new_val) diff --git a/test/test_db.py b/test/test_db.py index 36c0bda..3b8d831 100644 --- a/test/test_db.py +++ b/test/test_db.py @@ -173,3 +173,28 @@ def test_save_taxa__preserve_precomputed_cols(tmp_path): assert results[0].observations_count == 18017625 assert results[1].complete_species_count == 416 assert results[1].observations_count == 218279 + + +def test_save_taxa__partial(tmp_path): + db_path = tmp_path / 'observations.db' + taxon_1 = Taxon( + id=3, + name='Aves', + rank='class', + preferred_common_name='Birds', + complete_species_count=10672, + observations_count=18017625, + ) + + create_tables(db_path) + save_taxa([taxon_1], db_path=db_path) + + # Save with updated values for precomputed columns + taxon_1.preferred_common_name = 'updated!' + taxon_1._partial = True + save_taxa([taxon_1], db_path=db_path) + + # Only previously null values (taxon_2.observations_count) in DB should be updated + results = list(get_db_taxa(db_path)) + assert results[0].preferred_common_name == 'Birds' + assert results[0]._partial is False From 8a9350a66fd887eaa830c93dda16e99da93d2101 Mon Sep 17 00:00:00 2001 From: Jordan Cook Date: Wed, 3 Jul 2024 16:30:18 -0500 Subject: [PATCH 2/2] Add conservation status, establishment means, and wikipedia description + url to DbTaxon model --- pyinaturalist_convert/_models.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pyinaturalist_convert/_models.py b/pyinaturalist_convert/_models.py index ca2bdbb..c0b025b 100644 --- a/pyinaturalist_convert/_models.py +++ b/pyinaturalist_convert/_models.py @@ -175,6 +175,8 @@ class DbTaxon: id: int = sa_field(Integer, primary_key=True) ancestor_ids: str = sa_field(String, default=None) child_ids: str = sa_field(String, default=None) + conservation_status: str = sa_field(String, default=None) + establishment_means: str = sa_field(String, default=None) iconic_taxon_id: int = sa_field(Integer, default=0) is_active: bool = sa_field(Boolean, default=None) leaf_taxa_count: int = sa_field(Integer, default=0) @@ -182,11 +184,13 @@ class DbTaxon: observations_count_rg: int = sa_field(Integer, default=0) name: str = sa_field(String, default=None, index=True) parent_id: int = sa_field(ForeignKey('taxon.id'), default=None, index=True) - partial: int = sa_field(Boolean, default=False) + partial: bool = sa_field(Boolean, default=False) photo_urls: str = sa_field(String, default=None) preferred_common_name: str = sa_field(String, default=None) rank: str = sa_field(String, default=None) reference_url: str = sa_field(String, default=None) + wikipedia_summary: str = sa_field(String, default=None) + wikipedia_url: str = sa_field(String, default=None) @classmethod def from_model(cls, taxon: Taxon) -> 'DbTaxon': @@ -195,6 +199,8 @@ def from_model(cls, taxon: Taxon) -> 'DbTaxon': id=taxon.id, ancestor_ids=_join_list(taxon.ancestor_ids), child_ids=_join_list(taxon.child_ids), + conservation_status=taxon.conservation_status, + establishment_means=taxon.establishment_means, iconic_taxon_id=taxon.iconic_taxon_id, is_active=taxon.is_active, leaf_taxa_count=taxon.complete_species_count, @@ -206,6 +212,8 @@ def from_model(cls, taxon: Taxon) -> 'DbTaxon': rank=taxon.rank, reference_url=taxon.reference_url, photo_urls=photo_urls, + wikipedia_summary=taxon.wikipedia_summary, + wikipedia_url=taxon.wikipedia_url, ) def to_model(self) -> Taxon: @@ -214,6 +222,8 @@ def to_model(self) -> Taxon: id=self.id, ancestors=_get_taxa(self.ancestor_ids), children=_get_taxa(self.child_ids), + conservation_status=self.conservation_status, + establishment_means=self.establishment_means, default_photo=photos[0] if photos else None, iconic_taxon_id=self.iconic_taxon_id, is_active=self.is_active, @@ -226,6 +236,8 @@ def to_model(self) -> Taxon: rank=self.rank, reference_url=self.reference_url, taxon_photos=photos, + wikipedia_summary=self.wikipedia_summary, + wikipedia_url=self.wikipedia_url, ) def update(self, taxon: Taxon):