From 12f30a2daca02780faf1b5bb96f6571b59ddbbaf Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Wed, 16 Oct 2024 19:09:37 -0400 Subject: [PATCH 1/7] Update resolve_bill method --- openstates/importers/base.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index b50beca7..fa5ccffb 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -169,15 +169,28 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: if bill_transform_func: bill_id = bill_transform_func(bill_id) - # move the start_date up a bit in case the event is on the last day of a session to compare with end_date + # Some steps here to first find the session that match the incoming event using the event date + # If a unique session is not found, then use the session with the latest "start_date" date = datetime.fromisoformat(date) - new_date = date - timedelta(days=1) + legislative_session = LegislativeSession.objects.filter( + Q(end_date__gte=date) | Q(end_date=""), + start_date__lte=date, + jurisdiction_id=self.jurisdiction_id, + ) + session_ids = {each.id for each in legislative_session} + + if len(session_ids) == 1: + session_id = session_ids.pop() + else: + legislative_session = ( + LegislativeSession.objects.filter(jurisdiction_id=self.jurisdiction_id) + .order_by("-start_date") + .first() + ) + session_id = legislative_session.id objects = Bill.objects.filter( - Q(legislative_session__end_date__gte=new_date) - | Q(legislative_session__end_date=""), - legislative_session__start_date__lte=date, - legislative_session__jurisdiction_id=self.jurisdiction_id, + legislative_session__id=session_id, identifier=bill_id, ) ids = {each.id for each in objects} From 62c99b1feb733089b221db8b7629a255f9241ab5 Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Wed, 16 Oct 2024 19:31:40 -0400 Subject: [PATCH 2/7] Updates --- openstates/importers/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index fa5ccffb..ce317257 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -4,7 +4,7 @@ import json import logging import typing -from datetime import datetime, timedelta +from datetime import datetime from django.db.models import Q, Model from django.db.models.signals import post_save from .. import settings @@ -169,7 +169,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: if bill_transform_func: bill_id = bill_transform_func(bill_id) - # Some steps here to first find the session that match the incoming event using the event date + # Some steps here to first find the session that matches the incoming entity using the entity date # If a unique session is not found, then use the session with the latest "start_date" date = datetime.fromisoformat(date) legislative_session = LegislativeSession.objects.filter( @@ -187,7 +187,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: .order_by("-start_date") .first() ) - session_id = legislative_session.id + session_id = legislative_session.id if legislative_session else None objects = Bill.objects.filter( legislative_session__id=session_id, From 914c5569c99065cca0387d1f364ae6533b7459eb Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Thu, 17 Oct 2024 13:46:27 -0400 Subject: [PATCH 3/7] Add all_session_cache for event<>bill matching --- openstates/importers/base.py | 28 +++++++++++++++------------- openstates/importers/events.py | 1 + 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index ce317257..29ef2fe0 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -128,6 +128,7 @@ def __init__(self, jurisdiction_id: str, do_postimport=True) -> None: self.pseudo_id_cache: typing.Dict[str, typing.Optional[_ID]] = {} self.person_cache: typing.Dict[_PersonCacheKey, typing.Optional[str]] = {} self.session_cache: typing.Dict[str, LegislativeSession] = {} + self.all_sessions_cache: typing.List[LegislativeSession] = [] self.logger = logging.getLogger("openstates") self.info = self.logger.info self.debug = self.logger.debug @@ -139,6 +140,12 @@ def __init__(self, jurisdiction_id: str, do_postimport=True) -> None: if settings.IMPORT_TRANSFORMERS.get(self._type): self.cached_transformers = settings.IMPORT_TRANSFORMERS[self._type] + def get_all_sessions(self) -> None: + if not self.all_sessions_cache: + self.all_sessions_cache = LegislativeSession.objects.filter( + jurisdiction_id=self.jurisdiction_id + ).order_by("-start_date") + def get_session(self, identifier: str) -> LegislativeSession: if identifier not in self.session_cache: self.session_cache[identifier] = LegislativeSession.objects.get( @@ -171,23 +178,18 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: # Some steps here to first find the session that matches the incoming entity using the entity date # If a unique session is not found, then use the session with the latest "start_date" - date = datetime.fromisoformat(date) - legislative_session = LegislativeSession.objects.filter( - Q(end_date__gte=date) | Q(end_date=""), - start_date__lte=date, - jurisdiction_id=self.jurisdiction_id, - ) - session_ids = {each.id for each in legislative_session} + date = datetime.fromisoformat(date).strftime("%Y-%m-%d") + session_ids = [ + session.id + for session in self.all_sessions_cache + if session.start_date <= date + and (session.end_date >= date or not session.end_date) + ] if len(session_ids) == 1: session_id = session_ids.pop() else: - legislative_session = ( - LegislativeSession.objects.filter(jurisdiction_id=self.jurisdiction_id) - .order_by("-start_date") - .first() - ) - session_id = legislative_session.id if legislative_session else None + session_id = self.all_sessions_cache[0].id objects = Bill.objects.filter( legislative_session__id=session_id, diff --git a/openstates/importers/events.py b/openstates/importers/events.py index 90165a08..55055013 100644 --- a/openstates/importers/events.py +++ b/openstates/importers/events.py @@ -94,6 +94,7 @@ def get_chamber_name_from_event_name( return None def prepare_for_db(self, data: _JsonDict) -> _JsonDict: + self.get_all_sessions() data["jurisdiction_id"] = self.jurisdiction_id data["location"] = self.get_location(data["location"]) org_classification = self.get_chamber_name_from_event_name(data["name"]) From cd8c5ca8a78774651232a5310127bf6faf4a646e Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Thu, 17 Oct 2024 15:35:52 -0400 Subject: [PATCH 4/7] Move function around --- openstates/importers/base.py | 3 ++- openstates/importers/events.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index 29ef2fe0..6473dd6b 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -173,6 +173,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: bill_transform_func = settings.IMPORT_TRANSFORMERS.get("bill", {}).get( "identifier", None ) + self.get_all_sessions() if bill_transform_func: bill_id = bill_transform_func(bill_id) @@ -189,7 +190,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: if len(session_ids) == 1: session_id = session_ids.pop() else: - session_id = self.all_sessions_cache[0].id + session_id = self.all_sessions_cache[0].id if self.all_sessions_cache else None objects = Bill.objects.filter( legislative_session__id=session_id, diff --git a/openstates/importers/events.py b/openstates/importers/events.py index 55055013..90165a08 100644 --- a/openstates/importers/events.py +++ b/openstates/importers/events.py @@ -94,7 +94,6 @@ def get_chamber_name_from_event_name( return None def prepare_for_db(self, data: _JsonDict) -> _JsonDict: - self.get_all_sessions() data["jurisdiction_id"] = self.jurisdiction_id data["location"] = self.get_location(data["location"]) org_classification = self.get_chamber_name_from_event_name(data["name"]) From fa3452e17dab07ca520e4d1856c1c913f6da34c5 Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Fri, 18 Oct 2024 09:06:46 -0400 Subject: [PATCH 5/7] Bump version to 6.20.8 --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65094c9a..88f53a41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 6.20.8 - Oct 18, 2024q + +* Improve resolve bill to capture at least the most recent session + ## 6.20.7 - Oct 10, 2024 * Fix matching committee organizations when chamber is specified for an organizational bill sponsor diff --git a/pyproject.toml b/pyproject.toml index d2419e80..18cb49bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openstates" -version = "6.20.7" +version = "6.20.8" description = "core infrastructure for the openstates project" authors = ["James Turk "] license = "MIT" From d9de13bbf997547803f3eb383ef57007005cf4ca Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Fri, 18 Oct 2024 15:46:41 -0400 Subject: [PATCH 6/7] Return value for get_all_session() --- openstates/importers/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index 6473dd6b..47fde7f0 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -140,11 +140,12 @@ def __init__(self, jurisdiction_id: str, do_postimport=True) -> None: if settings.IMPORT_TRANSFORMERS.get(self._type): self.cached_transformers = settings.IMPORT_TRANSFORMERS[self._type] - def get_all_sessions(self) -> None: + def get_all_sessions(self) -> typing.List[LegislativeSession]: if not self.all_sessions_cache: self.all_sessions_cache = LegislativeSession.objects.filter( jurisdiction_id=self.jurisdiction_id ).order_by("-start_date") + return self.all_sessions_cache def get_session(self, identifier: str) -> LegislativeSession: if identifier not in self.session_cache: @@ -173,7 +174,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: bill_transform_func = settings.IMPORT_TRANSFORMERS.get("bill", {}).get( "identifier", None ) - self.get_all_sessions() + all_sessions = self.get_all_sessions() if bill_transform_func: bill_id = bill_transform_func(bill_id) @@ -182,7 +183,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: date = datetime.fromisoformat(date).strftime("%Y-%m-%d") session_ids = [ session.id - for session in self.all_sessions_cache + for session in all_sessions if session.start_date <= date and (session.end_date >= date or not session.end_date) ] @@ -190,7 +191,7 @@ def resolve_bill(self, bill_id: str, *, date: str) -> typing.Optional[_ID]: if len(session_ids) == 1: session_id = session_ids.pop() else: - session_id = self.all_sessions_cache[0].id if self.all_sessions_cache else None + session_id = all_sessions[0].id if all_sessions else None objects = Bill.objects.filter( legislative_session__id=session_id, From ee73cf2ada8a52297390b5113a00aa062b3d106f Mon Sep 17 00:00:00 2001 From: alexobaseki Date: Fri, 18 Oct 2024 16:03:36 -0400 Subject: [PATCH 7/7] Add some comment --- openstates/importers/base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openstates/importers/base.py b/openstates/importers/base.py index 47fde7f0..23bea243 100644 --- a/openstates/importers/base.py +++ b/openstates/importers/base.py @@ -128,6 +128,9 @@ def __init__(self, jurisdiction_id: str, do_postimport=True) -> None: self.pseudo_id_cache: typing.Dict[str, typing.Optional[_ID]] = {} self.person_cache: typing.Dict[_PersonCacheKey, typing.Optional[str]] = {} self.session_cache: typing.Dict[str, LegislativeSession] = {} + # Get all_session_cache is a list of all sessions available for this jurisdiction. + # It is different from session_cache: which is a dictionary session(s) that is loaded a session + # session_cache may not contain all jurisdiction legislative sessions while all_session_cache will. self.all_sessions_cache: typing.List[LegislativeSession] = [] self.logger = logging.getLogger("openstates") self.info = self.logger.info