Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add organization_classification to resolve_person context #140

Merged
merged 8 commits into from
Sep 16, 2024
Merged

Add organization_classification to resolve_person context #140

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
21d8e9a
Add organization_classification to resolve_person context
alexobaseki Aug 30, 2024
2256024
Update test to use an actual jurisdiction_id
alexobaseki Aug 30, 2024
da35821
Update chamber name lookup
alexobaseki Aug 30, 2024
5613565
Test event name is a tuple??
alexobaseki Aug 30, 2024
aee6111
add second jurisdiction id
alexobaseki Aug 30, 2024
0ed7718
Update typing hint
alexobaseki Sep 3, 2024
448f5c2
Improve get_chamber_name logic
alexobaseki Sep 4, 2024
d574ac6
Split first word from title
alexobaseki Sep 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions openstates/importers/events.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import typing

from .base import BaseImporter
from ._types import _JsonDict
from ..utils import get_pseudo_id
Expand All @@ -13,6 +15,7 @@
)
from .organizations import OrganizationImporter
from .vote_events import VoteEventImporter
from ..metadata import lookup


class EventImporter(BaseImporter):
Expand Down Expand Up @@ -71,9 +74,24 @@ def get_location(self, location_data: _JsonDict) -> EventLocation:
)
return obj

def get_chamber_name_from_event_name(self, name: str) -> typing.Union[str, None]:
if isinstance(name, tuple):
possible_chamber_name = name[0].split()[0].lower()
alexobaseki marked this conversation as resolved.
Show resolved Hide resolved
else:
possible_chamber_name = name.split()[0].lower()

state = lookup(jurisdiction_id=self.jurisdiction_id)

if state.lower and state.lower.name.lower() == possible_chamber_name:
return state.lower.chamber_type
elif state.upper and state.upper.name.lower() == possible_chamber_name:
return state.upper.chamber_type
return None

def prepare_for_db(self, data: _JsonDict) -> _JsonDict:
data["jurisdiction_id"] = self.jurisdiction_id
data["location"] = self.get_location(data["location"])
org_classification = self.get_chamber_name_from_event_name(data["name"])

# all objects being inserted should be non-deleted
data["deleted"] = False
Expand All @@ -88,7 +106,9 @@ def prepare_for_db(self, data: _JsonDict) -> _JsonDict:
participant["person_id"] = participant.pop("legislator_id")

if "person_id" in participant:
participant["person_id"] = self.resolve_person(participant["person_id"])
participant["person_id"] = self.resolve_person(
participant["person_id"], org_classification=org_classification
)
elif "organization_id" in participant:
participant["organization_id"] = self.org_importer.resolve_json_id(
participant["organization_id"], allow_no_match=True
Expand All @@ -97,7 +117,9 @@ def prepare_for_db(self, data: _JsonDict) -> _JsonDict:
for item in data["agenda"]:
for entity in item["related_entities"]:
if "person_id" in entity:
entity["person_id"] = self.resolve_person(entity["person_id"])
entity["person_id"] = self.resolve_person(
entity["person_id"], org_classification=org_classification
)
elif "organization_id" in entity:
entity["organization_id"] = self.org_importer.resolve_json_id(
entity["organization_id"], allow_no_match=True
Expand Down
71 changes: 37 additions & 34 deletions openstates/importers/tests/test_event_importer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,19 @@
Event,
)

jid = "ocd-jurisdiction/country:us/state:ne/government"
ojid = "ocd-jurisdiction/country:us/state:ca/government"


def create_jurisdiction():
Division.objects.create(id="ocd-division/country:us", name="USA")
j = Jurisdiction.objects.create(id="jid", division_id="ocd-division/country:us")
j = Jurisdiction.objects.create(id=jid, division_id="ocd-division/country:us")
return j


def create_other_jurisdiction():
Division.objects.create(id="ocd-division/country:ca", name="USA")
j = Jurisdiction.objects.create(id="ojid", division_id="ocd-division/country:ca")
j = Jurisdiction.objects.create(id=ojid, division_id="ocd-division/country:ca")
return j


Expand All @@ -39,8 +42,8 @@ def ge():
return event


bi = BillImporter("jid")
vei = VoteEventImporter("jid", bi)
bi = BillImporter(jid)
vei = VoteEventImporter(jid, bi)


@pytest.mark.django_db
Expand All @@ -49,7 +52,7 @@ def test_related_people_event():
george = Person.objects.create(id="gw", name="George Washington")
john = Person.objects.create(id="jqp", name="John Q. Public")
o = Organization.objects.create(
name="Merica", jurisdiction_id="jid", classification="legislature"
name="Merica", jurisdiction_id=jid, classification="legislature"
)

Membership.objects.create(person=george, organization=o)
Expand All @@ -63,10 +66,10 @@ def test_related_people_event():
item.add_person(person="John Q. Public")
event.add_person("George Washington")

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1

assert (
Expand Down Expand Up @@ -106,10 +109,10 @@ def test_related_vote_event():
item = event.add_agenda_item("Cookies will be served")
item.add_vote_event(vote_event="Roll no. 12")

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1

assert (
Expand All @@ -136,10 +139,10 @@ def test_related_bill_event():
item = event.add_agenda_item("Cookies will be served")
item.add_bill(bill="HB 101")

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1

assert (
Expand Down Expand Up @@ -168,7 +171,7 @@ def test_related_bill_event_duplicate_id_use_session():
item = event.add_agenda_item("Cookies will be served")
item.add_bill(bill="HB 101")

result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1

assert (
Expand Down Expand Up @@ -202,10 +205,10 @@ def test_related_committee_event():
item = event.add_agenda_item("Cookies will be served")
item.add_committee(committee="Fiscal Committee")

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1

assert (
Expand All @@ -231,10 +234,10 @@ def test_media_event():
url="http://hello.world/foo",
)

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1


Expand All @@ -249,33 +252,33 @@ def test_media_document():
note="Presentation", url="http://example.com/presentation.pdf"
)

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1


@pytest.mark.django_db
def test_full_event():
create_jurisdiction()
george = Person.objects.create(id="gw", name="George Washington")
o = Organization.objects.create(name="Merica", jurisdiction_id="jid")
o = Organization.objects.create(name="Merica", jurisdiction_id=jid)
Membership.objects.create(person=george, organization=o)

event = ge()

result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1

event = ge()

result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["noop"] == 1

event = ge()
event.location["name"] = "United States of America"
result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["update"] == 1


Expand All @@ -284,28 +287,28 @@ def test_dedupe_key_event():
create_jurisdiction()
create_other_jurisdiction()
george = Person.objects.create(id="gw", name="George Washington")
o = Organization.objects.create(name="Merica", jurisdiction_id="jid")
o = Organization.objects.create(name="Merica", jurisdiction_id=jid)
Membership.objects.create(person=george, organization=o)

event = ge()
event.dedupe_key = "foo"

result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["noop"] == 1

event.name = ("America's Anniversary",)
event.location["name"] = "United States of America"
result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["update"] == 1

event.dedupe_key = "bar"
result = EventImporter("jid", vei).import_data([event.as_dict()])
result = EventImporter(jid, vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("ojid", vei).import_data([event.as_dict()])
result = EventImporter(ojid, vei).import_data([event.as_dict()])
assert result["event"]["insert"] == 1


Expand Down Expand Up @@ -337,10 +340,10 @@ def test_top_level_media_event():
media_type="application/octet-stream",
)

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

result = EventImporter("jid", vei).import_data([event2.as_dict()])
result = EventImporter(jid, vei).import_data([event2.as_dict()])
assert result["event"]["noop"] == 1


Expand All @@ -352,7 +355,7 @@ def test_event_agenda_item():
agenda = event1.add_agenda_item("first item")
agenda["extras"] = {"one": 1, "two": [2]}

result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["insert"] == 1

e = Event.objects.get()
Expand All @@ -366,19 +369,19 @@ def test_event_soft_deletion():
event1 = ge()
event2 = ge()
event2.name = "Other Event"
result = EventImporter("jid", vei).import_data([event1.as_dict(), event2.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict(), event2.as_dict()])
assert result["event"]["insert"] == 2
assert Event.objects.count() == 2

# delete
result = EventImporter("jid", vei).import_data([event1.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict()])
assert result["event"]["noop"] == 1
# TODO: assert result["event"]["deleted"] == 1
assert Event.objects.count() == 2
assert Event.objects.get(deleted=True).name == "Other Event"

# undelete
result = EventImporter("jid", vei).import_data([event1.as_dict(), event2.as_dict()])
result = EventImporter(jid, vei).import_data([event1.as_dict(), event2.as_dict()])
assert result["event"]["update"] == 1
assert result["event"]["noop"] == 1
assert Event.objects.count() == 2
Expand Down
Loading