From 2dc19771ad9b98e50e7d986ae641ca938d30eb8e Mon Sep 17 00:00:00 2001 From: Sym Roe Date: Wed, 29 May 2024 18:35:01 +0100 Subject: [PATCH] Allow swapping EE backends based on USE_LOCAL_PARQUET_ELECTIONS --- polling_stations/apps/api/postcode.py | 6 +- .../data_finder/helpers/baked_data_helper.py | 73 +++++++++---------- .../data_finder/helpers/every_election.py | 29 ++++++-- .../apps/data_finder/helpers/routing.py | 25 +++++-- polling_stations/apps/data_finder/views.py | 3 +- polling_stations/settings/base.py | 1 + .../settings/constants/elections.py | 11 ++- polling_stations/settings/testing.py | 3 - 8 files changed, 92 insertions(+), 59 deletions(-) diff --git a/polling_stations/apps/api/postcode.py b/polling_stations/apps/api/postcode.py index c85ec074e8..6973de9530 100644 --- a/polling_stations/apps/api/postcode.py +++ b/polling_stations/apps/api/postcode.py @@ -1,5 +1,4 @@ from data_finder.helpers import ( - EveryElectionWrapper, PostcodeError, RoutingHelper, geocode, @@ -7,7 +6,6 @@ ) from data_finder.helpers.every_election import ( EmptyEveryElectionWrapper, - StaticElectionsAPIElectionWrapper, ) from data_finder.views import LogLookUpMixin, polling_station_current from django.core.exceptions import ObjectDoesNotExist @@ -61,12 +59,12 @@ def get_ee_wrapper(self, postcode, rh, query_params): if rh.route_type == "multiple_addresses": return EmptyEveryElectionWrapper() if rh.elections_response: - return StaticElectionsAPIElectionWrapper(rh.elections_response) + return rh.elections_backend.ee_wrapper(rh.elections_response) kwargs = {} query_params = parse_qs_to_python(query_params) if include_current := query_params.get("include_current", False): kwargs["include_current"] = any(include_current) - return EveryElectionWrapper(postcode, **kwargs) + return rh.elections_backend.ee_wrapper(postcode, **kwargs) @extend_schema( parameters=[ diff --git a/polling_stations/apps/data_finder/helpers/baked_data_helper.py b/polling_stations/apps/data_finder/helpers/baked_data_helper.py index d11e64e22e..6e9dedf2c2 100644 --- a/polling_stations/apps/data_finder/helpers/baked_data_helper.py +++ b/polling_stations/apps/data_finder/helpers/baked_data_helper.py @@ -1,10 +1,12 @@ import abc import enum from pathlib import Path -from typing import List, Optional, Tuple, Type +from typing import List, Tuple from urllib.parse import urljoin import polars +from data_finder.helpers import EveryElectionWrapper +from data_finder.helpers.every_election import StaticElectionsAPIElectionWrapper from django.conf import settings from requests import Session from uk_geo_utils.helpers import Postcode @@ -25,7 +27,7 @@ def ballot_paper_id_to_static_url(ballot_paper_id): return urljoin(settings.WCIVF_BALLOT_CACHE_URL, path) -class BaseBakedElectionsStrategy(abc.ABC): +class BaseBakedElectionsHelper(abc.ABC): def __init__(self, **kwargs): ... @@ -34,25 +36,23 @@ def get_response_for_postcode(self, postcode: Postcode): raise NotImplementedError -class RemoteBakedElectionsStrategy(BaseBakedElectionsStrategy): - def __init__(self, api_key=None, **kwargs): - self.api_key = api_key or getattr(settings, "DEVS_DC_API_KEY", None) - if not self.api_key: - raise ValueError("API key required for remote backend") - super().__init__(**kwargs) +class NoOpElectionsHelper(BaseBakedElectionsHelper): + """ + Just returns nothing, causing the app to use EE as it previously did. + + This maintains the previous default behaviour for looking up elections + + """ + + ee_wrapper = EveryElectionWrapper def get_response_for_postcode(self, postcode: Postcode): - # TODO: pull into a constant - devs_dc_base = "https://developers.democracyclub.org.uk" - req = session.get( - urljoin(devs_dc_base, f"/api/v1/elections/postcode/{postcode.with_space}/"), - params={"auth_token": self.api_key}, - ) - req.raise_for_status() - return req.json() + return {} -class LocalParquetElectionsStrategy(BaseBakedElectionsStrategy): +class LocalParquetElectionsHelper(BaseBakedElectionsHelper): + ee_wrapper = StaticElectionsAPIElectionWrapper + def __init__(self, elections_parquet_path: Path = None, **kwargs): self.elections_parquet_path = elections_parquet_path or getattr( settings, "ELECTION_PARQUET_DATA_PATH", None @@ -144,27 +144,20 @@ def get_ballot_list(self, postcode: Postcode) -> Tuple[bool, List]: return is_split, df["current_elections"][0].split(",") -class BakedElectionsHelper: - """ - Helper to get election data from "baked" data: a cut of EveryElection that's been - joined with AddressBase and stored in parquet files. - - We host these files and proxy them via an API. - - This helper allows us to consume the data either via parquet files or the API - - """ - - def __init__( - self, - strategy: Optional[ - Type[BaseBakedElectionsStrategy] - ] = LocalParquetElectionsStrategy, - **kwargs, - ): - self.strategy = strategy(**kwargs) - +class RemoteBakedElectionsHelper(BaseBakedElectionsHelper): + def __init__(self, api_key=None, **kwargs): + self.api_key = api_key or getattr(settings, "DEVS_DC_API_KEY", None) + if not self.api_key: + raise ValueError("API key required for remote backend") + super().__init__(**kwargs) -if __name__ == "__main__": - helper = BakedElectionsHelper(strategy=None, api_key="asd") - helper.strategy.get_response_for_postcode(Postcode("GL5 1NA")) + def get_response_for_postcode(self, postcode: Postcode): + req = session.get( + urljoin( + settings.DEVS_DC_BASE, + f"/api/v1/elections/postcode/{postcode.with_space}/", + ), + params={"auth_token": self.api_key}, + ) + req.raise_for_status() + return req.json() diff --git a/polling_stations/apps/data_finder/helpers/every_election.py b/polling_stations/apps/data_finder/helpers/every_election.py index c1d141b8a4..e21bce549a 100644 --- a/polling_stations/apps/data_finder/helpers/every_election.py +++ b/polling_stations/apps/data_finder/helpers/every_election.py @@ -1,4 +1,5 @@ import datetime +from functools import cached_property from typing import List, Optional from urllib.parse import urlencode, urljoin @@ -312,6 +313,19 @@ def get_ballots_for_next_date() -> List: def get_all_ballots() -> List: return [] + @staticmethod + def multiple_elections(): + return False + + def get_explanations(self): + return None + + def get_voter_id_status(self): + return None + + def get_cancelled_election_info(self): + return {} + class StaticElectionsAPIElectionWrapper: def __init__(self, elections_response): @@ -323,7 +337,6 @@ def __init__(self, elections_response): # so rename it back to what EE calls it ballot["election_title"] = ballot["election_name"] self.ballots.append(ballot) - ... def has_election(self, future_only=True): if not self.elections_response["dates"]: @@ -367,12 +380,18 @@ def get_cancelled_election_info(self): if not rec["cancelled"]: return rec - cancelled_ballot = self.cancelled_ballots[0] - if len(self.cancelled_ballots) == 1: - rec["name"] = cancelled_ballot["election_title"] - rec["metadata"] = cancelled_ballot["metadata"] + cancelled_ballots = self.cancelled_ballots + if cancelled_ballots: + cancelled_ballot = cancelled_ballots[0] + if len(cancelled_ballots) == 1: + rec["name"] = cancelled_ballot["election_title"] + rec["metadata"] = cancelled_ballot["metadata"] return rec + @cached_property + def cancelled_ballots(self): + return [b for b in self.ballots if b["cancelled"]] + def get_voter_id_status(self) -> Optional[str]: """ For a given election, determine whether any ballots require photo ID diff --git a/polling_stations/apps/data_finder/helpers/routing.py b/polling_stations/apps/data_finder/helpers/routing.py index a801243e83..e508639fa8 100644 --- a/polling_stations/apps/data_finder/helpers/routing.py +++ b/polling_stations/apps/data_finder/helpers/routing.py @@ -4,7 +4,11 @@ # use a postcode to decide which endpoint the user should be directed to from councils.models import CouncilGeography -from data_finder.helpers.baked_data_helper import BakedElectionsHelper +from data_finder.helpers.baked_data_helper import ( + LocalParquetElectionsHelper, + NoOpElectionsHelper, +) +from django.conf import settings from django.urls import reverse from django.utils.functional import cached_property from uk_geo_utils.helpers import Postcode @@ -21,7 +25,13 @@ class RoutingHelper: def __init__(self, postcode): self.postcode = Postcode(postcode) self.addresses = self.get_addresses() - self.elections_response = None + self.elections_backend = self.get_elections_backend() + self._elections_response = None + + def get_elections_backend(self): + if getattr(settings, "USE_LOCAL_PARQUET_ELECTIONS", False): + return LocalParquetElectionsHelper + return NoOpElectionsHelper def get_addresses(self): return Address.objects.filter(postcode=self.postcode.with_space).select_related( @@ -82,12 +92,17 @@ def addresses_have_single_station(self): return bool(list(self.polling_stations)[0]) return False + @property + def elections_response(self): + if not self._elections_response: + self.lookup_elections() + return self._elections_response + def lookup_elections(self): - helper = BakedElectionsHelper() - self.elections_response = helper.strategy.get_response_for_postcode( + self._elections_response = self.elections_backend().get_response_for_postcode( self.postcode ) - return self.elections_response + return self._elections_response @property def split_elections(self): diff --git a/polling_stations/apps/data_finder/views.py b/polling_stations/apps/data_finder/views.py index 24a62be85d..38c1653f44 100644 --- a/polling_stations/apps/data_finder/views.py +++ b/polling_stations/apps/data_finder/views.py @@ -300,7 +300,6 @@ def get_ee_wrapper(self, rh=None): class ExamplePostcodeView(BasePollingStationView): - """ This class presents a hard-coded example of what our website does without having to worry about having any data imported @@ -308,6 +307,7 @@ class ExamplePostcodeView(BasePollingStationView): """ def get(self, request, *args, **kwargs): + kwargs["rh"] = RoutingHelper("BS4 4NL") context = self.get_context_data(**kwargs) return self.render_to_response(context) @@ -357,6 +357,7 @@ def get(self, request, *args, **kwargs): kwargs={"postcode": self.postcode.without_space}, ) ) + kwargs["rh"] = rh context = self.get_context_data(**kwargs) return self.render_to_response(context) diff --git a/polling_stations/settings/base.py b/polling_stations/settings/base.py index 4e8ba08497..ff497fe37b 100644 --- a/polling_stations/settings/base.py +++ b/polling_stations/settings/base.py @@ -395,6 +395,7 @@ def repo_root(*x): from .constants.tiles import * # noqa from .constants.uploads import * # noqa + # Import .local.py last - settings in local.py override everything else # only if we're not testing try: diff --git a/polling_stations/settings/constants/elections.py b/polling_stations/settings/constants/elections.py index fc474243e1..3dd841fa09 100644 --- a/polling_stations/settings/constants/elections.py +++ b/polling_stations/settings/constants/elections.py @@ -1,4 +1,5 @@ import os +from pathlib import Path EE_BASE = os.environ.get("EE_BASE", "https://elections.democracyclub.org.uk/") """ @@ -18,7 +19,15 @@ "local.epping-forest.moreton-and-fyfield.by.2018-05-03" # uncontested ] - NEXT_CHARISMATIC_ELECTION_DATES = [] SHOW_GB_ID_MESSAGING = True + +if data_path := os.environ.get("ELECTION_PARQUET_DATA_PATH", False): + ELECTION_PARQUET_DATA_PATH = Path(data_path) + USE_LOCAL_PARQUET_ELECTIONS = True +WCIVF_BALLOT_CACHE_URL = ( + "https://wcivf-ballot-cache.s3.eu-west-2.amazonaws.com/ballot_data/" +) + +DEVS_DC_BASE = os.environ.get("DEVS_DC_BASE", "https://developers.democracyclub.org.uk") diff --git a/polling_stations/settings/testing.py b/polling_stations/settings/testing.py index bc6cbac0c7..cb84e9c23c 100644 --- a/polling_stations/settings/testing.py +++ b/polling_stations/settings/testing.py @@ -2,7 +2,6 @@ from dc_logging_client import DCWidePostcodeLoggingClient import os - EVERY_ELECTION["CHECK"] = True # noqa NEXT_CHARISMATIC_ELECTION_DATES = [] DISABLE_GA = True # don't log to Google Analytics when we are running tests @@ -15,7 +14,6 @@ GOOGLE_API_KEYS = [] MAPBOX_API_KEY = "" - STATICFILES_STORAGE = "pipeline.storage.PipelineStorage" RUNNING_TESTS = True @@ -23,7 +21,6 @@ POSTCODE_LOGGER = DCWidePostcodeLoggingClient(fake=True) EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" - os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true" READ_ONLY_API_AUTH_TOKENS = [