From 83e8701698bbb19adc0443ddccd93cecb2bd30cd Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 12:53:46 +0200 Subject: [PATCH 01/11] Fix broken CRON --- app.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.json b/app.json index 1540dab3..166d4b2c 100644 --- a/app.json +++ b/app.json @@ -1,8 +1,8 @@ { "cron": [ { - "command": "curl api.pmg.org.za/v2/bill-tracker/update/ > /var/log/cron-bill-tracker", - "schedule": "0 1 * * *" + "command": "curl https://api.pmg.org.za/v2/bill-tracker/update/", + "schedule": "@daily" } ] } \ No newline at end of file From a707b94a3f6bb204b5c9e26e86949c607ee43c19 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 12:55:03 +0200 Subject: [PATCH 02/11] Add Bill Tracker update request to post-deploy task so transient file is re-generated --- app.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app.json b/app.json index 166d4b2c..3c1107db 100644 --- a/app.json +++ b/app.json @@ -4,5 +4,8 @@ "command": "curl https://api.pmg.org.za/v2/bill-tracker/update/", "schedule": "@daily" } - ] + ], + "scripts": { + "postdeploy": "curl https://api.pmg.org.za/v2/bill-tracker/update/" + } } \ No newline at end of file From f3c8839986cc9348c9d19440244c976ed17d60fa Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 13:07:00 +0200 Subject: [PATCH 03/11] Basic postdeploy script did not work. Trying dokku.postdeploy script. --- app.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app.json b/app.json index 3c1107db..fa2e0de1 100644 --- a/app.json +++ b/app.json @@ -1,11 +1,13 @@ { + "scripts": { + "dokku": { + "postdeploy": "curl https://api.pmg.org.za/v2/bill-tracker/update/" + } + }, "cron": [ { "command": "curl https://api.pmg.org.za/v2/bill-tracker/update/", "schedule": "@daily" } - ], - "scripts": { - "postdeploy": "curl https://api.pmg.org.za/v2/bill-tracker/update/" - } + ] } \ No newline at end of file From 69520457b73ec22ee4d140cd299c6a094a52722c Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 23:26:28 +0200 Subject: [PATCH 04/11] Fallbacks for old event committee meetings without linked committee --- pmg/templates/committee_layout.html | 17 ++++++++++++++--- pmg/templates/committee_meeting.html | 6 +++++- pmg/views.py | 8 ++++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pmg/templates/committee_layout.html b/pmg/templates/committee_layout.html index c0a60738..5832559b 100644 --- a/pmg/templates/committee_layout.html +++ b/pmg/templates/committee_layout.html @@ -1,16 +1,25 @@ {% extends "base.html" %} -{% block title %}{{ committee.name }} | PMG{% endblock %} +{% block title %} + {% if committee %} + {{ committee.name }} | + {% endif %} + PMG +{% endblock %} {% block breadcrumb %}
    {% block committee_breadcrumb %} {% if not from_page %}
  1. Committees
    2. -
  2. {{ committee.house.name }}
    3. + {% if committee %} +
  3. {{ committee.house.name }}
    4. + {% endif %} {% else %}
  4. Provincial Legislatures
    5. -
  5. {{ committee.house.name }}
    6. + {% if committee %} +
  6. {{ committee.house.name }}
    7. + {% endif %}
  7. Committees
    8. {% endif %} {% endblock %} @@ -22,6 +31,7 @@
    {% block committee_title %} + {% if committee %}
    {% if not committee.premium %}

    {{ committee.name }}

    @@ -36,6 +46,7 @@

    {{ committee.house.name }}

    {{ committee.house.name }} Committee

    {% endif %}
    {% include '_social_sharing.html' %}
    + {% endif %} {% endblock %}
    diff --git a/pmg/templates/committee_meeting.html b/pmg/templates/committee_meeting.html index 7044e71e..be1111ac 100644 --- a/pmg/templates/committee_meeting.html +++ b/pmg/templates/committee_meeting.html @@ -5,7 +5,9 @@ {% block committee_breadcrumb %} {{ super() }} -
  8. {{ committee.name }}
    9. + {% if committee %} +
  9. {{ committee.name }}
    10. + {% endif %} {% endblock %} {% block javascript %} @@ -20,7 +22,9 @@

    {{ event.title }}

    {% if event.premium_but_free %}
    This premium content has been made freely available
    {% endif %} + {% if committee %}

    {{ committee.name }}

    + {% endif %} {% if event.date %}
    {{ event.date | pretty_date("long") }}
    {% endif %} diff --git a/pmg/views.py b/pmg/views.py index cff788b5..a7af0537 100644 --- a/pmg/views.py +++ b/pmg/views.py @@ -799,10 +799,14 @@ def committee_meeting(event_id): ) + sorted( [a for a in attendance if not a["chairperson"]], key=sorter ) # noqa + if event["committee"]: + event_committee_name = event["committee"]["name"] + else: + event_committee_name = "[UNKNOWN COMMITTEE]" if event["chairperson"]: social_summary = ( "A meeting of the " - + event["committee"]["name"] + + event_committee_name + " committee held on " + pretty_date(event["date"], "long") + ", lead by " @@ -811,7 +815,7 @@ def committee_meeting(event_id): else: social_summary = ( "A meeting of the " - + event["committee"]["name"] + + event_committee_name + " committee held on " + pretty_date(event["date"], "long") + "." From 5a3cb7129a1b40ddfba060bbca0609255073150d Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 23:53:42 +0200 Subject: [PATCH 05/11] Fallback for committee question with no asked by name --- pmg/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmg/views.py b/pmg/views.py index a7af0537..0097ed15 100644 --- a/pmg/views.py +++ b/pmg/views.py @@ -601,13 +601,17 @@ def committee_question(question_id): question = load_from_api("v2/minister-questions", question_id)["result"] minister = question["minister"] committee = minister.get("committee", {"house": {}, "id": 0}) + if question["asked_by_name"]: + asked_by_name = question["asked_by_name"] + else: + asked_by_name = "" social_summary = ( "A question to the " + question["question_to_name"] + ", asked on " + pretty_date(question["date"], "long") + " by " - + question["asked_by_name"] + + asked_by_name ) return render_template( From 55d1e84187a693dd2c9dc607792f343af8dd3bd1 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Wed, 14 Aug 2024 23:54:28 +0200 Subject: [PATCH 06/11] Fallback for committee question with no asked by name --- pmg/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmg/views.py b/pmg/views.py index 0097ed15..2c12acb2 100644 --- a/pmg/views.py +++ b/pmg/views.py @@ -604,7 +604,7 @@ def committee_question(question_id): if question["asked_by_name"]: asked_by_name = question["asked_by_name"] else: - asked_by_name = "" + asked_by_name = "[UNKNOWN]" social_summary = ( "A question to the " + question["question_to_name"] From dbdfe47015c01db592645c389d8b912b73648951 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Thu, 15 Aug 2024 11:34:49 +0200 Subject: [PATCH 07/11] Do not try to show file bytes size if not available --- docker-compose.yml | 2 +- pmg/templates/bills/detail.html | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 997fa597..1081004e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: volumes: - .:/app environment: - - FLASK_DEBUG_CACHE=true + - FLASK_DEBUG_CACHE=false - REDIS_URL=redis://redis:6379/0 - SQLALCHEMY_DATABASE_URI=postgresql://pmg:pmg@postgres/pmg?client_encoding=utf8 - ES_SERVER=http://elastic:9200 diff --git a/pmg/templates/bills/detail.html b/pmg/templates/bills/detail.html index 40b2135d..1c3c973f 100644 --- a/pmg/templates/bills/detail.html +++ b/pmg/templates/bills/detail.html @@ -152,13 +152,13 @@

    Bill Text

    {% for version in bill.versions %}
    - Download ({{ version.file.file_bytes | filesizeformat }}) + Download{% if (version.file.file_bytes) %} ({{ (version.file.file_bytes | filesizeformat) }}) {% endif %}

    {{ version.title }}

    {% if version.file.file_mime == "application/pdf" %}
    - +
    {% else %}

    A preview of this file is not available. Please download it instead.

    From 63be406383d9ada468a0615b153ad43737b888d4 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Thu, 15 Aug 2024 11:57:43 +0200 Subject: [PATCH 08/11] Return None committee if there is none --- pmg/views.py | 79 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/pmg/views.py b/pmg/views.py index 2c12acb2..5c04b33b 100644 --- a/pmg/views.py +++ b/pmg/views.py @@ -20,7 +20,7 @@ redirect, Response, jsonify, - send_from_directory + send_from_directory, ) from flask_security import current_user from flask_mail import Message @@ -77,7 +77,7 @@ def page_not_found(error): dest = Redirect.for_url(request.path) if dest: return redirect(dest, code=302) - + return render_template("404.html"), 404 @@ -113,7 +113,7 @@ def shortcircuit_wget(): @app.before_request def redirect_legacy_domains(): - """ Redirect legacy domains to the primary domain. """ + """Redirect legacy domains to the primary domain.""" parts = urlparse(request.url) if parts.netloc in LEGACY_DOMAINS: parts = list(parts) @@ -129,7 +129,7 @@ def update_last_login(): def classify_attachments(files): - """ Return an (audio_files, related_docs) tuple. """ + """Return an (audio_files, related_docs) tuple.""" audio = [] related = [] @@ -141,6 +141,7 @@ def classify_attachments(files): return audio, related + @cache.cached(timeout=10800, key_prefix="get_featured_content") def get_featured_content(): info = {} @@ -415,15 +416,19 @@ def committee_detail(committee_id): current_attendance_summary = CommitteeMeetingAttendance.committee_attendence_trends( committee_id, "current" ) - historical_attendance_summary = CommitteeMeetingAttendance.committee_attendence_trends( - committee_id, "historical" + historical_attendance_summary = ( + CommitteeMeetingAttendance.committee_attendence_trends( + committee_id, "historical" + ) ) if current_attendance_summary and committee["house"]["short_name"] != "Joint": year = current_attendance_summary[-1].year cte = Committee.query.get(committee_id) - attendance_rank = CommitteeMeetingAttendance.annual_attendance_rank_for_committee( - cte, int(year) + attendance_rank = ( + CommitteeMeetingAttendance.annual_attendance_rank_for_committee( + cte, int(year) + ) ) else: attendance_rank = None @@ -596,18 +601,24 @@ def attendance_overview(): @app.route("/committee-question//") def committee_question(question_id): - """ Display a single committee question. - """ + """Display a single committee question.""" question = load_from_api("v2/minister-questions", question_id)["result"] minister = question["minister"] - committee = minister.get("committee", {"house": {}, "id": 0}) + if minister: + committee = minister.get("committee", {"house": {}, "id": 0}) + else: + committee = None + if question["question_to_name"]: + question_to_name = question["question_to_name"] + else: + question_to_name = "[UNKNOWN]" if question["asked_by_name"]: asked_by_name = question["asked_by_name"] else: asked_by_name = "[UNKNOWN]" social_summary = ( "A question to the " - + question["question_to_name"] + + question_to_name + ", asked on " + pretty_date(question["date"], "long") + " by " @@ -824,21 +835,19 @@ def committee_meeting(event_id): + pretty_date(event["date"], "long") + "." ) - - return render_template( - "committee_meeting.html", - event=event, - committee=event["committee"], - audio=audio, - related_docs=related_docs, - attendance=attendance, - premium_committees=premium_committees, - content_date=event["date"], - social_summary=social_summary, - admin_edit_url=admin_url("committee-meeting", event_id), - SOUNDCLOUD_APP_KEY_ID=app.config["SOUNDCLOUD_APP_KEY_ID"] + "committee_meeting.html", + event=event, + committee=event["committee"], + audio=audio, + related_docs=related_docs, + attendance=attendance, + premium_committees=premium_committees, + content_date=event["date"], + social_summary=social_summary, + admin_edit_url=admin_url("committee-meeting", event_id), + SOUNDCLOUD_APP_KEY_ID=app.config["SOUNDCLOUD_APP_KEY_ID"], ) @@ -1122,8 +1131,7 @@ def gazette(gazette_id): @app.route("/members/") def members(): - """ All MPs. - """ + """All MPs.""" members = load_from_api("v2/members", return_everything=True)["results"] # partition by house @@ -2007,18 +2015,25 @@ def stats_review(stat): } return render_template(stat_group[stat]) + @app.route("/6th-parliament-review", methods=["GET"]) def pr6(): return render_template("pr6/landing.html") + @app.route("/6th-parliament-review/
    /", methods=["GET"]) -def pr6_articles(section,slug): - return render_template("pr6/article.html",section=section, article=slug) +def pr6_articles(section, slug): + return render_template("pr6/article.html", section=section, article=slug) + -@app.route('/favicon.ico') +@app.route("/favicon.ico") def favicon(): - return send_from_directory(os.path.join(app.root_path, 'static'), - 'favicon.ico', mimetype='image/vnd.microsoft.icon') + return send_from_directory( + os.path.join(app.root_path, "static"), + "favicon.ico", + mimetype="image/vnd.microsoft.icon", + ) + # Test to make sure sentry is working @app.route("/debug-sentry") From bc9713a4d6c15c1977daf284ee2324a58840831d Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Thu, 15 Aug 2024 12:56:02 +0200 Subject: [PATCH 09/11] Restored email template deleted in 2022 --- pmg/templates/saved_search_alert.html | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 pmg/templates/saved_search_alert.html diff --git a/pmg/templates/saved_search_alert.html b/pmg/templates/saved_search_alert.html new file mode 100644 index 00000000..965e8d1f --- /dev/null +++ b/pmg/templates/saved_search_alert.html @@ -0,0 +1,32 @@ +

    +Hello, +

    + +

    +We found {{ results|length }} recently updated item{% if results|length > 1 %}s{% endif %} for your search +{{ search.search }}{% +if search.content_type %} in {{ search.friendly_content_type }}{% endif %}: +

    + +
      +{% for result in results %} +
    • +

      {{ result._source.title }} - {{ result._source.date|pretty_date }}

      + {% if result.highlight %} +

      + {% if result.highlight.description %} + {{ result.highlight.description|search_snippet(mark='b')|ellipse|safe }} + {% elif result.highlight.fulltext %} + {{ result.highlight.fulltext|search_snippet(mark='b')|ellipse|safe }} + {% elif result.highlight.attachments %} + {{ result.highlight.attachments|search_snippet(mark='b')|ellipse|safe }} + {% endif %} +

      + {% endif %} +
      • +{% endfor %} +
    + +

    +Click here to manage your email alerts. +

    \ No newline at end of file From efa38d820aec1b956159beebd64f3b53c5a99c82 Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Mon, 19 Aug 2024 12:28:39 +0200 Subject: [PATCH 10/11] Add api.v2/elasticsearch-health route helper, like PA has --- pmg/api/v2.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pmg/api/v2.py b/pmg/api/v2.py index eb95c6ea..49e7d806 100644 --- a/pmg/api/v2.py +++ b/pmg/api/v2.py @@ -4,6 +4,8 @@ from sqlalchemy.orm import defer, noload from sqlalchemy.sql.expression import nullslast from pmg.bill_tracker import produce_bill_tracker_json +from pmg import app +import requests from pmg import cache, cache_key, should_skip_cache from pmg.models import ( @@ -265,3 +267,10 @@ def bill_tracker(): def bill_tracker_update(): produce_bill_tracker_json() return "/v2/bill-tracker JSON updated" + + +@api.route("/elasticsearch-health") +def elasticsearch_health(): + r = requests.get("%s/_cluster/health?pretty=true" % app.config["ES_SERVER"]) + r.raise_for_status() + return r.json() From faedbc7d13b4fc3df9053b455110a40fad21f91f Mon Sep 17 00:00:00 2001 From: paulmwatson Date: Tue, 20 Aug 2024 11:52:18 +0200 Subject: [PATCH 11/11] return [] if exception when getting version file bytes size --- pmg/models/resources.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pmg/models/resources.py b/pmg/models/resources.py index a4f3df82..c2d5a0bc 100644 --- a/pmg/models/resources.py +++ b/pmg/models/resources.py @@ -187,7 +187,10 @@ def latest_version_for_indexing(self): if not version: # don't return None return [] - return base64.encodestring(version.file.get_bytes()) + try: + return base64.encodestring(version.file.get_bytes()) + except AttributeError as e: + return [] def to_dict(self, include_related=False): tmp = serializers.model_to_dict(self, include_related=include_related)