Merge branch 'master' into move-renamed-committees-link

app.json

@@ -1,8 +1,13 @@
 {
+    "scripts": {
+        "dokku": {
+            "postdeploy": "curl https://api.pmg.org.za/v2/bill-tracker/update/"
+        }
+    },
     "cron": [
         {
-            "command": "curl api.pmg.org.za/v2/bill-tracker/update/ > /var/log/cron-bill-tracker",
-            "schedule": "0 1 * * *"
+            "command": "curl https://api.pmg.org.za/v2/bill-tracker/update/",
+            "schedule": "@daily"
         }
     ]
 }

docker-compose.yml

@@ -23,7 +23,7 @@ services:
     volumes:
       - .:/app
     environment:
-      - FLASK_DEBUG_CACHE=true
+      - FLASK_DEBUG_CACHE=false
       - REDIS_URL=redis://redis:6379/0
       - SQLALCHEMY_DATABASE_URI=postgresql://pmg:pmg@postgres/pmg?client_encoding=utf8
       - ES_SERVER=http://elastic:9200

pmg/api/v2.py

@@ -4,6 +4,8 @@
 from sqlalchemy.orm import defer, noload
 from sqlalchemy.sql.expression import nullslast
 from pmg.bill_tracker import produce_bill_tracker_json
+from pmg import app
+import requests
 
 from pmg import cache, cache_key, should_skip_cache
 from pmg.models import (
@@ -265,3 +267,10 @@ def bill_tracker():
 def bill_tracker_update():
     produce_bill_tracker_json()
     return "/v2/bill-tracker JSON updated"
+
+
+@api.route("/elasticsearch-health")
+def elasticsearch_health():
+    r = requests.get("%s/_cluster/health?pretty=true" % app.config["ES_SERVER"])
+    r.raise_for_status()
+    return r.json()

pmg/models/resources.py

@@ -187,7 +187,10 @@ def latest_version_for_indexing(self):
         if not version:
             # don't return None
             return []
-        return base64.encodestring(version.file.get_bytes())
+        try:
+            return base64.encodestring(version.file.get_bytes())
+        except AttributeError as e:
+            return []
 
     def to_dict(self, include_related=False):
         tmp = serializers.model_to_dict(self, include_related=include_related)

pmg/templates/bills/detail.html

@@ -152,13 +152,13 @@ <h2>Bill Text</h2>
     {% for version in bill.versions %}
     <div role="tabpanel" class="tab-pane {% if loop.first %}active{% endif %}" id="bill-version-{{ version.id }}" data-url="{{ version.file.url }}">
       <div class="clearfix">
-        <a href="{{ version.file.url }}" class="btn btn-default pull-right"><i class="fa fa-file-text"></i> Download ({{ version.file.file_bytes | filesizeformat }})</a>
+        <a href="{{ version.file.url }}" class="btn btn-default pull-right"><i class="fa fa-file-text"></i> Download{% if (version.file.file_bytes) %} ({{ (version.file.file_bytes | filesizeformat) }}) {% endif %}</a>
         <h4>{{ version.title }}</h4>
       </div>
 
       {% if version.file.file_mime == "application/pdf" %}
         <div class="bill-version-wrapper">
-          <button class="btn btn-primary load-pdf">Show PDF ({{ version.file.file_bytes | filesizeformat }})</button>
+          <button class="btn btn-primary load-pdf">Show PDF{% if (version.file.file_bytes) %} ({{ (version.file.file_bytes | filesizeformat) }}){% endif %}</button>
         </div>
       {% else %}
         <p class="alert alert-info"><i>A preview of this file is not available. Please download it instead.</i></p>

pmg/templates/committee_layout.html

@@ -1,16 +1,25 @@
 {% extends "base.html" %}
 
-{% block title %}{{ committee.name }} | PMG{% endblock %}
+{% block title %}
+  {% if committee %}
+    {{ committee.name }} | 
+  {% endif %}
+  PMG
+{% endblock %}
 
 {% block breadcrumb %}
 <ol class="breadcrumb">
   {% block committee_breadcrumb %}
   {% if not from_page %}
   <li><a href="{{ url_for('committees') }}">Committees</a></li>
-  <li><a href="/committees#{% if committee.house.sphere == 'provincial' %}prov{% elif committee.ad_hoc %}adhoc{% else %}{{ committee.house.short_name }}{% endif %}">{{ committee.house.name }}</a></li>
+    {% if committee %}
+    <li><a href="/committees#{% if committee.house.sphere == 'provincial' %}prov{% elif committee.ad_hoc %}adhoc{% else %}{{ committee.house.short_name }}{% endif %}">{{ committee.house.name }}</a></li>
+    {% endif %}
   {% else %}
   <li><a href="{{ url_for('provincial_legislatures_list') }}">Provincial Legislatures</a></li>
-  <li><a href="{{ url_for('provincial_legislatures_detail', slug=from_page) }}">{{ committee.house.name }}</a></li>
+    {% if committee %}
+    <li><a href="{{ url_for('provincial_legislatures_detail', slug=from_page) }}">{{ committee.house.name }}</a></li>
+    {% endif %}
   <li><a href="{{ url_for('provincial_committees', slug=from_page) }}">Committees</a></li>
   {% endif %}
   {% endblock %}
@@ -22,6 +31,7 @@
     <div class="col-sm-8">
       <header class="committee-header mb-3">
         {% block committee_title %}
+          {% if committee %}
           <div class="row">
               {% if not committee.premium %}
               <h1 class="col-xs-12 committee-name">{{ committee.name }}</h1>

pmg/templates/committee_meeting.html

@@ -5,7 +5,9 @@
 
 {% block committee_breadcrumb %}
   {{ super() }}
-  <li><a href="{{ url_for('committee_detail', committee_id=committee.id) }}">{{ committee.name }}</a></li>
+  {% if committee %}
+    <li><a href="{{ url_for('committee_detail', committee_id=committee.id) }}">{{ committee.name }}</a></li>
+    {% endif %}
 {% endblock %}
 
 {% block javascript %}
@@ -20,7 +22,9 @@ <h1>{{ event.title }}</h1>
       {% if event.premium_but_free %}
         <div class="premium-but-free"><i class="fa fa-fw fa-unlock-alt" aria-hidden="true"></i> This premium content has been made freely available</div>
       {% endif %}
+      {% if committee %}
       <h4><a href="{{ url_for('committee_detail', committee_id=committee.id) }}">{{ committee.name }}</a></h4>
+      {% endif %}
       {% if event.date %}
         <h5 class="date">{{ event.date | pretty_date("long") }}</h5>
       {% endif %}

pmg/templates/saved_search_alert.html

@@ -0,0 +1,32 @@
+<p>
+Hello,
+</p>
+
+<p>
+We found <b>{{ results|length }} recently updated item{% if results|length > 1 %}s{% endif %}</b> for your search
+<a href="{{ search.url(_external=True) }}">{{ search.search }}</a>{%
+if search.content_type %} in {{ search.friendly_content_type }}{% endif %}:
+</p>
+
+<ul>
+{% for result in results %}
+    <li style="padding-bottom: 1em">
+    <h4><a href="{{ result._source.url }}">{{ result._source.title }}</a> - {{ result._source.date|pretty_date }}</h4>
+    {% if result.highlight %}
+    <p>
+        {% if result.highlight.description %}
+        {{ result.highlight.description|search_snippet(mark='b')|ellipse|safe }}
+        {% elif result.highlight.fulltext %}
+        {{ result.highlight.fulltext|search_snippet(mark='b')|ellipse|safe }}
+        {% elif result.highlight.attachments %}
+        {{ result.highlight.attachments|search_snippet(mark='b')|ellipse|safe }}
+        {% endif %}
+    </p>
+    {% endif %}
+    </li>
+{% endfor %}
+</ul>
+
+<p>
+Click here to <a href="{{ url_for('email_alerts', _external=True) }}">manage your email alerts</a>.
+</p>

pmg/views.py

@@ -20,7 +20,7 @@
     redirect,
     Response,
     jsonify,
-    send_from_directory
+    send_from_directory,
 )
 from flask_security import current_user
 from flask_mail import Message
@@ -77,7 +77,7 @@ def page_not_found(error):
     dest = Redirect.for_url(request.path)
     if dest:
         return redirect(dest, code=302)
-    
+
     return render_template("404.html"), 404
 
 
@@ -113,7 +113,7 @@ def shortcircuit_wget():
 
 @app.before_request
 def redirect_legacy_domains():
-    """ Redirect legacy domains to the primary domain. """
+    """Redirect legacy domains to the primary domain."""
     parts = urlparse(request.url)
     if parts.netloc in LEGACY_DOMAINS:
         parts = list(parts)
@@ -129,7 +129,7 @@ def update_last_login():
 
 
 def classify_attachments(files):
-    """ Return an (audio_files, related_docs) tuple. """
+    """Return an (audio_files, related_docs) tuple."""
     audio = []
     related = []
 
@@ -141,6 +141,7 @@ def classify_attachments(files):
 
     return audio, related
 
+
 @cache.cached(timeout=10800, key_prefix="get_featured_content")
 def get_featured_content():
     info = {}
@@ -415,15 +416,19 @@ def committee_detail(committee_id):
     current_attendance_summary = CommitteeMeetingAttendance.committee_attendence_trends(
         committee_id, "current"
     )
-    historical_attendance_summary = CommitteeMeetingAttendance.committee_attendence_trends(
-        committee_id, "historical"
+    historical_attendance_summary = (
+        CommitteeMeetingAttendance.committee_attendence_trends(
+            committee_id, "historical"
+        )
     )
 
     if current_attendance_summary and committee["house"]["short_name"] != "Joint":
         year = current_attendance_summary[-1].year
         cte = Committee.query.get(committee_id)
-        attendance_rank = CommitteeMeetingAttendance.annual_attendance_rank_for_committee(
-            cte, int(year)
+        attendance_rank = (
+            CommitteeMeetingAttendance.annual_attendance_rank_for_committee(
+                cte, int(year)
+            )
         )
     else:
         attendance_rank = None
@@ -596,18 +601,28 @@ def attendance_overview():
 
 @app.route("/committee-question/<int:question_id>/")
 def committee_question(question_id):
-    """ Display a single committee question.
-    """
+    """Display a single committee question."""
     question = load_from_api("v2/minister-questions", question_id)["result"]
     minister = question["minister"]
-    committee = minister.get("committee", {"house": {}, "id": 0})
+    if minister:
+        committee = minister.get("committee", {"house": {}, "id": 0})
+    else:
+        committee = None
+    if question["question_to_name"]:
+        question_to_name = question["question_to_name"]
+    else:
+        question_to_name = "[UNKNOWN]"
+    if question["asked_by_name"]:
+        asked_by_name = question["asked_by_name"]
+    else:
+        asked_by_name = "[UNKNOWN]"
     social_summary = (
         "A question to the "
-        + question["question_to_name"]
+        + question_to_name
         + ", asked on "
         + pretty_date(question["date"], "long")
         + " by "
-        + question["asked_by_name"]
+        + asked_by_name
     )
 
     return render_template(
@@ -799,10 +814,14 @@ def committee_meeting(event_id):
     ) + sorted(
         [a for a in attendance if not a["chairperson"]], key=sorter
     )  # noqa
+    if event["committee"]:
+        event_committee_name = event["committee"]["name"]
+    else:
+        event_committee_name = "[UNKNOWN COMMITTEE]"
     if event["chairperson"]:
         social_summary = (
             "A meeting of the "
-            + event["committee"]["name"]
+            + event_committee_name
             + " committee held on "
             + pretty_date(event["date"], "long")
             + ", lead by "
@@ -811,26 +830,24 @@ def committee_meeting(event_id):
     else:
         social_summary = (
             "A meeting of the "
-            + event["committee"]["name"]
+            + event_committee_name
             + " committee held on "
             + pretty_date(event["date"], "long")
             + "."
         )
-
-
 
     return render_template(
-            "committee_meeting.html",
-            event=event,
-            committee=event["committee"],
-            audio=audio,
-            related_docs=related_docs,
-            attendance=attendance,
-            premium_committees=premium_committees,
-            content_date=event["date"],
-            social_summary=social_summary,
-            admin_edit_url=admin_url("committee-meeting", event_id),
-            SOUNDCLOUD_APP_KEY_ID=app.config["SOUNDCLOUD_APP_KEY_ID"]
+        "committee_meeting.html",
+        event=event,
+        committee=event["committee"],
+        audio=audio,
+        related_docs=related_docs,
+        attendance=attendance,
+        premium_committees=premium_committees,
+        content_date=event["date"],
+        social_summary=social_summary,
+        admin_edit_url=admin_url("committee-meeting", event_id),
+        SOUNDCLOUD_APP_KEY_ID=app.config["SOUNDCLOUD_APP_KEY_ID"],
     )
 
 
@@ -1114,8 +1131,7 @@ def gazette(gazette_id):
 
 @app.route("/members/")
 def members():
-    """ All MPs.
-    """
+    """All MPs."""
     members = load_from_api("v2/members", return_everything=True)["results"]
 
     # partition by house
@@ -1999,18 +2015,25 @@ def stats_review(stat):
     }
     return render_template(stat_group[stat])
 
+
 @app.route("/6th-parliament-review", methods=["GET"])
 def pr6():
     return render_template("pr6/landing.html")
 
+
 @app.route("/6th-parliament-review/<section>/<slug>", methods=["GET"])
-def pr6_articles(section,slug):
-    return render_template("pr6/article.html",section=section, article=slug)
+def pr6_articles(section, slug):
+    return render_template("pr6/article.html", section=section, article=slug)
+
 
-@app.route('/favicon.ico')
+@app.route("/favicon.ico")
 def favicon():
-    return send_from_directory(os.path.join(app.root_path, 'static'),
-                               'favicon.ico', mimetype='image/vnd.microsoft.icon')
+    return send_from_directory(
+        os.path.join(app.root_path, "static"),
+        "favicon.ico",
+        mimetype="image/vnd.microsoft.icon",
+    )
+
 
 # Test to make sure sentry is working
 @app.route("/debug-sentry")