Merge pull request #104 from GSA/add-gunicorn-server

Adds Gunicorn WSGI web server

.env

@@ -13,3 +13,11 @@ ISSUER=https://idp.int.identitysandbox.gov
 REDIRECT_URI=http://localhost:8080/callback
 FLASK_APP_SECRET_KEY=Px4o0MPEsCzHJRRSUDyt1VRi-lYF2W5VEzR13lylAgI
 OPENID_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDM3NixEAT9x6jhG38OKRc9bkt1Pku2iZGQdBpHHNN3X6tyasOICP1sB42638N48w3vZAu39m6cZbRAsAxzgm73CwT1WnCILb3GAMumL+w75nAZAfwr/l7DTDCmxY55G7Hij9cW58JeKJtzXLpBU/KwpwsRbPlHHnBRefFbP8mRVoxL7SOlAD9ASVUWODbLKQn/dvzN1OZrx2mIHpHX1G93rBZaU4HZBjf1xfKrGHo22N2n5047hXIhd+MEN7oygSQzB5UCSyYihPe7E2+gr97e05qrYfWjUztPrBRZV9hN+pzppgKWo7JPfIG7rTrd+y6l0zuPBrFX4Q9+AJJud/kdAgMBAAECggEABopJZ7OzeReh8lfSoVTZ/1XRVmOf/qqbKVGvkhQfQykce9lhNytUTdna4jupDagyynD2989H/jm3/LWyHBDusnfZV2zk+U/+hj75wWs3vzzO2NmJn6zuAQ2iYvydHQwvjGRbsUlCWQeyWMc90f6YWZ/nmE2SkrzryIocVLWvXXKnhgaDilMlbhxiUDyNjIX8AzuwABzcCJ/PMSPAvgBrQ6zfTysQA1/anjrdBGD+edNxS5g5fQKO7U9zjpYzkvb6583FMDHczY48XJ8TRByNHEf/F/RtRKk6afFtRkj4V7iDAP5Eh/ZjtHmpDw0dhPNfc9EoXV4uVOdn//txCUNYsQKBgQDm/36Pch4KsfJX5hO3pfWnup+IGSc7/0FpCB61l9TNKwlMBVbaBvLO2oGTPgGaI7IiMtMJKoou2/9wAb8LvrdHT1HcB2z2oelnkZbecNnQY3yGhpm1hZDxYA6IEyakt/PWyQaYtqj6vslmf5LXNVQGFBQW7XK3Bk8uCXBsukcYzQKBgQDjCTFNCahE2++6w7Cmx4hqxtIa1DPH6e1ii1L8PuCtBkt9s42vz+PA8rVO2QlvEffistYKnMxodALpYn1PspTOgiudJe5eEagF4uUK6JYBp6aQfocw129iHGrV0KOWyy1xDwAxxufyCZqpWzixxnW9BoW0hOFAgpsJ4Iy4FAKhkQKBgHGPBM7MEQy88iBYETVlNuDgE/E6PFl4YNWbkk0/ePPrxkRDE7FXXSJntO+Hug3monnBdpG5AfmLWYXctei2Ny+tRNbgJ6cPbcjmf76Wx910KxGPNh1QrjKbQEmQ8JxlelAKv11v3p52EFh9OtjldIsQb9fdn1lulppRNZ2d7bm5AoGAd04SWUBU63KEBKnm/41lE29NY4nVeB1mgF1ki2djW6kuLG0ZbKUbMv6X5kpJGdPgZoHB1qvVAqLMEOaPoTzf07E1yAvxuf8fQ5Ca+eOFSHNYJSBIXJMpVe5G+fr8rDYyt/HBEWdL5bXET3C/VzRKFL4DrvwQ/nxjioo0iFsDkvECgYAoMrrB8HIfExCAssbqWaFLOvgIH5bWq6Qs6wEhpwryktAnsmDx5BkXlTDA5vijk/aTqsCHvaBeP8omLu/WQITNft14pPFGZzoscwOU/512VGblny4RZjpCm/MB4tJYU6Po/tqcbeLHl3jqItoR9XHGA87+sZRYV9ZzVhqHtEANTA==\n-----END PRIVATE KEY-----\n"
+
+# New Relic
+NEW_RELIC_LICENSE_KEY=
+NEW_RELIC_APP_NAME=
+NEW_RELIC_MONITOR_MODE=false
+NEW_RELIC_LOG=/var/log/new_relic.log
+NEW_RELIC_LOG_LEVEL=info
+NEW_RELIC_HOST=gov-collector.newrelic.com

.profile

@@ -32,6 +32,10 @@ export OPENID_PRIVATE_KEY=$(vcap_get_service secrets .credentials.OPENID_PRIVATE
 
 export CKAN_API_TOKEN=$(vcap_get_service secrets .credentials.CKAN_API_TOKEN)
 
+# New Relic
+export NEW_RELIC_LICENSE_KEY=$(vcap_get_service secrets .credentials.NEW_RELIC_LICENSE_KEY)
+
+
 if [[ $REAL_NAME = "datagov-harvest-admin" ]]; then
   flask db upgrade
 fi

Makefile

@@ -3,7 +3,7 @@ all: help
 pypi-upload: build-dist  ## Uploads new package to PyPi after clean, build
 	poetry publish
 
-deps-update: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml
+update-dependencies: ## Updates requirements.txt and requirements_dev.txt from pyproject.toml
 	poetry export --without-hashes --without=dev --format=requirements.txt > requirements.txt
 	poetry export --without-hashes --only=dev --format=requirements.txt > requirements-dev.txt
 
@@ -56,9 +56,12 @@ down: ## Tears down the flask and harvester containers
 	docker compose down
 	docker compose -p harvest-app down
 
-up-debug: ## Sets up local docker environment
+up-debug: ## Sets up local docker environment with VSCODE debug support enabled
 	docker compose -f docker-compose.yml -f docker-compose_debug.yml up -d
 
+up-prod: ## Sets up local flask env running gunicorn instead of standard dev server
+	docker compose -f docker-compose.yml -f docker-compose_prod.yml up -d
+
 clean: ## Cleans docker images
 	docker compose down -v --remove-orphans
 	docker compose -p harvest-app down -v --remove-orphans

app/routes.py

@@ -461,10 +461,24 @@ def add_harvest_source():
 @mod.route("/harvest_source/<source_id>", methods=["GET"])
 def view_harvest_source_data(source_id: str):
     source = db.get_harvest_source(source_id)
-    jobs = db.get_all_harvest_jobs_by_filter({"harvest_source_id": source.id})
-    records = db.get_harvest_record_by_source(source.id)
-    ckan_records = [record for record in records if record.ckan_id is not None]
-    error_records = [record for record in records if record.status == "error"]
+    records_count = db.get_harvest_records_by_source(
+        count=True,
+        skip_pagination=True,
+        source_id=source.id,
+    )
+    ckan_records_count = db.get_harvest_records_by_source(
+        count=True,
+        skip_pagination=True,
+        source_id=source.id,
+        facets=["ckan_id != null"],
+    )
+    error_records_count = db.get_harvest_records_by_source(
+        count=True,
+        skip_pagination=True,
+        source_id=source.id,
+        facets=["status = 'error'"],
+    )
+    # TODO: wire in paginated jobs query
     jobs = db.get_all_harvest_jobs_by_filter({"harvest_source_id": source.id})
     next_job = "N/A"
     future_jobs = db.get_new_harvest_jobs_by_source_in_future(source.id)
@@ -502,9 +516,9 @@ def view_harvest_source_data(source_id: str):
     data = {
         "harvest_source": source,
         "harvest_source_dict": db._to_dict(source),
-        "total_records": len(records),
-        "records_with_ckan_id": len(ckan_records),
-        "records_with_error": len(error_records),
+        "total_records": records_count,
+        "records_with_ckan_id": ckan_records_count,
+        "records_with_error": error_records_count,
         "harvest_jobs": jobs,
         "chart": chartdata,
         "next_job": next_job,
@@ -709,28 +723,29 @@ def get_harvest_errors_by_job(job_id, error_type):
 
 ## Harvest Record
 ### Get record
-@mod.route("/harvest_record/", methods=["GET"])
 @mod.route("/harvest_record/<record_id>", methods=["GET"])
-def get_harvest_record(record_id=None):
-    if record_id:
-        record = db.get_harvest_record(record_id)
-        return db._to_dict(record) if record else ("Not Found", 404)
+def get_harvest_record(record_id):
+    record = db.get_harvest_record(record_id)
+    return db._to_dict(record) if record else ("Not Found", 404)
+
 
+### Get records
+@mod.route("/harvest_records/", methods=["GET"])
+def get_harvest_records():
     job_id = request.args.get("harvest_job_id")
     source_id = request.args.get("harvest_source_id")
+    page = request.args.get("page")
     if job_id:
-        record = db.get_harvest_record_by_job(job_id)
-        if not record:
+        records = db.get_harvest_records_by_job(job_id, page)
+        if not records:
             return "No harvest records found for this harvest job", 404
     elif source_id:
-        record = db.get_harvest_record_by_source(source_id)
-        if not record:
+        records = db.get_harvest_records_by_source(source_id, page)
+        if not records:
             return "No harvest records found for this harvest source", 404
     else:
-        # TODO for test, will remove later
-        record = db.pget_harvest_records()
-
-    return db._to_dict(record)
+        records = db.pget_harvest_records(page)
+    return db._to_dict(records)
 
 @mod.route("/harvest_record/<record_id>/raw", methods=["GET"])
 def get_harvest_record_raw(record_id=None):

config/newrelic.ini

@@ -0,0 +1,26 @@
+# ---------------------------------------------------------------------------
+
+#
+# This file configures the New Relic Python Agent.
+#
+# The path to the configuration file should be supplied to the function
+# newrelic.agent.initialize() when the agent is being initialized.
+#
+# The configuration file follows a structure similar to what you would
+# find for Microsoft Windows INI files. For further information on the
+# configuration file format see the Python ConfigParser documentation at:
+#
+#    http://docs.python.org/library/configparser.html
+#
+# For further discussion on the behaviour of the Python agent that can
+# be configured via this configuration file see:
+#
+#    http://newrelic.com/docs/python/python-agent-configuration
+#
+
+# ---------------------------------------------------------------------------
+
+# Here are the settings that are common to all environments.
+
+[newrelic]
+error_collector.expected_classes = werkzeug.exceptions:Forbidden werkzeug.exceptions:MethodNotAllowed

database/interface.py

@@ -507,16 +507,6 @@ def update_harvest_record(self, record_id, updates):
     def get_harvest_record(self, record_id):
         return self.db.query(HarvestRecord).filter_by(id=record_id).first()
 
-    def get_harvest_record_by_job(self, job_id):
-        harvest_records = self.db.query(HarvestRecord).filter_by(harvest_job_id=job_id)
-        return [rcd for rcd in harvest_records]
-
-    def get_harvest_record_by_source(self, source_id, filters={}):
-        harvest_records = self.db.query(HarvestRecord).filter_by(
-            harvest_source_id=source_id, **filters
-        )
-        return [rcd for rcd in harvest_records]
-
     def get_latest_harvest_records_by_source(self, source_id):
         # datetimes are returned as datetime objs not strs
         sql = text(
@@ -610,7 +600,7 @@ def verify_user(self, usr_data):
             print("Error:", e)
             return False
 
-    #### PAGINATED QUERIES
+    #### PAGINATED QUERIES ####
     @count
     @paginate
     def pget_harvest_jobs(self, filter=text(""), **kwargs):
@@ -631,21 +621,11 @@ def pget_harvest_job_errors(self, filter=text(""), **kwargs):
     def pget_harvest_record_errors(self, filter=text(""), **kwargs):
         return self.db.query(HarvestRecordError).filter(filter)
 
-    ##### TEST INTERFACES BELOW #####
-    ######## TO BE REMOVED ##########
-    def get_all_harvest_jobs(self):
-        harvest_jobs = self.db.query(HarvestJob).all()
-        harvest_jobs_data = [job for job in harvest_jobs]
-        return harvest_jobs_data
-
-    def get_all_harvest_errors(self):
-        job_errors = self.db.query(HarvestJobError).all()
-        record_errors = self.db.query(HarvestRecordError).all()
-        return [*job_errors, *record_errors]
-
-    def delete_all_harvest_records(self):
-        harvest_records = self.db.query(HarvestRecord).all()
-        for record in harvest_records:
-            self.db.delete(record)
-        self.db.commit()
-        return "Records deleted successfully"
+    #### FACETED BUILDER QUERIES ####
+    def get_harvest_records_by_job(self, job_id, facets=[], **kwargs):
+        filter_string = " AND ".join([f"harvest_job_id = '{job_id}'"] + facets)
+        return self.pget_harvest_records(filter=text(filter_string), **kwargs)
+
+    def get_harvest_records_by_source(self, source_id, facets=[], **kwargs):
+        filter_string = " AND ".join([f"harvest_source_id = '{source_id}'"] + facets)
+        return self.pget_harvest_records(filter=text(filter_string), **kwargs)

docker-compose_prod.yml

@@ -0,0 +1,3 @@
+services:
+  app:
+    command: ./harvest-admin-start.sh

gunicorn.conf.py

@@ -0,0 +1,5 @@
+# gunicorn prior to version 20.1.0 discloses "server" header
+# https://github.com/GSA/datagov-deploy/issues/2826
+import gunicorn
+
+gunicorn.SERVER_SOFTWARE = ""

harvest-admin-start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+DIR="$(dirname "${BASH_SOURCE[0]}")"
+
+exec newrelic-admin run-program gunicorn "wsgi:application" --config "$DIR/gunicorn.conf.py" -b "0.0.0.0:$PORT" --chdir $DIR --timeout 120 --worker-class gevent --workers 8 --forwarded-allow-ips='*'

manifest.yml

@@ -18,7 +18,11 @@ applications:
       CLIENT_ID: ((CLIENT_ID))
       ISSUER: ((ISSUER))
       REDIRECT_URI: ((REDIRECT_URI))
-    command: flask run --host=0.0.0.0 --port=8080
+      NEW_RELIC_APP_NAME: ((app_name))-admin-((space_name))
+      NEW_RELIC_HOST: gov-collector.newrelic.com
+      NEW_RELIC_MONITOR_MODE: ((new_relic_monitor_mode))
+      NEW_RELIC_CONFIG_FILE: /home/vcap/app/config/newrelic.ini
+    command: ./harvest-admin-start.sh
   - name: ((app_name))-runner
     buildpacks:
       - python_buildpack