Merge pull request #80 from GSA/update-models

organize by model; update with new models definitions

app/routes.py

@@ -207,9 +207,7 @@ def edit_harvest_source(source_id=None):
 
 @mod.route("/harvest_source/harvest/<source_id>", methods=["GET"])
 def trigger_harvest_source(source_id):
-    job = db.add_harvest_job(
-        {"harvest_source_id": source_id, "status": "pending_manual"}
-    )
+    job = db.add_harvest_job({"harvest_source_id": source_id, "status": "manual"})
     if job:
         flash(f"Triggered harvest of source with ID: {source_id}")
     else:
@@ -232,24 +230,12 @@ def delete_harvest_source(source_id):
 
 
 ## Harvest Job
-@mod.route("/harvest_job/add", methods=["POST"])
-def add_harvest_job():
-    if request.is_json:
-        job = db.add_harvest_job(request.json)
-        if job:
-            return jsonify({"message": f"Added new harvest job with ID: {job.id}"})
-        else:
-            return jsonify({"error": "Failed to add harvest job."}), 400
-    else:
-        return jsonify({"Please provide harvest job with json format."})
-
-
 @mod.route("/harvest_job/", methods=["GET"])
 @mod.route("/harvest_job/<job_id>", methods=["GET"])
 def get_harvest_job(job_id=None):
     try:
         if job_id:
-            job = db.get_harvest_job(job_id)
+            job = HarvesterDBInterface._to_dict(db.get_harvest_job(job_id))
             return jsonify(job) if job else ("Not Found", 404)
 
         source_id = request.args.get("harvest_source_id")
@@ -265,6 +251,22 @@ def get_harvest_job(job_id=None):
         return "Please provide correct job_id or harvest_source_id"
 
 
+# all errors associated with a job ( job & records )
+@mod.route("/harvest_job/<job_id>/errors", methods=["GET"])
+@mod.route("/harvest_job/<job_id>/errors/<error_type>", methods=["GET"])
+def get_all_errors_of_harvest_job(job_id, error_type="all"):
+    try:
+        all_errors = db.get_all_errors_of_job(job_id)
+        if error_type == "job":
+            return all_errors[0]
+        elif error_type == "record":
+            return all_errors[1]
+        else:
+            return all_errors
+    except Exception:
+        return "Please provide correct job_id"
+
+
 @mod.route("/harvest_job/<job_id>", methods=["PUT"])
 def update_harvest_job(job_id):
     result = db.update_harvest_job(job_id, request.json)
@@ -277,54 +279,19 @@ def delete_harvest_job(job_id):
     return result
 
 
-## Harvest Error
-@mod.route("/harvest_error/add", methods=["POST", "GET"])
-def add_harvest_error():
+@mod.route("/harvest_job/add", methods=["POST"])
+def add_harvest_job():
     if request.is_json:
-        error = db.add_harvest_error(request.json)
-        if error:
-            return jsonify({"message": f"Added new harvest error with ID: {error.id}"})
+        job = db.add_harvest_job(request.json)
+        if job:
+            return jsonify({"message": f"Added new harvest job with ID: {job.id}"})
         else:
-            return jsonify({"error": "Failed to add harvest error."}), 400
+            return jsonify({"error": "Failed to add harvest job."}), 400
     else:
-        return jsonify({"Please provide harvest error with json format."})
-
-
-@mod.route("/harvest_error/", methods=["GET"])
-@mod.route("/harvest_error/<error_id>", methods=["GET"])
-def get_harvest_error(error_id=None):
-    try:
-        if error_id:
-            error = db.get_harvest_error(error_id)
-            return jsonify(error) if error else ("Not Found", 404)
-
-        job_id = request.args.get("harvest_job_id")
-        if job_id:
-            error = db.get_harvest_errors_by_job(job_id)
-            if not error:
-                return "No harvest errors found for this harvest job", 404
-        else:
-            # for test, will remove later
-            error = db.get_all_harvest_errors()
-
-        return jsonify(error)
-    except Exception:
-        return "Please provide correct error_id or harvest_job_id"
+        return jsonify({"Please provide harvest job with json format."})
 
 
 ## Harvest Record
-@mod.route("/harvest_record/add", methods=["POST", "GET"])
-def add_harvest_record():
-    if request.is_json:
-        record = db.add_harvest_record(request.json)
-        if record:
-            return jsonify({"message": f"Added new record with ID: {record.id}"})
-        else:
-            return jsonify({"error": "Failed to add harvest record."}), 400
-    else:
-        return jsonify({"Please provide harvest record with json format."})
-
-
 @mod.route("/harvest_record/", methods=["GET"])
 @mod.route("/harvest_record/<record_id>", methods=["GET"])
 def get_harvest_record(record_id=None):
@@ -352,6 +319,42 @@ def get_harvest_record(record_id=None):
         return "Please provide correct record_id or harvest_job_id"
 
 
+@mod.route("/harvest_record/<record_id>/errors", methods=["GET"])
+def get_all_harvest_record_errors(record_id: str) -> list:
+    try:
+        record_errors = db.get_harvest_record_errors(record_id)
+        return record_errors if record_errors else ("Not Found", 404)
+    except Exception:
+        return "Please provide correct record_id"
+
+
+@mod.route("/harvest_error/<error_id>", methods=["GET"])
+def get_harvest_record_error(error_id: str) -> dict:
+    # retrieves the given error ( either job or record )
+    try:
+        job_error = db.get_harvest_job_error(error_id)
+        if job_error:
+            return job_error
+        record_error = db.get_harvest_record_error(error_id)
+        if record_error:
+            return record_error
+        return ("Not Found", 404)
+    except Exception:
+        return "Please provide correct record_id and record_error_id"
+
+
+@mod.route("/harvest_record/add", methods=["POST", "GET"])
+def add_harvest_record():
+    if request.is_json:
+        record = db.add_harvest_record(request.json)
+        if record:
+            return jsonify({"message": f"Added new record with ID: {record.id}"})
+        else:
+            return jsonify({"error": "Failed to add harvest record."}), 400
+    else:
+        return jsonify({"Please provide harvest record with json format."})
+
+
 ## Test interface, will remove later
 # TODO: remove / improve
 @mod.route("/get_data_sources", methods=["GET"])

app/scripts/load_manager.py

@@ -24,7 +24,7 @@ def create_task(jobId):
 
 
 def sort_jobs(jobs):
-    return sorted(jobs, key=lambda x: x["status"], reverse=True)
+    return sorted(jobs, key=lambda x: x["status"])
 
 
 def load_manager():
@@ -39,10 +39,8 @@ def load_manager():
         print("CF_INSTANCE_INDEX is not set or not equal to zero")
         return
 
-    # filter harvestjobs by pending / pending_manual
-    jobs = interface.get_harvest_jobs_by_faceted_filter(
-        "status", ["pending", "pending_manual"]
-    )
+    # filter harvestjobs by new (automated) & manual
+    jobs = interface.get_harvest_jobs_by_faceted_filter("status", ["new", "manual"])
 
     # get current list of all tasks
     current_tasks = cf_handler.get_all_app_tasks(LM_RUNNER_APP_GUID)
@@ -56,7 +54,7 @@ def load_manager():
     else:
         slots = LM_MAX_TASKS_COUNT - running_tasks
 
-    # sort jobs by pending_manual first
+    # sort jobs by manual first
     sorted_jobs = sort_jobs(jobs)
 
     # slice off jobs to invoke

database/interface.py

@@ -1,11 +1,19 @@
 import os
 import uuid
+import itertools
 
 from sqlalchemy import create_engine, inspect, or_, text
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.orm import scoped_session, sessionmaker
 
-from .models import HarvestError, HarvestJob, HarvestRecord, HarvestSource, Organization
+from .models import (
+    HarvestJobError,
+    HarvestRecordError,
+    HarvestJob,
+    HarvestRecord,
+    HarvestSource,
+    Organization,
+)
 
 DATABASE_URI = os.getenv("DATABASE_URI")
 
@@ -142,8 +150,7 @@ def add_harvest_job(self, job_data):
             return None
 
     def get_harvest_job(self, job_id):
-        result = self.db.query(HarvestJob).filter_by(id=job_id).first()
-        return HarvesterDBInterface._to_dict(result)
+        return self.db.query(HarvestJob).filter_by(id=job_id).first()
 
     def get_harvest_jobs_by_filter(self, filter):
         harvest_jobs = self.db.query(HarvestJob).filter_by(**filter).all()
@@ -187,9 +194,14 @@ def delete_harvest_job(self, job_id):
         self.db.commit()
         return "Harvest job deleted successfully"
 
-    def add_harvest_error(self, error_data):
+    def add_harvest_error(self, error_data: dict, error_type: str):
         try:
-            new_error = HarvestError(**error_data)
+            if error_type is None:
+                return "Must indicate what type of error to add"
+            if error_type == "job":
+                new_error = HarvestJobError(**error_data)
+            else:
+                new_error = HarvestRecordError(**error_data)
             self.db.add(new_error)
             self.db.commit()
             self.db.refresh(new_error)
@@ -199,19 +211,36 @@ def add_harvest_error(self, error_data):
             self.db.rollback()
             return None
 
-    def get_harvest_error(self, error_id):
-        result = self.db.query(HarvestError).filter_by(id=error_id).first()
-        return HarvesterDBInterface._to_dict(result)
+    def get_all_errors_of_job(self, job_id: str) -> [[dict], [dict]]:
+        job = self.get_harvest_job(job_id)
+        job_errors = list(map(HarvesterDBInterface._to_dict, job.errors))
+
+        # itertools.chain flattens a list of lists into a 1 dimensional list
+        record_errors = itertools.chain(*[record.errors for record in job.records])
+        record_errors = list(map(HarvesterDBInterface._to_dict, record_errors))
 
-    def get_harvest_errors_by_job(self, job_id: str) -> list:
-        harvest_errors = self.db.query(HarvestError).filter_by(harvest_job_id=job_id)
-        return [HarvesterDBInterface._to_dict(err) for err in harvest_errors]
+        return [job_errors, record_errors]
+
+    def get_harvest_job_error(self, job_id: str) -> dict:
+        result = self.db.query(HarvestJobError).filter_by(harvest_job_id=job_id).first()
+        return HarvesterDBInterface._to_dict(result)
 
-    def get_harvest_errors_by_record_id(self, record_id: str) -> list:
-        harvest_errors = self.db.query(HarvestError).filter_by(
+    def get_harvest_record_errors(self, record_id: str):
+        # TODO: paginate this
+        errors = self.db.query(HarvestRecordError).filter_by(
             harvest_record_id=record_id
         )
-        return [HarvesterDBInterface._to_dict(err) for err in harvest_errors]
+        return [HarvesterDBInterface._to_dict(err) for err in errors]
+
+    def get_harvest_record_error(self, error_id: str) -> dict:
+        error = (
+            self.db.query(HarvestRecordError)
+            .filter_by(
+                id=error_id,
+            )
+            .first()
+        )
+        return HarvesterDBInterface._to_dict(error)
 
     def add_harvest_record(self, record_data):
         try:
@@ -323,10 +352,3 @@ def get_all_harvest_records(self):
             HarvesterDBInterface._to_dict(err) for err in harvest_records
         ]
         return harvest_records_data
-
-    def get_all_harvest_errors(self):
-        harvest_errors = self.db.query(HarvestError).all()
-        harvest_errors_data = [
-            HarvesterDBInterface._to_dict(err) for err in harvest_errors
-        ]
-        return harvest_errors_data