update harvest error route; add job errors db func; add job-record re…

…lationship; add job errors test

app/routes.py

@@ -235,7 +235,7 @@ def delete_harvest_source(source_id):
 def get_harvest_job(job_id=None):
     try:
         if job_id:
-            job = db.get_harvest_job(job_id)
+            job = HarvesterDBInterface._to_dict(db.get_harvest_job(job_id))
             return jsonify(job) if job else ("Not Found", 404)
 
         source_id = request.args.get("harvest_source_id")
@@ -251,11 +251,18 @@ def get_harvest_job(job_id=None):
         return "Please provide correct job_id or harvest_source_id"
 
 
-@mod.route("/harvest_job/<job_id>/error", methods=["GET"])
-def get_harvest_job_error(job_id):
+# all errors associated with a job ( job & records )
+@mod.route("/harvest_job/<job_id>/errors", methods=["GET"])
+@mod.route("/harvest_job/<job_id>/errors/<error_type>", methods=["GET"])
+def get_all_errors_of_harvest_job(job_id, error_type="all"):
     try:
-        job_error = db.get_harvest_job_error(job_id)
-        return jsonify(job_error) if job_id else ("Not Found", 404)
+        all_errors = db.get_all_errors_of_job(job_id)
+        if error_type == "job":
+            return all_errors[0]
+        elif error_type == "record":
+            return all_errors[1]
+        else:
+            return all_errors
     except Exception:
         return "Please provide correct job_id"
 

database/interface.py

@@ -1,5 +1,6 @@
 import os
 import uuid
+import itertools
 
 from sqlalchemy import create_engine, inspect, or_, text
 from sqlalchemy.exc import NoResultFound
@@ -149,8 +150,7 @@ def add_harvest_job(self, job_data):
             return None
 
     def get_harvest_job(self, job_id):
-        result = self.db.query(HarvestJob).filter_by(id=job_id).first()
-        return HarvesterDBInterface._to_dict(result)
+        return self.db.query(HarvestJob).filter_by(id=job_id).first()
 
     def get_harvest_jobs_by_filter(self, filter):
         harvest_jobs = self.db.query(HarvestJob).filter_by(**filter).all()
@@ -196,6 +196,8 @@ def delete_harvest_job(self, job_id):
 
     def add_harvest_error(self, error_data: dict, error_type: str):
         try:
+            if error_type is None:
+                return "Must indicate what type of error to add"
             if error_type == "job":
                 new_error = HarvestJobError(**error_data)
             else:
@@ -209,6 +211,16 @@ def add_harvest_error(self, error_data: dict, error_type: str):
             self.db.rollback()
             return None
 
+    def get_all_errors_of_job(self, job_id: str) -> [[dict], [dict]]:
+        job = self.get_harvest_job(job_id)
+        job_errors = list(map(HarvesterDBInterface._to_dict, job.errors))
+
+        # itertools.chain flattens a list of lists into a 1 dimensional list
+        record_errors = itertools.chain(*[record.errors for record in job.records])
+        record_errors = list(map(HarvesterDBInterface._to_dict, record_errors))
+
+        return [job_errors, record_errors]
+
     def get_harvest_job_error(self, job_id: str) -> dict:
         result = self.db.query(HarvestJobError).filter_by(harvest_job_id=job_id).first()
         return HarvesterDBInterface._to_dict(result)

database/models.py

@@ -77,6 +77,7 @@ class HarvestJob(db.Model):
     errors = db.relationship(
         "HarvestJobError", backref="job", cascade="all, delete-orphan", lazy=True
     )
+    records = db.relationship("HarvestRecord", backref="job", lazy=True)
 
 
 class HarvestRecord(db.Model):

tests/integration/database/test_db.py

@@ -118,6 +118,27 @@ def test_add_harvest_job_error(
         assert isinstance(harvest_job_error, HarvestJobError)
         assert harvest_job_error.message == job_error_data["message"]
 
+    def test_get_all_errors_of_job(
+        self,
+        interface,
+        organization_data,
+        source_data_dcatus,
+        job_data_dcatus,
+        job_error_data,
+    ):
+        interface.add_organization(organization_data)
+        interface.add_harvest_source(source_data_dcatus)
+        interface.add_harvest_job(job_data_dcatus)
+        interface.add_harvest_error(job_error_data, "job")
+
+        all_errors = interface.get_all_errors_of_job(job_data_dcatus["id"])
+
+        assert len(all_errors) == 2
+        assert len(all_errors[0]) == 1  # job error
+        assert len(all_errors[1]) == 0  # record errors
+        assert all_errors[0][0]["type"] == "ExtractInternalException"
+        assert len(all_errors[1]) == 0
+
     def test_add_harvest_record(
         self,
         interface,