[MDS-6260] Added support to permit condition extraction pipeline to c…

…reate identified reports (#3339)

* [MDS-6260] Added support to permit condition extraction pipeline to create identified reports

* Use auto-reloadable celery for core api worker

* Add meta column to permit_conditions_version

* Update fastapi version

* Added testkey for documentintelligence

* MDS-6260 Cleanup

.github/workflows/permit-service.unit.yaml

@@ -28,8 +28,8 @@ jobs:
         env:
           DOCKER_BUILDKIT: 1
         run: |
-          docker compose -f docker-compose.yaml run -e AZURE_API_KEY=testkey haystack coverage run -m pytest
-          docker compose -f docker-compose.yaml run -e AZURE_API_KEY=testkey haystack coverage xml
+          docker compose -f docker-compose.yaml run -e AZURE_API_KEY=testkey -e DOCUMENTINTELLIGENCE_API_KEY=testkey haystack coverage run -m pytest
+          docker compose -f docker-compose.yaml run -e AZURE_API_KEY=testkey -e DOCUMENTINTELLIGENCE_API_KEY=testkey haystack coverage xml
           sed -i "s/<source>\/code/<source>\/github\/workspace\/services\/permits/g" services/permits/coverage.xml
       - name: Upload test coverage results
         uses: actions/upload-artifact@v4

docker-compose.yaml

@@ -134,7 +134,7 @@ services:
     container_name: core_api_celery
     build:
       context: services/core-api
-    entrypoint: ./celery.sh
+    entrypoint: ./celery_dev.sh
     ports:
       - 5556:5555
     volumes:

migrations/sql/V2024.12.03.13.1__add_permit_condition_meta_column.sql

@@ -0,0 +1 @@
+ALTER TABLE permit_conditions ADD COLUMN IF NOT EXISTS meta JSONB NULL;

migrations/sql/V2024.12.06.11.1__add_mine_report_definition_report_name_column.sql

@@ -0,0 +1 @@
+ALTER TABLE mine_report_permit_requirement ADD COLUMN report_name VARCHAR(255) NULL;

migrations/sql/V2024.12.11.15.1__add_permit_condition_history_meta_column.sql

@@ -0,0 +1 @@
+ALTER TABLE permit_conditions_version ADD COLUMN IF NOT EXISTS meta JSONB NULL;

services/core-api/Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get install build-essential -y
 # Install the requirements
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install watchdog
 
 COPY . .
 

services/core-api/app/api/mines/permits/permit_conditions/models/permit_conditions.py

@@ -1,54 +1,68 @@
-import uuid
-from datetime import datetime
-
 from app.api.utils.field_template import FieldTemplate
 from app.api.utils.list_lettering_helpers import num_to_letter, num_to_roman
 from app.api.utils.models_mixins import AuditMixin, Base, SoftDeleteMixin
 from app.extensions import db
-from marshmallow import fields, validate
-from sqlalchemy.dialects.postgresql import UUID
+from marshmallow import fields
+from sqlalchemy.dialects.postgresql import JSONB, UUID
 from sqlalchemy.ext.hybrid import hybrid_property
-from sqlalchemy.orm import backref, validates
+from sqlalchemy.orm import backref
 from sqlalchemy.schema import FetchedValue
 
 
 class PermitConditions(SoftDeleteMixin, AuditMixin, Base):
-    __tablename__ = 'permit_conditions'
+    __tablename__ = "permit_conditions"
 
     class _ModelSchema(Base._ModelSchema):
         permit_condition_id = fields.Integer(dump_only=True)
         permit_condition_guid = fields.UUID(dump_only=True)
         condition_category_code = FieldTemplate(
-            field=fields.String, one_of='PermitConditionCategory')
-        condition_type_code = FieldTemplate(field=fields.String, one_of='PermitConditionType')
+            field=fields.String, one_of="PermitConditionCategory"
+        )
+        condition_type_code = FieldTemplate(
+            field=fields.String, one_of="PermitConditionType"
+        )
 
     permit_condition_id = db.Column(db.Integer, primary_key=True)
     permit_amendment_id = db.Column(
-        db.Integer, db.ForeignKey('permit_amendment.permit_amendment_id'), nullable=False)
-    permit_amendment = db.relationship('PermitAmendment', lazy='select', back_populates='conditions')
+        db.Integer,
+        db.ForeignKey("permit_amendment.permit_amendment_id"),
+        nullable=False,
+    )
+    permit_amendment = db.relationship(
+        "PermitAmendment", lazy="select", back_populates="conditions"
+    )
     permit_condition_guid = db.Column(UUID(as_uuid=True), server_default=FetchedValue())
     condition = db.Column(db.String, nullable=False)
     condition_category_code = db.Column(
         db.String,
-        db.ForeignKey('permit_condition_category.condition_category_code'),
-        nullable=False)
-
-    condition_category = db.relationship('PermitConditionCategory', lazy='select')
+        db.ForeignKey("permit_condition_category.condition_category_code"),
+        nullable=False,
+    )
+
+    condition_category = db.relationship("PermitConditionCategory", lazy="select")
 
     condition_type_code = db.Column(
-        db.String, db.ForeignKey('permit_condition_type.condition_type_code'), nullable=False)
-    parent_permit_condition_id = db.Column(db.Integer,
-                                           db.ForeignKey('permit_conditions.permit_condition_id'))
+        db.String,
+        db.ForeignKey("permit_condition_type.condition_type_code"),
+        nullable=False,
+    )
+    parent_permit_condition_id = db.Column(
+        db.Integer, db.ForeignKey("permit_conditions.permit_condition_id")
+    )
     display_order = db.Column(db.Integer, nullable=False)
-    _step = db.Column('step', db.String, nullable=True)
+
+    meta = db.Column(JSONB(astext_type=db.Text()), nullable=True)
+
+    _step = db.Column("step", db.String, nullable=True)
 
     __versioned__ = {}
 
     all_sub_conditions = db.relationship(
-        'PermitConditions',
-        lazy='joined',
-        order_by='asc(PermitConditions.display_order)',
-        backref=backref('parent', remote_side=[permit_condition_id]))
+        "PermitConditions",
+        lazy="joined",
+        order_by="asc(PermitConditions.display_order)",
+        backref=backref("parent", remote_side=[permit_condition_id]),
+    )
 
     @hybrid_property
     def sub_conditions(self):
@@ -68,64 +82,80 @@ def step(self):
         if self._step:
             # Format the first level with a trailing dot - A. B. C. and the rest with () - (a), (i), (ii)
             if depth == 0:
-                return f'{self._step}.'
-            return f'({self._step})'
-        if self._step == '':
-            return ''
+                return f"{self._step}."
+            return f"({self._step})"
+        if self._step == "":
+            return ""
 
         step_format = depth % 3
         if step_format == 0:
-            return str(self.display_order) + '.'
+            return str(self.display_order) + "."
         elif step_format == 1:
-            return num_to_letter(self.display_order) + '.'
+            return num_to_letter(self.display_order) + "."
         elif step_format == 2:
-            return num_to_roman(self.display_order) + '.'
+            return num_to_roman(self.display_order) + "."
 
     def __repr__(self):
-        return '<PermitConditions %r, %r, %r>' % (self.permit_condition_id,
-                                                  self.permit_condition_guid, self.display_order)
+        return "<PermitConditions %r, %r, %r>" % (
+            self.permit_condition_id,
+            self.permit_condition_guid,
+            self.display_order,
+        )
 
     @classmethod
-    def create(cls,
-               condition_category_code,
-               condition_type_code,
-               permit_amendment_id,
-               condition,
-               display_order,
-               sub_conditions,
-               parent=None):
+    def create(
+        cls,
+        condition_category_code,
+        condition_type_code,
+        permit_amendment_id,
+        condition,
+        display_order,
+        sub_conditions,
+        parent=None,
+    ):
         permit_condition = cls(
             condition_category_code=condition_category_code,
             condition_type_code=condition_type_code,
             permit_amendment_id=permit_amendment_id,
             condition=condition,
             display_order=display_order,
-            parent=parent)
+            parent=parent,
+        )
 
         permit_condition.save(commit=False)
         for condition in sub_conditions:
-            PermitConditions.create(condition.condition_category_code,
-                                    condition.condition_type_code, permit_amendment_id,
-                                    condition.condition, condition.display_order,
-                                    condition.sub_conditions, permit_condition)
+            PermitConditions.create(
+                condition.condition_category_code,
+                condition.condition_type_code,
+                permit_amendment_id,
+                condition.condition,
+                condition.display_order,
+                condition.sub_conditions,
+                permit_condition,
+            )
         return permit_condition
 
-
     @classmethod
     def delete_all_by_permit_amendment_id(cls, permit_amendment_id, commit=False):
-        parent_conditions = cls.query.filter_by(
-            permit_amendment_id=permit_amendment_id,
-            parent_permit_condition_id=None,
-            deleted_ind=False).order_by(cls.display_order).all()
+        parent_conditions = (
+            cls.query.filter_by(
+                permit_amendment_id=permit_amendment_id,
+                parent_permit_condition_id=None,
+                deleted_ind=False,
+            )
+            .order_by(cls.display_order)
+            .all()
+        )
         for condition in parent_conditions:
             condition.delete_condition(commit=commit)
             if commit:
                 condition.save()
 
-
     def delete_condition(self, commit=False):
         if self.all_sub_conditions is not None:
-            subconditions = [c for c in self.all_sub_conditions if c.deleted_ind == False]
+            subconditions = [
+                c for c in self.all_sub_conditions if c.deleted_ind == False
+            ]
             if len(subconditions) > 0:
                 for item in subconditions:
                     item.deleted_ind = True
@@ -134,25 +164,32 @@ def delete_condition(self, commit=False):
                         item.save()
         self.deleted_ind = True
 
-
     @classmethod
     def find_all_by_permit_amendment_id(cls, permit_amendment_id):
-        return cls.query.filter_by(
-            permit_amendment_id=permit_amendment_id,
-            parent_permit_condition_id=None,
-            deleted_ind=False).order_by(cls.display_order).all()
+        return (
+            cls.query.filter_by(
+                permit_amendment_id=permit_amendment_id,
+                parent_permit_condition_id=None,
+                deleted_ind=False,
+            )
+            .order_by(cls.display_order)
+            .all()
+        )
 
     @classmethod
     def find_by_permit_condition_guid(cls, permit_condition_guid):
         return cls.query.filter_by(
-            permit_condition_guid=permit_condition_guid, deleted_ind=False).first()
+            permit_condition_guid=permit_condition_guid, deleted_ind=False
+        ).first()
 
     @classmethod
     def find_by_permit_condition_id(cls, permit_condition_id):
         return cls.query.filter_by(
-            permit_condition_id=permit_condition_id, deleted_ind=False).first()
+            permit_condition_id=permit_condition_id, deleted_ind=False
+        ).first()
 
     @classmethod
     def find_by_condition_category_code(cls, condition_category_code):
         return cls.query.filter_by(
-            condition_category_code=condition_category_code, deleted_ind=False).all()
+            condition_category_code=condition_category_code, deleted_ind=False
+        ).all()