From d224b443dc9d177384cbaba87239b71516167f2c Mon Sep 17 00:00:00 2001 From: Andrea Petrucci Date: Wed, 15 Nov 2023 14:25:22 +0100 Subject: [PATCH 1/4] added cascade delete on some models --- core-engine/src/pipeline_steps/models.py | 1 + core-engine/src/services/models.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core-engine/src/pipeline_steps/models.py b/core-engine/src/pipeline_steps/models.py index 83aeaced..09d1952a 100644 --- a/core-engine/src/pipeline_steps/models.py +++ b/core-engine/src/pipeline_steps/models.py @@ -41,6 +41,7 @@ class PipelineStep( pipeline_id: UUID | None = Field(foreign_key="pipelines.id") pipeline: "Pipeline" = Relationship(back_populates="steps") # noqa F821 pipeline_executions: List["PipelineExecution"] = Relationship( + sa_relationship_kwargs={"cascade": "delete"}, back_populates="current_pipeline_step" ) # noqa F821 service_id: UUID = Field(nullable=False, foreign_key="services.id") diff --git a/core-engine/src/services/models.py b/core-engine/src/services/models.py index 4e321114..1cd1582e 100644 --- a/core-engine/src/services/models.py +++ b/core-engine/src/services/models.py @@ -24,7 +24,10 @@ class Service(ServiceBase, table=True): __tablename__ = "services" id: UUID = Field(default_factory=uuid4, primary_key=True) - tasks: List["Task"] = Relationship(back_populates="service") # noqa F821 + tasks: List["Task"] = Relationship( + sa_relationship_kwargs={"cascade": "delete"}, + back_populates="service" + ) # noqa F821 pipeline_steps: List["PipelineStep"] = Relationship(back_populates="service") # noqa F821 From 470a1a0dee8fe5a482016110d0482ea6e0c74f53 Mon Sep 17 00:00:00 2001 From: Andrea Petrucci Date: Wed, 22 Nov 2023 09:48:50 +0100 Subject: [PATCH 2/4] updated deletion routines --- core-engine/src/common/exceptions.py | 8 ++++++++ core-engine/src/pipeline_executions/models.py | 1 + core-engine/src/pipelines/models.py | 5 ++++- core-engine/src/pipelines/service.py | 2 +- core-engine/src/services/controller.py | 4 +++- core-engine/src/services/service.py | 13 +++++++++---- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core-engine/src/common/exceptions.py b/core-engine/src/common/exceptions.py index fd0449cd..11dd875d 100644 --- a/core-engine/src/common/exceptions.py +++ b/core-engine/src/common/exceptions.py @@ -54,3 +54,11 @@ def __init__(self, message, message_to_send, linked_id): self.message_to_send = message_to_send self.linked_id = linked_id super().__init__(self.message) + + +class ConstraintException(Exception): + """Exception raised when a constraint is not respected.""" + + def __init__(self, message): + self.message = message + super().__init__(self.message) diff --git a/core-engine/src/pipeline_executions/models.py b/core-engine/src/pipeline_executions/models.py index 42873ec1..e28add56 100644 --- a/core-engine/src/pipeline_executions/models.py +++ b/core-engine/src/pipeline_executions/models.py @@ -37,6 +37,7 @@ class PipelineExecution(PipelineExecutionBase, table=True): pipeline: "Pipeline" = Relationship(back_populates="pipeline_executions") current_pipeline_step: Union["PipelineStep", None] = Relationship(back_populates="pipeline_executions") tasks: List[Task] = Relationship( + sa_relationship_kwargs={"cascade": "delete"}, back_populates="pipeline_execution", ) files: List[FileKeyReference] | None = Field(sa_column=Column(JSON), default=None, nullable=True) diff --git a/core-engine/src/pipelines/models.py b/core-engine/src/pipelines/models.py index 34b525c6..336d3e22 100644 --- a/core-engine/src/pipelines/models.py +++ b/core-engine/src/pipelines/models.py @@ -23,7 +23,10 @@ class Pipeline(PipelineBase, table=True): __tablename__ = "pipelines" id: UUID = Field(default_factory=uuid4, primary_key=True) - pipeline_executions: List["PipelineExecution"] = Relationship(back_populates="pipeline") # noqa F821 + pipeline_executions: List["PipelineExecution"] = Relationship( + sa_relationship_kwargs={"cascade": "delete"}, + back_populates="pipeline" + ) # noqa F821 steps: List[PipelineStep] = Relationship( sa_relationship_kwargs={"cascade": "delete"}, back_populates="pipeline" diff --git a/core-engine/src/pipelines/service.py b/core-engine/src/pipelines/service.py index 6c7f8637..475e5cec 100644 --- a/core-engine/src/pipelines/service.py +++ b/core-engine/src/pipelines/service.py @@ -308,7 +308,7 @@ def update(self, app: FastAPI, pipeline_id: UUID, pipeline: PipelineUpdate): needs=pipeline_step.needs, condition=pipeline_step.condition, inputs=pipeline_step.inputs, - service_id=pipeline_step.service.id, + service_id=service.id, ) pipeline_step_create = PipelineStep.from_orm(new_pipeline_step) self.session.add(new_pipeline_step) diff --git a/core-engine/src/services/controller.py b/core-engine/src/services/controller.py index 4b5f246b..cd97b127 100644 --- a/core-engine/src/services/controller.py +++ b/core-engine/src/services/controller.py @@ -1,6 +1,6 @@ from typing import List from fastapi import APIRouter, Depends, HTTPException, Request -from common.exceptions import NotFoundException, ConflictException +from common.exceptions import NotFoundException, ConflictException, ConstraintException from execution_units.enums import ExecutionUnitStatus from services.service import ServicesService from common.query_parameters import QueryParameters @@ -144,3 +144,5 @@ def delete( services_service.delete(service_id, request.app) except NotFoundException as e: raise HTTPException(status_code=404, detail=str(e)) + except ConstraintException as e: + raise HTTPException(status_code=409, detail=str(e)) diff --git a/core-engine/src/services/service.py b/core-engine/src/services/service.py index f90eb38e..cceeec25 100644 --- a/core-engine/src/services/service.py +++ b/core-engine/src/services/service.py @@ -1,4 +1,5 @@ from inspect import Parameter, Signature +from sqlalchemy.exc import IntegrityError from common_code.common.models import ExecutionUnitTag from common_code.common.enums import ExecutionUnitTagName, ExecutionUnitTagAcronym from fastapi import FastAPI, UploadFile, Depends, HTTPException @@ -15,7 +16,7 @@ from common_code.logger.logger import Logger, get_logger from config import Settings, get_settings from services.models import Service, ServiceUpdate, ServiceTask -from common.exceptions import NotFoundException, ConflictException, UnreachableException +from common.exceptions import NotFoundException, ConflictException, UnreachableException, ConstraintException from http_client import HttpClient from fastapi.encoders import jsonable_encoder from httpx import HTTPError @@ -292,9 +293,13 @@ def delete(self, service_id: UUID, app: FastAPI): current_service = self.session.get(Service, service_id) if not current_service: raise NotFoundException("Service Not Found") - self.session.delete(current_service) - self.remove_route(app, current_service.slug) - self.session.commit() + try: + self.session.delete(current_service) + self.remove_route(app, current_service.slug) + self.session.commit() + except IntegrityError: + raise ConstraintException( + "Service is linked to a pipeline, please update the related step in the pipeline first.") self.logger.debug(f"Deleted service with id {current_service.id}") def remove_route(self, app: FastAPI, slug: str): From 87b1beff3d9961af4102d7878be6062a270389c5 Mon Sep 17 00:00:00 2001 From: Andrea Petrucci Date: Wed, 22 Nov 2023 14:14:23 +0100 Subject: [PATCH 3/4] correction of engine stats if no task has been created yet --- webapp/src/components/EngineStats/EngineStats.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/components/EngineStats/EngineStats.tsx b/webapp/src/components/EngineStats/EngineStats.tsx index 7f16607a..65781b52 100644 --- a/webapp/src/components/EngineStats/EngineStats.tsx +++ b/webapp/src/components/EngineStats/EngineStats.tsx @@ -98,7 +98,7 @@ export const EngineStats: React.FC<{ const loadStats = async () => { const stats = await getStats(); - if (stats.total) { + if (stats.hasOwnProperty("total")) { setStats(stats); } else { toast(`Error loading engine stats: ${stats.error}`, {type: "error"}); From b983478d126cbabead824792cec905c79c51389a Mon Sep 17 00:00:00 2001 From: Andrea Petrucci Date: Wed, 22 Nov 2023 14:17:08 +0100 Subject: [PATCH 4/4] remove legacy peer deps from npm since extension needing that is not used anymore --- .github/workflows/webapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/webapp.yml b/.github/workflows/webapp.yml index 6619dd29..d5855892 100644 --- a/.github/workflows/webapp.yml +++ b/.github/workflows/webapp.yml @@ -28,7 +28,7 @@ jobs: - name: Install dependencies working-directory: webapp - run: npm ci --legacy-peer-deps + run: npm ci - name: Build app for dev # Only run on main