Merge branch 'develop' into enhancement-oep-django-5-migration

OpenEnergyPlatform · Nov 17, 2024 · 4cd7b6a · 4cd7b6a
2 parents 1068a8a + e80deab
commit 4cd7b6a
Show file tree

Hide file tree

Showing 68 changed files with 653 additions and 207 deletions.
diff --git a/.gitignore b/.gitignore
@@ -84,6 +84,8 @@ venv*/
 .env*
 /fuseki
 apache*
+/oep-django-5
+
 
 .DS_Store
 
@@ -103,3 +105,7 @@ ontologies/
 scripts/
 
 site/
+
+# Jenna fuseki graph db installation
+/apache-jena-fuseki*/
+/apache-jena-fuseki*.tar.gz*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,4 +1,4 @@
-exclude: ^.*(.min.js|.min.css|.html|.js|.js.map)$
+exclude: ^.*(.min.js|.min.css|.html|.js|.js.map|docs/*|mkdocs.yml)$
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0

diff --git a/.prettierignore b/.prettierignore
@@ -1,3 +1,5 @@
 # Ignore all HTML files, because they contain
 # django markup that will not be broken by prettier
 **/*.html
+
+docs/*
diff --git a/api/actions.py b/api/actions.py
@@ -857,11 +857,15 @@ def table_create(schema, table, column_definitions, constraints_definitions):
 
         # check for duplicate column names
         if col.name in columns_by_name:
-            raise APIError("Duplicate column name: %s" % col.name)
+            error = APIError("Duplicate column name: %s" % col.name)
+            logger.error(error)
+            raise error
         columns_by_name[col.name] = col
         if col.primary_key:
             if primary_key_col_names:
-                raise APIError("Multiple definitions of primary key")
+                error = APIError("Multiple definitions of primary key")
+                logger.error(error)
+                raise error
             primary_key_col_names = [col.name]
 
     constraints = []
@@ -887,7 +891,7 @@ def table_create(schema, table, column_definitions, constraints_definitions):
                 # to column level PK, both must be the same (#1110)
                 if set(ccolumns) == set(primary_key_col_names):
                     continue
-                raise APIError("Multiple definitions of primary key")
+                raise APIError("Multiple definitions of primary key.")
             primary_key_col_names = ccolumns
 
             const = sa.schema.PrimaryKeyConstraint(*ccolumns, **kwargs)

diff --git a/api/tests/test_regression/test_issue_1885_table_creation_fail_behavior.py b/api/tests/test_regression/test_issue_1885_table_creation_fail_behavior.py
@@ -0,0 +1,43 @@
+from api import actions
+from api.actions import has_table
+from api.tests import APITestCase
+
+
+class TestTableNameUnique(APITestCase):
+    schema_sandbox = "sandbox"
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        actions.perform_sql(f"DROP SCHEMA IF EXISTS {cls.schema_sandbox} CASCADE")
+        actions.perform_sql(f"CREATE SCHEMA {cls.schema_sandbox}")
+        actions.perform_sql(f"DROP SCHEMA IF EXISTS _{cls.schema_sandbox} CASCADE")
+        actions.perform_sql(f"CREATE SCHEMA _{cls.schema_sandbox}")
+
+    def test_tables_should_not_exists_on_error(self):
+        test_duplicate_column_table_name = "table_column_duplicate"
+        # create table with duplicated column names will should an error
+        duplicate_field_error_data_struct = {
+            "columns": [
+                {"name": "id", "data_type": "bigint"},
+                {"name": "id", "data_type": "bigint"},
+            ]
+        }
+        # create table in default (test) schema (django_db)
+        self.assertRaises(
+            AssertionError,
+            self.create_table,
+            table=test_duplicate_column_table_name,
+            structure=duplicate_field_error_data_struct,
+            schema=self.schema_sandbox,
+        )
+
+        # also check: table should not have been created in oedb
+        self.assertFalse(
+            has_table(
+                {
+                    "table": test_duplicate_column_table_name,
+                    "schema": self.schema_sandbox,
+                }
+            )
+        )
diff --git a/api/views.py b/api/views.py
@@ -14,6 +14,7 @@
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.contrib.postgres.search import TrigramSimilarity
 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
+from django.db import DatabaseError, transaction
 from django.db.models import Q
 from django.db.utils import IntegrityError
 from django.http import (
@@ -484,6 +485,100 @@ def validate_column_names(self, column_definitions):
             if len(colname) > MAX_COL_NAME_LENGTH:
                 raise APIError(f"Column name is too long! {err_msg}")
 
+    def oep_create_table_transaction(
+        self,
+        django_schema_object,
+        schema,
+        table,
+        column_definitions,
+        constraint_definitions,
+    ):
+        """
+        This method handles atomic table creation transactions on the OEP. It
+        attempts to create first the django table objects and stored it in
+        dataedit_tables table. Then it attempts to create the OEDB table.
+        If there is an error raised during the first two steps the function
+        will cleanup any table object or table artifacts created during the
+        process. The order of execution matters, it should always first
+        create the django table object.
+
+        Params:
+            django_schema_object: The schema object stored in the django
+                database
+            schema:
+            table
+            column_definitions
+            constraint_definitions
+
+        returns:
+            table_object: The django table objects that was created
+        """
+
+        try:
+            with transaction.atomic():
+                # First create the table object in the django database.
+                table_object = self._create_table_object(django_schema_object, table)
+            # Then attempt to create the OEDB table to check
+            # if creation will succeed - action includes checks
+            # and will raise api errors
+            actions.table_create(
+                schema, table, column_definitions, constraint_definitions
+            )
+        except DatabaseError as e:
+            # remove any oedb table artifacts left after table creation
+            # transaction failed
+            self.__remove_oedb_table_on_exception_raised_during_creation_transaction(
+                table, schema
+            )
+
+            # also remove any django table object
+            # find the created django table object
+            object_to_delete = DBTable.objects.filter(
+                name=table, schema=django_schema_object
+            )
+            # delete it if it exists
+            if object_to_delete.exists():
+                object_to_delete.delete()
+
+            raise APIError(
+                message="Error during table creation transaction. All table fragments"
+                f"have been removed. For further details see: {e}"
+            )
+
+        # for now only return the django table object
+        # TODO: Check if is necessary to return the response dict returned by the oedb
+        # table creation function
+        return table_object
+
+    def __remove_oedb_table_on_exception_raised_during_creation_transaction(
+        self, table, schema
+    ):
+        """
+        This private method handles removing a table form the OEDB only for the case
+        where an error was raised during table creation. It specifically will delete
+        the OEDB table created by the user and also the edit_ meta(revision) table
+        that is automatically created in the background.
+        """
+        # find the created oedb table
+        if actions.has_table({"table": table, "schema": schema}):
+            # get table and schema names, also for meta(revision) tables
+            schema, table = actions.get_table_name(schema, table)
+            meta_schema = actions.get_meta_schema_name(schema)
+
+            # drop the revision table with edit_ prefix
+            edit_table = actions.get_edit_table_name(schema, table)
+            actions._get_engine().execute(
+                'DROP TABLE "{schema}"."{table}" CASCADE;'.format(
+                    schema=meta_schema, table=edit_table
+                )
+            )
+            # drop the data table
+            actions._get_engine().execute(
+                'DROP TABLE "{schema}"."{table}" CASCADE;'.format(
+                    schema=schema, table=table
+                )
+            )
+
     @load_cursor()
     def __create_table(
         self,
@@ -512,11 +607,13 @@ def __create_table(
             raise embargo_error
 
         if embargo_payload_check:
-            table_object = self._create_table_object(schema_object, table)
-            actions.table_create(
-                schema, table, column_definitions, constraint_definitions
+            table_object = self.oep_create_table_transaction(
+                django_schema_object=schema_object,
+                table=table,
+                schema=schema,
+                column_definitions=column_definitions,
+                constraint_definitions=constraint_definitions,
             )
-
             self._apply_embargo(table_object, embargo_data)
 
             if metadata:
@@ -536,9 +633,12 @@ def __create_table(
                 )
 
         else:
-            table_object = self._create_table_object(schema_object, table)
-            actions.table_create(
-                schema, table, column_definitions, constraint_definitions
+            table_object = self.oep_create_table_transaction(
+                django_schema_object=schema_object,
+                table=table,
+                schema=schema,
+                column_definitions=column_definitions,
+                constraint_definitions=constraint_definitions,
             )
             self._assign_table_holder(request.user, schema, table)
 

diff --git a/base/static/css/bootstrap.min.css b/base/static/css/bootstrap.min.css
diff --git a/dataedit/static/metaedit/schema.json b/dataedit/static/metaedit/schema.json
@@ -4,6 +4,8 @@
   "$id": "https://raw.githubusercontent.com/OpenEnergyPlatform/oemetadata/master/oemetadata/latest/schema.json",
   "description": "Application to create oemetadata in JSON format and download the JSON file to use it later or edit metadata on existing data tables. The builder currently uses the OEMetadata v1.6 schema. See the Open Energy Platform metadata standard: https://github.com/OpenEnergyPlatform/oemetadata",
   "type": "object",
+  "format": "categories",
+  "basicCategoryTitle": "General",
   "properties": {
     "name": {
       "description": "File name or database table name",
@@ -873,56 +875,6 @@
       "options": {
         "hidden": true
       }
-    },
-    "_comment": {
-      "description": "Object. The “_comment”-section is used as a self-description of the final metadata-file. It is text, intended for humans and can include a link to the metadata documentation(s), required value formats and similar remarks. The comment section has no fix structure or mandatory values, but a useful self-description, similar to the one depicted here, is encouraged.",
-      "type": "object",
-      "properties": {
-        "metadata": {
-          "description": "Reference to the metadata documentation in use. Example: Metadata documentation and explanation (https://github.com/OpenEnergyPlatform/organisation/wiki/metadata)",
-          "type": ["string", "null"],
-          "title": "Metadata"
-        },
-        "dates": {
-          "description": "Comment on data/time format. Example: Dates and time must follow the ISO8601 including time zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)",
-          "type": ["string", "null"],
-          "title": "Dates"
-        },
-        "units": {
-          "description": "Comment on units. Example: If you must use units in cells (which is discouraged), leave a space between numbers and units (100 m)",
-          "type": ["string", "null"],
-          "title": "Units"
-        },
-        "languages": {
-          "description": "Comment on language format. Example: Languages must follow the IETF (BCP47) format (en-GB, en-US, de-DE)",
-          "type": ["string", "null"],
-          "title": "Languages"
-        },
-        "licenses": {
-          "description": "Reference to license format. Example: License name must follow the SPDX License List (https://spdx.org/licenses/)",
-          "type": ["string", "null"],
-          "title": "Licenses"
-        },
-        "review": {
-          "description": "Reference to review documentation. Example: Following the OEP Data Review (https://github.com/OpenEnergyPlatform/data-preprocessing/wiki)",
-          "type": ["string", "null"],
-          "title": "Review"
-        },
-        "null": {
-          "description": "Feel free to add more descriptive comments. Like \"null\". Example: If a field is not applicable just enter \"null\"",
-          "type": ["string", "null"],
-          "title": "Null"
-        },
-        "todo": {
-          "description": "If an applicable value is not yet available and will be inserted later on use: 'todo' ",
-          "type": ["string", "null"],
-          "title": "Todo"
-        }
-      },
-      "title": "_comment",
-      "options": {
-        "hidden": true
-      }
     }
   },
   "additionalProperties": false

diff --git a/dataedit/static/wizard/wizard.js b/dataedit/static/wizard/wizard.js
@@ -715,19 +715,20 @@ function calculateEmbargoPeriod(embargoValue) {
   function showCreate() {
     // create default id column
     addColumn({"name": "id", "data_type": "bigint", "is_nullable": false, "is_pk": true});
-    $("#wizard-container-upload").collapse("hide");
-    $("#wizard-container-create").collapse("show");
+    new bootstrap.Collapse('#wizard-container-create', {'toggle': false}).show();
+    new bootstrap.Collapse('#wizard-container-upload', {'toggle': false}).hide();
     $("#wizard-table-delete").hide();
     $("#wizard-container-upload").find(".btn").hide();
     $("#wizard-container-upload").find("input").prop("readonly", true);
   }
 
 
   function showUpload() {
-    $("#wizard-container-create").collapse("hide");
-    $("#wizard-container-upload").collapse("show");
-    $("#wizard-container-create").find(".btn").hide();
+    new bootstrap.Collapse('#wizard-container-create', {'toggle': false}).hide();
+    new bootstrap.Collapse('#wizard-container-upload', {'toggle': false}).show();
     $("#wizard-table-delete").show();
+
+    $("#wizard-container-create").find(".btn").hide();
     $("#wizard-container-create").find("input").prop("readonly", true);
     $("#wizard-container-create").find("input,select,.combobox-container").not("[type=text]").prop("disabled", true);
     if (!state.canAdd) {
@@ -831,7 +832,6 @@ function calculateEmbargoPeriod(embargoValue) {
       });
       $("#wizard-confirm-delete-delete").bind("click", deleteTable);
 
-
       showUpload();
     } else {
       showCreate();

diff --git a/dataedit/templates/messages.html b/dataedit/templates/messages.html
@@ -2,12 +2,12 @@
   {% for message in messages %}
     <div class="alert alert-dismissible alert-success">
       <h4>
-        Sucessuflly updated table tags!
+        Successfully updated table tags!
       </h4>
       <button type="button" class="btn-close" data-bs-dismiss="alert">
       ×
       </button>
-      <p class="mb-0" style="font-size: 30px;">
+      <p class="mb-0">
       {{message}}
       </p>
     </div>

diff --git a/dataedit/views.py b/dataedit/views.py
@@ -1558,7 +1558,7 @@ def update_table_tags(request):
                 table=table, schema=schema, metadata=metadata, cursor=con
             )
 
-    messasge = messages.success(
+    message = messages.success(
         request,
         'Please note that OEMetadata keywords and table tags are synchronized. When submitting new tags, you may notice automatic changes to the table tags on the OEP and/or the "Keywords" field in the metadata.',  # noqa
         # noqa
@@ -1567,7 +1567,7 @@ def update_table_tags(request):
     return render(
         request,
         "dataedit/dataview.html",
-        {"messages": messasge, "table": table, "schema": schema},
+        {"messages": message, "table": table, "schema": schema},
     )
 
 

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -17,8 +17,7 @@ COPY ./docker/apache2.conf /etc/apache2/conf-enabled/oeplatform.conf
 COPY . /app
 COPY ./docker/docker-entrypoint.sh /app/docker-entrypoint.sh
 
-RUN mkdir -p /app/ontologies/oeo && cd /app/ontologies/oeo && wget https://github.com/OpenEnergyPlatform/ontology/releases/download/v2.4.0/build-files.zip && unzip build-files.zip && mv build-files 2.4.0 && rm -r __MACOSX && rm build-files.zip
-
+RUN mkdir -p /app/ontologies && cd /app/ontologies/ && wget https://github.com/OpenEnergyPlatform/ontology/releases/download/v2.5.0/build-files.zip && unzip build-files.zip
 RUN mkdir -p /app/media/oeo_ext && cp /app/oeo_ext/oeo_extended_store/oeox_template/oeo_ext_template_empty.owl /app/media/oeo_ext/oeo_ext.owl
 
 RUN cp /app/oeplatform/securitysettings.py.default /app/oeplatform/securitysettings.py && python manage.py collectstatic --noinput && rm /app/oeplatform/securitysettings.py

diff --git a/docker/USAGE.md → docker/README.md b/docker/USAGE.md → docker/README.md