feat(iatlas): update iatlas-data with the latest improvements made to…

… GitLab (ARCH-315, ARCH-317) (#2900)

apps/iatlas/data/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+exclude =
+	.git,
+	__pycache__,
+	build,
+	dist,
+	.tox,
+	venv,
+	.venv,
+	.pytest_cache
+max-line-length = 120

apps/iatlas/data/.python-version

@@ -1 +1 @@
-3.10.13
+3.11.10

apps/iatlas/data/Dockerfile

@@ -1,33 +1,78 @@
-FROM python:3.10.13
+FROM python:3.11.10
+
+ARG USERNAME=app
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+ENV APP_DIR=/opt/app \
+    APP_USERNAME=${USERNAME}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-RUN apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
+RUN groupadd --gid "$USER_GID" "$USERNAME" \
+  && useradd --uid "$USER_UID" --gid "$USER_GID" -m "$USERNAME" \
+  # Install sudo
+  && apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
   && apt-get install --no-install-recommends -qq -y \
-    wget \
+    gosu wget \
   # Install the GPG key for the Postgres repo
   && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
   # Add the repo
   && echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
   && apt-get update -qq -y \
-  # Install the Postgres 12 client
+  # Install the Postgres 15 client
   && apt-get install --no-install-recommends -qq -y \
-    postgresql-client-12 \
-  && mkdir -p /home/gitlab/sage-iatlas-data \
+    postgresql-client-15 \
   && apt-get -y autoclean \
   && apt-get -y autoremove \
   && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /
-COPY requirements.txt requirements.txt
-COPY src/ /src
+WORKDIR ${APP_DIR}
+COPY src src/
+COPY pyproject.toml poetry.lock ./
 
-RUN pip install --no-cache-dir -r requirements.txt
+# Use the version of Poetry installed in the dev container.
+# See /workspaces/sage-monorepo/tools/devcontainers/sage/.devcontainer/Dockerfile
+RUN pip install --no-cache-dir poetry==1.8.3 \
+  && poetry config --local virtualenvs.create false \
+  && poetry install --no-root --no-interaction --no-ansi \
+  && pip cache purge
 
+WORKDIR /
 COPY docker-entrypoint.sh ./
 RUN chmod +x docker-entrypoint.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 
-# Set up the iAtlas database
-CMD ["python", "/src/build_database.py"]
+CMD ["python", "src/build_database.py"]
+
+
+
+# RUN apt-get update -qq -y && export DEBIAN_FRONTEND=noninteractive \
+#   && apt-get install --no-install-recommends -qq -y \
+#     wget \
+#   # Install the GPG key for the Postgres repo
+#   && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+#   # Add the repo
+#   && echo "deb http://apt.postgresql.org/pub/repos/apt/ bookworm-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
+#   && apt-get update -qq -y \
+#   # Install the Postgres 15 client
+#   && apt-get install --no-install-recommends -qq -y \
+#     postgresql-client-15 \
+#   && mkdir -p /home/gitlab/sage-iatlas-data \
+#   && apt-get -y autoclean \
+#   && apt-get -y autoremove \
+#   && rm -rf /var/lib/apt/lists/*
+
+# WORKDIR /
+# COPY requirements.txt requirements.txt
+# COPY src/ /src
+
+# RUN pip install --no-cache-dir -r requirements.txt
+
+# COPY docker-entrypoint.sh ./
+# RUN chmod +x docker-entrypoint.sh
+
+# ENTRYPOINT ["/docker-entrypoint.sh"]
+
+# # Set up the iAtlas database
+# CMD ["python", "/src/build_database.py"]

apps/iatlas/data/README.md

@@ -4,3 +4,9 @@
 
 Creates a Docker container that download iAtlas data and data schema from Synapse before seeding the
 iAtlas DB.
+
+## Prepare the project
+
+```console
+nx prepare iatlas-data
+```

apps/iatlas/data/prepare-python.sh → apps/iatlas/data/install.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-PYTHON_VERSION="3.10.13"
+PYTHON_VERSION="3.11.10"
 
 pyenv install --skip-existing $PYTHON_VERSION
 
@@ -9,6 +9,5 @@ pyenv install --skip-existing $PYTHON_VERSION
 eval "$(pyenv init -)"
 
 pyenv local $PYTHON_VERSION
-poetry env use $PYTHON_VERSION
-poetry install
-# poetry install --with prod,dev
+poetry env use $(pyenv which python)
+poetry install --with dev

apps/iatlas/data/poetry.toml

@@ -1,3 +1,2 @@
 [virtualenvs]
 in-project = true
-create = true

apps/iatlas/data/project.json

@@ -1,35 +1,52 @@
 {
   "name": "iatlas-data",
   "$schema": "../../../node_modules/nx/schemas/project-schema.json",
-  "sourceRoot": "apps/iatlas/data/scripts",
   "projectType": "application",
+  "sourceRoot": "apps/iatlas/data/src",
   "targets": {
-    "create-config": {
+    "lock": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "cp -n .env.example .env",
+        "command": "poetry lock --no-update",
         "cwd": "{projectRoot}"
       }
     },
+    "build": {
+      "executor": "@nxlv/python:build",
+      "outputs": ["{projectRoot}/dist"],
+      "options": {
+        "outputPath": "{projectRoot}/dist",
+        "publish": false,
+        "lockedVersions": true,
+        "bundleLocalDependencies": true
+      }
+    },
     "prepare": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "./prepare-python.sh",
+        "command": "./install.sh",
         "cwd": "{projectRoot}"
       }
     },
-    "serve": {
+    "update": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "docker compose up",
-        "cwd": "apps/iatlas/data"
-      },
-      "dependsOn": []
+        "command": "poetry update",
+        "cwd": "{projectRoot}"
+      }
+    },
+    "lint": {
+      "executor": "@nxlv/python:flake8",
+      "outputs": ["{workspaceRoot}/reports/{projectRoot}/pylint.txt"],
+      "options": {
+        "outputFile": "reports/{projectRoot}/pylint.txt"
+      }
     },
-    "serve-detach": {
+    "serve": {
       "executor": "nx:run-commands",
       "options": {
-        "command": "docker/iatlas/serve-detach.sh {projectName}"
+        "command": "poetry run python src/build_database.py",
+        "cwd": "{projectRoot}"
       }
     },
     "build-image": {
@@ -42,27 +59,12 @@
         },
         "push": false
       }
-    },
-    "publish-image": {
-      "executor": "@nx-tools/nx-container:build",
-      "options": {
-        "context": "apps/iatlas/data",
-        "metadata": {
-          "images": ["ghcr.io/sage-bionetworks/{projectName}"],
-          "tags": ["type=edge,branch=main", "type=sha"]
-        },
-        "push": true
-      },
-      "dependsOn": ["build-image"]
-    },
-    "scan-image": {
-      "executor": "nx:run-commands",
-      "options": {
-        "command": "trivy image ghcr.io/sage-bionetworks/{projectName}:local --quiet",
-        "color": true
-      }
     }
   },
-  "tags": ["type:db", "scope:backend", "language:python"],
-  "implicitDependencies": []
+  "tags": ["language:python", "package-manager:poetry"],
+  "release": {
+    "version": {
+      "generator": "@nxlv/python:release-version"
+    }
+  }
 }

apps/iatlas/data/pyproject.toml

@@ -1,15 +1,37 @@
+[tool.coverage.run]
+branch = true
+source = [ "src" ]
+
+[tool.coverage.report]
+exclude_lines = [ "if TYPE_CHECKING:" ]
+show_missing = true
+
+[tool.pytest.ini_options]
+addopts = "--cov --cov-report html:'../../../coverage/apps/iatlas/data/html' --cov-report xml:'../../../coverage/apps/iatlas/data/coverage.xml' --html='../../../reports/apps/iatlas/data/unittests/html/index.html' --junitxml='../../../reports/apps/iatlas/data/unittests/junit.xml'"
+
 [tool.poetry]
 name = "iatlas-data"
-version = "0.1.0"
-description = ""
-authors = ["andrewelamb <[email protected]>"]
+version = "1.0.0"
+description = "Automatically generated by Nx."
+authors = [ ]
+license = "Proprietary"
 readme = "README.md"
 
-[tool.poetry.dependencies]
-python = "^3.9"
-schematic-db = {version = "0.0.36", extras = ["postgres", "synapse"]}
+  [[tool.poetry.packages]]
+  include = "src"
+
+  [tool.poetry.dependencies]
+  python = "3.11.10"
+  schematic-db = {version = "0.1.5", extras = ["postgres", "synapse"]}
 
+[tool.poetry.group.dev.dependencies]
+autopep8 = "2.0.2"
+flake8 = "6.0.0"
+pytest = "7.3.1"
+pytest-sugar = "0.9.7"
+pytest-cov = "4.1.0"
+pytest-html = "3.2.0"
 
 [build-system]
-requires = ["poetry-core"]
+requires = [ "poetry-core" ]
 build-backend = "poetry.core.masonry.api"

apps/iatlas/data/pyrightconfig.json

@@ -0,0 +1,7 @@
+{
+  "include": ["src"],
+  "exclude": ["tests"],
+  "venvPath": ".",
+  "venv": ".venv",
+  "pythonVersion": "3.11"
+}

apps/iatlas/data/requirements.txt

@@ -1,47 +1,51 @@
-attrs==23.2.0 ; python_version >= "3.9" and python_version < "4.0"
-backoff==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2024.2.2 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "4.0"
-click==8.1.7 ; python_version >= "3.9" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
-decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
-deprecated==1.2.14 ; python_version >= "3.9" and python_version < "4.0"
-deprecation==2.1.0 ; python_version >= "3.9" and python_version < "4.0"
-googleapis-common-protos==1.62.0 ; python_version >= "3.9" and python_version < "4.0"
-greenlet==3.0.3 ; python_version >= "3.9" and python_version < "4.0" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32")
-idna==3.6 ; python_version >= "3.9" and python_version < "4.0"
-importlib-metadata==6.11.0 ; python_version >= "3.9" and python_version < "4.0"
-interrogate==1.5.0 ; python_version >= "3.9" and python_version < "4.0"
-nest-asyncio==1.6.0 ; python_version >= "3.9" and python_version < "4.0"
-networkx==2.8.8 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.26.4 ; python_version >= "3.9" and python_version <= "3.11" or python_version >= "3.12" and python_version < "4.0"
-opentelemetry-api==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
-opentelemetry-exporter-otlp-proto-common==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
-opentelemetry-exporter-otlp-proto-http==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
-opentelemetry-proto==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
-opentelemetry-sdk==1.21.0 ; python_version >= "3.9" and python_version < "4.0"
-opentelemetry-semantic-conventions==0.42b0 ; python_version >= "3.9" and python_version < "4.0"
-packaging==23.2 ; python_version >= "3.9" and python_version < "4.0"
-pandas==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
-protobuf==4.25.3 ; python_version >= "3.9" and python_version < "4.0"
-psycopg2-binary==2.9.9 ; python_version >= "3.9" and python_version < "4.0"
-py==1.11.0 ; python_version >= "3.9" and python_version < "4.0"
-pydantic==1.10.14 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2024.1 ; python_version >= "3.9" and python_version < "4.0"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.31.0 ; python_version >= "3.9" and python_version < "4.0"
-schematic-db[postgres,synapse]==0.0.41 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-sqlalchemy-utils==0.41.1 ; python_version >= "3.9" and python_version < "4.0"
-sqlalchemy==2.0.27 ; python_version >= "3.9" and python_version < "4.0"
-synapseclient==4.1.1 ; python_version >= "3.9" and python_version < "4.0"
-tabulate==0.9.0 ; python_version >= "3.9" and python_version < "4.0"
-tenacity==8.2.3 ; python_version >= "3.9" and python_version < "4.0"
-toml==0.10.2 ; python_version >= "3.9" and python_version < "4.0"
-typing-extensions==4.10.0 ; python_version >= "3.9" and python_version < "4.0"
-tzdata==2024.1 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.18 ; python_version >= "3.9" and python_version < "4.0"
-validators==0.20.0 ; python_version >= "3.9" and python_version < "4.0"
-wrapt==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-zipp==3.17.0 ; python_version >= "3.9" and python_version < "4.0"
+annotated-types==0.7.0 ; python_version >= "3.11" and python_version < "4.0"
+anyio==4.4.0 ; python_version >= "3.11" and python_version < "4.0"
+async-lru==2.0.4 ; python_version >= "3.11" and python_version < "4.0"
+asyncio-atexit==1.0.1 ; python_version >= "3.11" and python_version < "4.0"
+backoff==2.2.1 ; python_version >= "3.11" and python_version < "4.0"
+certifi==2024.7.4 ; python_version >= "3.11" and python_version < "4.0"
+charset-normalizer==3.3.2 ; python_version >= "3.11" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows"
+deprecated==1.2.14 ; python_version >= "3.11" and python_version < "4.0"
+deprecation==2.1.0 ; python_version >= "3.11" and python_version < "4.0"
+googleapis-common-protos==1.63.2 ; python_version >= "3.11" and python_version < "4.0"
+greenlet==3.0.3 ; python_version < "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version >= "3.11"
+h11==0.14.0 ; python_version >= "3.11" and python_version < "4.0"
+httpcore==1.0.5 ; python_version >= "3.11" and python_version < "4.0"
+httpx==0.27.0 ; python_version >= "3.11" and python_version < "4.0"
+idna==3.7 ; python_version >= "3.11" and python_version < "4.0"
+importlib-metadata==6.11.0 ; python_version >= "3.11" and python_version < "4.0"
+nest-asyncio==1.6.0 ; python_version >= "3.11" and python_version < "4.0"
+networkx==3.3 ; python_version >= "3.11" and python_version < "4.0"
+numpy==2.0.1 ; python_version >= "3.12" and python_version < "4.0" or python_version == "3.11"
+opentelemetry-api==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
+opentelemetry-exporter-otlp-proto-common==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
+opentelemetry-exporter-otlp-proto-http==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
+opentelemetry-proto==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
+opentelemetry-sdk==1.21.0 ; python_version >= "3.11" and python_version < "4.0"
+opentelemetry-semantic-conventions==0.42b0 ; python_version >= "3.11" and python_version < "4.0"
+packaging==24.1 ; python_version >= "3.11" and python_version < "4.0"
+pandas==2.2.2 ; python_version >= "3.11" and python_version < "4.0"
+protobuf==4.25.4 ; python_version >= "3.11" and python_version < "4.0"
+psutil==5.9.8 ; python_version >= "3.11" and python_version < "4.0"
+psycopg2-binary==2.9.9 ; python_version >= "3.11" and python_version < "4.0"
+pydantic-core==2.20.1 ; python_version >= "3.11" and python_version < "4.0"
+pydantic==2.8.2 ; python_version >= "3.11" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.11" and python_version < "4.0"
+pytz==2024.1 ; python_version >= "3.11" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.11" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.11" and python_version < "4.0"
+schematic-db[postgres,synapse]==0.1.5 ; python_version >= "3.11" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.11" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.11" and python_version < "4.0"
+sqlalchemy-utils==0.41.2 ; python_version >= "3.11" and python_version < "4.0"
+sqlalchemy==2.0.31 ; python_version >= "3.11" and python_version < "4.0"
+synapseclient==4.4.0 ; python_version >= "3.11" and python_version < "4.0"
+tenacity==9.0.0 ; python_version >= "3.11" and python_version < "4.0"
+tqdm==4.66.4 ; python_version >= "3.11" and python_version < "4.0"
+typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "4.0"
+tzdata==2024.1 ; python_version >= "3.11" and python_version < "4.0"
+urllib3==1.26.19 ; python_version >= "3.11" and python_version < "4.0"
+validators==0.29.0 ; python_version >= "3.11" and python_version < "4.0"
+wrapt==1.16.0 ; python_version >= "3.11" and python_version < "4.0"
+zipp==3.19.2 ; python_version >= "3.11" and python_version < "4.0"