From 74f534813c2f8d111f1b5795370813d279b183b5 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Fri, 25 Oct 2024 14:45:35 +0200 Subject: [PATCH 1/9] Fix docker compose to make it running with older docker images --- docker-compose.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4be2b490..48edf593 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: user: 901:999 volumes: - ./projects:/data + - ./server/entrypoint.sh:/app/entrypoint.sh env_file: - .prod.env depends_on: @@ -38,8 +39,11 @@ services: celery-beat: image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: celery-beat + restart: always env_file: - .prod.env + volumes: + - ./server/entrypoint.sh:/app/entrypoint.sh depends_on: - redis - server-gunicorn @@ -49,8 +53,13 @@ services: celery-worker: image: lutraconsulting/merginmaps-backend:2024.2.2 container_name: celery-worker + restart: always + user: 901:999 env_file: - .prod.env + volumes: + - ./projects:/data + - ./server/entrypoint.sh:/app/entrypoint.sh depends_on: - redis - server-gunicorn From 728c9d7edbb114117ec787c1c1109a6cbe018033 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 28 Oct 2024 16:07:38 +0100 Subject: [PATCH 2/9] Switch workspace with project change --- web-app/packages/lib/src/modules/project/store.ts | 9 +++++++++ web-app/packages/lib/src/modules/user/store.ts | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/web-app/packages/lib/src/modules/project/store.ts b/web-app/packages/lib/src/modules/project/store.ts index fad35874..1c8c2a41 100644 --- a/web-app/packages/lib/src/modules/project/store.ts +++ b/web-app/packages/lib/src/modules/project/store.ts @@ -129,6 +129,7 @@ export const useProjectStore = defineStore('projectModule', { }, setProject(payload: { project: ProjectDetail }) { + const userStore = useUserStore() let enhancedProject: EnhancedProjectDetail = null if (payload.project) { enhancedProject = { @@ -147,6 +148,14 @@ export const useProjectStore = defineStore('projectModule', { } } this.project = enhancedProject + + // Update workspace + if (enhancedProject) { + userStore.setWorkspace({ + workspaceId: enhancedProject?.workspace_id, + skipStorage: false + }) + } }, setProjects(payload: ProjectsPayload) { this.projects = payload.projects diff --git a/web-app/packages/lib/src/modules/user/store.ts b/web-app/packages/lib/src/modules/user/store.ts index 7a2b9beb..83f2fb9b 100644 --- a/web-app/packages/lib/src/modules/user/store.ts +++ b/web-app/packages/lib/src/modules/user/store.ts @@ -135,7 +135,7 @@ export const useUserStore = defineStore('userModule', { this.workspaces = payload.workspaces }, - setWorkspaceId(payload) { + setWorkspaceId(payload: { id: number }) { this.workspaceId = payload.id }, From 0d7d566315f54f0c25cab6d37c926f796a4c187a Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 29 Oct 2024 10:34:58 +0100 Subject: [PATCH 3/9] Enable coveralls --- .github/workflows/auto_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/auto_tests.yml b/.github/workflows/auto_tests.yml index ffb6478f..c99c44a7 100644 --- a/.github/workflows/auto_tests.yml +++ b/.github/workflows/auto_tests.yml @@ -38,7 +38,7 @@ jobs: cd server pipenv run pytest -v --cov=mergin mergin/tests - # - name: Coveralls - # uses: coverallsapp/github-action@v2 - # with: - # base-path: server + - name: Coveralls + uses: coverallsapp/github-action@v2 + with: + base-path: server From 55d17242e1179c930b612dabcff0261bcd12cdac Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 29 Oct 2024 10:49:42 +0100 Subject: [PATCH 4/9] added format for coveralls --- .github/workflows/auto_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto_tests.yml b/.github/workflows/auto_tests.yml index c99c44a7..7eb7e22b 100644 --- a/.github/workflows/auto_tests.yml +++ b/.github/workflows/auto_tests.yml @@ -36,7 +36,7 @@ jobs: - name: Run tests run: | cd server - pipenv run pytest -v --cov=mergin mergin/tests + pipenv run pytest -v --cov=mergin mergin/tests --format=python - name: Coveralls uses: coverallsapp/github-action@v2 From 476944490a86e1a40d05af5cb64826aadc1673d4 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 29 Oct 2024 10:56:07 +0100 Subject: [PATCH 5/9] Some upgrades --- .github/workflows/auto_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/auto_tests.yml b/.github/workflows/auto_tests.yml index 7eb7e22b..04435adc 100644 --- a/.github/workflows/auto_tests.yml +++ b/.github/workflows/auto_tests.yml @@ -36,9 +36,10 @@ jobs: - name: Run tests run: | cd server - pipenv run pytest -v --cov=mergin mergin/tests --format=python + pipenv run pytest -v --cov=mergin --cov-report=xml mergin/tests - name: Coveralls uses: coverallsapp/github-action@v2 with: base-path: server + format: python From 8b1b38c8262471cd16de01d3d146fceadedd00f8 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 29 Oct 2024 11:06:27 +0100 Subject: [PATCH 6/9] lcov report --- .github/workflows/auto_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/auto_tests.yml b/.github/workflows/auto_tests.yml index 04435adc..31c5d379 100644 --- a/.github/workflows/auto_tests.yml +++ b/.github/workflows/auto_tests.yml @@ -36,10 +36,10 @@ jobs: - name: Run tests run: | cd server - pipenv run pytest -v --cov=mergin --cov-report=xml mergin/tests + pipenv run pytest -v --cov=mergin --cov-report=lcov mergin/tests - name: Coveralls uses: coverallsapp/github-action@v2 with: base-path: server - format: python + format: lcov From 23274e1faca2b01f32d1d15b6696730cb973def7 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 29 Oct 2024 11:40:10 +0100 Subject: [PATCH 7/9] Add space after workspace name --- .../lib/src/modules/project/components/ProjectsTable.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web-app/packages/lib/src/modules/project/components/ProjectsTable.vue b/web-app/packages/lib/src/modules/project/components/ProjectsTable.vue index 1fffa5ba..85c02b5b 100644 --- a/web-app/packages/lib/src/modules/project/components/ProjectsTable.vue +++ b/web-app/packages/lib/src/modules/project/components/ProjectsTable.vue @@ -50,7 +50,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial }" > {{ slotProps.data.namespace }} / {{ slotProps.data.name }} Date: Tue, 29 Oct 2024 13:49:47 +0100 Subject: [PATCH 8/9] Config cleanup (env vars) and make sure use_x_accel works properly with nginx --- .prod.env | 17 ++++++++++++----- docker-compose.yml | 2 ++ nginx.conf | 2 +- server/mergin/config.py | 24 ++++++++++-------------- 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.prod.env b/.prod.env index 911d7d84..a9ef451f 100644 --- a/.prod.env +++ b/.prod.env @@ -14,10 +14,6 @@ LOCAL_PROJECTS=/data #MAINTENANCE_FILE=os.path.join(LOCAL_PROJECTS, 'MAINTENANCE') # locking file when backups are created MAINTENANCE_FILE=/data/MAINTENANCE -#PROXY_FIX=True - -#PUBLIC_DIR=os.path.join(config_dir, os.pardir, 'build', 'static') - #SECRET_KEY=NODEFAULT SECRET_KEY=fixme @@ -110,7 +106,11 @@ MAIL_USERNAME=fixme #MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 # max total files size for archive download #USE_X_ACCEL=False # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/) +USE_X_ACCEL=1 +# where geodiff lib copies working files +#GEODIFF_WORKING_DIR=$LOCAL_PROJECTS/geodiff_tmp +GEODIFF_WORKING_DIR=/data/geodiff # celery @@ -133,6 +133,8 @@ CELERYD_CONCURRENCY=2 #CELERYD_PREFETCH_MULTIPLIER=4 +#CELERY_ROUTES={} # split tasks into separate queues + # various life times #CLOSED_ACCOUNT_EXPIRATION=5 # time in days after user closed his account to all projects and files are permanently deleted @@ -167,6 +169,11 @@ GLOBAL_STORAGE=10737418240 # GLOBAL_ADMIN False -# Gunicorn server socket +# toggle registration form to create new users +#USER_SELF_REGISTRATION=False + +# what type of server is running, e.g. community edition or enterprise edition +#SERVER_TYPE=ce +# Gunicorn server socket PORT=5000 diff --git a/docker-compose.yml b/docker-compose.yml index 48edf593..f38ca2dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,8 @@ services: image: nginxinc/nginx-unprivileged:1.25.5 container_name: merginmaps-proxy restart: always + # run nginx as built-in user but with group mergin-family for files permissions + user: 101:999 ports: - "8080:8080" volumes: diff --git a/nginx.conf b/nginx.conf index 539d086a..ed4512ad 100644 --- a/nginx.conf +++ b/nginx.conf @@ -49,6 +49,6 @@ server { location /download/ { internal; - alias /data; # we need to mount data from mergin server here + alias /data/; # we need to mount data from mergin server here } } diff --git a/server/mergin/config.py b/server/mergin/config.py index fedeb31d..177063b5 100644 --- a/server/mergin/config.py +++ b/server/mergin/config.py @@ -10,17 +10,15 @@ class Configuration(object): + # flask/connexion variables DEBUG = config("FLASK_DEBUG", default=False, cast=bool) TESTING = config("TESTING", default=False, cast=bool) SECRET_KEY = config("SECRET_KEY") - PROXY_FIX = config("PROXY_FIX", default=True, cast=bool) - SWAGGER_UI = config( - "SWAGGER_UI", default=False, cast=bool - ) # to enable swagger UI console (for tests only) - VERSION = config("VERSION", default=get_version()) - PUBLIC_DIR = config( - "PUBLIC_DIR", default=os.path.join(config_dir, os.pardir, "build", "static") - ) + # to enable swagger UI console (for tests only) + SWAGGER_UI = config("SWAGGER_UI", default=False, cast=bool) + # expiration time in seconds + WTF_CSRF_TIME_LIMIT = config("WTF_CSRF_TIME_LIMIT", default=3600 * 24, cast=int) + WTF_CSRF_ENABLED = config("WTF_CSRF_ENABLED", default=True, cast=bool) # Mergin DB related SQLALCHEMY_TRACK_MODIFICATIONS = config( @@ -43,11 +41,6 @@ class Configuration(object): f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_DATABASE}?application_name={DB_APPLICATION_NAME}", ) - WTF_CSRF_TIME_LIMIT = config( - "WTF_CSRF_TIME_LIMIT", default=3600 * 24, cast=int - ) # in seconds - WTF_CSRF_ENABLED = config("WTF_CSRF_ENABLED", default=True, cast=bool) - # for flask mail MAIL_SERVER = config("MAIL_SERVER", default="localhost") MAIL_PORT = config("MAIL_PORT", default=587, cast=int) @@ -97,8 +90,11 @@ class Configuration(object): GLOBAL_WRITE = config("GLOBAL_WRITE", default=False, cast=bool) GLOBAL_ADMIN = config("GLOBAL_ADMIN", default=False, cast=bool) - SERVER_TYPE = config("SERVER_TYPE", default="") # can users create their own account or is it reserved for superuser only USER_SELF_REGISTRATION = config("USER_SELF_REGISTRATION", default=False, cast=bool) + # build hash number BUILD_HASH = config("BUILD_HASH", default="") + # backend version + VERSION = config("VERSION", default=get_version()) + SERVER_TYPE = config("SERVER_TYPE", default="ce") From b0d4be82952743b0dcf8076b58bbc9ef1ef1b169 Mon Sep 17 00:00:00 2001 From: Marcel Kocisek Date: Wed, 30 Oct 2024 15:29:13 +0100 Subject: [PATCH 9/9] Use . noation for ProjectFile class instead dict getter (#315) --- server/mergin/sync/commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/mergin/sync/commands.py b/server/mergin/sync/commands.py index 327bc560..a07d5966 100644 --- a/server/mergin/sync/commands.py +++ b/server/mergin/sync/commands.py @@ -75,13 +75,13 @@ def download(project_name, version, directory): # pylint: disable=W0612 os.mkdir(directory) files = pv.files for f in files: - project.storage.restore_versioned_file(f["path"], version) - f_dir = os.path.dirname(f["path"]) + project.storage.restore_versioned_file(f.path, version) + f_dir = os.path.dirname(f.path) if f_dir: os.makedirs(os.path.join(directory, f_dir), exist_ok=True) shutil.copy( - os.path.join(project.storage.project_dir, f["location"]), - os.path.join(directory, f["path"]), + os.path.join(project.storage.project_dir, f.location), + os.path.join(directory, f.path), ) print("Project downloaded successfully")