From 9350e75c523972bee24bb048b40b4a099ffdbf3b Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 28 Sep 2024 19:49:59 +1000 Subject: [PATCH 1/5] Fix playbook.yml files --- ansible/dev.yml | 2 +- ansible/prod.yml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ansible/dev.yml b/ansible/dev.yml index f001565..eaa735b 100644 --- a/ansible/dev.yml +++ b/ansible/dev.yml @@ -5,7 +5,7 @@ vars_files: - group_vars/VAULT - group_vars/webservers.yml - - host_vars/labs.neoformit.com.yml + - host_vars/dev-labs.gvl.org.au.yml roles: - role: geerlingguy.docker tags: init diff --git a/ansible/prod.yml b/ansible/prod.yml index 53dffea..e7d4d2c 100644 --- a/ansible/prod.yml +++ b/ansible/prod.yml @@ -3,7 +3,10 @@ remote_user: ubuntu become: true vars_files: - - group_vars/secrets.yml + - group_vars/VAULT - group_vars/webservers.yml + - host_vars/labs.usegalaxy.org.au.yml roles: + - role: geerlingguy.docker + tags: init - galaxy_labs_engine From e735d022c32c9687e9be05f916683d233fbbf9a9 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 28 Sep 2024 19:50:33 +1000 Subject: [PATCH 2/5] Debug labs_engine role --- .../roles/galaxy_labs_engine/tasks/main.yml | 22 +++++++++---------- .../templates/nginx.conf.j2 | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ansible/roles/galaxy_labs_engine/tasks/main.yml b/ansible/roles/galaxy_labs_engine/tasks/main.yml index 32f7ec7..fcd22fb 100644 --- a/ansible/roles/galaxy_labs_engine/tasks/main.yml +++ b/ansible/roles/galaxy_labs_engine/tasks/main.yml @@ -7,7 +7,7 @@ - name: clone git repository for galaxy-labs-engine ansible.builtin.git: repo: https://github.com/neoformit/galaxy-labs-engine.git - branch: "{{ git_branch }}" + version: "{{ git_branch }}" dest: "{{ project_root }}" clone: yes force: yes @@ -50,14 +50,6 @@ loop: "{{ labs_engine.files }}" tags: config -- name: Django create django_cache table with 'manage.py createcachetable' - shell: > - docker compose --profile prod run --rm labs-engine - python manage.py createcachetable - args: - chdir: "{{ config_root }}" - tags: django - - name: update media file ownership file: dest={{ django_root }}/app/media owner=www-data group=www-data mode=u=rwX,g=rwX,o=rwX recurse=yes tags: @@ -75,6 +67,7 @@ tags: - permissions - update + ignore_errors: yes - name: update sqlite3 database directory permissions file: @@ -86,6 +79,14 @@ - permissions - update +- name: Django create django_cache table with 'manage.py createcachetable' + shell: > + docker compose --profile prod run --rm labs-engine + python manage.py createcachetable + args: + chdir: "{{ config_root }}" + tags: django + - name: Django run migrations shell: > docker compose --profile prod run --rm @@ -100,12 +101,11 @@ - name: Django collect static files shell: > - docker compose --profile prod run --rm + docker compose --profile prod run --rm --user root labs-engine python manage.py collectstatic --noinput args: chdir: "{{ config_root }}" - when: git.changed tags: always - name: Django ensure superuser login diff --git a/ansible/roles/galaxy_labs_engine/templates/nginx.conf.j2 b/ansible/roles/galaxy_labs_engine/templates/nginx.conf.j2 index d582108..d77dc27 100644 --- a/ansible/roles/galaxy_labs_engine/templates/nginx.conf.j2 +++ b/ansible/roles/galaxy_labs_engine/templates/nginx.conf.j2 @@ -74,7 +74,7 @@ server { # Redirect all other traffic to HTTPS, but not the challenge location / { - if ($host = labs.neoformit.com) { + if ($host = {{ inventory_hostname }}) { return 301 https://$host$request_uri; } } From 73f6b41835b37dd6f6da583440623d5c59478598 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 28 Sep 2024 19:51:00 +1000 Subject: [PATCH 3/5] Fix lab_export.py _validate_url --- app/labs/lab_export.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/app/labs/lab_export.py b/app/labs/lab_export.py index 05b704c..e1673ea 100644 --- a/app/labs/lab_export.py +++ b/app/labs/lab_export.py @@ -140,12 +140,9 @@ def looks_like_a_webpage(response): def _validate_url(self, url, expected_type): """Validate URL to prevent circular request.""" - if ( - '/lab/export' in url - and url.split('/')[1, 2] == ['lab', 'export'] - ): + if url.strip('/').split('/')[-1] == settings.HOSTNAME: raise LabBuildError( - "URL cannot contain '/lab/export'.", + "URL cannot match the root URL of this server", url=url, source=expected_type, ) From f6ffd8349a240031ecde8a2c6a0b282e93e48953 Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sat, 28 Sep 2024 19:51:40 +1000 Subject: [PATCH 4/5] Replace /lab/export URL references with root URL --- app/labs/static/labs/content/docs/base.yml | 2 +- .../static/labs/content/docs/templates/intro.html | 14 +++++++------- app/labs/static/labs/content/simple/base.yml | 2 +- .../labs/content/simple/templates/conclusion.md | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/labs/static/labs/content/docs/base.yml b/app/labs/static/labs/content/docs/base.yml index 72ebd31..5828893 100644 --- a/app/labs/static/labs/content/docs/base.yml +++ b/app/labs/static/labs/content/docs/base.yml @@ -1,6 +1,6 @@ # Default spec for an exported lab landing page -# You can test with: /lab/export?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml +# You can test with: /?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml site_name: Antarctica lab_name: Galaxy Lab Pages diff --git a/app/labs/static/labs/content/docs/templates/intro.html b/app/labs/static/labs/content/docs/templates/intro.html index 2d3669a..967fca0 100644 --- a/app/labs/static/labs/content/docs/templates/intro.html +++ b/app/labs/static/labs/content/docs/templates/intro.html @@ -14,18 +14,18 @@
  • See a full working example of an exported Galaxy Lab page - here. + here.
  • See a minimal working example of a lab page - here. + here.

    • This custom introductory text was pulled down from GitHub and rendered in real time. Updates to - + this text on the remote will be reflected on this page. Note that GitHub raw content is @@ -59,8 +59,8 @@

    For example, we can explicitly request this page with:
    - - http://{{ HOSTNAME }}/lab/export?content_root={{ GITHUB_CONTENT_ROOT_BASE_URL }}/docs/base.yml + + http://{{ HOSTNAME }}/?content_root={{ GITHUB_CONTENT_ROOT_BASE_URL }}/docs/base.yml @@ -110,7 +110,7 @@


    - http://{{ HOSTNAME }}/lab/export?content_root=https://raw.githubusercontent.com/myusername/myrepo/path/to/content/base.yml + http://{{ HOSTNAME }}/?content_root=https://raw.githubusercontent.com/myusername/myrepo/path/to/content/base.yml @@ -192,7 +192,7 @@

    Galaxy {{ site_name }}
    const exportInfoButton = $(` diff --git a/app/labs/static/labs/content/simple/base.yml b/app/labs/static/labs/content/simple/base.yml index d1d3379..18f72a0 100644 --- a/app/labs/static/labs/content/simple/base.yml +++ b/app/labs/static/labs/content/simple/base.yml @@ -1,6 +1,6 @@ # Default spec for an exported lab landing page -# You can test with: /lab/export?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml +# You can test with: /?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml site_name: Archaeology lab_name: Archaeology Lab diff --git a/app/labs/static/labs/content/simple/templates/conclusion.md b/app/labs/static/labs/content/simple/templates/conclusion.md index 1c3b104..7013122 100644 --- a/app/labs/static/labs/content/simple/templates/conclusion.md +++ b/app/labs/static/labs/content/simple/templates/conclusion.md @@ -1,3 +1,3 @@ And that's the end of the example Galaxy Lab! -Visit the [docs](/lab/export) to learn how to build your own. +Visit the [docs](/) to learn how to build your own. From 2e4e99a2df42c155783049aafe38f19ddd42164f Mon Sep 17 00:00:00 2001 From: Cameron Hyde Date: Sun, 29 Sep 2024 09:03:24 +1000 Subject: [PATCH 5/5] Lab content URLs defined in settings --- README.md | 4 +- app/app/settings.py | 123 ------------------ app/app/settings/base.py | 37 +++++- app/labs/context_processors.py | 3 - app/labs/lab_export.py | 4 +- app/labs/static/labs/content/docs/base.yml | 2 +- .../labs/content/docs/templates/intro.html | 26 ++-- app/labs/static/labs/content/simple/base.yml | 2 +- .../labs/content/simple/templates/intro.md | 2 +- app/labs/views.py | 7 +- 10 files changed, 59 insertions(+), 151 deletions(-) delete mode 100644 app/app/settings.py diff --git a/README.md b/README.md index fbfc315..db45956 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Create a lab content directory like And then request your page anywhere like this: -https://labs.usegalaxy.org.au/export/?content_root=https://raw.githubusercontent.com/my-username/my-labs-repo/my-lab/base.yml +https://labs.usegalaxy.org.au/?content_root=https://raw.githubusercontent.com/my-username/my-labs-repo/my-lab/base.yml To get a webpage like -[this](https://labs.usegalaxy.org.au/export/?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/simple/main.yml). +[this](https://labs.usegalaxy.org.au/?content_root=https://raw.githubusercontent.com/usegalaxy-au/galaxy-labs-engine/dev/app/labs/content/simple/main.yml). diff --git a/app/app/settings.py b/app/app/settings.py deleted file mode 100644 index 3a184a4..0000000 --- a/app/app/settings.py +++ /dev/null @@ -1,123 +0,0 @@ -""" -Django settings for app project. - -Generated by 'django-admin startproject' using Django 5.0.7. - -For more information on this file, see -https://docs.djangoproject.com/en/5.0/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/5.0/ref/settings/ -""" - -from pathlib import Path - -# Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = Path(__file__).resolve().parent.parent - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = 'django-insecure-g33bcvq6_*x)*(7h)-%^pn#&qa1neoiyx1z)z4rn(4@h@)xw%k' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', -] - -MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', -] - -ROOT_URLCONF = 'app.urls' - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], - }, - }, -] - -WSGI_APPLICATION = 'app.wsgi.application' - - -# Database -# https://docs.djangoproject.com/en/5.0/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', - } -} - - -# Password validation -# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/5.0/topics/i18n/ - -LANGUAGE_CODE = 'en-us' - -TIME_ZONE = 'UTC' - -USE_I18N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/5.0/howto/static-files/ - -STATIC_URL = 'static/' - -# Default primary key field type -# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field - -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/app/app/settings/base.py b/app/app/settings/base.py index a97b56f..3ed2bf5 100644 --- a/app/app/settings/base.py +++ b/app/app/settings/base.py @@ -44,9 +44,40 @@ LOG_ROOT = ensure_dir(BASE_DIR / 'app/logs') DEFAULT_EXPORTED_LAB_CONTENT_ROOT = ( f'http://{HOSTNAME}/static/labs/content/docs/base.yml') -GITHUB_CONTENT_ROOT_BASE_URL = ( - "https://raw.githubusercontent.com/neoformit" - "/galaxy-labs-engine/dev/app/labs/content") + +CODEX_REPO = 'galaxyproject/galaxy_codex' +CODEX_GITHUB_URL = f'https://github.com/{CODEX_REPO}' +LABS_ENGINE_GITHUB_REPO = 'usegalaxy-au/galaxy-labs-engine' +LABS_ENGINE_GITHUB_URL = f'https://github.com/{LABS_ENGINE_GITHUB_REPO}' +EXAMPLE_LABS = { + 'FULL': { + 'RAW_URL': ( + f'https://raw.githubusercontent.com/{CODEX_REPO}/refs/heads/main' + '/communities/genome/lab/base.yml'), + 'WEB_DIR_URL': ( + f'{CODEX_GITHUB_URL}/blob/main' + '/communities/genome/lab'), + }, + 'DOCS': { + 'RAW_URL': ( + f'https://raw.githubusercontent.com/{LABS_ENGINE_GITHUB_REPO}/refs' + '/heads/main/app/labs/static/labs/content/docs/base.yml'), + 'WEB_DIR_URL': ( + f'{LABS_ENGINE_GITHUB_URL}/tree/dev/app/labs/static/labs/content' + '/docs'), + 'WEB_DIR_ROOT': ( + f'{LABS_ENGINE_GITHUB_URL}/blob/dev/app/labs/static/labs/content' + '/docs'), + }, + 'SIMPLE': { + 'RAW_URL': ( + f'https://raw.githubusercontent.com/{LABS_ENGINE_GITHUB_REPO}' + '/refs/heads/main/app/labs/static/labs/content/simple/base.yml'), + 'WEB_DIR_URL': ( + f'{LABS_ENGINE_GITHUB_URL}/blob/dev/app/labs/static/labs/content' + '/simple'), + }, +} # Hostnames ALLOWED_HOSTS = [ diff --git a/app/labs/context_processors.py b/app/labs/context_processors.py index f8a3580..79792b4 100644 --- a/app/labs/context_processors.py +++ b/app/labs/context_processors.py @@ -6,8 +6,5 @@ def settings(request): """Include some settings variables.""" return { - 'title': None, # prevents variable not found templating error 'HOSTNAME': app_settings.HOSTNAME, - "GITHUB_CONTENT_ROOT_BASE_URL": - app_settings.GITHUB_CONTENT_ROOT_BASE_URL, } diff --git a/app/labs/lab_export.py b/app/labs/lab_export.py index e1673ea..d72004e 100644 --- a/app/labs/lab_export.py +++ b/app/labs/lab_export.py @@ -1,10 +1,10 @@ """Exported landing pages to be requested by external Galaxy servers. Example with local YAML: -http://127.0.0.1:8000/landing/export?content_root=http://127.0.0.1:8000/static/labs/content/genome/main.yml +http://127.0.0.1:8000/?content_root=http://127.0.0.1:8000/static/labs/content/simple/base.yml Example URL with remote YAML: -http://127.0.0.1:8000/landing/export?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/... base.yml # noqa +http://127.0.0.1:8000/?content_root=https://raw.githubusercontent.com/usegalaxy-au/galaxy-labs-engine/blob/main/app/labs/content/simple/base.yml """ diff --git a/app/labs/static/labs/content/docs/base.yml b/app/labs/static/labs/content/docs/base.yml index 5828893..24be751 100644 --- a/app/labs/static/labs/content/docs/base.yml +++ b/app/labs/static/labs/content/docs/base.yml @@ -1,6 +1,6 @@ # Default spec for an exported lab landing page -# You can test with: /?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml +# You can test with: /?content_root=https://raw.githubusercontent.com/usegalaxy-au/galaxy-labs-engine/dev/app/labs/content/docs/base.yml site_name: Antarctica lab_name: Galaxy Lab Pages diff --git a/app/labs/static/labs/content/docs/templates/intro.html b/app/labs/static/labs/content/docs/templates/intro.html index 967fca0..bc38f4e 100644 --- a/app/labs/static/labs/content/docs/templates/intro.html +++ b/app/labs/static/labs/content/docs/templates/intro.html @@ -8,24 +8,24 @@

    This custom introductory text was pulled down from GitHub and rendered in real time. Updates to - + this text on the remote will be reflected on this page. Note that GitHub raw content is @@ -48,7 +48,7 @@

    @@ -96,7 +96,7 @@

    1. Copy our - + example content directory to your own github repository. @@ -139,7 +139,7 @@

      defined there. This means you can have webforms, interactive elements, and more. Check out the Genome Lab conclusion.html @@ -170,7 +170,7 @@
      Galaxy {{ site_name }}

      This is not a real Galaxy Lab page! It's a documentation page, to show you how to create your own Galaxy Lab pages using the - + Galaxy Labs Engine.

      @@ -192,7 +192,7 @@

      Galaxy {{ site_name }}
      const exportInfoButton = $(` diff --git a/app/labs/static/labs/content/simple/base.yml b/app/labs/static/labs/content/simple/base.yml index 18f72a0..694753c 100644 --- a/app/labs/static/labs/content/simple/base.yml +++ b/app/labs/static/labs/content/simple/base.yml @@ -1,6 +1,6 @@ # Default spec for an exported lab landing page -# You can test with: /?content_root=https://raw.githubusercontent.com/neoformit/galaxy-labs-engine/dev/app/labs/content/docs/base.yml +# You can test with: /?content_root=https://raw.githubusercontent.com/usegalaxy-au/galaxy-labs-engine/dev/app/labs/content/docs/base.yml site_name: Archaeology lab_name: Archaeology Lab diff --git a/app/labs/static/labs/content/simple/templates/intro.md b/app/labs/static/labs/content/simple/templates/intro.md index 6bea140..ea2e198 100644 --- a/app/labs/static/labs/content/simple/templates/intro.md +++ b/app/labs/static/labs/content/simple/templates/intro.md @@ -1,5 +1,5 @@ -##### This is an example of a simple Galaxy Lab page, you can check out the [code on GitHub](https://github.com/neoformit/galaxy-labs-engine/tree/dev/app/labs/content/simple) +##### This is an example of a simple Galaxy Lab page, you can check out the [code on GitHub](https://github.com/usegalaxy-au/galaxy-labs-engine/tree/dev/app/labs/content/simple)
      Welcome to **Galaxy {{ site_name }}**! diff --git a/app/labs/views.py b/app/labs/views.py index af97d76..6316eef 100644 --- a/app/labs/views.py +++ b/app/labs/views.py @@ -35,10 +35,13 @@ def export_lab(request): else: context = ExportLabContext( settings.DEFAULT_EXPORTED_LAB_CONTENT_ROOT) - context['HOSTNAME'] = settings.HOSTNAME + context.update({ + 'LABS_ENGINE_GITHUB_URL': settings.LABS_ENGINE_GITHUB_URL, + 'EXAMPLE_LABS': settings.EXAMPLE_LABS, + }) context.validate() except LabBuildError as exc: - return render(request, 'labs//export-error.html', { + return render(request, 'labs/export-error.html', { 'exc': exc, }, status=400)