diff --git a/ansible.cfg b/ansible.cfg index d41c2e5..a7e8e88 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -4,7 +4,7 @@ inventory = hosts retry_files_enabled = false vault_password_file = .vault-password.txt gather_subset=!hardware -no_log = True +no_log = False # Show diffs of changes [diff] diff --git a/eu_tool_list.yaml b/eu_tool_list.yaml index 236ed07..c60ab30 100644 --- a/eu_tool_list.yaml +++ b/eu_tool_list.yaml @@ -293,6 +293,7 @@ tools: revisions: - f46f0e4f75c4 - d698c222f354 + - fbf99087e067 tool_panel_section_label: Text Manipulation tool_shed_url: toolshed.g2.bx.psu.edu - name: query_tabular @@ -521,6 +522,7 @@ tools: owner: iuc revisions: - 374cb875d38a + - a513e3fbb4c5 tool_panel_section_label: Join, Subtract and Group tool_shed_url: toolshed.g2.bx.psu.edu - name: datamash_reverse @@ -5553,12 +5555,12 @@ tools: - fd1f57782683 tool_panel_section_label: Peak Calling tool_shed_url: toolshed.g2.bx.psu.edu -- name: jamm - owner: messersc - revisions: - - 243f75d0ed6e - tool_panel_section_label: Peak Calling - tool_shed_url: testtoolshed.g2.bx.psu.edu +#- name: jamm +# owner: messersc +# revisions: +# - 243f75d0ed6e +# tool_panel_section_label: Peak Calling +# tool_shed_url: testtoolshed.g2.bx.psu.edu - name: piranha owner: rnateam revisions: diff --git a/files/galaxy/tools/testing.xml b/files/galaxy/tools/testing.xml index 3a5e766..0b43be7 100644 --- a/files/galaxy/tools/testing.xml +++ b/files/galaxy/tools/testing.xml @@ -1,15 +1,12 @@ - + - - + - - + diff --git a/galaxy.yml b/galaxy.yml index 5900143..4065387 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -29,7 +29,7 @@ mode: 0640 - name: Install Dependencies package: - name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools'] + name: ['acl', 'bzip2', 'git', 'make', 'tar', 'python3-bioblend', 'python3-venv', 'python3-setuptools', 'squashfs-tools'] - name: Install RHEL/CentOS/Rocky specific dependencies package: name: ['tmpwatch'] @@ -100,3 +100,28 @@ path: "{{ galaxy_root }}/server/lib/galaxy/webapps/galaxy/api/job_files.py" regexp: '^(\s*?)shutil\.move\(input_file\.name,\s*path\)\s*$' replace: '\1os.chmod(shutil.move(input_file.name, path), 0o664)' + + ## Install required Galaxy tools + - name: Install required tools - create Ephemeris venv + ansible.builtin.shell: + cmd: python3 -m venv ~/ephemeris_venv + tags: tools + - name: Install required tools - install Ephemeris + ansible.builtin.shell: + cmd: source ~/ephemeris_venv/bin/activate && pip install ephemeris + tags: tools + - name: Install required tools - install Galaxy tools + ansible.builtin.shell: + cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t regalaxy_tool_list.yaml + when: "'galaxy-re' in inventory_hostname" + tags: tools + - name: Install required tools - install Galaxy tools + ansible.builtin.shell: + cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t umsa_tool_list.yaml + when: "'umsa' in inventory_hostname" + tags: tools + - name: Install required tools - install Galaxy tools + ansible.builtin.shell: + cmd: ~/ephemeris_venv/bin/shed-tools install -g https://{{ inventory_hostname }} -a {{ api_key }} -t eu_tool_list.yaml + when: "'usegalaxy' in inventory_hostname or 'galaxy-cz' in inventory_hostname" + tags: tools diff --git a/group_vars/dbservers.yml b/group_vars/dbservers.yml index 2504faf..8aaed70 100644 --- a/group_vars/dbservers.yml +++ b/group_vars/dbservers.yml @@ -35,6 +35,6 @@ postgresql_objects_privileges: privs: USAGE,SELECT # PostgreSQL Backups -postgresql_backup_dir: /data/backups +postgresql_backup_dir: /rbd/data/backups postgresql_backup_local_dir: "{{ '~postgres' | expanduser }}/backups" diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index 2885a5b..7cfa97f 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -30,16 +30,6 @@ galaxy_job_config: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 manager: _default_ - pulsar_gpu_runner: - load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner - galaxy_url: "https://{{ rabbitmq_hostname }}" - amqp_url: "pyamqp://pulsar:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar?ssl=1" - amqp_acknowledge: true - amqp_ack_republish_time: 1200 - amqp_consumer_timeout: 2 - amqp_publish_retry: true - amqp_publish_retry_max_retries: 60 - manager: gpu pulsar_tpv_runner: load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner galaxy_url: "https://{{ rabbitmq_hostname }}" @@ -86,9 +76,6 @@ galaxy_job_config: singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR" ## Following configuration works! # singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro" - container_resolvers: - - type: explicit_singularity - - type: mulled_singularity env: # Ensuring a consistent collation environment is good for reproducibility. - name: LC_ALL @@ -99,43 +86,6 @@ galaxy_job_config: value: "$SCRATCHDIR" - name: TEMP value: "$SCRATCHDIR" - pulsar_gpu: - runner: pulsar_gpu_runner - default_file_action: remote_transfer - transport: curl - dependency_resolution: remote - jobs_directory: "/storage/praha5-elixir/home/galaxyeu/pulsar-cz/files/staging" - persistence_directory: "/opt/pulsar/files/persistent" - remote_metadata: false - rewrite_parameters: true - outputs_to_working_directory: false - singularity_enabled: true - singularity_run_extra_arguments: '--nv' - singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data:ro' - env: - # The cache directory holds the docker containers that get converted - - name: SINGULARITY_CACHEDIR - value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache" - - name: APPTAINER_CACHEDIR - value: "/storage/praha5-elixir/home/galaxyeu/singularity/cache" - # Singularity uses a temporary directory to build the squashfs filesystem - - name: SINGULARITY_TMPDIR - value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp" - - name: APPTAINER_TMPDIR - value: "/storage/praha5-elixir/home/galaxyeu/singularity/tmp" - # Alphafold specific variables - - name: ALPHAFOLD_DB - value: "/storage/brno11-elixir/projects/alphafold/alphafold.db-2.3.1" - # Default variables - - name: TMPDIR - value: "$SCRATCHDIR" - - name: TMP - value: "$SCRATCHDIR" - - name: TEMP - value: "$SCRATCHDIR" - # Ensuring a consistent collation environment is good for reproducibility. - - name: LC_ALL - value: C tpv_dispatcher: runner: dynamic type: python @@ -149,6 +99,13 @@ galaxy_job_config: groups: default: [] testing: [cores, time] + limits: + - type: registered_user_concurrent_jobs + value: 10 + - type: anonymous_user_concurrent_jobs + value: 1 + - type: output_size + value: 50GB tools: - class: local # these special tools that aren't parameterized for remote execution - expression tools, upload, etc environment: local_env @@ -178,10 +135,12 @@ galaxy_config: cleanup_job: onsuccess smtp_server: "rs.cesnet.cz:25" error_email_to: "regalaxy@rt.cesnet.cz" - allow_user_creation: False + email_from: "regalaxy@rt.cesnet.cz" + allow_user_creation: false + require_login: true database_connection: "postgresql:///{{ galaxy_db_name }}?host=/var/run/postgresql" - file_path: /mnt/data/datasets - job_working_directory: /mnt/data/jobs + file_path: /rbd/data/datasets + job_working_directory: /rbd/data/jobs cluster_files_directory: "/rbd/data/pbs" short_term_storage_dir: "/rbd/data/short_term_web_storage" object_store_store_by: uuid @@ -221,7 +180,16 @@ galaxy_config: tool_data_table_config_path: /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml,/cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml # Tool Dependencies dependency_resolvers_config_file: "{{ galaxy_config_dir }}/dependency_resolvers_conf.xml" - containers_resolvers_config_file: "{{ galaxy_config_dir }}/container_resolvers_conf.yml" + container_resolvers: + - type: explicit_singularity +# - cache_directory: /cvmfs/singularity.galaxyproject.org/all/ + - cache_directory: /rbd/data/singularity + type: cached_mulled_singularity + - auto_install: true + type: mulled_singularity + - auto_install: false + type: build_mulled_singularity + cache_directory: /rbd/data/singularity/mulled # Data Library Directories # library_import_dir: "/rbd/data/library_import_dir" # likely to be here library_import_dir: /libraries/admin @@ -235,7 +203,7 @@ galaxy_config: statsd_host: localhost statsd_influxdb: true # FTP - ftp_upload_dir: /mnt/data/uploads + ftp_upload_dir: /rbd/data/uploads ftp_upload_site: "{{ inventory_hostname }}" gravity: process_manager: systemd @@ -297,12 +265,8 @@ galaxy_config_files_public: galaxy_config_files: - src: files/galaxy/themes.yml dest: "{{ galaxy_config.galaxy.themes_config_file }}" - - src: files/galaxy/config/tpv_rules_local.yml - dest: "{{ tpv_mutable_dir }}/tpv_rules_local.yml" galaxy_config_templates: - - src: templates/galaxy/config/container_resolvers_conf.yml.j2 - dest: "{{ galaxy_config.galaxy.containers_resolvers_config_file }}" - src: templates/galaxy/config/dependency_resolvers_conf.xml dest: "{{ galaxy_config.galaxy.dependency_resolvers_config_file }}" - src: templates/galaxy/config/job_resource_params_conf.xml.j2 @@ -319,10 +283,11 @@ galaxy_config_templates: dest: "{{ galaxy_config.galaxy.tool_config_file[0] }}" - src: templates/galaxy/config/local_tool_conf.xml.j2 dest: "{{ galaxy_config.galaxy.tool_config_file[1] }}" + - src: templates/galaxy/config/tpv_rules_local.yml.j2 + dest: "{{ tpv_mutable_dir }}/tpv_rules_local.yml" galaxy_extra_dirs: - - /mnt/data - "{{ galaxy_config_dir }}/{{ tpv_config_dir_name }}" galaxy_extra_privsep_dirs: @@ -365,7 +330,7 @@ nginx_servers: - redirect-ssl nginx_ssl_servers: - galaxy - # - sentry +# - sentry nginx_enable_default_server: false nginx_conf_http: client_max_body_size: 1g @@ -448,7 +413,7 @@ rabbitmq_users: # TUS galaxy_tusd_port: 1080 -galaxy_tus_upload_store: /rbd/data/tus # /mnt/data/tus +galaxy_tus_upload_store: /rbd/data/tus #Redis galaxy_additional_venv_packages: diff --git a/requirements.yml b/requirements.yml index f49675a..a5084c0 100644 --- a/requirements.yml +++ b/requirements.yml @@ -19,7 +19,7 @@ version: 0.0.1 # CVMFS Support role - src: galaxyproject.cvmfs - version: 0.2.21 + version: 0.2.22 # Singularity/Apptainer - src: usegalaxy_eu.apptainer version: 0.0.1 diff --git a/roles/galaxyproject.cvmfs/.github/workflows/lint.yml b/roles/galaxyproject.cvmfs/.github/workflows/lint.yml index 65cb931..f3a5b42 100644 --- a/roles/galaxyproject.cvmfs/.github/workflows/lint.yml +++ b/roles/galaxyproject.cvmfs/.github/workflows/lint.yml @@ -7,11 +7,9 @@ jobs: name: ansible-lint runs-on: ubuntu-latest steps: - - name: Check out the codebase. - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - - name: Set up Python 3. - uses: actions/setup-python@v3 + - uses: actions/setup-python@v2 with: python-version: '3.x' diff --git a/roles/galaxyproject.cvmfs/.github/workflows/release.yml b/roles/galaxyproject.cvmfs/.github/workflows/release.yml index 773c89e..954e00d 100644 --- a/roles/galaxyproject.cvmfs/.github/workflows/release.yml +++ b/roles/galaxyproject.cvmfs/.github/workflows/release.yml @@ -18,10 +18,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v2 - name: Set up Python 3. - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: '3.x' diff --git a/roles/galaxyproject.cvmfs/.github/workflows/slugger.yml b/roles/galaxyproject.cvmfs/.github/workflows/slugger.yml index bb4ca7d..c73006a 100644 --- a/roles/galaxyproject.cvmfs/.github/workflows/slugger.yml +++ b/roles/galaxyproject.cvmfs/.github/workflows/slugger.yml @@ -24,12 +24,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the codebase. - uses: actions/checkout@v3 + uses: actions/checkout@v2 with: fetch-depth: 0 - name: Set up Python 3. - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: '3.x' diff --git a/roles/galaxyproject.cvmfs/meta/.galaxy_install_info b/roles/galaxyproject.cvmfs/meta/.galaxy_install_info index 50ecfb5..8ef769b 100644 --- a/roles/galaxyproject.cvmfs/meta/.galaxy_install_info +++ b/roles/galaxyproject.cvmfs/meta/.galaxy_install_info @@ -1,2 +1,2 @@ -install_date: 'Wed 06 Mar 2024 01:49:34 PM ' -version: 0.2.21 +install_date: 'Sun 26 May 2024 09:19:28 PM ' +version: 0.2.22 diff --git a/roles/galaxyproject.cvmfs/tasks/localproxy.yml b/roles/galaxyproject.cvmfs/tasks/localproxy.yml index 47d4425..ee460fb 100644 --- a/roles/galaxyproject.cvmfs/tasks/localproxy.yml +++ b/roles/galaxyproject.cvmfs/tasks/localproxy.yml @@ -17,7 +17,7 @@ # {{ inventory_hostname }} 127.0.0.1 # dest: /etc/squid/hosts # notify: -# - restart squid +# - Restart squid - name: Include firewall tasks ansible.builtin.include_tasks: firewall.yml diff --git a/roles/galaxyproject.cvmfs/tasks/squid.yml b/roles/galaxyproject.cvmfs/tasks/squid.yml index b994e61..43ecbe9 100644 --- a/roles/galaxyproject.cvmfs/tasks/squid.yml +++ b/roles/galaxyproject.cvmfs/tasks/squid.yml @@ -6,7 +6,7 @@ backup: true mode: 0644 notify: - - restart squid + - Restart squid - name: Fix cache directory permission ansible.builtin.file: @@ -23,7 +23,7 @@ become_user: "{{ cvmfs_squid_user }}" ansible.builtin.command: squid -z args: - creates: "{{ cvmfs_stratum1_cache_dir }}/00" + creates: "{{ cvmfs_stratum1_cache_dir.dir }}/00" when: cvmfs_stratum1_cache_dir is defined - name: Ensure squid is enabled and started diff --git a/roles/galaxyproject.cvmfs/tasks/stratum0.yml b/roles/galaxyproject.cvmfs/tasks/stratum0.yml index 7b555d6..9139252 100644 --- a/roles/galaxyproject.cvmfs/tasks/stratum0.yml +++ b/roles/galaxyproject.cvmfs/tasks/stratum0.yml @@ -46,7 +46,7 @@ creates: /srv/cvmfs/{{ item.repository }} with_items: "{{ cvmfs_repositories }}" notify: - - restart apache + - Restart apache - name: Ensure repositories are imported ansible.builtin.command: | @@ -55,7 +55,7 @@ creates: /etc/cvmfs/repositories.d/{{ item.repository }} with_items: "{{ cvmfs_repositories }}" notify: - - restart apache + - Restart apache - name: Include repository server options tasks ansible.builtin.include_tasks: options.yml diff --git a/roles/galaxyproject.cvmfs/tasks/stratum1.yml b/roles/galaxyproject.cvmfs/tasks/stratum1.yml index 220ba2d..6a502ae 100644 --- a/roles/galaxyproject.cvmfs/tasks/stratum1.yml +++ b/roles/galaxyproject.cvmfs/tasks/stratum1.yml @@ -16,7 +16,7 @@ backup: true when: cvmfs_config_apache notify: - - reload apache + - Reload apache - name: Include stratumN tasks ansible.builtin.include_tasks: stratumN.yml @@ -53,7 +53,7 @@ loop: "{{ cvmfs_repositories }}" register: __cvmfs_add_replica_result notify: - - restart apache + - Restart apache # Ideally we could use item.stratum0_url_scheme directly in `cvmfs_server add-replica` above, but it appears not to # support it, so we instead have to change it after the fact diff --git a/templates/galaxy/config/container_resolvers_conf.xml.j2 b/templates/galaxy/config/container_resolvers_conf.xml.j2 deleted file mode 100644 index 85e8e0b..0000000 --- a/templates/galaxy/config/container_resolvers_conf.xml.j2 +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/templates/galaxy/config/container_resolvers_conf.yml.j2 b/templates/galaxy/config/container_resolvers_conf.yml.j2 deleted file mode 100644 index 788d613..0000000 --- a/templates/galaxy/config/container_resolvers_conf.yml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -container_resolvers: - - type: cached_explicit_singularity - cache_directory: "{{ galaxy_mutable_data_dir }}/cache/singularity/explicit/" - - type: cached_mulled_singularity - cache_directory: "{{ galaxy_mutable_data_dir }}/cache/singularity/mulled/" - - type: mulled_singularity - auto_install: False - cache_directory: "{{ galaxy_mutable_data_dir }}/cache/singularity/mulled/" - - type: build_mulled_singularity - auto_install: False - cache_directory: "{{ galaxy_mutable_data_dir }}/cache/singularity/built/" diff --git a/files/galaxy/config/tpv_rules_local.yml b/templates/galaxy/config/tpv_rules_local.yml.j2 similarity index 63% rename from files/galaxy/config/tpv_rules_local.yml rename to templates/galaxy/config/tpv_rules_local.yml.j2 index c571070..20e4bab 100644 --- a/files/galaxy/config/tpv_rules_local.yml +++ b/templates/galaxy/config/tpv_rules_local.yml.j2 @@ -3,11 +3,13 @@ global: tools: default: -# abstract: true cores: 1 mem: cores * 4 - env: {} - params: {} + env: + - name: GALAXY_SLOTS + value: "{cores}" + - name: GALAXY_MEMORY_MB + value: "{int(mem)*1000}" context: walltime: 24 scratch: 50 @@ -44,11 +46,24 @@ tools: .*/alphafold/.*: cores: 1 - scheduling: + mem: 120 + gpus: 1 + context: + walltime: 24 + scratch: 100 + gpu_mem: 8 + scheduling: require: - - pulsar - - singularity - - gpu + - alphafold + + .*/goenrichment/.*: + scheduling: + require: + - nasty-java + + .*/bedtools_annotatebed/.*: + context: + walltime: 48 toolshed.g2.bx.psu.edu/repos/iuc/megahit/megahit/.*: cores: 16 @@ -57,6 +72,20 @@ tools: walltime: 96 scratch: 100 + CONVERTER_.*: + scheduling: + require: +# - local +# reject: + - pulsar + + __SET_.*: + scheduling: + require: + - local + reject: + - pulsar + roles: training.*: max_cores: 2 @@ -85,23 +114,20 @@ destinations: max_accepted_mem: 4 params: singularity_enabled: true + singularity_default_container_id: "/cvmfs/singularity.galaxyproject.org/all/python:3.8.3" env: # Ensuring a consistent collation environment is good for reproducibility. LC_ALL: C # The cache directory holds the docker containers that get converted - SINGULARITY_CACHEDIR: "/srv/galaxy/var/cache/singularity" +# SINGULARITY_CACHEDIR: "/cvmfs/singularity.galaxyproject.org/all/" + SINGULARITY_CACHEDIR: "/rbd/data/singularity/" # Singularity uses a temporary directory to build the squashfs filesystem - SINGULARITY_TMPDIR: /tmp - container_resolvers: - - type: explicit_singularity - - type: cached_mulled_singularity - cache_directory: "/srv/galaxy/var/cache/singularity/mulled" - - type: mulled_singularity + SINGULARITY_TMPDIR: /rbd/data//tmp scheduling: + require: + - local prefer: - singularity - reject: - - pulsar tpv_pulsar: runner: pulsar_tpv_runner max_accepted_cores: 128 @@ -119,64 +145,49 @@ destinations: rewrite_parameters: true transport: curl outputs_to_working_directory: false - submit_native_specification: "-l select=1:ncpus={cores}:mem={mem}gb:scratch_local={scratch}gb -l walltime={walltime}:00:00 -q galaxyeu@elixir-pbs.elixir-czech.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}" + submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb -l walltime={int(walltime)}:00:00 -q galaxycz@pbs-m1.metacentrum.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}" + singularity_enabled: true + singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR:rw" + singularity_default_container_id: "/cvmfs/singularity.galaxyproject.org/all/python:3.8.3" + singularity_run_extra_arguments: >- + --env JAVA_OPTS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR" + --env JAVA_TOOL_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR" env: LC_ALL: C TMPDIR: $SCRATCHDIR TMP: $SCRATCHDIR TEMP: $SCRATCHDIR - GALAXY_SLOTS: "{cores}" - GALAXY_MEMORY_MB: "{int(mem)*1000}" + SINGULARITY_CACHEDIR: "/cvmfs/singularity.galaxyproject.org/all/" + SINGULARITY_TMPDIR: "$SCRATCHDIR" + XDG_CACHE_HOME: "$SCRATCHDIR" scheduling: require: - pulsar - reject: - - singularity - tpv_pulsar_singularity: + tpv_pulsar_nasty_java: inherits: tpv_pulsar runner: pulsar_tpv_runner params: - singularity_enabled: true - singularity_volumes: "$job_directory:rw,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,/cvmfs/data.galaxyproject.org:ro,$SCRATCHDIR:rw" - singularity_default_container_id: "/cvmfs/singularity.galaxyproject.org/all/python:3.8.3" - env: - SINGULARITY_CACHEDIR: "/auto/praha5-elixir/home/galaxyeu/pulsar-cz/files/singularity_cache" -# SINGULARITY_CACHEDIR: "/auto/praha5-elixir/home/galaxyeu/singularity/cache" - SINGULARITY_TMPDIR: "$SCRATCHDIR/tmp" -# SINGULARITY_TMPDIR: "/auto/praha5-elixir/home/galaxyeu/singularity/tmp" - XDG_CACHE_HOME: "$SCRATCHDIR" - container_resolvers: - - type: explicit_singularity -# cache_directory: "/auto/praha5-elixir/home/galaxyeu/pulsar-cz/singularity" - - type: cached_mulled_singularity -# cache_directory: "/auto/praha5-elixir/home/galaxyeu/pulsar-cz/files/singularity_cache" - - type: mulled_singularity -# cache_directory: "/auto/praha5-elixir/home/galaxyeu/pulsar-cz/singularity" + singularity_run_extra_arguments: '--env _JAVA_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR"' scheduling: require: - - pulsar - - singularity - reject: - tpv_pulsar_gpu: - inherits: tpv_pulsar_singularity - runner: pulsar_gpu_runner + - nasty-java + tpv_pulsar_alphafold: + inherits: tpv_pulsar + runner: pulsar_tpv_runner params: singularity_run_extra_arguments: '--nv' singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro' - submit_native_specification: "" + submit_native_specification: "-l select=1:ncpus={int(cores)}:mem={int(mem)}gb:scratch_local={int(scratch)}gb:ngpus={int(gpus)}:gpu_mem={int(gpu_mem)}gb -l walltime={int(walltime)}:00:00 -q galaxy_gpu@pbs-m1.metacentrum.cz -N pulsar_cz_j{job.id}__{tool.id if '/' not in tool.id else tool.id.split('/')[-2]+'_v'+tool.id.split('/')[-1]}__{user.username if user and hasattr(user, 'username') else 'anonymous'}" env: ALPHAFOLD_DB: "/auto/brno11-elixir/projects/alphafold/alphafold.db-2.3.1" ALPHAFOLD_USE_GPU: True scheduling: require: - - pulsar - - singularity - - gpu + - alphafold reject: - pulsar-training: - inherits: tpv_pulsar_singularity + tpv_pulsar_training: + inherits: tpv_pulsar runner: pulsar_tpv_runner scheduling: require: - - pulsar - training diff --git a/templates/nginx/galaxy.j2 b/templates/nginx/galaxy.j2 index fd78f81..60bc03b 100644 --- a/templates/nginx/galaxy.j2 +++ b/templates/nginx/galaxy.j2 @@ -18,7 +18,13 @@ server { error_log syslog:server=unix:/dev/log; # client_body_temp_path /rbd/data/nginx_upload_store; client_body_buffer_size 1024m; - client_max_body_size 20G; + client_max_body_size 50G; + + # smaller timeout interrupted transimition of bigger files + send_timeout 300s; + proxy_read_timeout 600s; + client_body_timeout 3600s; + client_header_timeout 600s; # The most important location block, by default all requests are sent to gunicorn # If you serve galaxy at a path like /galaxy, change that below (and all other locations!) diff --git a/templates/nginx/redirect-ssl.j2 b/templates/nginx/redirect-ssl.j2 index 182c85e..12473b2 100644 --- a/templates/nginx/redirect-ssl.j2 +++ b/templates/nginx/redirect-ssl.j2 @@ -4,12 +4,12 @@ server { server_name "{{ inventory_hostname }}"; - location /.well-known/ { + location /.well-known/acme-challenge/ { root {{ certbot_well_known_root }}; } location / { - return 302 https://$host$request_uri; + return 302 https://{{ inventory_hostname }}$request_uri; } } @@ -21,7 +21,13 @@ server { server_name "galaxy.metacentrum.cz"; - return 308 "https://{{ inventory_hostname }}$request_uri"; + location /.well-known/acme-challenge/ { + root /srv/nginx/_well-known_root; + } + + location / { + return 308 "https://{{ inventory_hostname }}$request_uri"; + } } server { @@ -32,6 +38,11 @@ server { server_name "galaxy-cz.grid.cesnet.cz"; - return 308 "https://{{ inventory_hostname }}$request_uri"; -} + location /.well-known/acme-challenge/ { + root /srv/nginx/_well-known_root; + } + location / { + return 308 "https://{{ inventory_hostname }}$request_uri"; + } +}