model search with model upload to best run #114
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration tests | |
| on: [push] | |
| env: | |
| AIRFLOW__CORE__DAGS_FOLDER: $GITHUB_WORKSPACE | |
| AIRFLOW__CORE__LOAD_EXAMPLES: False | |
| AIRFLOW_CONN_SSHX_SERVER: 'ssh://[email protected]:2222?private_key=-----BEGIN+OPENSSH+PRIVATE+KEY-----%0Ab3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcn%0ANhAAAAAwEAAQAAAIEAuq%2Bua51m1MHDqdSgZ2w%2FoJOcQrAyXMUUAtDNi7%2FOyyrVDMqHp3rR%0AuNGCuzKhztwwQ8ATtoHtQADH%2FCYw8%2FT1EtaDPSvqZRmWHEujPTwsrgtulpRkBcaMO35lB2%0AfENBMLRgA9cmmyTcduTqnbYL7UdMKoYhcw0yTgLlNK9H0vnbUAAAIA%2B0Tb%2FftE2%2F0AAAAH%0Ac3NoLXJzYQAAAIEAuq%2Bua51m1MHDqdSgZ2w%2FoJOcQrAyXMUUAtDNi7%2FOyyrVDMqHp3rRuN%0AGCuzKhztwwQ8ATtoHtQADH%2FCYw8%2FT1EtaDPSvqZRmWHEujPTwsrgtulpRkBcaMO35lB2fE%0ANBMLRgA9cmmyTcduTqnbYL7UdMKoYhcw0yTgLlNK9H0vnbUAAAADAQABAAAAgQCvBjSNuk%0AWFZKBP4gP80rUYlCulLlIZPb%2FUH%2FUFd2%2BmdOLHmj3yXCixkONzJDYlnbQ2YKdarZdEMTdN%0AhHTS067LPzBJ0wOwt%2BQ2ZoVnJ%2F%2FGlp1JoYX4ldEfGT0Ihmqn3wJscHwIGeWbM4AD%2FZIa9H%0A8BnMWq8d4x7W4aLOd11xxpYQAAAEALdgo2Ti1lmq9rmyAj8s5OjeJxwzdFUsuMPAGgD%2BD4%0AqowTAehb7aLqkjtZyJn2HliLVODrNohW%2BuFsciQsaRLmAAAAQQDkycH%2BAJ7kHW4wcb8RLr%0A%2FHU52EVKIcCv0bwqwnATKQIbz8fqCldhdD8O47rpBsD7Iyvp9PDjPFTiBXhBfqwGydAAAA%0AQQDQ4%2FzEgPFngsuI%2Be4j1H4oSmLgAXsw1kcDNYq7a%2FJK7R7kGnIhPX7tI0qiFrq0h9iKip%0AiKOaW0KG6obaAX6g35AAAACWpqQGpqLTkwOAE%3D%0A-----END+OPENSSH+PRIVATE+KEY-----%0A' | |
| AIRFLOW_CONN_DEFAULT_WEBDAV: 'https://alice:[email protected]/' | |
| jobs: | |
| runner-job: | |
| runs-on: ubuntu-latest | |
| services: | |
| openssh-server: | |
| image: lscr.io/linuxserver/openssh-server:latest | |
| ports: | |
| - 2222:2222 | |
| volumes: | |
| - /home/runner/sshdir/sshdir:/tmps/ | |
| env: | |
| PASSWORD_ACCESS: false | |
| PUBLIC_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6r65rnWbUwcOp1KBnbD+gk5xCsDJcxRQC0M2Lv87LKtUMyoenetG40YK7MqHO3DBDwBO2ge1AAMf8JjDz9PUS1oM9K+plGZYcS6M9PCyuC26WlGQFxow7fmUHZ8Q0EwtGAD1yabJNx25OqdtgvtR0wqhiFzDTJOAuU0r0fS+dtQ==" | |
| USER_NAME: eflows | |
| webdav-server: | |
| image: k3vmcd/webdav:latest | |
| ports: | |
| - 443:443 | |
| volumes: | |
| - /home/runner/webdav/:/var/lib/dav/ | |
| env: | |
| AUTH_TYPE: Basic | |
| USERNAME: alice | |
| PASSWORD: secret1234 | |
| SSL_CERT: selfsigned | |
| unicore-server: | |
| image: ghcr.io/unicore-eu/unicore-testing-all | |
| ports: | |
| - 8080:8080 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt --constraint constraints-3.9.txt ; fi | |
| airflow db init | |
| - name: Verify airflow | |
| run: | | |
| echo "Workspace: $GITHUB_WORKSPACE" | |
| echo "ENV: $AIRFLOW__CORE__DAGS_FOLDER" | |
| airflow dags list | |
| - name: Setup access | |
| run: | | |
| docker ps -a | |
| did=`docker ps -f "ancestor=lscr.io/linuxserver/openssh-server" --format '{{ .ID }} '` | |
| docker exec $did chown -R eflows /tmps/ | |
| docker exec $did /bin/sh -c 'mkdir -p /tmps/source/sub /tmps/target' | |
| docker exec $did /bin/sh -c 'echo "Some random text" > /tmps/source/some1' | |
| docker exec $did /bin/sh -c 'echo "Some random other text" > /tmps/source/some2' | |
| docker exec $did /bin/sh -c 'echo "Some random other other text" > /tmps/source/sub/some12' | |
| docker exec $did chown -R eflows /tmps/ | |
| did=`docker ps -f "ancestor=k3vmcd/webdav:latest" --format '{{ .ID }} '` | |
| docker exec $did /bin/sh -c 'mkdir -p /var/lib/dav/data/dls' | |
| docker exec $did /bin/sh -c 'chown -R www-data /var/lib/dav/data/dls' | |
| - name: Connect to ssh | |
| run: | | |
| cd integration | |
| python tester.py --target /tmps/file | |
| ls /home/runner/sshdir/sshdir | |
| cat /home/runner/sshdir/sshdir/file | |
| diff /home/runner/sshdir/sshdir/file data.txt | |
| - name: SSH2SSH test | |
| run: | | |
| pwd | |
| airflow dags test ssh2ssh -c '{"source_connection_id": "sshx_server", "source_path": "/tmps/source/", "target_connection_id": "sshx_server", "target_path": "/tmps/target/"}' | |
| - name: WebDav test | |
| run: | | |
| airflow dags test webdav_stageout -c '{"connection_id": "sshx_server", "path":"/tmps/source/", "verify_webdav_cert": false }' | |
| ls /home/runner/webdav/data/ | |
| - name: WebDav stage out file test | |
| run: | | |
| airflow dags test webdav_stageout -c '{"connection_id": "sshx_server", "path":"/tmps/source/some1", "verify_webdav_cert": false }' | |
| ls /home/runner/webdav/data/ | |
| - name: Unicore test | |
| run: | | |
| airflow dags test webdav2unicore -c '{"oid": "notimportant", "site_url": "https://localhost:8080/DEMO-SITE/rest/core", "user": "demouser", "password": "test123", "verify_webdav_cert": false}' | |
| - name: gitlab test | |
| run: | | |
| airflow dags test git2ssh -c '{"connection_id": "sshx_server", "target": "/tmps/gits", "gitlab_url": "https://gitlab.jsc.fz-juelich.de", "gitlab_repo": "DataCatalog", "host": "localhost", "port": 2222}' | |
| ls /home/runner/sshdir/sshdir/gits/ | |