Build & test Airflow #23
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build & test Airflow | |
on: | |
workflow_dispatch: | |
jobs: | |
build: | |
timeout-minutes: 15 | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python 3.9 | |
uses: actions/setup-python@v3 | |
with: | |
python-version: "3.9" | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install -r requirements.txt build | |
python -m build | |
- name: Login to Docker Hub | |
uses: docker/login-action@v3 | |
with: | |
username: ${{ secrets.DUSERNAME }} | |
password: ${{ secrets.DPASSWORD }} | |
- name: Install & Run Airflow | |
run: | | |
PYPACKAGE=$(ls dist | grep .whl) | |
cp "dist/${PYPACKAGE}" tests_integration/plugins/ | |
cd tests_integration | |
echo "FROM apache/airflow:2.9.1" > Dockerfile | |
echo "ADD plugins/${PYPACKAGE} ." >> Dockerfile | |
echo "RUN pip install ${PYPACKAGE}" >> Dockerfile | |
echo -e "AIRFLOW_UID=$(id -u)" > .env | |
docker compose up -d airflow-init | |
docker compose up -d --build | |
- name: Add TM1 connection to Airflow | |
run: | | |
DOCKID=$(docker ps -q --filter "name=tests_integration-airflow-worker-1") | |
docker exec -i $DOCKID /bin/bash <<-EOF | |
airflow connections get tm1_conn | |
exit_status=$? | |
if [ $exit_status -eq 0 ]; then | |
docker exec $DOCKID airflow connections add tm1_conn --conn-type tm1 --conn-login admin --conn-password admin --conn-host tm1-centos7 --conn-port 5360 --conn-schema "" --conn-extra "" | |
else | |
echo "Connection already added" | |
fi | |
EOF | |
- name: Test Airflow TM1 provider | |
run: | | |
cd tests_integration | |
echo "Running tests..." | |
DOCKID=$(docker ps -q --filter "name=tests_integration-airflow-worker-1") | |
docker exec -i $DOCKID /bin/bash <<-EOF | |
airflow dags test airflow_test_success_dag > logs/integration_test_airflow_test_success_dag.log | |
airflow dags test airflow_test_aborted_dag >> logs/integration_test_airflow_test_aborted_dag.log | |
airflow dags test airflow_test_data_error_dag >> logs/integration_test_airflow_test_data_error_dag.log | |
airflow dags test airflow_test_timeout_dag >> logs/integration_test_airflow_test_timeout_dag.log | |
EOF | |
- name: Archive production artifacts | |
if: always() | |
uses: actions/upload-artifact@v4 | |
with: | |
name: integration-test-logs | |
path: | | |
tests_integration/logs/* | |
- name: Stop docker containers | |
if: always() | |
run: | | |
cd tests_integration | |
docker compose down | |