Merge pull request #172 from gdcc/local-testing

Provide local testing functionality

.gitignore

@@ -1,3 +1,10 @@
+# Local testing dv artifacts
+dv
+solr
+
+# Apple artifacts
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -9,6 +9,48 @@ It helps to access the Dataverse [API's](http://guides.dataverse.org/en/latest/a
 
 **Find out more: [Read the Docs](https://pydataverse.readthedocs.io/en/latest/)**
 
+# Running tests
+
+In order to run the tests, you need to have a Dataverse instance running. We have prepared a shell script that will start a Dataverse instance using Docker that runs all tests in a clean environment. To run the tests, execute the following command:
+
+```bash
+# Defaults to Python 3.11
+./run_tests.sh
+
+# To run the tests with a specific Python version
+./run_tests.sh -p 3.8
+```
+
+Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal.
+
+## Manual setup
+
+If you want to run single tests you need to manually set up the environment and set up the necessary environment variables. Please follow the instructions below.
+
+**1. Start the Dataverse instance**
+
+```bash
+docker compose \
+    -f ./docker/docker-compose-base.yml \
+    --env-file local-test.env \
+    up -d
+```
+
+**2. Set up the environment variables**
+
+```bash
+export BASE_URL=http://localhost:8080
+export DV_VERSION=6.2 # or any other version
+export $(grep "API_TOKEN" "dv/bootstrap.exposed.env")
+export API_TOKEN_SUPERUSER=$API_TOKEN
+```
+
+**3. Run the test(s) with pytest**
+
+```bash
+python -m pytest -v
+```
+
 ## Chat with us!
 
 If you are interested in the development of pyDataverse, we invite you to join us for a chat on our [Zulip Channel](https://dataverse.zulipchat.com/#narrow/stream/377090-python). This is the perfect place to discuss and exchange ideas about the development of pyDataverse. Whether you need help or have ideas to share, feel free to join us!

docker/docker-compose-base.yml

@@ -0,0 +1,137 @@
+version: "2.4"
+name: pydataverse
+services:
+  dataverse:
+    container_name: "dataverse"
+    hostname: dataverse
+    image: ${DATAVERSE_IMAGE}
+    restart: on-failure
+    user: payara
+    environment:
+      - DATAVERSE_DB_HOST=postgres
+      - DATAVERSE_DB_USER=${DATAVERSE_DB_USER}
+      - DATAVERSE_DB_PASSWORD=${DATAVERSE_DB_PASSWORD}
+      - JVM_ARGS=-Ddataverse.pid.providers=fake
+        -Ddataverse.pid.default-provider=fake
+        -Ddataverse.pid.fake.type=FAKE
+        -Ddataverse.pid.fake.label=FakeDOIProvider
+        -Ddataverse.pid.fake.authority=10.5072
+        -Ddataverse.pid.fake.shoulder=FK2/
+    ports:
+      - "8080:8080"
+    networks:
+      - dataverse
+    depends_on:
+      postgres:
+        condition: service_started
+      solr:
+        condition: service_started
+      dv_initializer:
+        condition: service_completed_successfully
+    volumes:
+      - ${PWD}/dv/data:/dv
+      - ${PWD}:/secrets
+    tmpfs:
+      - /dumps:mode=770,size=2052M,uid=1000,gid=1000
+      - /tmp:mode=770,size=2052M,uid=1000,gid=1000
+    mem_limit: 2147483648 # 2 GiB
+    mem_reservation: 1024m
+    privileged: false
+    healthcheck:
+      test: curl --fail http://dataverse:8080/api/info/version || exit 1
+      interval: 10s
+      retries: 20
+      start_period: 20s
+      timeout: 240s
+
+  dv_initializer:
+    container_name: "dv_initializer"
+    image: ${CONFIGBAKER_IMAGE}
+    restart: "no"
+    command:
+      - sh
+      - -c
+      - "fix-fs-perms.sh dv"
+    volumes:
+      - ${PWD}/dv/data:/dv
+
+  postgres:
+    container_name: "postgres"
+    hostname: postgres
+    image: postgres:${POSTGRES_VERSION}
+    restart: on-failure
+    environment:
+      - POSTGRES_USER=${DATAVERSE_DB_USER}
+      - POSTGRES_PASSWORD=${DATAVERSE_DB_PASSWORD}
+    ports:
+      - "5432:5432"
+    networks:
+      - dataverse
+
+  solr_initializer:
+    container_name: "solr_initializer"
+    image: ${CONFIGBAKER_IMAGE}
+    restart: "no"
+    command:
+      - sh
+      - -c
+      - "fix-fs-perms.sh solr && cp -a /template/* /solr-template"
+    volumes:
+      - ${PWD}/solr/data:/var/solr
+      - ${PWD}/solr/conf:/solr-template
+
+  solr:
+    container_name: "solr"
+    hostname: "solr"
+    image: solr:${SOLR_VERSION}
+    depends_on:
+      solr_initializer:
+        condition: service_completed_successfully
+    restart: on-failure
+    ports:
+      - "8983:8983"
+    networks:
+      - dataverse
+    command:
+      - "solr-precreate"
+      - "collection1"
+      - "/template"
+    volumes:
+      - ${PWD}/solr/data:/var/solr
+      - ${PWD}/solr/conf:/template
+
+  smtp:
+    container_name: "smtp"
+    hostname: "smtp"
+    image: maildev/maildev:2.0.5
+    restart: on-failure
+    expose:
+      - "25" # smtp server
+    environment:
+      - MAILDEV_SMTP_PORT=25
+      - MAILDEV_MAIL_DIRECTORY=/mail
+    networks:
+      - dataverse
+    tmpfs:
+      - /mail:mode=770,size=128M,uid=1000,gid=1000
+
+  bootstrap:
+    container_name: "bootstrap"
+    hostname: "bootstrap"
+    image: ${CONFIGBAKER_IMAGE}
+    restart: "no"
+    networks:
+      - dataverse
+    volumes:
+      - ${PWD}/dv/bootstrap.exposed.env:/.env
+    command:
+      - sh
+      - -c
+      - "bootstrap.sh -e /.env dev"
+    depends_on:
+      dataverse:
+        condition: service_healthy
+
+networks:
+  dataverse:
+    driver: bridge

docker/docker-compose-test-all.yml

@@ -0,0 +1,31 @@
+version: "2.4"
+services:
+  unit-tests:
+    container_name: unit-tests
+    image: python:${PYTHON_VERSION}-slim
+    environment:
+      BASE_URL: http://dataverse:8080
+      DV_VERSION: 6.2
+    networks:
+      - dataverse
+    volumes:
+      - ${PWD}:/pydataverse
+      - ../dv:/dv
+    command:
+      - sh
+      - -c
+      - |
+        # Fetch the API Token from the local file
+        export $(grep "API_TOKEN" "dv/bootstrap.exposed.env")
+        export API_TOKEN_SUPERUSER=$$API_TOKEN
+        cd /pydataverse
+
+        # Run the unit tests
+        python3 -m pip install --upgrade pip
+        python3 -m pip install pytest pytest-cov
+        python3 -m pip install -e .
+        python3 -m pytest > /dv/unit-tests.log
+
+    depends_on:
+      bootstrap:
+        condition: service_completed_successfully

local-test.env

@@ -0,0 +1,9 @@
+# Dataverse
+DATAVERSE_IMAGE=docker.io/gdcc/dataverse:unstable
+DATAVERSE_DB_USER=dataverse
+DATAVERSE_DB_PASSWORD=secret
+CONFIGBAKER_IMAGE=docker.io/gdcc/configbaker:unstable
+
+# Services
+POSTGRES_VERSION=15
+SOLR_VERSION=9.3.0

run-tests.sh

@@ -0,0 +1,87 @@
+#!bin/bash
+
+# Parse arguments
+usage() {
+    echo "Usage: $0 [-p Python version (e.g. 3.10, 3.11, ...)]" 1>&2
+    exit 1
+}
+
+while getopts ":p:d:" o; do
+    case "${o}" in
+    p)
+        p=${OPTARG}
+        ;;
+    *) ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+# Fall back to Python 3.11 if no Python version is specified
+if [ -z "${p}" ]; then
+    printf "\n⚠️  No Python version specified falling back to '3.11'\n"
+    p=3.11
+fi
+
+# Validate Python version
+if [[ ! "${p}" =~ ^3\.[0-9]+$ ]]; then
+    echo "\n❌ Invalid Python version. Please specify a valid Python version (e.g. 3.10, 3.11, ...)\n"
+    exit 1
+fi
+
+# Check if Docker is installed
+if ! command -v docker &>/dev/null; then
+    echo "✋ Docker is not installed. Please install Docker before running this script."
+    exit 1
+fi
+
+# Prepare the environment for the test
+mkdir dv >>/dev/null 2>&1
+touch dv/bootstrap.exposed.env >>/dev/null 2>&1
+
+# Add python version to the environment
+export PYTHON_VERSION=${p}
+
+printf "\n🚀 Preparing containers\n"
+printf "   Using PYTHON_VERSION=${p}\n\n"
+
+# Run all containers
+docker compose \
+    -f docker/docker-compose-base.yml \
+    -f ./docker/docker-compose-test-all.yml \
+    --env-file local-test.env \
+    up -d
+
+printf "\n🔎 Running pyDataverse tests\n"
+printf "   Logs will be printed once finished...\n\n"
+
+# Check if "unit-test" container has finished
+while [ -n "$(docker ps -f "name=unit-tests" -f "status=running" -q)" ]; do
+    printf "   Waiting for unit-tests container to finish...\n"
+    sleep 5
+done
+
+# Check if "unit-test" container has failed
+if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then
+    printf "\n❌ Unit tests failed. Printing logs...\n"
+    docker logs unit-tests
+    printf "\n   Stopping containers\n"
+    docker compose \
+        -f docker/docker-compose-base.yml \
+        -f ./docker/docker-compose-test-all.yml \
+        --env-file local-test.env \
+        down
+    exit 1
+fi
+
+# Print test results
+printf "\n"
+cat dv/unit-tests.log
+printf "\n\n✅ Unit tests passed\n\n"
+
+# Stop all containers
+docker compose \
+    -f docker/docker-compose-base.yml \
+    -f ./docker/docker-compose-test-all.yml \
+    --env-file local-test.env \
+    down
+printf "\n🎉 Done\n\n"