diff --git a/.gitignore b/.gitignore index 5610c23..6a67685 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# Local testing dv artifacts +dv +solr + +# Apple artifacts +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index fe8fbb0..75c97c2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![PyPI](https://img.shields.io/pypi/v/pyDataverse.svg)](https://pypi.org/project/pyDataverse/) ![Build Status](https://github.com/gdcc/pyDataverse/actions/workflows/test_build.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/gdcc/pyDataverse/badge.svg)](https://coveralls.io/github/gdcc/pyDataverse) [![Documentation Status](https://readthedocs.org/projects/pydataverse/badge/?version=latest)](https://pydataverse.readthedocs.io/en/latest) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pydataverse.svg) [![GitHub](https://img.shields.io/github/license/gdcc/pydataverse.svg)](https://opensource.org/licenses/MIT) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4664557.svg)](https://doi.org/10.5281/zenodo.4664557) +[![PyPI](https://img.shields.io/pypi/v/pyDataverse.svg)](https://pypi.org/project/pyDataverse/) ![Build Status](https://github.com/gdcc/pyDataverse/actions/workflows/test_build.yml/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/gdcc/pyDataverse/badge.svg)](https://coveralls.io/github/gdcc/pyDataverse) [![Documentation Status](https://readthedocs.org/projects/pydataverse/badge/?version=latest)](https://pydataverse.readthedocs.io/en/latest) PyPI - Python Version [![GitHub](https://img.shields.io/github/license/gdcc/pydataverse.svg)](https://opensource.org/licenses/MIT) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4664557.svg)](https://doi.org/10.5281/zenodo.4664557) # pyDataverse @@ -9,6 +9,48 @@ It helps to access the Dataverse [API's](http://guides.dataverse.org/en/latest/a **Find out more: [Read the Docs](https://pydataverse.readthedocs.io/en/latest/)** +# Running tests + +In order to run the tests, you need to have a Dataverse instance running. We have prepared a shell script that will start a Dataverse instance using Docker that runs all tests in a clean environment. To run the tests, execute the following command: + +```bash +# Defaults to Python 3.11 +./run_tests.sh + +# To run the tests with a specific Python version +./run_tests.sh -p 3.8 +``` + +Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal. + +## Manual setup + +If you want to run single tests you need to manually set up the environment and set up the necessary environment variables. Please follow the instructions below. + +**1. Start the Dataverse instance** + +```bash +docker compose \ + -f ./docker/docker-compose-base.yml \ + --env-file local-test.env \ + up -d +``` + +**2. Set up the environment variables** + +```bash +export BASE_URL=http://localhost:8080 +export DV_VERSION=6.2 # or any other version +export $(grep "API_TOKEN" "dv/bootstrap.exposed.env") +export API_TOKEN_SUPERUSER=$API_TOKEN +``` + +**3. Run the test(s) with pytest** + +```bash +python -m pytest -v +``` + ## Chat with us! If you are interested in the development of pyDataverse, we invite you to join us for a chat on our [Zulip Channel](https://dataverse.zulipchat.com/#narrow/stream/377090-python). This is the perfect place to discuss and exchange ideas about the development of pyDataverse. Whether you need help or have ideas to share, feel free to join us! diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.yml new file mode 100644 index 0000000..196a1a3 --- /dev/null +++ b/docker/docker-compose-base.yml @@ -0,0 +1,137 @@ +version: "2.4" +name: pydataverse +services: + dataverse: + container_name: "dataverse" + hostname: dataverse + image: ${DATAVERSE_IMAGE} + restart: on-failure + user: payara + environment: + - DATAVERSE_DB_HOST=postgres + - DATAVERSE_DB_USER=${DATAVERSE_DB_USER} + - DATAVERSE_DB_PASSWORD=${DATAVERSE_DB_PASSWORD} + - JVM_ARGS=-Ddataverse.pid.providers=fake + -Ddataverse.pid.default-provider=fake + -Ddataverse.pid.fake.type=FAKE + -Ddataverse.pid.fake.label=FakeDOIProvider + -Ddataverse.pid.fake.authority=10.5072 + -Ddataverse.pid.fake.shoulder=FK2/ + ports: + - "8080:8080" + networks: + - dataverse + depends_on: + postgres: + condition: service_started + solr: + condition: service_started + dv_initializer: + condition: service_completed_successfully + volumes: + - ${PWD}/dv/data:/dv + - ${PWD}:/secrets + tmpfs: + - /dumps:mode=770,size=2052M,uid=1000,gid=1000 + - /tmp:mode=770,size=2052M,uid=1000,gid=1000 + mem_limit: 2147483648 # 2 GiB + mem_reservation: 1024m + privileged: false + healthcheck: + test: curl --fail http://dataverse:8080/api/info/version || exit 1 + interval: 10s + retries: 20 + start_period: 20s + timeout: 240s + + dv_initializer: + container_name: "dv_initializer" + image: ${CONFIGBAKER_IMAGE} + restart: "no" + command: + - sh + - -c + - "fix-fs-perms.sh dv" + volumes: + - ${PWD}/dv/data:/dv + + postgres: + container_name: "postgres" + hostname: postgres + image: postgres:${POSTGRES_VERSION} + restart: on-failure + environment: + - POSTGRES_USER=${DATAVERSE_DB_USER} + - POSTGRES_PASSWORD=${DATAVERSE_DB_PASSWORD} + ports: + - "5432:5432" + networks: + - dataverse + + solr_initializer: + container_name: "solr_initializer" + image: ${CONFIGBAKER_IMAGE} + restart: "no" + command: + - sh + - -c + - "fix-fs-perms.sh solr && cp -a /template/* /solr-template" + volumes: + - ${PWD}/solr/data:/var/solr + - ${PWD}/solr/conf:/solr-template + + solr: + container_name: "solr" + hostname: "solr" + image: solr:${SOLR_VERSION} + depends_on: + solr_initializer: + condition: service_completed_successfully + restart: on-failure + ports: + - "8983:8983" + networks: + - dataverse + command: + - "solr-precreate" + - "collection1" + - "/template" + volumes: + - ${PWD}/solr/data:/var/solr + - ${PWD}/solr/conf:/template + + smtp: + container_name: "smtp" + hostname: "smtp" + image: maildev/maildev:2.0.5 + restart: on-failure + expose: + - "25" # smtp server + environment: + - MAILDEV_SMTP_PORT=25 + - MAILDEV_MAIL_DIRECTORY=/mail + networks: + - dataverse + tmpfs: + - /mail:mode=770,size=128M,uid=1000,gid=1000 + + bootstrap: + container_name: "bootstrap" + hostname: "bootstrap" + image: ${CONFIGBAKER_IMAGE} + restart: "no" + networks: + - dataverse + volumes: + - ${PWD}/dv/bootstrap.exposed.env:/.env + command: + - sh + - -c + - "bootstrap.sh -e /.env dev" + depends_on: + dataverse: + condition: service_healthy + +networks: + dataverse: + driver: bridge diff --git a/docker/docker-compose-test-all.yml b/docker/docker-compose-test-all.yml new file mode 100644 index 0000000..22274a2 --- /dev/null +++ b/docker/docker-compose-test-all.yml @@ -0,0 +1,31 @@ +version: "2.4" +services: + unit-tests: + container_name: unit-tests + image: python:${PYTHON_VERSION}-slim + environment: + BASE_URL: http://dataverse:8080 + DV_VERSION: 6.2 + networks: + - dataverse + volumes: + - ${PWD}:/pydataverse + - ../dv:/dv + command: + - sh + - -c + - | + # Fetch the API Token from the local file + export $(grep "API_TOKEN" "dv/bootstrap.exposed.env") + export API_TOKEN_SUPERUSER=$$API_TOKEN + cd /pydataverse + + # Run the unit tests + python3 -m pip install --upgrade pip + python3 -m pip install pytest pytest-cov + python3 -m pip install -e . + python3 -m pytest > /dv/unit-tests.log + + depends_on: + bootstrap: + condition: service_completed_successfully diff --git a/local-test.env b/local-test.env new file mode 100644 index 0000000..7400574 --- /dev/null +++ b/local-test.env @@ -0,0 +1,9 @@ +# Dataverse +DATAVERSE_IMAGE=docker.io/gdcc/dataverse:unstable +DATAVERSE_DB_USER=dataverse +DATAVERSE_DB_PASSWORD=secret +CONFIGBAKER_IMAGE=docker.io/gdcc/configbaker:unstable + +# Services +POSTGRES_VERSION=15 +SOLR_VERSION=9.3.0 \ No newline at end of file diff --git a/run-tests.sh b/run-tests.sh new file mode 100644 index 0000000..8c28614 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,87 @@ +#!bin/bash + +# Parse arguments +usage() { + echo "Usage: $0 [-p Python version (e.g. 3.10, 3.11, ...)]" 1>&2 + exit 1 +} + +while getopts ":p:d:" o; do + case "${o}" in + p) + p=${OPTARG} + ;; + *) ;; + esac +done +shift $((OPTIND - 1)) + +# Fall back to Python 3.11 if no Python version is specified +if [ -z "${p}" ]; then + printf "\nāš ļø No Python version specified falling back to '3.11'\n" + p=3.11 +fi + +# Validate Python version +if [[ ! "${p}" =~ ^3\.[0-9]+$ ]]; then + echo "\nāŒ Invalid Python version. Please specify a valid Python version (e.g. 3.10, 3.11, ...)\n" + exit 1 +fi + +# Check if Docker is installed +if ! command -v docker &>/dev/null; then + echo "āœ‹ Docker is not installed. Please install Docker before running this script." + exit 1 +fi + +# Prepare the environment for the test +mkdir dv >>/dev/null 2>&1 +touch dv/bootstrap.exposed.env >>/dev/null 2>&1 + +# Add python version to the environment +export PYTHON_VERSION=${p} + +printf "\nšŸš€ Preparing containers\n" +printf " Using PYTHON_VERSION=${p}\n\n" + +# Run all containers +docker compose \ + -f docker/docker-compose-base.yml \ + -f ./docker/docker-compose-test-all.yml \ + --env-file local-test.env \ + up -d + +printf "\nšŸ”Ž Running pyDataverse tests\n" +printf " Logs will be printed once finished...\n\n" + +# Check if "unit-test" container has finished +while [ -n "$(docker ps -f "name=unit-tests" -f "status=running" -q)" ]; do + printf " Waiting for unit-tests container to finish...\n" + sleep 5 +done + +# Check if "unit-test" container has failed +if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then + printf "\nāŒ Unit tests failed. Printing logs...\n" + docker logs unit-tests + printf "\n Stopping containers\n" + docker compose \ + -f docker/docker-compose-base.yml \ + -f ./docker/docker-compose-test-all.yml \ + --env-file local-test.env \ + down + exit 1 +fi + +# Print test results +printf "\n" +cat dv/unit-tests.log +printf "\n\nāœ… Unit tests passed\n\n" + +# Stop all containers +docker compose \ + -f docker/docker-compose-base.yml \ + -f ./docker/docker-compose-test-all.yml \ + --env-file local-test.env \ + down +printf "\nšŸŽ‰ Done\n\n"