ci-test.yml

# merged in python-package.yml workflow

# reference docs:
# https://blog.deepjyoti30.dev/tests-github-python
# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
# https://github.com/pypa/twine/blob/main/.github/workflows/main.yml

name: roundup-ci

on: 
    push:
         # skip if github.ref is 'refs/heads/maint-1.6'
         #   aka github.ref_name of 'maint-1.6'
         # see https://github.com/orgs/community/discussions/26253
         # for mechanism to control matrix based on branch
         branches: [ "*", '!maint-1.6' ]
#    pull_request:
#        branches: [ "master" ]
    schedule:
        # monthly build/check
        - cron: '23 17 1 * *'
    workflow_dispatch:
      inputs:
        debug_enabled:
          type: boolean
          description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
          required: false
          default: false

# GITHUB_TOKEN only has read repo context.
permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: CI build test

    #runs-on: ubuntu-latest
    # use below if running on multiple OS's.
    runs-on: ${{ matrix.os }}

    if: "!contains(github.event.head_commit.message, 'no-github-ci')"

    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        # Run in all these versions of Python
        python-version:
          # - "2.7"
          - "3.13"
          # - 3.6 run via include on ubuntu 20.04
          # - "3.7"
          # - "3.8" run via include for ubuntu-22.04
          # - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"

        # use for multiple os or ubuntu versions
        #os: [ubuntu-latest, macos-latest, windows-latest]
        # ubuntu latest 22.04 12/2022
        # ubuntu latest 24.04 12/2024
        os: [ubuntu-24.04, ubuntu-latest, ubuntu-20.04]

        # if the ones above fail. fail the build
        experimental: [ false ]

        include:
            # example: if 3.13 fails the jobs still succeeds
            # allow-prereleases in setup-python allows alpha/beta
            # releases to run
            #- python-version: 3.13
            #  os: ubuntu-22.04
            #  experimental: true
            
            # 3.6 not available on new 22.04 runners, so run on 20.04 ubuntu
            - python-version: 3.6
              os: ubuntu-20.04

            # 3.8 not available on new 24.04 runners, so run on 22.04 ubuntu
            - python-version: 3.8
              os: ubuntu-22.04

              # test on new default 24.04 runner 3.12 for 24.04
            - python-version: 3.12
              os: ubuntu-24.04

        exclude:
            # skip all python versions on 20.04/24.04 except
            # explicitly included
            - os: ubuntu-20.04
            # disable for testing when getting 24.04 running
            - os: ubuntu-22.04
            - os: ubuntu-24.04

    # run the finalizer for coveralls even if one or more
    # experimental matrix runs fail.
    # moving it above strategy produces unexpected value false
    # moving it below (here) produces unexpected value ''.
    # continue-on-error: ${{ matrix.experimental }}

    env:
      # get colorized pytest output even without a controlling tty
      PYTEST_ADDOPTS: "--color=yes"
      #  OS: ${{ matrix.os }}
      PYTHON_VERSION: ${{ matrix.python-version }}

    steps:
      # Checkout the latest code from the repo
      - name: Checkout source
        # example directives:
          # disable step
        # if: {{ false }}
          # continue running if step fails
        # continue-on-error: true
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

      # Setup version of Python to use
      - name: Set Up Python ${{ matrix.python-version }}
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: ${{ matrix.python-version }}
          allow-prereleases: true
          cache: 'pip'

      - name: Install build tools - setuptools
        run: pip install setuptools

      # Display the Python version being used
      - name: Display Python and key module versions
        run: |
          python -c "import sys; print('python version: ', sys.version)"
          python -c "import sqlite3; print('sqlite version: ', sqlite3.sqlite_version)"
          python -c "import setuptools; print('setuptools version: ', setuptools.__version__);"

      - name: Update pip
        run: python -m pip install --upgrade pip

      # note pytest-env is not needed for redis password as there is
      # no password on CI's redis.
      - name: Install pytest and other packages needed for running tests
        run: pip install flake8 hypothesis mock pytest pytest-cov requests sphinx-tabs

      # https://github.com/mxschmitt/action-tmate
      # allow remote ssh into the CI container. I need this to debug
      # some xfail cases 
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
        timeout-minutes: 10
        with:
          limit-access-to-actor: true

      - name: run flake8 - abort for syntax error, otherwise warn only
        run: |
            # stop the build for Python syntax errors or undefined names
            # talgettext is a utility function ignore it.
            flake8 roundup --count --select=E9,F63,F7,F82 --show-source --statistics --extend-exclude talgettext.py
            # exit-zero treats all errors as warnings.
            #   The GitHub editor is 127 chars wide
            flake8 roundup --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

      # Install the databases
      - name: Install mysql/mariadb
        run: |
          #set -xv
          # mysql is pre-installed and active but this is the install command
          # sudo apt-get install mysql-server mysql-client
          # set up mysql database
          sudo sed -i -e '/^\[mysqld\]/,/^\[mysql/s/^#* *max_allowed_packet.*/max_allowed_packet = 500M/' /etc/mysql/mysql.conf.d/mysqld.cnf; sleep 3
          #tail -n +0 /etc/mysql/my.cnf /etc/mysql/mysql.conf.d/mysqld.cnf
          #grep max_allowed /etc/mysql/mysql.conf.d/mysqld.cnf
          #ls  /etc/mysql/conf.d/  /etc/mysql/mysql.conf.d/
          #sleep 5
          # try to improve performance speed by disabling some ACID
          # settings and change some layout defaults.
          sudo sed -i -e '$a\innodb_flush_log_at_trx_commit = 2' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_file_per_table = OFF' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_doublewrite=OFF' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_fast_shutdown=2' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_log_file_size=1048576' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_flush_method=O_DIRECT' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_log_buffer_size=3M' /etc/mysql/mysql.conf.d/mysqld.cnf
          sudo sed -i -e '$a\innodb_buffer_pool_size=180M' /etc/mysql/mysql.conf.d/mysqld.cnf
          sleep 3
          sudo service mysql restart
          #sleep 10
          #ps -ef | grep mysqld
          #sudo netstat -anp | grep mysqld
          sudo mysql -u root -proot -e 'CREATE USER "rounduptest"@"localhost" IDENTIFIED WITH mysql_native_password BY "rounduptest"; GRANT ALL on rounduptest.* TO "rounduptest"@"localhost";'

      - name: Install postgres
        run: |
          sudo apt-get update && sudo apt-get install postgresql
          # Disable fsync, full page writes for speed,
          # don't care about data durability when testing
          sudo sed -i -e '$a\fsync = off' /etc/postgresql/*/*/postgresql.conf
          sudo sed -i -e '$a\full_page_writes = off' /etc/postgresql/*/*/postgresql.conf
          sudo sed -i -e '$a\synchronous_commit = off' /etc/postgresql/*/*/postgresql.conf
          sudo service postgresql restart; sleep 10
          # set up postgresql database
          sudo -u postgres psql -c "CREATE ROLE rounduptest WITH CREATEDB LOGIN PASSWORD 'rounduptest';" -U postgres
          sudo -u postgres psql -c "CREATE ROLE rounduptest_schema LOGIN PASSWORD 'rounduptest';" -U postgres
          sudo -u postgres psql -c "CREATE DATABASE rounduptest_schema;" -U postgres
          sudo -u postgres psql -c "GRANT CREATE ON DATABASE rounduptest_schema TO rounduptest_schema;" -U postgres

      - name: install redis
        run: |
          sudo apt-get install redis
          pip install redis

      - name: Install python db libraries
        run: |
          pip install mysqlclient
          pip install psycopg2

      - name: Install auxiliary packages
        run: |
          sudo apt-get install swig gpgsm libgpgme-dev libgpgme
          # pygments for markdown2 to highlight code blocks
          pip install markdown2 pygments
          # docutils for ReStructuredText
          pip install beautifulsoup4 brotli docutils gpg jinja2 \
            mistune==0.8.4 pyjwt pytz whoosh

      - name: Install aux packages that need versions differences
        # if zstd fails install, keep going with test, don't abort
        run: |
          set -xv
          pip install zstd || true
          if [[ "$PYTHON_VERSION" != "2."* ]]; then 
              pip install Markdown; fi

      - name: Install xapian
        if: matrix.python-version != '3.13'
        run: |
          set -xv
          sudo apt-get install libxapian-dev
          # Sphinx required to build the xapian python bindings. Use 1.8.5 on
          # older python and newest on newer.
          if [[ $PYTHON_VERSION == "2."* ]]; then pip install sphinx==1.8.5; fi
          if [[ $PYTHON_VERSION == '3.'* ]] ; then pip install sphinx; fi
          if [[ $PYTHON_VERSION != "3.13" ]]; then
             XAPIAN_VER=$(dpkg -l libxapian-dev | tail -n 1 | awk '{print $3}' | cut -d '-' -f 1); echo $XAPIAN_VER;
          else
             XAPIAN_VER="1.4.25"; echo $XAPIAN_VER;
          fi
          cd /tmp
          curl -s -O https://oligarchy.co.uk/xapian/$XAPIAN_VER/xapian-bindings-$XAPIAN_VER.tar.xz
          tar -Jxvf xapian-bindings-$XAPIAN_VER.tar.xz
          cd xapian-bindings-$XAPIAN_VER/
          if [[ $PYTHON_VERSION == "2."* ]]; then ./configure --prefix=$VIRTUAL_ENV --with-python --disable-documentation; fi
          # edit the configure script.
          # distutils.sysconfig.get_config_vars('SO')  doesn't work for
          # 3.11 or newer.
          # Change distutils.sysconfig... to just sysconfig and SO
          # to EXT_SUFFIX to get valid value.
          if [[ $PYTHON_VERSION == "3."* ]]; then \
            cp configure configure.FCS; \
            sed -i \
              -e '/PYTHON3_SO=/s/distutils\.//g' \
              -e '/PYTHON3_SO=/s/"SO"/"EXT_SUFFIX"/g' \
              -e '/PYTHON3_CACHE_TAG=/s/imp;print(imp.get_tag())/sys;print(sys.implementation.cache_tag)/' \
              -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\.get_tag()/sys.implementation.cache_tag/g' \
              -e '/PYTHON3_CACHE_OPT1_EXT=/s/imp\b/importlib/g' \
            configure; \
            diff -u configure.FCS configure || true; \
            ./configure --prefix=$VIRTUAL_ENV --with-python3 --disable-documentation; \
          fi
          case "$PYTHON_VERSION" in "3.13") echo skipping xapian build;; *) make && sudo make install; esac

      - name: Test build roundup and install locale so lang tests work.
        run: |
          sudo apt-get install gettext
          python setup.py build
          (cd locale; make local_install; ls -lR locale/de/LC_MESSAGES)

      # Run the tests using pytest with test files in tests directory.
      - name: Run tests
        run: |
          if [[ "$PYTHON_VERSION" != "2."* ]]; then 
            pytest -r a \
              --durations=20 \
              -W default \
              -W "ignore:SelectableGroups:DeprecationWarning" \
              -W "ignore:the imp module:DeprecationWarning:gpg.gpgme:15" \
              -W "ignore:'U' mode::docutils.io"  \
              -W "ignore:unclosed:ResourceWarning:roundup.roundup.demo" \
              -W "ignore:unclosed file:ResourceWarning:enum" \
              -v  test/ --cov=roundup
            if [[ "$PYTHON_VERSION" != "3.6" ]]; then
              # coverage before 3.6 doesn't support lcov output
              coverage lcov
            fi
          else
            # python2 case
            pytest -v -r a --durations=20 test/ --cov=roundup
          fi

      - name: Build docs
        run: python ./setup.py build_doc
        
      - name: Upload coverage to Codecov
        # see: https://github.com/codecov/codecov-action#usage
        uses: codecov/codecov-action@4fe8c5f003fae66aa5ebb77cfd3e7bfbbda0b6b0 # v3.1.5
        with:
          verbose: true
          token: ${{ secrets.CODECOV_TOKEN }}

      - name: Upload coverage to Coveralls
        # python 2.7 and 3.6 versions of coverage can't produce lcov files.
        if: matrix.python-version != '2.7' && matrix.python-version != '3.6'
        uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          path-to-lcov: coverage.lcov
          parallel: run-{{ matrix.python-version }}-{{ matrix.os }}

      #- name: test docker build current version ubuntu-latest
      #  if: matrix.python-version == '3.10' && matrix.os == 'ubuntu-latest'
      #  run: |
      #     docker build -t roundup-app-dev -f scripts/Docker/Dockerfile .
      #     mkdir tracker; chmod 777 tracker
      #     docker run -d --rm -p 9017:8080 \
      #         -v $PWD/tracker:/usr/src/app/tracker \
      #         roundup-app-dev:latest demo
            # expect 200
      #     curl --fail http://localhost:9017/demo/ > /dev/null

      #- name: test docker build released pip version
      #  run: |
      #     docker build -t roundup-app-rel --build-arg="source=pypi" \
      #         -f scripts/Docker/Dockerfile . 


      - name: run benchmarks
        if: "contains(github.event.head_commit.message, 'benchmark')"        
        run: INCI=1 python test/benchmark.py
        
  # in parallel build coveralls requires a finish step
  finish:
    needs: test
    runs-on: ubuntu-latest

    if: "!contains(github.event.head_commit.message, 'no-github-ci')"
    
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2.2.3
        with:
          github-token: ${{ secrets.github_token }}
          parallel-finished: true