diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 0000000..7a1af9c --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,73 @@ +name: Documentation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ '3.10' ] + + steps: + - uses: actions/checkout@v3 + + - name: Cache air + uses: actions/cache@v3 + with: + path: ~/audb/air/1.4.2 + key: air-1.4.2 + + - name: Cache cough-speech-sneeze + uses: actions/cache@v3 + with: + path: ~/audb/cough-speech-sneeze/2.0.1 + key: cough-speech-sneeze-2.0.1 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb/emodb/1.4.1 + key: emodb-1.4.1 + + - name: Cache micirp + uses: actions/cache@v3 + with: + path: ~/audb/micirp/1.0.0 + key: micirp-1.0.0 + + - name: Cache musan + uses: actions/cache@v3 + with: + path: ~/audb/musan/1.0.0 + key: musan-1.0.0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Ubuntu - install libsndfile + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Install docs requirements + run: pip install -r docs/requirements.txt + + - name: Test building documentation + run: python -m sphinx docs/ docs/_build/ -b html -W + + #- name: Check links in documentation + # run: python -m sphinx docs/ docs/_build/ -b linkcheck -W diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..1f5613e --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,29 @@ +name: Linter + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install pre-commit hooks + run: | + pip install pre-commit + pre-commit install --install-hooks + + - name: Code style check via pre-commit + run: | + pre-commit run --all-files diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..944f620 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,116 @@ +name: Publish + +on: + push: + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + deploy: + runs-on: ubuntu-latest + environment: release + permissions: + contents: write + id-token: write + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Cache air + uses: actions/cache@v3 + with: + path: ~/audb/air/1.4.2 + key: air-1.4.2 + + - name: Cache cough-speech-sneeze + uses: actions/cache@v3 + with: + path: ~/audb/cough-speech-sneeze/2.0.1 + key: cough-speech-sneeze-2.0.1 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb/emodb/1.4.1 + key: emodb-1.4.1 + + - name: Cache micirp + uses: actions/cache@v3 + with: + path: ~/audb/micirp/1.0.0 + key: micirp-1.0.0 + + - name: Cache musan + uses: actions/cache@v3 + with: + path: ~/audb/musan/1.0.0 + key: musan-1.0.0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build virtualenv + + # PyPI package + - name: Build Python package + run: python -m build + + - name: Publish Python package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # Remove apt repos that are known to break from time to time + # See https://github.com/actions/virtual-environments/issues/323 + - name: Remove broken apt repos + run: | + for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done + + # Documentation + - name: Install doc dependencies + run: | + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra + pip install -r requirements.txt + pip install -r docs/requirements.txt + + - name: Build documentation + run: | + python -m sphinx docs/ docs/_build/ -b html + + - name: Deploy documentation to Github pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/_build + + # Github release + - name: Read CHANGELOG + id: changelog + run: | + # Get bullet points from last CHANGELOG entry + CHANGELOG=$(git diff -U0 HEAD^ HEAD | grep '^[+][\* ]' | sed 's/\+//') + echo "Got changelog: $CHANGELOG" + # Support for multiline, see + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo 'body<> "$GITHUB_OUTPUT" + + - name: Create release on Github + id: create_release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + name: Release ${{ github.ref_name }} + body: ${{ steps.changelog.outputs.body }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7f6cf72 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ '3.10' ] + include: + - os: ubuntu-latest + python-version: '3.11' + tasks: tests + - os: ubuntu-latest + python-version: '3.9' + tasks: tests + + steps: + - uses: actions/checkout@v3 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb/emodb/1.4.1 + key: emodb-1.4.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Ubuntu - install libsndfile + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra + if: matrix.os == 'ubuntu-latest' + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + # TESTS + - name: Install tests requirements + run: pip install -r tests/requirements.txt + + - name: Test with pytest + run: python -m pytest + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + if: matrix.os == 'ubuntu-latest' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f8a3e68..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,93 +0,0 @@ -stages: - - prepare - - test - - doc - - pypi - -variables: - ARTIFACTORY_GROUPS: "readers,deployers" - ARTIFACTORY_TOKEN_VARS: "ARTIFACTORY_API_KEY,TWINE_PASSWORD" - ARTIFACTORY_USERNAME_VARS: "ARTIFACTORY_USERNAME,TWINE_USERNAME" - -include: - - project: 'devops/artifactory-token-helper' - file: '/.gitlab-ci-artifactory-token.yml' - -.pytest: - image: docker.audeering.com/audeering/python-3.10:latest - before_script: - - apt-get install -y ffmpeg sox libavcodec-extra59 - - python -V - script: - - pip install -r requirements.txt - - pip install -r tests/requirements.txt - - python -m pytest - -python-3.10: - extends: .pytest - stage: test - -linter: - stage: test - image: docker.audeering.com/audeering/python-3.10:latest - before_script: - - pip install pre-commit - - pre-commit install --install-hooks - script: - - pre-commit run --all-files - cache: - key: pre-commit - paths: - - .cache/pre-commit - -docs: - extends: .pytest - stage: test - script: - - pip install -r requirements.txt - - pip install -r docs/requirements.txt - # Create Artifactory config script to manage user access - - echo -e "[artifactory.audeering.com/artifactory]\nusername = ${ARTIFACTORY_USERNAME}\npassword = ${ARTIFACTORY_API_KEY}\n" >~/.artifactory_python.cfg - # Allow anonymous access - - unset ARTIFACTORY_USERNAME - - unset ARTIFACTORY_API_KEY - # Copy audb config file to correct location - - cp docs/audb.yaml ~/.audb.yaml - - python -m sphinx -W docs/ build/html -b html - # Disable linkcheck until new README links are public - # - python -m sphinx -W docs/ build/html -b linkcheck - artifacts: - paths: - - build/ - cache: - key: audb - paths: - - .cache/audb - -pages: - stage: doc - image: docker.audeering.com/alpine - variables: - GIT_STRATEGY: none # disable git fetch - script: - - cp -r build/html public - dependencies: - - docs - artifacts: - paths: - - public - only: - refs: - - tags - -pypi: - extends: .pytest - stage: pypi - before_script: - - pip install build twine virtualenv - script: - - python -m build - - python -m twine upload --repository local dist/* - only: - refs: - - tags diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8e87a94..ea33673 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,32 +1,40 @@ Contributing ============ -If you would like to add new functionality fell free to create a `merge -request`_ . If you find errors, omissions, inconsistencies or other things -that need improvement, please create an issue_. -Contributions are always welcome! +Everyone is invited to contribute to this project. +Feel free to create a `pull request`_ . +If you find errors, +omissions, +inconsistencies, +or other things +that need improvement, +please create an issue_. -.. _issue: - https://gitlab.audeering.com/tools/pyauglib/issues/new?issue%5BD= -.. _merge request: - https://gitlab.audeering.com/tools/pyauglib/merge_requests/new +.. _issue: https://github.com/audeering/auglib/issues/new/ +.. _pull request: https://github.com/audeering/auglib/compare/ Development Installation ------------------------ -Instead of pip-installing the latest release from PyPI, you should get the -newest development version from Gitlab_:: +Instead of pip-installing the latest release from PyPI_, +you should get the newest development version from Github_:: - git clone git@srv-app-01.audeering.local:tools/pyauglib.git - cd pyauglib - # Use virtual environment + git clone https://github.com/audeering/auglib/ + cd auglib + # Create virtual environment for this project + # e.g. + # virtualenv --python="python3" $HOME/.envs/auglib + # source $HOME/.envs/auglib/bin/activate pip install -r requirements.txt -.. _Gitlab: https://gitlab.audeering.com/tools/pyauglib -This way, your installation always stays up-to-date, even if you pull new -changes from the Gitlab repository. +This way, +your installation always stays up-to-date, +even if you pull new changes from the Github repository. + +.. _PyPI: https://pypi.org/project/auglib/ +.. _Github: https://github.com/audeering/auglib/ Coding Convention @@ -102,80 +110,9 @@ It can be installed with:: To execute the tests, simply run:: - pytest tests/ - -.. _pytest: https://pytest.org/ - - -Updating the C++ Library ------------------------- - -``auglib`` depends on the -`C++ auglib library`_, -which is included in ``auglib`` -with the help of a C-wrapper. - -To use a new version of the -`C++ auglib library`_, -you need to update the C-wrapper. -Install needed requirements by:: - - pip install -r cwrapper/requirements.txt - sudo apt-get install --yes cmake libtool automake patchelf - -Clone and install AMR codec:: - - cd cwrapper - git clone --depth 1 --branch master https://gitlab.audeering.com/tools/opencore-amr.git - cd opencore-amr - autoreconf --install - autoconf - ./configure - make - sudo make install - cd .. + python -m pytest -Setup Conan_ to find packages on Artifactory_ -by following the `Setup Conan`_ instructions. - -Clone and build a static version of the `C++ auglib library`_ -in release mode:: - - git clone --depth 1 --branch main https://gitlab.audeering.com/tools/auglib - cd auglib - bash build-soundtouch.sh - mkdir build - cd build - conan install .. - cmake .. -DCMAKE_BUILD_TYPE=Release -DOPENCOREAMR_SUPPORT=ON -DOPENCOREAMRNB_INCLUDE_PATH=/usr/local/include/opencore-amrnb - cd ../.. - -And finally build the C-wrapper:: - - export AUGLIB=./auglib - bash build.sh - -.. _C++ auglib library: https://gitlab.audeering.com/tools/auglib -.. _Conan: https://conan.io -.. _Artifactory: https://artifactory.audeering.com/ui/repos/tree/General/conan-local/ -.. _Setup Conan: https://gitlab.audeering.com/devops/conan/meta/-/blob/master/conan-setup.md - - -Including a New C++ Function ----------------------------- - -If you added a new function in the -`C++ auglib library`_ -and would like to include it in ``auglib`` -as well, -you need to include it in ``cwrapper/cauglib.cpp`` -and follow the steps -explained in the previous section -on updating the C++ library. - -Afterwards, -add the functions signature to ``auglib/core/api.py`` -as well. +.. _pytest: https://pytest.org Creating a New Release @@ -185,9 +122,5 @@ New releases are made using the following steps: #. Update ``CHANGELOG.rst`` #. Commit those changes as "Release X.Y.Z" -#. Create an (annotated) tag with ``git tag -a vX.Y.Z`` -#. Make sure you have an `artifactory-tokenizer`_ project -#. Push the commit and the tag to Gitlab - -.. _artifactory-tokenizer: - https://gitlab.audeering.com/devops/artifactory/tree/master/token +#. Create an (annotated) tag with ``git tag -a X.Y.Z`` +#. Push the commit and the tag to Github diff --git a/auglib/core/interface.py b/auglib/core/interface.py index bf12674..8b85636 100644 --- a/auglib/core/interface.py +++ b/auglib/core/interface.py @@ -117,29 +117,27 @@ class Augment(audinterface.Process, audobject.Object): >>> import audiofile >>> import auglib >>> db = audb.load( - ... "testdata", - ... version="1.5.0", - ... full_path=False, + ... "emodb", + ... version="1.4.1", + ... media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], ... verbose=False, ... ) >>> transform = auglib.transform.WhiteNoiseUniform() >>> augment = auglib.Augment(transform) >>> # Augment a numpy array - >>> file = os.path.join(db.root, db.files[0]) - >>> signal, sampling_rate = audiofile.read(file) + >>> signal, sampling_rate = audiofile.read(db.files[0]) >>> signal_augmented = augment(signal, sampling_rate) >>> # Augment (parts of) a database - >>> column = db["emotion.test.gold"]["emotion"].get() - >>> augmented_column = augment.augment( - ... column, + >>> df = db.get("emotion") + >>> df_augmented = augment.augment( + ... df, ... cache_root="cache", - ... data_root=db.root, + ... remove_root=db.root, ... ) - >>> label = augmented_column.iloc[0] - >>> file = augmented_column.index[0][0] - >>> file = file.replace(audeer.path("."), ".") # remove absolute path - >>> file, label # doctest: +SKIP - ('./cache/decdec83/-6100627981361312836/0/audio/006.wav', 'unhappy') + >>> label = df_augmented.iloc[0, 0] + >>> file = df_augmented.index[0][0] + >>> file, label + ('.../0/wav/03a01Fa.wav', 'happiness') """ # noqa: E501 diff --git a/docs/audb.yaml b/docs/audb.yaml deleted file mode 100644 index 2938aa5..0000000 --- a/docs/audb.yaml +++ /dev/null @@ -1,8 +0,0 @@ -cache_root: .cache/audb -repositories: - - name: data-public-local - backend: artifactory - host: https://artifactory.audeering.com/artifactory - - name: data-public - backend: artifactory - host: https://audeering.jfrog.io/artifactory diff --git a/pyproject.toml b/pyproject.toml index dbbcb33..9ebca2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,9 @@ classifiers = [ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.9', 'Topic :: Scientific/Engineering', ] dependencies = [ diff --git a/tests/conftest.py b/tests/conftest.py index 30d63ca..a1e1854 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,9 +20,12 @@ def _call(self, base: np.ndarray, *, sampling_rate: int = None): ) ] -db = audb.load("testdata", version="1.5.0") +db = audb.load( + "emodb", + version="1.4.1", + media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], + verbose=False, +) pytest.AUDB_ROOT = audb.default_cache_root() -pytest.DATA_COLUMN = db["happiness.dev.gold"]["happiness"].get() -pytest.DATA_FILES = db.files[:5] -pytest.DATA_TABLE = db["happiness.dev.gold"].get() +pytest.DATA_FILES = db.files pytest.TRANSFORM_ONES = Ones() diff --git a/tests/requirements.txt b/tests/requirements.txt index c386cd7..ddcc1ea 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ audb +audformat >=1.1.0 audmath >=1.1.0 # Allow the usage of `strict` in `np.testing.assert_array_equal()` numpy >=1.24.0 diff --git a/tests/test_interface.py b/tests/test_interface.py index c175a44..bda7c46 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -317,7 +317,7 @@ ( # Time values might change due to pandas.to_timedelta() # see - # https://gitlab.audeering.com/tools/pyauglib/-/merge_requests/206 + # https://github.com/pandas-dev/pandas/issues/56629 audformat.segmented_index( ["f1.wav"], [0], @@ -838,7 +838,8 @@ def test_augment_num_workers(tmpdir, transform): None, pytest.AUDB_ROOT, pytest.param( - "/invalid/directory", marks=pytest.mark.xfail(raises=RuntimeError) + "/invalid/directory", + marks=pytest.mark.xfail(raises=RuntimeError), ), pytest.param( pytest.AUDB_ROOT[: len(pytest.AUDB_ROOT) - 1],