From 371d2507e796c05df1aa79b359ab5e2d6087bcbb Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 10:09:06 +0100 Subject: [PATCH 01/13] CI: switch to Github --- .github/workflows/doc.yml | 50 +++++++++++++ .github/workflows/linter.yml | 29 ++++++++ .github/workflows/publish.yml | 86 ++++++++++++++++++++++ .github/workflows/test.yml | 62 ++++++++++++++++ CONTRIBUTING.rst | 135 +++++++++------------------------- docs/audb.yaml | 8 -- tests/test_interface.py | 2 +- 7 files changed, 262 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/doc.yml create mode 100644 .github/workflows/linter.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml delete mode 100644 docs/audb.yaml diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 0000000..1a88d51 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,50 @@ +name: Documentation + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ '3.10' ] + + steps: + - uses: actions/checkout@v3 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb + key: emodb-1.4.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Ubuntu - install libsndfile + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + # DOCS + - name: Install docs requirements + run: pip install -r docs/requirements.txt + + - name: Test building documentation + run: python -m sphinx docs/ docs/_build/ -b html -W + + #- name: Check links in documentation + # run: python -m sphinx docs/ docs/_build/ -b linkcheck -W diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml new file mode 100644 index 0000000..1f5613e --- /dev/null +++ b/.github/workflows/linter.yml @@ -0,0 +1,29 @@ +name: Linter + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install pre-commit hooks + run: | + pip install pre-commit + pre-commit install --install-hooks + + - name: Code style check via pre-commit + run: | + pre-commit run --all-files diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..72b490e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,86 @@ +name: Publish + +on: + push: + tags: + - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + deploy: + runs-on: ubuntu-latest + environment: release + permissions: + contents: write + id-token: write + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build virtualenv + + # PyPI package + - name: Build Python package + run: python -m build + + - name: Publish Python package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + # Remove apt repos that are known to break from time to time + # See https://github.com/actions/virtual-environments/issues/323 + - name: Remove broken apt repos + run: | + for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done + + # Docuemntation + - name: Install doc dependencies + run: | + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + pip install -r requirements.txt + pip install -r docs/requirements.txt + + - name: Build documentation + run: | + python -m sphinx docs/ docs/_build/ -b html + + - name: Deploy documentation to Github pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs/_build + + # Github release + - name: Read CHANGELOG + id: changelog + run: | + # Get bullet points from last CHANGELOG entry + CHANGELOG=$(git diff -U0 HEAD^ HEAD | grep '^[+][\* ]' | sed 's/\+//') + echo "Got changelog: $CHANGELOG" + # Support for multiline, see + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo 'body<> "$GITHUB_OUTPUT" + + - name: Create release on Github + id: create_release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + name: Release ${{ github.ref_name }} + body: ${{ steps.changelog.outputs.body }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..cd4e981 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,62 @@ +name: Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ '3.10' ] + include: + - os: ubuntu-latest + python-version: '3.8' + tasks: tests + - os: ubuntu-latest + python-version: '3.9' + tasks: tests + + steps: + - uses: actions/checkout@v3 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb + key: emodb-1.4.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Ubuntu - install libsndfile + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + if: matrix.os == 'ubuntu-latest' + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + # TESTS + - name: Install tests requirements + run: pip install -r tests/requirements.txt + + - name: Test with pytest + run: python -m pytest + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./coverage.xml + if: matrix.os == 'ubuntu-latest' diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 8e87a94..5d7d9b5 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,32 +1,40 @@ Contributing ============ -If you would like to add new functionality fell free to create a `merge -request`_ . If you find errors, omissions, inconsistencies or other things -that need improvement, please create an issue_. -Contributions are always welcome! +Everyone is invited to contribute to this project. +Feel free to create a `pull request`_ . +If you find errors, +omissions, +inconsistencies, +or other things +that need improvement, +please create an issue_. -.. _issue: - https://gitlab.audeering.com/tools/pyauglib/issues/new?issue%5BD= -.. _merge request: - https://gitlab.audeering.com/tools/pyauglib/merge_requests/new +.. _issue: https://github.com/audeering/auglib/issues/new/ +.. _pull request: https://github.com/audeering/auglib/compare/ Development Installation ------------------------ -Instead of pip-installing the latest release from PyPI, you should get the -newest development version from Gitlab_:: +Instead of pip-installing the latest release from PyPI_, +you should get the newest development version from Github_:: - git clone git@srv-app-01.audeering.local:tools/pyauglib.git - cd pyauglib - # Use virtual environment - pip install -r requirements.txt + git clone https://github.com/audeering/auglib/ + cd auglib + # Create virtual environment for this project + # e.g. + # virtualenv --python="python3" $HOME/.envs/auglib + # source $HOME/.envs/auglib/bin/activate + pip install -r requirements.txt -.. _Gitlab: https://gitlab.audeering.com/tools/pyauglib -This way, your installation always stays up-to-date, even if you pull new -changes from the Gitlab repository. +This way, +your installation always stays up-to-date, +even if you pull new changes from the Github repository. + +.. _PyPI: https://pypi.org/project/auglib/ +.. _Github: https://github.com/audeering/auglib/ Coding Convention @@ -76,20 +84,20 @@ If you make changes to the documentation, you can re-create the HTML pages using Sphinx_. You can install it and a few other necessary packages with:: - pip install -r docs/requirements.txt + pip install -r docs/requirements.txt To create the HTML pages, use:: - python -m sphinx docs/ build/sphinx/html -b html + python -m sphinx docs/ build/sphinx/html -b html The generated files will be available in the directory :file:`build/sphinx/html/`. It is also possible to automatically check if all links are still valid:: - python -m sphinx docs/ build/sphinx/html -b linkcheck + python -m sphinx docs/ build/sphinx/html -b linkcheck -.. _Sphinx: https://www.sphinx-doc.org +.. _Sphinx: http://sphinx-doc.org Running the Tests @@ -98,84 +106,13 @@ Running the Tests You'll need pytest_ for that. It can be installed with:: - pip install -r tests/requirements.txt + pip install -r tests/requirements.txt To execute the tests, simply run:: - pytest tests/ - -.. _pytest: https://pytest.org/ - - -Updating the C++ Library ------------------------- - -``auglib`` depends on the -`C++ auglib library`_, -which is included in ``auglib`` -with the help of a C-wrapper. - -To use a new version of the -`C++ auglib library`_, -you need to update the C-wrapper. -Install needed requirements by:: - - pip install -r cwrapper/requirements.txt - sudo apt-get install --yes cmake libtool automake patchelf - -Clone and install AMR codec:: - - cd cwrapper - git clone --depth 1 --branch master https://gitlab.audeering.com/tools/opencore-amr.git - cd opencore-amr - autoreconf --install - autoconf - ./configure - make - sudo make install - cd .. + python -m pytest -Setup Conan_ to find packages on Artifactory_ -by following the `Setup Conan`_ instructions. - -Clone and build a static version of the `C++ auglib library`_ -in release mode:: - - git clone --depth 1 --branch main https://gitlab.audeering.com/tools/auglib - cd auglib - bash build-soundtouch.sh - mkdir build - cd build - conan install .. - cmake .. -DCMAKE_BUILD_TYPE=Release -DOPENCOREAMR_SUPPORT=ON -DOPENCOREAMRNB_INCLUDE_PATH=/usr/local/include/opencore-amrnb - cd ../.. - -And finally build the C-wrapper:: - - export AUGLIB=./auglib - bash build.sh - -.. _C++ auglib library: https://gitlab.audeering.com/tools/auglib -.. _Conan: https://conan.io -.. _Artifactory: https://artifactory.audeering.com/ui/repos/tree/General/conan-local/ -.. _Setup Conan: https://gitlab.audeering.com/devops/conan/meta/-/blob/master/conan-setup.md - - -Including a New C++ Function ----------------------------- - -If you added a new function in the -`C++ auglib library`_ -and would like to include it in ``auglib`` -as well, -you need to include it in ``cwrapper/cauglib.cpp`` -and follow the steps -explained in the previous section -on updating the C++ library. - -Afterwards, -add the functions signature to ``auglib/core/api.py`` -as well. +.. _pytest: https://pytest.org Creating a New Release @@ -185,9 +122,5 @@ New releases are made using the following steps: #. Update ``CHANGELOG.rst`` #. Commit those changes as "Release X.Y.Z" -#. Create an (annotated) tag with ``git tag -a vX.Y.Z`` -#. Make sure you have an `artifactory-tokenizer`_ project -#. Push the commit and the tag to Gitlab - -.. _artifactory-tokenizer: - https://gitlab.audeering.com/devops/artifactory/tree/master/token +#. Create an (annotated) tag with ``git tag -a X.Y.Z`` +#. Push the commit and the tag to Github diff --git a/docs/audb.yaml b/docs/audb.yaml deleted file mode 100644 index 2938aa5..0000000 --- a/docs/audb.yaml +++ /dev/null @@ -1,8 +0,0 @@ -cache_root: .cache/audb -repositories: - - name: data-public-local - backend: artifactory - host: https://artifactory.audeering.com/artifactory - - name: data-public - backend: artifactory - host: https://audeering.jfrog.io/artifactory diff --git a/tests/test_interface.py b/tests/test_interface.py index c175a44..96cbc04 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -317,7 +317,7 @@ ( # Time values might change due to pandas.to_timedelta() # see - # https://gitlab.audeering.com/tools/pyauglib/-/merge_requests/206 + # https://github.com/pandas-dev/pandas/issues/56629 audformat.segmented_index( ["f1.wav"], [0], From 2adc31b6a2574ece8bbf5b5cd28a2c74303a1616 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 10:11:40 +0100 Subject: [PATCH 02/13] Try to fix libavcodec-extra dependency --- .github/workflows/doc.yml | 2 +- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 1a88d51..4b15f6b 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -32,7 +32,7 @@ jobs: - name: Ubuntu - install libsndfile run: | sudo apt-get update - sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra - name: Install package run: | diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 72b490e..0c5e10d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -46,7 +46,7 @@ jobs: # Docuemntation - name: Install doc dependencies run: | - sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra pip install -r requirements.txt pip install -r docs/requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd4e981..e241a37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - name: Ubuntu - install libsndfile run: | sudo apt-get update - sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra59 + sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra if: matrix.os == 'ubuntu-latest' - name: Install package From fd557fea28cd16befa98369aea61e947974ff13f Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 10:38:14 +0100 Subject: [PATCH 03/13] Remove testdata dataset --- .github/workflows/test.yml | 2 +- auglib/core/interface.py | 25 ++++++++++++------------- tests/conftest.py | 12 ++++++++---- tests/requirements.txt | 1 + tests/test_interface.py | 3 ++- 5 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e241a37..3a7f094 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: python-version: [ '3.10' ] include: - os: ubuntu-latest - python-version: '3.8' + python-version: '3.11' tasks: tests - os: ubuntu-latest python-version: '3.9' diff --git a/auglib/core/interface.py b/auglib/core/interface.py index bf12674..5fdab5a 100644 --- a/auglib/core/interface.py +++ b/auglib/core/interface.py @@ -117,29 +117,28 @@ class Augment(audinterface.Process, audobject.Object): >>> import audiofile >>> import auglib >>> db = audb.load( - ... "testdata", - ... version="1.5.0", - ... full_path=False, + ... "emodb", + ... version="1.4.1", + ... media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], ... verbose=False, ... ) >>> transform = auglib.transform.WhiteNoiseUniform() >>> augment = auglib.Augment(transform) >>> # Augment a numpy array - >>> file = os.path.join(db.root, db.files[0]) - >>> signal, sampling_rate = audiofile.read(file) + >>> signal, sampling_rate = audiofile.read(db.files[0]) >>> signal_augmented = augment(signal, sampling_rate) >>> # Augment (parts of) a database - >>> column = db["emotion.test.gold"]["emotion"].get() - >>> augmented_column = augment.augment( - ... column, + >>> df = db.get("emotion") + >>> df_augmented = augment.augment( + ... df, ... cache_root="cache", - ... data_root=db.root, + ... remove_root=db.root, ... ) - >>> label = augmented_column.iloc[0] - >>> file = augmented_column.index[0][0] + >>> label = df_augmented.iloc[0, 0] + >>> file = df_augmented.index[0][0] >>> file = file.replace(audeer.path("."), ".") # remove absolute path - >>> file, label # doctest: +SKIP - ('./cache/decdec83/-6100627981361312836/0/audio/006.wav', 'unhappy') + >>> file, label + ('./cache/db5496bf/-5167105955204987526/0/wav/03a01Fa.wav', 'happiness') """ # noqa: E501 diff --git a/tests/conftest.py b/tests/conftest.py index 30d63ca..bdd313a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,9 +20,13 @@ def _call(self, base: np.ndarray, *, sampling_rate: int = None): ) ] -db = audb.load("testdata", version="1.5.0") + +db = audb.load( + "emodb", + version="1.4.1", + media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], + verbose=False, +) pytest.AUDB_ROOT = audb.default_cache_root() -pytest.DATA_COLUMN = db["happiness.dev.gold"]["happiness"].get() -pytest.DATA_FILES = db.files[:5] -pytest.DATA_TABLE = db["happiness.dev.gold"].get() +pytest.DATA_FILES = db.files pytest.TRANSFORM_ONES = Ones() diff --git a/tests/requirements.txt b/tests/requirements.txt index c386cd7..ddcc1ea 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ audb +audformat >=1.1.0 audmath >=1.1.0 # Allow the usage of `strict` in `np.testing.assert_array_equal()` numpy >=1.24.0 diff --git a/tests/test_interface.py b/tests/test_interface.py index 96cbc04..bda7c46 100644 --- a/tests/test_interface.py +++ b/tests/test_interface.py @@ -838,7 +838,8 @@ def test_augment_num_workers(tmpdir, transform): None, pytest.AUDB_ROOT, pytest.param( - "/invalid/directory", marks=pytest.mark.xfail(raises=RuntimeError) + "/invalid/directory", + marks=pytest.mark.xfail(raises=RuntimeError), ), pytest.param( pytest.AUDB_ROOT[: len(pytest.AUDB_ROOT) - 1], From a061b88d7e0255f3dfa36ef66ddceb8d112ed3e8 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 11:10:41 +0100 Subject: [PATCH 04/13] Fix data_root --- auglib/core/interface.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auglib/core/interface.py b/auglib/core/interface.py index 5fdab5a..be65a5c 100644 --- a/auglib/core/interface.py +++ b/auglib/core/interface.py @@ -120,18 +120,21 @@ class Augment(audinterface.Process, audobject.Object): ... "emodb", ... version="1.4.1", ... media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], + ... full_path=False, ... verbose=False, ... ) >>> transform = auglib.transform.WhiteNoiseUniform() >>> augment = auglib.Augment(transform) >>> # Augment a numpy array - >>> signal, sampling_rate = audiofile.read(db.files[0]) + >>> file = os.path.join(db.root, db.files[0]) + >>> signal, sampling_rate = audiofile.read(file) >>> signal_augmented = augment(signal, sampling_rate) >>> # Augment (parts of) a database >>> df = db.get("emotion") >>> df_augmented = augment.augment( ... df, ... cache_root="cache", + ... data_root=db.root, ... remove_root=db.root, ... ) >>> label = df_augmented.iloc[0, 0] From 55fdd25668eccc974c883dbe820ffcc9cac3cc8c Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 11:42:07 +0100 Subject: [PATCH 05/13] Use ellipses in docstring output --- auglib/core/interface.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/auglib/core/interface.py b/auglib/core/interface.py index be65a5c..8b85636 100644 --- a/auglib/core/interface.py +++ b/auglib/core/interface.py @@ -120,28 +120,24 @@ class Augment(audinterface.Process, audobject.Object): ... "emodb", ... version="1.4.1", ... media=["wav/03a01Fa.wav", "wav/03a01Nc.wav", "wav/03a01Wa.wav"], - ... full_path=False, ... verbose=False, ... ) >>> transform = auglib.transform.WhiteNoiseUniform() >>> augment = auglib.Augment(transform) >>> # Augment a numpy array - >>> file = os.path.join(db.root, db.files[0]) - >>> signal, sampling_rate = audiofile.read(file) + >>> signal, sampling_rate = audiofile.read(db.files[0]) >>> signal_augmented = augment(signal, sampling_rate) >>> # Augment (parts of) a database >>> df = db.get("emotion") >>> df_augmented = augment.augment( ... df, ... cache_root="cache", - ... data_root=db.root, ... remove_root=db.root, ... ) >>> label = df_augmented.iloc[0, 0] >>> file = df_augmented.index[0][0] - >>> file = file.replace(audeer.path("."), ".") # remove absolute path >>> file, label - ('./cache/db5496bf/-5167105955204987526/0/wav/03a01Fa.wav', 'happiness') + ('.../0/wav/03a01Fa.wav', 'happiness') """ # noqa: E501 From 0fab66b4c56c54754a1646d8e8a9e864f5398382 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 11:48:28 +0100 Subject: [PATCH 06/13] Mention support for Python 3.9 and 3.11 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dbbcb33..9ebca2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,9 @@ classifiers = [ 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.9', 'Topic :: Scientific/Engineering', ] dependencies = [ From da3171fbd776d46aabc48b48625cf050ac29524b Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 11:57:17 +0100 Subject: [PATCH 07/13] Fix naming for caches and add for publish --- .github/workflows/doc.yml | 2 +- .github/workflows/publish.yml | 6 ++++++ .github/workflows/test.yml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 4b15f6b..9aefc40 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache emodb + - name: Cache audb uses: actions/cache@v3 with: path: ~/audb diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c5e10d..1f6dbe6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,6 +20,12 @@ jobs: with: fetch-depth: 2 + - name: Cache audb + uses: actions/cache@v3 + with: + path: ~/audb + key: emodb-1.4.1 + - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3a7f094..c688dc3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,7 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache emodb + - name: Cache audb uses: actions/cache@v3 with: path: ~/audb From d728b342afa2186f87da35e5ba77463b774b6247 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 11:59:05 +0100 Subject: [PATCH 08/13] Fix indention in CONTRIBUTING --- CONTRIBUTING.rst | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5d7d9b5..94b2cc4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -20,13 +20,13 @@ Development Installation Instead of pip-installing the latest release from PyPI_, you should get the newest development version from Github_:: - git clone https://github.com/audeering/auglib/ - cd auglib - # Create virtual environment for this project - # e.g. - # virtualenv --python="python3" $HOME/.envs/auglib - # source $HOME/.envs/auglib/bin/activate - pip install -r requirements.txt + git clone https://github.com/audeering/auglib/ + cd auglib + # Create virtual environment for this project + # e.g. + # virtualenv --python="python3" $HOME/.envs/auglib + # source $HOME/.envs/auglib/bin/activate + pip install -r requirements.txt This way, @@ -84,18 +84,18 @@ If you make changes to the documentation, you can re-create the HTML pages using Sphinx_. You can install it and a few other necessary packages with:: - pip install -r docs/requirements.txt + pip install -r docs/requirements.txt To create the HTML pages, use:: - python -m sphinx docs/ build/sphinx/html -b html + python -m sphinx docs/ build/sphinx/html -b html The generated files will be available in the directory :file:`build/sphinx/html/`. It is also possible to automatically check if all links are still valid:: - python -m sphinx docs/ build/sphinx/html -b linkcheck + python -m sphinx docs/ build/sphinx/html -b linkcheck .. _Sphinx: http://sphinx-doc.org @@ -106,11 +106,11 @@ Running the Tests You'll need pytest_ for that. It can be installed with:: - pip install -r tests/requirements.txt + pip install -r tests/requirements.txt To execute the tests, simply run:: - python -m pytest + python -m pytest .. _pytest: https://pytest.org From 2e172ac7d782a5f7d3941469bd0f3d6f0cc434d0 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 12:00:34 +0100 Subject: [PATCH 09/13] Update sphinx URL in CONTRIBUTING --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 94b2cc4..ea33673 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -97,7 +97,7 @@ It is also possible to automatically check if all links are still valid:: python -m sphinx docs/ build/sphinx/html -b linkcheck -.. _Sphinx: http://sphinx-doc.org +.. _Sphinx: https://www.sphinx-doc.org Running the Tests From 2682d44e54c9878b1b3ff8321c64a580ab0554c2 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 12:01:22 +0100 Subject: [PATCH 10/13] Remove empty line in conftest --- tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index bdd313a..a1e1854 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,6 @@ def _call(self, base: np.ndarray, *, sampling_rate: int = None): ) ] - db = audb.load( "emodb", version="1.4.1", From f510eb09ee0f74b4a61cb377f904984d0d726abd Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 12:14:42 +0100 Subject: [PATCH 11/13] Clean up action comments --- .github/workflows/doc.yml | 1 - .github/workflows/publish.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 9aefc40..1ddba25 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -39,7 +39,6 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - # DOCS - name: Install docs requirements run: pip install -r docs/requirements.txt diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1f6dbe6..0de3557 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -49,7 +49,7 @@ jobs: run: | for apt_file in `grep -lr microsoft /etc/apt/sources.list.d/`; do sudo rm $apt_file; done - # Docuemntation + # Documentation - name: Install doc dependencies run: | sudo apt-get install --no-install-recommends --yes libsndfile1 ffmpeg sox libavcodec-extra From 0e2c1165a3177ec00ca2f62be64546a26e500637 Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 12:24:01 +0100 Subject: [PATCH 12/13] Use dedicated dataset caches --- .github/workflows/doc.yml | 28 ++++++++++++++++++++++++++-- .github/workflows/publish.yml | 28 ++++++++++++++++++++++++++-- .github/workflows/test.yml | 4 ++-- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 1ddba25..7a1af9c 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -18,12 +18,36 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache audb + - name: Cache air uses: actions/cache@v3 with: - path: ~/audb + path: ~/audb/air/1.4.2 + key: air-1.4.2 + + - name: Cache cough-speech-sneeze + uses: actions/cache@v3 + with: + path: ~/audb/cough-speech-sneeze/2.0.1 + key: cough-speech-sneeze-2.0.1 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb/emodb/1.4.1 key: emodb-1.4.1 + - name: Cache micirp + uses: actions/cache@v3 + with: + path: ~/audb/micirp/1.0.0 + key: micirp-1.0.0 + + - name: Cache musan + uses: actions/cache@v3 + with: + path: ~/audb/musan/1.0.0 + key: musan-1.0.0 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0de3557..944f620 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -20,12 +20,36 @@ jobs: with: fetch-depth: 2 - - name: Cache audb + - name: Cache air uses: actions/cache@v3 with: - path: ~/audb + path: ~/audb/air/1.4.2 + key: air-1.4.2 + + - name: Cache cough-speech-sneeze + uses: actions/cache@v3 + with: + path: ~/audb/cough-speech-sneeze/2.0.1 + key: cough-speech-sneeze-2.0.1 + + - name: Cache emodb + uses: actions/cache@v3 + with: + path: ~/audb/emodb/1.4.1 key: emodb-1.4.1 + - name: Cache micirp + uses: actions/cache@v3 + with: + path: ~/audb/micirp/1.0.0 + key: micirp-1.0.0 + + - name: Cache musan + uses: actions/cache@v3 + with: + path: ~/audb/musan/1.0.0 + key: musan-1.0.0 + - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c688dc3..7f6cf72 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,10 +25,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Cache audb + - name: Cache emodb uses: actions/cache@v3 with: - path: ~/audb + path: ~/audb/emodb/1.4.1 key: emodb-1.4.1 - name: Set up Python ${{ matrix.python-version }} From 85d1841dee939bc2f660cbf07fee5b37dfb9c5fb Mon Sep 17 00:00:00 2001 From: Hagen Wierstorf Date: Wed, 17 Jan 2024 12:44:27 +0100 Subject: [PATCH 13/13] Remove Gitlab CI file --- .gitlab-ci.yml | 93 -------------------------------------------------- 1 file changed, 93 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index f8a3e68..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,93 +0,0 @@ -stages: - - prepare - - test - - doc - - pypi - -variables: - ARTIFACTORY_GROUPS: "readers,deployers" - ARTIFACTORY_TOKEN_VARS: "ARTIFACTORY_API_KEY,TWINE_PASSWORD" - ARTIFACTORY_USERNAME_VARS: "ARTIFACTORY_USERNAME,TWINE_USERNAME" - -include: - - project: 'devops/artifactory-token-helper' - file: '/.gitlab-ci-artifactory-token.yml' - -.pytest: - image: docker.audeering.com/audeering/python-3.10:latest - before_script: - - apt-get install -y ffmpeg sox libavcodec-extra59 - - python -V - script: - - pip install -r requirements.txt - - pip install -r tests/requirements.txt - - python -m pytest - -python-3.10: - extends: .pytest - stage: test - -linter: - stage: test - image: docker.audeering.com/audeering/python-3.10:latest - before_script: - - pip install pre-commit - - pre-commit install --install-hooks - script: - - pre-commit run --all-files - cache: - key: pre-commit - paths: - - .cache/pre-commit - -docs: - extends: .pytest - stage: test - script: - - pip install -r requirements.txt - - pip install -r docs/requirements.txt - # Create Artifactory config script to manage user access - - echo -e "[artifactory.audeering.com/artifactory]\nusername = ${ARTIFACTORY_USERNAME}\npassword = ${ARTIFACTORY_API_KEY}\n" >~/.artifactory_python.cfg - # Allow anonymous access - - unset ARTIFACTORY_USERNAME - - unset ARTIFACTORY_API_KEY - # Copy audb config file to correct location - - cp docs/audb.yaml ~/.audb.yaml - - python -m sphinx -W docs/ build/html -b html - # Disable linkcheck until new README links are public - # - python -m sphinx -W docs/ build/html -b linkcheck - artifacts: - paths: - - build/ - cache: - key: audb - paths: - - .cache/audb - -pages: - stage: doc - image: docker.audeering.com/alpine - variables: - GIT_STRATEGY: none # disable git fetch - script: - - cp -r build/html public - dependencies: - - docs - artifacts: - paths: - - public - only: - refs: - - tags - -pypi: - extends: .pytest - stage: pypi - before_script: - - pip install build twine virtualenv - script: - - python -m build - - python -m twine upload --repository local dist/* - only: - refs: - - tags