feat: Update dependencies to latest versions (#972)

Update dependencies to their latest versions as of 2024-04-15 (apart from `conda-forge-pinning` and `python`) and adjust code accordingly. --------- feat: Update dependencies to latest versions Remove outdated workaround in container creation Avoid removed DataFrame.append in test fixture Use update_index from conda-index Use python=3.10 for now We can switch to >=3.11 later, e.g., when we build for >=3.11 generally and/or the conda-forge/linux-anvil-* base containers use python>3.10. Use upstream ruamel.yaml package ruamel_yaml was only meant as a temporary workaround and we don't update it anymore. Run conda index instead of making invalid repodata feat: Update dependencies to latest versions chore: CI: Use conda for test env creation chore: CI: Run conda create quietly; list packages chore: CI: Run conda info --verbose conda info --all got deprecated in favor of --verbose. fix: Workaround conda-mambabuild needs local channel prepopulated if referenced via "-c local" for newer conda/conda-build versions (not sure why, TBH). --------- Signed-off-by: Marcel Bargull <[email protected]>
bioconda · Apr 15, 2024 · dc6ca07 · dc6ca07
1 parent 18aa598
commit dc6ca07
Show file tree

Hide file tree

Showing 14 changed files with 112 additions and 113 deletions.
diff --git a/.github/workflows/GithubActionTests.yml b/.github/workflows/GithubActionTests.yml
@@ -29,15 +29,16 @@ jobs:
           wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
           bash install-and-set-up-conda.sh
           eval "$(conda shell.bash hook)"
-          mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
           conda activate bioconda
+          conda list
           python setup.py install
 
       - name: Build docker container
         run: |
           docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./
           docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest
-          docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list'
+          docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info --verbose && conda list'
           docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./
 
       - name: Run tests '${{ matrix.py_test_marker }}'
@@ -66,7 +67,8 @@ jobs:
           wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
           bash install-and-set-up-conda.sh
           eval "$(conda shell.bash hook)"
-          mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda list
           conda activate bioconda
           python setup.py install
 
@@ -94,8 +96,9 @@ jobs:
           wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
           bash install-and-set-up-conda.sh
           eval "$(conda shell.bash hook)"
-          mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
           conda activate bioconda
+          conda list
           python setup.py install
 
       - name: Test autobump

diff --git a/Dockerfile b/Dockerfile
@@ -51,14 +51,10 @@ RUN . /opt/conda/etc/profile.d/conda.sh  && conda activate base && \
     sed -nE \
     '/^conda([><!=~ ].+)?$/p' \
     /opt/bioconda-utils/bioconda_utils-requirements.txt \
-    | xargs -r mamba install --yes && \
-    # FIXME: "remove truststore" only necessary due to python downgrade.
-    #        Updating requirements should fix that.
-    #        (Also this removal will break in future.)
-    mamba remove --yes truststore && \
-    mamba install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
+    | xargs -r conda install --yes && \
+    conda install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
     pip install --no-deps --find-links /opt/bioconda-utils bioconda_utils && \
-    mamba clean --yes --index --tarballs && \
+    conda clean --yes --index --tarballs && \
     # Find files that are not already in group "lucky" and change their group and mode.
     find /opt/conda \
     \! -group lucky \

diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt
@@ -2,60 +2,72 @@
 # in ../docs/source/user/versions.rst
 # so please keep that documentation up to date as they change
 
-# basics
-python>=3.7
-conda=23.3.*
-boa=0.15.*
-conda-build=3.24.*
-conda-verify=3.1.*
-argh=0.26.*          # CLI
-colorlog=4.8.*       # Logging
-tqdm>=4.26           # Progress monitor
-yaspin =2
-ruamel_yaml=0.15.*   # Recipe YAML parsing
-pyaml=17.12.*        # Faster YAML parser (deprecate?)
-networkx=2.*
-pandas=1.4.*
-numpy=1.19.*         # Avoid breaking pandas on OSX
-libblas=*=*openblas  # Avoid large mkl package (pulled in by pandas)
-boltons=23.*
-jsonschema=3.2.*     # JSON schema verification
-pyopenssl>=22.1      # Stay compatible with cryptography
-
 # pinnings
 conda-forge-pinning=2023.05.06.13.08.41
 
-# tools
-anaconda-client=1.12.*  # anaconda_upload
-involucro=1.1.*        # mulled test and container build
-skopeo=1.11.*          # docker upload
-git=2.*                # well - git
+# basics
+python=3.10.*
+conda=24.3.*
+conda-libmamba-solver=24.1.*
+conda-build=24.3.*
+conda-index=0.4.*
+mamba=1.5.*
+boa=0.17.*
+
+argh=0.31.*                   # CLI
+colorlog=6.*                  # Logging
+tqdm=4.66.*                   # Progress monitor
+yaspin=2.*                    #
+ruamel.yaml=0.18.*            # Recipe YAML parsing
+pyaml=23.12.*                 # Faster YAML parser (deprecate?)
+networkx=3.3.*                # (networkx>3.3 needs python>=3.10)
+pandas=2.2.*                  #
+libblas=*=*openblas           # Avoid large mkl package (pulled in by pandas)
+boltons=24.0.*                #
+jsonschema=4.21.*             # JSON schema verification
+jinja2=3.1.*                  #
+
+anaconda-client=1.12.*        # anaconda_upload
+galaxy-tool-util=24.*         # mulled test and container build
+involucro=1.1.*               # mulled test and container build
+skopeo=1.15.*                 # docker upload
+git=2.*                       # well - git
 
 # hosters - special regex not supported by RE
-regex=2022.7.9
+regex=2023.*                  #
 
 # asyncio
-aiohttp=3.8.*      # HTTP lib
-aiohttp-jinja2     # jinja2 renderer for aiohttp.web
-aiohttp-session
-aiohttp-security
-aiofiles=0.8.*     # async open
-aioftp=0.12.*      # FTP lib
-backoff=2.2.*      # retrying of failed API calls
-cachetools=3.0.*   # request caching (NEEDED?)
+aiohttp=3.9.*                 # HTTP lib
+aiohttp-jinja2                # jinja2 renderer for aiohttp.web
+aiohttp-session               #
+aiohttp-security              #
+aiofiles=23.*                 # async open
+aioftp=0.22.*                 # FTP lib
+backoff=2.2.*                 # retrying of failed API calls
+cachetools=5.3.*              # request caching (NEEDED?)
 
 # client API wrappers
-gitpython>=3.0.8,3.0.*    # githandler
-                          # needs >=3.0.8 due to https://github.com/conda-forge/staged-recipes/issues/10874
-gidgethub=3.0.*           # githubhandler
-pyjwt>=2.4.0              # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217
+gitpython=3.1.*               # githandler
+gidgethub=5.*                 # githubhandler
+pyjwt>=2.4.0                  # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217
+
+# bioconductor-skeleton
+beautifulsoup4=4.*            #
 
-# unknown
-beautifulsoup4=4.12.*
-galaxy-lib>=18.9.1
-jinja2>=2.10.1,<3
-markupsafe<2.1           # markupsafe 2.1 breaks jinja2
+# The bioconductor skeleton needs this
+requests=2.31.*               #
+
+# merge handling
+pygithub                      #
+
+# caching
+diskcache=5.*                 #
+platformdirs=4.*              #
 
+# build failure output
+tabulate=0.9.*                #
+
+# TODO: Remove these from general requirements into a doc-only-requirements file.
 # docs
 sphinx>=4.1
 celery
@@ -64,16 +76,3 @@ alabaster=0.7.*
 docutils
 markdown
 graphviz
-
-# The bioconductor skeleton needs this
-requests=2.29.*
-
-# merge handling
-pygithub
-
-# caching
-diskcache =5.*
-appdirs =1.*
-
-# build failure output
-tabulate =0.9
diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py
@@ -17,7 +17,7 @@
 from conda_build.exceptions import DependencyNeedsBuildingError
 import networkx as nx
 import pandas
-from ruamel_yaml import YAML
+from ruamel.yaml import YAML
 
 from . import utils
 from . import docker_utils

diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py
@@ -56,7 +56,7 @@ def decorator(func):
         @arg('--logfile', help="Write log to file")
         @arg('--logfile-level', help="Log level for log file")
         @arg('--log-command-max-lines', help="Limit lines emitted for commands executed")
-        @utils.wraps(func)
+        @utils.wraps(func, hide_wrapped=True)
         def wrapper(*args, loglevel=default_loglevel, logfile=None,
                     logfile_level=default_file_loglevel,
                     log_command_max_lines=None, **kwargs):
@@ -125,7 +125,7 @@ def decorator(func):
                                  for field in allow_missing_for or []]
         except ValueError:
             sys.exit(f"Function {func} must have 'recipe_folder' and 'config' args")
-        @arg('recipe_folder', nargs='?',
+        @arg('recipe-folder', nargs='?',
              help='Path to folder containing recipes (default: recipes/)')
         @arg('config', nargs='?',
              help='Path to Bioconda config (default: config.yml)')
@@ -763,7 +763,7 @@ def dependent(recipe_folder, config, restrict=False,
      is specified, then all packages in a given bioconductor release will be
      created/updated (--force is then implied).''')
 @recipe_folder_and_config()
-@arg('bioc_data_packages', nargs='?',
+@arg('bioc-data-packages', nargs='?',
      help='''Path to folder containing the recipe for the bioconductor-data-packages
      (default: recipes/bioconductor-data-packages)''')
 @arg('--versioned', action='store_true', help='''If specified, recipe will be
@@ -854,8 +854,6 @@ def clean_cran_skeleton(recipe, no_windows=False):
     cran_skeleton.clean_skeleton_files(recipe, no_windows=no_windows)
 
 
-@arg('recipe_folder', help='Path to recipes directory')
-@arg('config', help='Path to yaml file specifying the configuration')
 @recipe_folder_and_config()
 @arg('--packages', nargs="+",
      help='Glob(s) for package[s] to scan. Can be specified more than once')

diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py
@@ -94,10 +94,13 @@
 # will exist in the container but will be empty.  Channels expect at least
 # a linux-64/linux-aarch64 and noarch directory within that directory, so we
 # make sure it exists before adding the channel.
-mkdir -p {self.container_staging}/linux-64
-mkdir -p {self.container_staging}/linux-aarch64
-mkdir -p {self.container_staging}/noarch
-touch {self.container_staging}/noarch/repodata.json
+# Also ensure conda-build's local channel directory exists the same way.
+for local_channel in '/opt/conda/conda-bld' '{self.container_staging}'; do
+  mkdir -p "${{local_channel}}"/linux-64
+  mkdir -p "${{local_channel}}"/linux-aarch64
+  mkdir -p "${{local_channel}}"/noarch
+  conda index "${{local_channel}}"
+done
 conda config --add channels file://{self.container_staging} 2> >(
     grep -vF "Warning: 'file://{self.container_staging}' already in 'channels' list, moving to the top" >&2
 )

diff --git a/bioconda_utils/lint/__init__.py b/bioconda_utils/lint/__init__.py
@@ -105,7 +105,6 @@
 
 import pandas as pd
 from bioconda_utils.skiplist import Skiplist
-import ruamel_yaml as yaml
 import networkx as nx
 
 from .. import utils

diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py
@@ -12,8 +12,8 @@
 
 from . import utils
 
-import conda_build.api
 from conda_build.metadata import MetaData
+from conda_index.index import update_index
 
 logger = logging.getLogger(__name__)
 
@@ -130,7 +130,7 @@ def test_package(
 
     conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path)))
 
-    conda_build.api.update_index([conda_bld_dir])
+    update_index(conda_bld_dir)
 
     spec = get_image_name(path)
 

diff --git a/bioconda_utils/recipe.py b/bioconda_utils/recipe.py
@@ -28,14 +28,9 @@
 
 import jinja2
 
-try:
-    from ruamel.yaml import YAML
-    from ruamel.yaml.constructor import DuplicateKeyError
-    from ruamel.yaml.error import YAMLError
-except ModuleNotFoundError:
-    from ruamel_yaml import YAML
-    from ruamel_yaml.constructor import DuplicateKeyError
-    from ruamel_yaml.error import YAMLError
+from ruamel.yaml import YAML
+from ruamel.yaml.constructor import DuplicateKeyError
+from ruamel.yaml.error import YAMLError
 
 from . import utils
 from .aiopipe import EndProcessingItem

diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py
@@ -34,7 +34,7 @@
 from yaspin import yaspin, Spinner
 from yaspin.spinners import Spinners
 from urllib3 import Retry
-import appdirs
+import platformdirs
 import diskcache
 
 from github import Github
@@ -66,7 +66,7 @@
 
 logger = logging.getLogger(__name__)
 
-disk_cache = diskcache.Cache(appdirs.user_cache_dir("bioconda-utils"))
+disk_cache = diskcache.Cache(platformdirs.user_cache_dir("bioconda-utils"))
 
 
 class TqdmHandler(logging.StreamHandler):
@@ -120,7 +120,7 @@ def ensure_list(obj):
     return [obj]
 
 
-def wraps(func):
+def wraps(func, hide_wrapped=False):
     """Custom wraps() function for decorators
 
     This one differs from functiools.wraps and boltons.funcutils.wraps in
@@ -148,7 +148,10 @@ def wrapper_wrapper(wrapper_func):
         fb.body = 'return _call(%s)' % fb.get_invocation_str()
         execdict = dict(_call=wrapper_func, _func=func)
         fully_wrapped = fb.get_func(execdict)
-        fully_wrapped.__wrapped__ = func
+        if not hide_wrapped:
+            fully_wrapped.__wrapped__ = func
+        elif hasattr(fully_wrapped, '__wrapped__'):
+            del fully_wrapped.__dict__['__wrapped__']
         return fully_wrapped
 
     return wrapper_wrapper

diff --git a/test/conftest.py b/test/conftest.py
@@ -6,7 +6,7 @@
 import tempfile
 from copy import deepcopy
 
-from ruamel_yaml import YAML
+from ruamel.yaml import YAML
 import pandas as pd
 import pytest
 import py
@@ -65,22 +65,25 @@ def mock_repodata(repodata, case):
     else:
         data = repodata
 
-    dataframe = pd.DataFrame(columns=utils.RepoData.columns)
-    for channel, packages in data.items():
-        for name, versions in packages.items():
-            for item in versions:
-                pkg = {
-                    'channel': channel,
-                    'name': name,
-                    'build': '',
-                    'build_number': 0,
-                    'version': 0,
-                    'depends': [],
-                    'subdir': '',
-                    'platform': 'noarch',
-                }
-                pkg.update(item)
-                dataframe = dataframe.append(pkg, ignore_index=True)
+    dataframe = pd.DataFrame(
+        (
+            {
+                'channel': channel,
+                'name': name,
+                'build': '',
+                'build_number': 0,
+                'version': 0,
+                'depends': [],
+                'subdir': '',
+                'platform': 'noarch',
+                **item,
+            }
+            for channel, packages in data.items()
+            for name, versions in packages.items()
+            for item in versions
+        ),
+        columns=utils.RepoData.columns,
+    )
 
     backup = utils.RepoData()._df, utils.RepoData()._df_ts
     utils.RepoData()._df = dataframe