diff --git a/.github/workflows/GithubActionTests.yml b/.github/workflows/GithubActionTests.yml index 7208e0cf57..c4577ebddc 100644 --- a/.github/workflows/GithubActionTests.yml +++ b/.github/workflows/GithubActionTests.yml @@ -29,15 +29,16 @@ jobs: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh bash install-and-set-up-conda.sh eval "$(conda shell.bash hook)" - mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt + conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt conda activate bioconda + conda list python setup.py install - name: Build docker container run: | docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./ docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest - docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list' + docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info --verbose && conda list' docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./ - name: Run tests '${{ matrix.py_test_marker }}' @@ -66,7 +67,8 @@ jobs: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh bash install-and-set-up-conda.sh eval "$(conda shell.bash hook)" - mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt + conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt + conda list conda activate bioconda python setup.py install @@ -94,8 +96,9 @@ jobs: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh bash install-and-set-up-conda.sh eval "$(conda shell.bash hook)" - mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt + conda create -n bioconda --quiet --yes --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt conda activate bioconda + conda list python setup.py install - name: Test autobump diff --git a/Dockerfile b/Dockerfile index 2ac4fe4177..e1247b19e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,14 +51,10 @@ RUN . /opt/conda/etc/profile.d/conda.sh && conda activate base && \ sed -nE \ '/^conda([>=3.7 -conda=23.3.* -boa=0.15.* -conda-build=3.24.* -conda-verify=3.1.* -argh=0.26.* # CLI -colorlog=4.8.* # Logging -tqdm>=4.26 # Progress monitor -yaspin =2 -ruamel_yaml=0.15.* # Recipe YAML parsing -pyaml=17.12.* # Faster YAML parser (deprecate?) -networkx=2.* -pandas=1.4.* -numpy=1.19.* # Avoid breaking pandas on OSX -libblas=*=*openblas # Avoid large mkl package (pulled in by pandas) -boltons=23.* -jsonschema=3.2.* # JSON schema verification -pyopenssl>=22.1 # Stay compatible with cryptography - # pinnings conda-forge-pinning=2023.05.06.13.08.41 -# tools -anaconda-client=1.12.* # anaconda_upload -involucro=1.1.* # mulled test and container build -skopeo=1.11.* # docker upload -git=2.* # well - git +# basics +python=3.10.* +conda=24.3.* +conda-libmamba-solver=24.1.* +conda-build=24.3.* +conda-index=0.4.* +mamba=1.5.* +boa=0.17.* + +argh=0.31.* # CLI +colorlog=6.* # Logging +tqdm=4.66.* # Progress monitor +yaspin=2.* # +ruamel.yaml=0.18.* # Recipe YAML parsing +pyaml=23.12.* # Faster YAML parser (deprecate?) +networkx=3.3.* # (networkx>3.3 needs python>=3.10) +pandas=2.2.* # +libblas=*=*openblas # Avoid large mkl package (pulled in by pandas) +boltons=24.0.* # +jsonschema=4.21.* # JSON schema verification +jinja2=3.1.* # + +anaconda-client=1.12.* # anaconda_upload +galaxy-tool-util=24.* # mulled test and container build +involucro=1.1.* # mulled test and container build +skopeo=1.15.* # docker upload +git=2.* # well - git # hosters - special regex not supported by RE -regex=2022.7.9 +regex=2023.* # # asyncio -aiohttp=3.8.* # HTTP lib -aiohttp-jinja2 # jinja2 renderer for aiohttp.web -aiohttp-session -aiohttp-security -aiofiles=0.8.* # async open -aioftp=0.12.* # FTP lib -backoff=2.2.* # retrying of failed API calls -cachetools=3.0.* # request caching (NEEDED?) +aiohttp=3.9.* # HTTP lib +aiohttp-jinja2 # jinja2 renderer for aiohttp.web +aiohttp-session # +aiohttp-security # +aiofiles=23.* # async open +aioftp=0.22.* # FTP lib +backoff=2.2.* # retrying of failed API calls +cachetools=5.3.* # request caching (NEEDED?) # client API wrappers -gitpython>=3.0.8,3.0.* # githandler - # needs >=3.0.8 due to https://github.com/conda-forge/staged-recipes/issues/10874 -gidgethub=3.0.* # githubhandler -pyjwt>=2.4.0 # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217 +gitpython=3.1.* # githandler +gidgethub=5.* # githubhandler +pyjwt>=2.4.0 # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217 + +# bioconductor-skeleton +beautifulsoup4=4.* # -# unknown -beautifulsoup4=4.12.* -galaxy-lib>=18.9.1 -jinja2>=2.10.1,<3 -markupsafe<2.1 # markupsafe 2.1 breaks jinja2 +# The bioconductor skeleton needs this +requests=2.31.* # + +# merge handling +pygithub # + +# caching +diskcache=5.* # +platformdirs=4.* # +# build failure output +tabulate=0.9.* # + +# TODO: Remove these from general requirements into a doc-only-requirements file. # docs sphinx>=4.1 celery @@ -64,16 +76,3 @@ alabaster=0.7.* docutils markdown graphviz - -# The bioconductor skeleton needs this -requests=2.29.* - -# merge handling -pygithub - -# caching -diskcache =5.* -appdirs =1.* - -# build failure output -tabulate =0.9 \ No newline at end of file diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index f1d3aecfbf..4c9ef5872d 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -17,7 +17,7 @@ from conda_build.exceptions import DependencyNeedsBuildingError import networkx as nx import pandas -from ruamel_yaml import YAML +from ruamel.yaml import YAML from . import utils from . import docker_utils diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 24bca16bff..7a5cfada6b 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -56,7 +56,7 @@ def decorator(func): @arg('--logfile', help="Write log to file") @arg('--logfile-level', help="Log level for log file") @arg('--log-command-max-lines', help="Limit lines emitted for commands executed") - @utils.wraps(func) + @utils.wraps(func, hide_wrapped=True) def wrapper(*args, loglevel=default_loglevel, logfile=None, logfile_level=default_file_loglevel, log_command_max_lines=None, **kwargs): @@ -125,7 +125,7 @@ def decorator(func): for field in allow_missing_for or []] except ValueError: sys.exit(f"Function {func} must have 'recipe_folder' and 'config' args") - @arg('recipe_folder', nargs='?', + @arg('recipe-folder', nargs='?', help='Path to folder containing recipes (default: recipes/)') @arg('config', nargs='?', help='Path to Bioconda config (default: config.yml)') @@ -763,7 +763,7 @@ def dependent(recipe_folder, config, restrict=False, is specified, then all packages in a given bioconductor release will be created/updated (--force is then implied).''') @recipe_folder_and_config() -@arg('bioc_data_packages', nargs='?', +@arg('bioc-data-packages', nargs='?', help='''Path to folder containing the recipe for the bioconductor-data-packages (default: recipes/bioconductor-data-packages)''') @arg('--versioned', action='store_true', help='''If specified, recipe will be @@ -854,8 +854,6 @@ def clean_cran_skeleton(recipe, no_windows=False): cran_skeleton.clean_skeleton_files(recipe, no_windows=no_windows) -@arg('recipe_folder', help='Path to recipes directory') -@arg('config', help='Path to yaml file specifying the configuration') @recipe_folder_and_config() @arg('--packages', nargs="+", help='Glob(s) for package[s] to scan. Can be specified more than once') diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index 52aae0879e..482eedb128 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -94,10 +94,13 @@ # will exist in the container but will be empty. Channels expect at least # a linux-64/linux-aarch64 and noarch directory within that directory, so we # make sure it exists before adding the channel. -mkdir -p {self.container_staging}/linux-64 -mkdir -p {self.container_staging}/linux-aarch64 -mkdir -p {self.container_staging}/noarch -touch {self.container_staging}/noarch/repodata.json +# Also ensure conda-build's local channel directory exists the same way. +for local_channel in '/opt/conda/conda-bld' '{self.container_staging}'; do + mkdir -p "${{local_channel}}"/linux-64 + mkdir -p "${{local_channel}}"/linux-aarch64 + mkdir -p "${{local_channel}}"/noarch + conda index "${{local_channel}}" +done conda config --add channels file://{self.container_staging} 2> >( grep -vF "Warning: 'file://{self.container_staging}' already in 'channels' list, moving to the top" >&2 ) diff --git a/bioconda_utils/lint/__init__.py b/bioconda_utils/lint/__init__.py index 82be133ea3..bce452aaca 100644 --- a/bioconda_utils/lint/__init__.py +++ b/bioconda_utils/lint/__init__.py @@ -105,7 +105,6 @@ import pandas as pd from bioconda_utils.skiplist import Skiplist -import ruamel_yaml as yaml import networkx as nx from .. import utils diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py index efeb802e67..3520ebf8f1 100644 --- a/bioconda_utils/pkg_test.py +++ b/bioconda_utils/pkg_test.py @@ -12,8 +12,8 @@ from . import utils -import conda_build.api from conda_build.metadata import MetaData +from conda_index.index import update_index logger = logging.getLogger(__name__) @@ -130,7 +130,7 @@ def test_package( conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path))) - conda_build.api.update_index([conda_bld_dir]) + update_index(conda_bld_dir) spec = get_image_name(path) diff --git a/bioconda_utils/recipe.py b/bioconda_utils/recipe.py index 2eae36f2c7..fbfebdac0a 100644 --- a/bioconda_utils/recipe.py +++ b/bioconda_utils/recipe.py @@ -28,14 +28,9 @@ import jinja2 -try: - from ruamel.yaml import YAML - from ruamel.yaml.constructor import DuplicateKeyError - from ruamel.yaml.error import YAMLError -except ModuleNotFoundError: - from ruamel_yaml import YAML - from ruamel_yaml.constructor import DuplicateKeyError - from ruamel_yaml.error import YAMLError +from ruamel.yaml import YAML +from ruamel.yaml.constructor import DuplicateKeyError +from ruamel.yaml.error import YAMLError from . import utils from .aiopipe import EndProcessingItem diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index cc6be6ec09..c591034d38 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -34,7 +34,7 @@ from yaspin import yaspin, Spinner from yaspin.spinners import Spinners from urllib3 import Retry -import appdirs +import platformdirs import diskcache from github import Github @@ -66,7 +66,7 @@ logger = logging.getLogger(__name__) -disk_cache = diskcache.Cache(appdirs.user_cache_dir("bioconda-utils")) +disk_cache = diskcache.Cache(platformdirs.user_cache_dir("bioconda-utils")) class TqdmHandler(logging.StreamHandler): @@ -120,7 +120,7 @@ def ensure_list(obj): return [obj] -def wraps(func): +def wraps(func, hide_wrapped=False): """Custom wraps() function for decorators This one differs from functiools.wraps and boltons.funcutils.wraps in @@ -148,7 +148,10 @@ def wrapper_wrapper(wrapper_func): fb.body = 'return _call(%s)' % fb.get_invocation_str() execdict = dict(_call=wrapper_func, _func=func) fully_wrapped = fb.get_func(execdict) - fully_wrapped.__wrapped__ = func + if not hide_wrapped: + fully_wrapped.__wrapped__ = func + elif hasattr(fully_wrapped, '__wrapped__'): + del fully_wrapped.__dict__['__wrapped__'] return fully_wrapped return wrapper_wrapper diff --git a/test/conftest.py b/test/conftest.py index b2318667bc..03dafe519c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,7 @@ import tempfile from copy import deepcopy -from ruamel_yaml import YAML +from ruamel.yaml import YAML import pandas as pd import pytest import py @@ -65,22 +65,25 @@ def mock_repodata(repodata, case): else: data = repodata - dataframe = pd.DataFrame(columns=utils.RepoData.columns) - for channel, packages in data.items(): - for name, versions in packages.items(): - for item in versions: - pkg = { - 'channel': channel, - 'name': name, - 'build': '', - 'build_number': 0, - 'version': 0, - 'depends': [], - 'subdir': '', - 'platform': 'noarch', - } - pkg.update(item) - dataframe = dataframe.append(pkg, ignore_index=True) + dataframe = pd.DataFrame( + ( + { + 'channel': channel, + 'name': name, + 'build': '', + 'build_number': 0, + 'version': 0, + 'depends': [], + 'subdir': '', + 'platform': 'noarch', + **item, + } + for channel, packages in data.items() + for name, versions in packages.items() + for item in versions + ), + columns=utils.RepoData.columns, + ) backup = utils.RepoData()._df, utils.RepoData()._df_ts utils.RepoData()._df = dataframe diff --git a/test/helpers.py b/test/helpers.py index 35a78b07ba..dc630d616a 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -4,7 +4,7 @@ import os import subprocess as sp -import conda_build.api +from conda_index.index import update_index def ensure_missing(package): @@ -25,7 +25,7 @@ def ensure_missing(package): if os.path.exists(package): os.unlink(package) assert not os.path.exists(package) - conda_build.api.update_index([os.path.dirname(os.path.dirname(package))]) + update_index(os.path.dirname(os.path.dirname(package))) class Recipes(object): diff --git a/test/test_lint.py b/test/test_lint.py index 48b459c6fa..ad37c44d49 100644 --- a/test/test_lint.py +++ b/test/test_lint.py @@ -1,5 +1,5 @@ import os.path as op -from ruamel_yaml import YAML +from ruamel.yaml import YAML import pytest diff --git a/test/test_recipe.py b/test/test_recipe.py index e3822ce7a9..ff6f26602a 100644 --- a/test/test_recipe.py +++ b/test/test_recipe.py @@ -3,7 +3,7 @@ import pytest -from ruamel_yaml import YAML +from ruamel.yaml import YAML yaml = YAML(typ="rt") # pylint: disable=invalid-name from bioconda_utils.recipe import (