Merge pull request #2075 from jaimergp/micromamba

Add micromamba as a possible install tool and make it default on macOS and Windows

.github/workflows/_proxy-file-for-dependabot-tests.yml

@@ -11,5 +11,4 @@ jobs:
     if: false
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
-      - uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0

conda_smithy/configure_feedstock.py

@@ -2337,17 +2337,24 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
         config["noarch_platforms"]
     )
 
-    # NOTE: Currently assuming these dependencies are name-only (no version constraints)
+    # NOTE: We are not shell escaping, so no ><! symbols in the version constraints, just =
+    python_version = "3.12"  # make it match latest Miniforge, if possible
     if config["conda_build_tool"] == "mambabuild":
-        config["conda_build_tool_deps"] = "conda-build boa"
+        config["conda_build_tool_deps"] = (
+            f"python={python_version} conda-build boa"
+        )
     elif config["conda_build_tool"] == "conda-build+conda-libmamba-solver":
-        config["conda_build_tool_deps"] = "conda-build conda-libmamba-solver"
+        config["conda_build_tool_deps"] = (
+            f"python={python_version} conda-build conda-libmamba-solver"
+        )
     elif config["conda_build_tool"] == "rattler-build":
         config["conda_build_tool_deps"] = "rattler-build"
     else:
-        config["conda_build_tool_deps"] = "conda-build"
+        config["conda_build_tool_deps"] = (
+            f"python={python_version} conda-build"
+        )
 
-    # NOTE: Currently assuming these dependencies are name-only (no version constraints)
+    # NOTE: We are not shell escaping, so no ><! symbols in the version constraints, just =
     if config["conda_install_tool"] == "mamba":
         config["conda_install_tool_deps"] = "mamba"
     elif config["conda_install_tool"] in "conda":

conda_smithy/data/conda-forge.json

@@ -1767,16 +1767,17 @@
         {
           "enum": [
             "conda",
-            "mamba"
+            "mamba",
+            "micromamba"
           ],
           "type": "string"
         },
         {
           "type": "null"
         }
       ],
-      "default": "mamba",
-      "description": "Use this option to choose which tool is used to provision the tooling in your\nfeedstock.",
+      "default": "micromamba",
+      "description": "Use this option to choose which tool is used to provision the tooling in your\nfeedstock. Defaults to micromamba.\n\nIf conda or mamba are chosen, the latest Miniforge will be used to\nprovision the base environment. If micromamba is chosen, Miniforge\nis not involved; the environment is created directly by micromamba.\n\nNote: micromamba is only used on macOS and Windows for now.\nOn Linux, mamba (as provided in the Docker images) will still be used\neven if micromamba is chosen.",
       "title": "Conda Install Tool"
     },
     "conda_forge_output_validation": {

conda_smithy/data/conda-forge.yml

@@ -72,7 +72,7 @@ conda_build:
   zstd_compression_level: 16
 conda_build_tool: conda-build
 conda_forge_output_validation: false
-conda_install_tool: mamba
+conda_install_tool: micromamba
 conda_solver: libmamba
 config_version: '2'
 docker:

conda_smithy/schema.py

@@ -626,14 +626,24 @@ class ConfigModel(BaseModel):
         ),
     )
 
-    conda_install_tool: Optional[Literal["conda", "mamba"]] = Field(
-        default="mamba",
-        description=cleandoc(
-            """
-        Use this option to choose which tool is used to provision the tooling in your
-        feedstock.
-        """
-        ),
+    conda_install_tool: Optional[Literal["conda", "mamba", "micromamba"]] = (
+        Field(
+            default="micromamba",
+            description=cleandoc(
+                """
+                Use this option to choose which tool is used to provision the tooling in your
+                feedstock. Defaults to micromamba.
+
+                If conda or mamba are chosen, the latest Miniforge will be used to
+                provision the base environment. If micromamba is chosen, Miniforge
+                is not involved; the environment is created directly by micromamba.
+
+                Note: micromamba is only used on macOS and Windows for now.
+                On Linux, mamba (as provided in the Docker images) will still be used
+                even if micromamba is chosen.
+                """
+            ),
+        )
     )
 
     conda_forge_output_validation: Optional[bool] = Field(

conda_smithy/templates/azure-pipelines-win.yml.tmpl

@@ -16,27 +16,11 @@ jobs:
       displayName: "Install Chocolatey Package: {{ choco_pkg }}"
 {% endfor %}
 
-    - task: PythonScript@0
-      displayName: 'Download Miniforge'
-      inputs:
-        scriptSource: inline
-        script: |
-          import urllib.request
-          url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe'
-          path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe"
-          urllib.request.urlretrieve(url, path)
-
-    - script: |
-        start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=$(MINIFORGE_HOME)
-      displayName: Install Miniforge
-
-    - powershell: Write-Host "##vso[task.prependpath]$(MINIFORGE_HOME)\Scripts"
-      displayName: Add conda to PATH
-
     - script: |
         call ".scripts\run_win_build.bat"
       displayName: Run Windows build
       env:
+        MINIFORGE_HOME: $(MINIFORGE_HOME)
         PYTHONUNBUFFERED: 1
         CONFIG: $(CONFIG)
         CI: azure
@@ -54,6 +38,7 @@ jobs:
 
 {%- if azure.store_build_artifacts %}
     - script: |
+        set MINIFORGE_HOME=$(MINIFORGE_HOME)
         set CI=azure
         set CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt)
         set FEEDSTOCK_NAME=$(build.Repository.Name)

conda_smithy/templates/build_steps.sh.tmpl

@@ -47,6 +47,9 @@ CONDARC
 {%- if conda_solver == "libmamba" %}
 export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1
 {%- endif %}
+{%- if conda_install_tool == "micromamba" %}
+{%- set conda_install_tool = "mamba" %}
+{%- endif %}
 
 {{ conda_install_tool }} install --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \
     pip {{ conda_install_tool_deps }} {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }}

conda_smithy/templates/github-actions.yml.tmpl

@@ -164,12 +164,6 @@ jobs:
         choco install {{ choco_pkg }} -fdv -y --debug
 {% endfor %}
 
-    - name: Install Miniconda for windows
-      uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3.0.4
-      with:
-        miniforge-version: latest
-      if: matrix.os == 'windows'
-
     - name: Build on windows
       shell: cmd
       run: |
@@ -178,6 +172,7 @@ jobs:
         set "sha=%GITHUB_SHA%"
         call ".scripts\run_win_build.bat"
       env:
+        MINIFORGE_HOME: D:\Miniforge
         PYTHONUNBUFFERED: 1
         CONFIG: {% raw %}${{ matrix.CONFIG }}{% endraw %}
         CI: github_actions
@@ -199,6 +194,7 @@ jobs:
         CONFIG: {% raw %}${{ matrix.CONFIG }}{% endraw %}
         SHORT_CONFIG: {% raw %}${{ matrix.SHORT_CONFIG }}{% endraw %}
         OS: {% raw %}${{ matrix.os }}{% endraw %}
+        MINIFORGE_HOME_WIN: D:\Miniforge
       run: |
         export CI=github_actions
         export CI_RUN_ID=$GITHUB_RUN_ID
@@ -207,7 +203,7 @@ jobs:
         if [ $OS == "macos" ]; then
           export CONDA_BLD_DIR="${MINIFORGE_HOME:-${HOME}/miniforge3}/conda-bld"
         elif [ $OS == "windows" ]; then
-          export CONDA_BLD_DIR="${CONDA//\\//}/conda-bld"
+          export CONDA_BLD_DIR="${MINIFORGE_HOME_WIN//\\//}/conda-bld"
         else
           export CONDA_BLD_DIR="build_artifacts"
         fi

conda_smithy/templates/run_osx_build.sh.tmpl

@@ -7,16 +7,42 @@ source .scripts/logging_utils.sh
 set -xe
 
 MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3}
+MINIFORGE_HOME=${MINIFORGE_HOME%/} # remove trailing slash
+{%- if conda_install_tool == "micromamba" %}
 
+( startgroup "Provisioning base env with micromamba" ) 2> /dev/null
+MICROMAMBA_VERSION="1.5.10-0"
+if [[ "$(uname -m)" == "arm64" ]]; then
+  osx_arch="osx-arm64"
+else
+  osx_arch="osx-64"
+fi
+MICROMAMBA_URL="https://github.com/mamba-org/micromamba-releases/releases/download/${MICROMAMBA_VERSION}/micromamba-${osx_arch}"
+MAMBA_ROOT_PREFIX="${MINIFORGE_HOME}-micromamba-$(date +%s)"
+echo "Downloading micromamba ${MICROMAMBA_VERSION}"
+micromamba_exe="$(mktemp -d)/micromamba"
+curl -L -o "${micromamba_exe}" "${MICROMAMBA_URL}"
+chmod +x "${micromamba_exe}"
+echo "Creating environment"
+"${micromamba_exe}" create --yes --root-prefix "${MAMBA_ROOT_PREFIX}" --prefix "${MINIFORGE_HOME}" \
+  --channel conda-forge \
+  pip {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }}
+echo "Moving pkgs cache from ${MAMBA_ROOT_PREFIX} to ${MINIFORGE_HOME}"
+mv "${MAMBA_ROOT_PREFIX}/pkgs" "${MINIFORGE_HOME}"
+echo "Cleaning up micromamba"
+rm -rf "${MAMBA_ROOT_PREFIX}" "${micromamba_exe}" || true
+( endgroup "Provisioning base env with micromamba" ) 2> /dev/null
+{%- else %}
 ( startgroup "Installing a fresh version of Miniforge" ) 2> /dev/null
 
 MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download"
 MINIFORGE_FILE="Miniforge3-MacOSX-$(uname -m).sh"
 curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}"
-rm -rf ${MINIFORGE_HOME}
-bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME}
+rm -rf "${MINIFORGE_HOME}"
+bash "${MINIFORGE_FILE}" -b -p "${MINIFORGE_HOME}"
 
 ( endgroup "Installing a fresh version of Miniforge" ) 2> /dev/null
+{%- endif %}
 
 ( startgroup "Configuring conda" ) 2> /dev/null
 {%- if conda_build_tool == "mambabuild" %}
@@ -31,7 +57,7 @@ bash $MINIFORGE_FILE -b -p ${MINIFORGE_HOME}
 {%- set BUILD_CMD="conda-build" %}
 {%- endif %}
 
-source ${MINIFORGE_HOME}/etc/profile.d/conda.sh
+source "${MINIFORGE_HOME}/etc/profile.d/conda.sh"
 conda activate base
 {%- if conda_solver %}
 export CONDA_SOLVER="{{ conda_solver }}"
@@ -40,11 +66,9 @@ export CONDA_SOLVER="{{ conda_solver }}"
 export CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1
 {%- endif %}
 
+{% if conda_install_tool in ("conda", "mamba") -%}
 {{ conda_install_tool }} install --update-specs --quiet --yes --channel conda-forge --strict-channel-priority \
     pip {{ conda_install_tool_deps }} {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }}
-{%- if conda_install_tool == "mamba" and (conda_build_tool_deps != "" or conda_install_tool_deps != "") %}
-{{ conda_install_tool }} update --update-specs --yes --quiet --channel conda-forge --strict-channel-priority \
-    pip {{ conda_install_tool_deps }} {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup_update) }}
 {%- endif %}
 
 {% if local_ci_setup %}

conda_smithy/templates/run_win_build.bat.tmpl

@@ -3,20 +3,60 @@
 :: changes to this script, consider a proposal to conda-smithy so that other feedstocks can also
 :: benefit from the improvement.
 
-:: Note: we assume a Miniforge installation is available
-
 :: INPUTS (required environment variables)
 :: CONFIG: name of the .ci_support/*.yaml file for this job
 :: CI: azure, github_actions, or unset
+:: MINIFORGE_HOME: where to install the base conda environment
 :: UPLOAD_PACKAGES: true or false
 :: UPLOAD_ON_BRANCH: true or false
 
 setlocal enableextensions enabledelayedexpansion
 
+if "%MINIFORGE_HOME%"=="" set "MINIFORGE_HOME=%USERPROFILE%\Miniforge3"
+:: Remove trailing backslash, if present
+if "%MINIFORGE_HOME:~-1%"=="\" set "MINIFORGE_HOME=%MINIFORGE_HOME:~0,-1%"
+
+{%- if conda_install_tool == "micromamba" %}
+call :start_group "Provisioning base env with micromamba"
+set "MAMBA_ROOT_PREFIX=%MINIFORGE_HOME%-micromamba-%RANDOM%"
+set "MICROMAMBA_VERSION=1.5.10-0"
+set "MICROMAMBA_URL=https://github.com/mamba-org/micromamba-releases/releases/download/%MICROMAMBA_VERSION%/micromamba-win-64"
+set "MICROMAMBA_TMPDIR=%TMP%\micromamba-%RANDOM%"
+set "MICROMAMBA_EXE=%MICROMAMBA_TMPDIR%\micromamba.exe"
+
+echo Downloading micromamba %MICROMAMBA_VERSION%
+if not exist "%MICROMAMBA_TMPDIR%" mkdir "%MICROMAMBA_TMPDIR%"
+certutil -urlcache -split -f "%MICROMAMBA_URL%" "%MICROMAMBA_EXE%"
+if !errorlevel! neq 0 exit /b !errorlevel!
+
+echo Creating environment
+call "%MICROMAMBA_EXE%" create --yes --root-prefix "%MAMBA_ROOT_PREFIX%" --prefix "%MINIFORGE_HOME%" ^
+    --channel conda-forge ^
+    pip {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }}
+if !errorlevel! neq 0 exit /b !errorlevel!
+echo Moving pkgs cache from %MAMBA_ROOT_PREFIX% to %MINIFORGE_HOME%
+move /Y "%MAMBA_ROOT_PREFIX%\pkgs" "%MINIFORGE_HOME%"
+if !errorlevel! neq 0 exit /b !errorlevel!
+echo Removing %MAMBA_ROOT_PREFIX%
+del /S /Q "%MAMBA_ROOT_PREFIX%"
+del /S /Q "%MICROMAMBA_TMPDIR%"
+call :end_group
+{%- else %}
+call :start_group "Installing a fresh version of Miniforge"
+set "MINIFORGE_URL=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe"
+set "MINIFORGE_EXE=Miniforge3-Windows-x86_64.exe"
+echo Downloading Miniforge
+certutil -urlcache -split -f "%MINIFORGE_URL%" "%MINIFORGE_EXE%"
+if !errorlevel! neq 0 exit /b !errorlevel!
+echo Installing Miniforge
+start /wait "" %MINIFORGE_EXE% /InstallationType=JustMe /RegisterPython=0 /S /D=%MINIFORGE_HOME%
+call :end_group
+{%- endif %}
+
 call :start_group "Configuring conda"
 
 :: Activate the base conda environment
-call activate base
+call "%MINIFORGE_HOME%\Scripts\activate.bat"
 
 {%- if conda_solver %}
 :: Configure the solver
@@ -27,10 +67,12 @@ if !errorlevel! neq 0 exit /b !errorlevel!
 set "CONDA_LIBMAMBA_SOLVER_NO_CHANNELS_FROM_INSTALLED=1"
 {%- endif %}
 
+{%- if conda_install_tool in ("conda", "mamba") %}
 :: Provision the necessary dependencies to build the recipe later
 echo Installing dependencies
 {{ conda_install_tool }}.exe install pip {{ conda_install_tool_deps }} {{ conda_build_tool_deps }} {{ " ".join(remote_ci_setup) }} -c conda-forge --strict-channel-priority --yes
 if !errorlevel! neq 0 exit /b !errorlevel!
+{%- endif %}
 
 {%- if local_ci_setup %}
 echo Overriding conda-forge-ci-setup with local version

news/gh2075-micromamba-install-tool.rst

@@ -0,0 +1,23 @@
+**Added:**
+
+* Added ``micromamba`` as the default install tool on macOS and Windows. (#2075)
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>