Merge pull request #1577 from dbast/main

Burn in flow_run_id, remote_url, sha into package meta data
conda-forge · Nov 6, 2023 · df653f1 · df653f1
2 parents 5d113a9 + 593ffa1
commit df653f1
Show file tree

Hide file tree

Showing 14 changed files with 82 additions and 4 deletions.
diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl
@@ -66,7 +66,10 @@ install:
 build: off
 
 test_script:
-    - cmd: conda.exe build {{ recipe_dir }} -m .ci_support\%CONFIG%.yaml
+    - set "flow_run_id=appveyor_%APPVEYOR_BUILD_ID%"
+    - set "remote_url=https://github.com/%APPVEYOR_REPO_NAME%"
+    - set "sha=%APPVEYOR_REPO_COMMIT%"
+    - cmd: conda.exe build {{ recipe_dir }} -m .ci_support\%CONFIG%.yaml --extra-meta flow_run_id=%flow_run_id% remote_url=%remote_url% sha=%sha%
 {%- if conda_forge_output_validation %}
     - set "FEEDSTOCK_NAME=%APPVEYOR_REPO_NAME:*/=%"
     - cmd: validate_recipe_outputs "%FEEDSTOCK_NAME%"

diff --git a/conda_smithy/templates/azure-pipelines-linux.yml.tmpl b/conda_smithy/templates/azure-pipelines-linux.yml.tmpl
@@ -58,6 +58,9 @@ jobs:
 
   - script: |
         export CI=azure
+        export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt)
+        export remote_url=$(Build.Repository.Uri)
+        export sha=$(Build.SourceVersion)
         export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME
         export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME})
 {%- if upload_on_branch %}

diff --git a/conda_smithy/templates/azure-pipelines-osx.yml.tmpl b/conda_smithy/templates/azure-pipelines-osx.yml.tmpl
@@ -13,6 +13,9 @@ jobs:
 {%- endif %}
   - script: |
       export CI=azure
+      export flow_run_id=azure_$(Build.BuildNumber).$(System.JobAttempt)
+      export remote_url=$(Build.Repository.Uri)
+      export sha=$(Build.SourceVersion)
       export OSX_FORCE_SDK_DOWNLOAD="1"
       export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME
       export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME})

diff --git a/conda_smithy/templates/azure-pipelines-win.yml.tmpl b/conda_smithy/templates/azure-pipelines-win.yml.tmpl
@@ -40,6 +40,9 @@ jobs:
         PYTHONUNBUFFERED: 1
         CONFIG: $(CONFIG)
         CI: azure
+        flow_run_id: azure_$(Build.BuildNumber).$(System.JobAttempt)
+        remote_url: $(Build.Repository.Uri)
+        sha: $(Build.SourceVersion)
         UPLOAD_PACKAGES: $(UPLOAD_PACKAGES)
         UPLOAD_TEMP: $(UPLOAD_TEMP)
 {%- if upload_on_branch %}

diff --git a/conda_smithy/templates/build_steps.sh.tmpl b/conda_smithy/templates/build_steps.sh.tmpl
@@ -83,6 +83,12 @@ if [[ -f "${FEEDSTOCK_ROOT}/LICENSE.txt" ]]; then
   cp "${FEEDSTOCK_ROOT}/LICENSE.txt" "${RECIPE_ROOT}/recipe-scripts-license.txt"
 fi
 
+if [[ "${sha:-}" == "" ]]; then
+  pushd ${FEEDSTOCK_ROOT}
+  sha=$(git rev-parse HEAD)
+  popd
+fi
+
 if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then
     if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then
         EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}"
@@ -96,7 +102,8 @@ if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then
 else
     {{ BUILD_CMD }} "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \
         --suppress-variables ${EXTRA_CB_OPTIONS:-} \
-        --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml"
+        --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \
+        --extra-meta flow_run_id="${flow_run_id:-}" remote_url="${remote_url:-}" sha="${sha:-}"
 
     {%- if conda_forge_output_validation %}
     ( startgroup "Validating outputs" ) 2> /dev/null

diff --git a/conda_smithy/templates/circle.yml.tmpl b/conda_smithy/templates/circle.yml.tmpl
@@ -47,6 +47,9 @@ jobs:
 {%- if data.platform.startswith('linux') %}
           command: |
             export CI=circle
+            export flow_run_id="circle_$CIRCLE_WORKFLOW_ID"
+            export remote_url="$CIRCLE_REPOSITORY_URL"
+            export sha="$CIRCLE_SHA1"
             export FEEDSTOCK_NAME=$(basename ${CIRCLE_PROJECT_REPONAME})
             if [[ "${CIRCLE_PR_NUMBER:-}" == "" ]]; then
               export IS_PR_BUILD="False"
@@ -57,6 +60,9 @@ jobs:
 {%- else %}
           command: |
             export CI=circle
+            export flow_run_id="circle_$CIRCLE_WORKFLOW_ID"
+            export remote_url="$CIRCLE_REPOSITORY_URL"
+            export sha="$CIRCLE_SHA1"
             export FEEDSTOCK_NAME=$(basename ${CIRCLE_PROJECT_REPONAME})
             if [[ "${CIRCLE_PR_NUMBER:-}" == "" ]]; then
               export IS_PR_BUILD="False"

diff --git a/conda_smithy/templates/drone.yml.tmpl b/conda_smithy/templates/drone.yml.tmpl
@@ -35,6 +35,9 @@ steps:
     - export RECIPE_ROOT="$FEEDSTOCK_ROOT/{{ recipe_dir }}"
     - export CI=drone
     - export GIT_BRANCH="$DRONE_BRANCH"
+    - export flow_run_id="drone_$DRONE_BUILD_NUMBER"
+    - export remote_url="$DRONE_REPO_LINK"
+    - export sha="$DRONE_COMMIT_SHA"
     - export FEEDSTOCK_NAME=$(basename ${DRONE_REPO_NAME})
 {%- if upload_on_branch %}
     - export UPLOAD_ON_BRANCH="{{ upload_on_branch }}"

diff --git a/conda_smithy/templates/github-actions.tmpl b/conda_smithy/templates/github-actions.tmpl
@@ -125,6 +125,9 @@ jobs:
       run: |
         echo "::group::Configure binfmt_misc"
         docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
+        export flow_run_id="github_$GITHUB_RUN_ID"
+        export remote_url="https://github.com/$GITHUB_REPOSITORY"
+        export sha="$GITHUB_SHA"
         export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
         export GIT_BRANCH="$(basename $GITHUB_REF)"
         if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
@@ -149,6 +152,9 @@ jobs:
 {%- endfor %}
       shell: bash
       run: |
+        export flow_run_id="github_$GITHUB_RUN_ID"
+        export remote_url="https://github.com/$GITHUB_REPOSITORY"
+        export sha="$GITHUB_SHA"
         export FEEDSTOCK_NAME="$(basename $GITHUB_REPOSITORY)"
         export GIT_BRANCH="$(basename $GITHUB_REF)"
         if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
@@ -175,6 +181,9 @@ jobs:
     - name: Build on windows
       shell: cmd
       run: |
+        set "flow_run_id=github_%GITHUB_RUN_ID%"
+        set "remote_url=https://github.com/%GITHUB_REPOSITORY%"
+        set "sha=%GITHUB_SHA%"
         call ".scripts\run_win_build.bat"
       env:
         PYTHONUNBUFFERED: 1

diff --git a/conda_smithy/templates/run_docker_build.sh.tmpl b/conda_smithy/templates/run_docker_build.sh.tmpl
@@ -91,6 +91,9 @@ export IS_PR_BUILD="${IS_PR_BUILD:-False}"
            -e CPU_COUNT \
            -e BUILD_WITH_CONDA_DEBUG \
            -e BUILD_OUTPUT_ID \
+           -e flow_run_id \
+           -e remote_url \
+           -e sha \
 {%- for secret in secrets %}
            -e {{ secret }} \
 {%- endfor %}

diff --git a/conda_smithy/templates/run_osx_build.sh.tmpl b/conda_smithy/templates/run_osx_build.sh.tmpl
@@ -65,6 +65,10 @@ else
   echo -e "\n\nNot mangling homebrew as we are not running in CI"
 fi
 
+if [[ "${sha:-}" == "" ]]; then
+  sha=$(git rev-parse HEAD)
+fi
+
 {% if build_setup -%}
 echo -e "\n\nRunning the build setup script."
 {{ build_setup }}
@@ -97,7 +101,8 @@ else
 {% endif %}
     {{ BUILD_CMD }} ./{{ recipe_dir }} -m ./.ci_support/${CONFIG}.yaml \
         --suppress-variables ${EXTRA_CB_OPTIONS:-} \
-        --clobber-file ./.ci_support/clobber_${CONFIG}.yaml
+        --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \
+        --extra-meta flow_run_id="$flow_run_id" remote_url="$remote_url" sha="$sha"
 
 
     {%- if conda_forge_output_validation %}

diff --git a/conda_smithy/templates/run_win_build.bat.tmpl b/conda_smithy/templates/run_win_build.bat.tmpl
@@ -62,6 +62,10 @@ if NOT [%HOST_PLATFORM%] == [%BUILD_PLATFORM%] (
 )
 {%- endif %}
 
+if NOT [%flow_run_id%] == [] (
+    set "EXTRA_CB_OPTIONS=%EXTRA_CB_OPTIONS% --extra-meta flow_run_id=%flow_run_id% remote_url=%remote_url% sha=%sha%"
+)
+
 call :end_group
 
 :: Build the recipe

diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl
@@ -51,6 +51,9 @@ matrix:
 script:
   - export CI=travis
   - export GIT_BRANCH="$TRAVIS_BRANCH"
+  - export flow_run_id="travis_$TRAVIS_JOB_ID"
+  - export sha="$TRAVIS_COMMIT"
+  - export remote_url="https://github.com/$TRAVIS_REPO_SLUG"
   - export FEEDSTOCK_NAME=$(basename ${TRAVIS_REPO_SLUG})
 {%- if upload_on_branch %}
   - export UPLOAD_ON_BRANCH="{{ upload_on_branch }}"

diff --git a/environment.yml b/environment.yml
@@ -16,7 +16,7 @@ dependencies:
   - pytest-cov
   # Runtime dependencies, duplicated in conda_smithy.recipe/meta.yaml
   - conda >=4.2
-  - conda-build >=3.18.3
+  - conda-build >=3.21.8
   - conda-package-handling >=1.9.0
   - jinja2
   - requests

diff --git a/news/burn_in_ids.rst b/news/burn_in_ids.rst
@@ -0,0 +1,26 @@
+**Added:**
+
+* Add ``flow_run_id`` (CI provider specific), ``remote_url`` and ``sha`` as extra-meta data to packages.
+  Enables tracing back packages to a specific commit in a feedstock and to a specific CI run.
+  When packages are built using ``build-locally.py`` only ``sha`` will have a non-empty value.
+  Requires ``conda-build >=3.21.8``. (#1577)
+
+**Changed:**
+
+* <news item>
+
+**Deprecated:**
+
+* <news item>
+
+**Removed:**
+
+* <news item>
+
+**Fixed:**
+
+* <news item>
+
+**Security:**
+
+* <news item>