[ci] further simplify CI configurations (#6463)

.appveyor.yml

@@ -2,15 +2,13 @@ version: 4.3.0.99.{build}
 
 image: Visual Studio 2015
 platform: x64
-configuration:  # a trick to construct a build matrix with multiple Python versions
+configuration:
   - '3.8'
 
-# only build pull requests and
-# commits to 'master' or any branch starting with 'release'
+# only build on 'master' and pull requests targeting it
 branches:
   only:
     - master
-    - /^release/
 
 environment:
   matrix:
@@ -25,8 +23,9 @@ install:
   - git submodule update --init --recursive  # get `external_libs` folder
   - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
-  - set CONDA_ENV="test-env"
   - ps: |
+      $env:ALLOW_SKIP_ARROW_TESTS = "1"
+      $env:APPVEYOR = "true"
       $env:CMAKE_BUILD_PARALLEL_LEVEL = 4
       $env:MINICONDA = "C:\Miniconda3-x64"
       $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH"

.ci/check_python_dists.sh

@@ -26,11 +26,12 @@ fi
 PY_MINOR_VER=$(python -c "import sys; print(sys.version_info.minor)")
 if [ $PY_MINOR_VER -gt 7 ]; then
     echo "pydistcheck..."
-    pip install pydistcheck
+    pip install 'pydistcheck>=0.7.0'
     if { test "${TASK}" = "cuda" || test "${METHOD}" = "wheel"; }; then
         pydistcheck \
             --inspect \
-            --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
+            --ignore 'compiled-objects-have-debug-symbols'\
+            --ignore 'distro-too-large-compressed' \
             --max-allowed-size-uncompressed '100M' \
             --max-allowed-files 800 \
             ${DIST_DIR}/* || exit 1

.ci/get_workflow_status.py

@@ -10,11 +10,7 @@
 from os import environ
 from sys import argv, exit
 from time import sleep
-
-try:
-    from urllib import request
-except ImportError:
-    import urllib2 as request
+from urllib import request
 
 
 def get_runs(trigger_phrase):

.ci/setup.sh

@@ -30,10 +30,6 @@ if [[ $OS_NAME == "macos" ]]; then
     if [[ $TASK == "swig" ]]; then
         brew install swig
     fi
-    curl \
-        -sL \
-        -o miniforge.sh \
-        https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-${ARCH}.sh
 else  # Linux
     if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then
         # fixes error "unable to initialize frontend: Dialog"
@@ -144,16 +140,14 @@ else  # Linux
         apt-get install --no-install-recommends -y \
             cmake
     fi
-    if [[ $SETUP_CONDA != "false" ]]; then
-        curl \
-            -sL \
-            -o miniforge.sh \
-            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${ARCH}.sh
-    fi
 fi
 
 if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then
     if [[ $SETUP_CONDA != "false" ]]; then
+        curl \
+            -sL \
+            -o miniforge.sh \
+            https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh
         sh miniforge.sh -b -p $CONDA
     fi
     conda config --set always_yes yes --set changeps1 no

.ci/test.sh

@@ -3,6 +3,7 @@
 set -e -E -o -u pipefail
 
 # defaults
+CONDA_ENV="test-env"
 IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"}
 METHOD=${METHOD:-""}
 PRODUCES_ARTIFACTS=${PRODUCES_ARTIFACTS:-"false"}

.ci/test_r_package_valgrind.sh

@@ -72,10 +72,14 @@ bytes_possibly_lost=$(
     | tr -d ","
 )
 echo "valgrind found ${bytes_possibly_lost} bytes possibly lost"
-if [[ ${bytes_possibly_lost} -gt 1056 ]]; then
+if [[ ${bytes_possibly_lost} -gt 1104 ]]; then
     exit 1
 fi
 
+# ensure 'grep --count' doesn't cause failures
+set +e
+
+echo "checking for invalid reads"
 invalid_reads=$(
   cat ${VALGRIND_LOGS_FILE} \
     | grep --count -i "Invalid read"
@@ -85,6 +89,7 @@ if [[ ${invalid_reads} -gt 0 ]]; then
     exit 1
 fi
 
+echo "checking for invalid writes"
 invalid_writes=$(
   cat ${VALGRIND_LOGS_FILE} \
     | grep --count -i "Invalid write"

.ci/test_windows.ps1

@@ -6,14 +6,9 @@ function Check-Output {
   }
 }
 
+$env:CONDA_ENV = "test-env"
 $env:LGB_VER = (Get-Content $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim()
 
-# unify environment variable for Azure DevOps and AppVeyor
-if (Test-Path env:APPVEYOR) {
-  $env:APPVEYOR = "true"
-  $env:ALLOW_SKIP_ARROW_TESTS = "1"
-}
-
 if ($env:TASK -eq "r-package") {
   & $env:BUILD_SOURCESDIRECTORY\.ci\test_r_package_windows.ps1 ; Check-Output $?
   Exit 0

.github/workflows/cuda.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
     - master
-    - release/*
   # Run manually by clicking a button in the UI
   workflow_dispatch:
     inputs:
@@ -59,16 +58,15 @@ jobs:
         run: |
           exit 0
   test:
-    name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
+    name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }})
     runs-on: [self-hosted, linux]
     needs: [restart-docker]
     container:
-      image: ${{ matrix.image }}
+      image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }}
       env:
         CMAKE_BUILD_PARALLEL_LEVEL: 4
         COMPILER: ${{ matrix.compiler }}
         CONDA: /tmp/miniforge
-        CONDA_ENV: test-env
         DEBIAN_FRONTEND: noninteractive
         METHOD: ${{ matrix.method }}
         OS_NAME: linux
@@ -84,19 +82,19 @@ jobs:
             compiler: gcc
             python_version: "3.10"
             cuda_version: "11.8.0"
-            image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu20.04
+            linux_version: "ubuntu20.04"
             task: cuda
           - method: source
             compiler: gcc
             python_version: "3.12"
             cuda_version: "12.2.0"
-            image: nvcr.io/nvidia/cuda:12.2.0-devel-ubuntu22.04
+            linux_version: "ubuntu22.04"
             task: cuda
           - method: pip
             compiler: clang
             python_version: "3.11"
             cuda_version: "11.8.0"
-            image: nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu20.04
+            linux_version: "ubuntu20.04"
             task: cuda
     steps:
       - name: Install latest git

.github/workflows/linkchecker.yml

@@ -8,7 +8,6 @@ on:
     - cron: '0 8 * * *'
 
 env:
-  CONDA_ENV: test-env
   OS_NAME: 'linux'
   PYTHON_VERSION: '3.12'
   TASK: 'check-links'

.github/workflows/optional_checks.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - master
-      - release/*
 
 jobs:
   all-optional-checks-successful:

.github/workflows/python_package.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
     - master
-    - release/*
 
 # automatically cancel in-progress builds if another commit is pushed
 concurrency:
@@ -16,7 +15,6 @@ concurrency:
 
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 4
-  CONDA_ENV: test-env
 
 jobs:
   test:

.github/workflows/r_package.yml

@@ -7,7 +7,6 @@ on:
   pull_request:
     branches:
     - master
-    - release/*
 
 # automatically cancel in-progress builds if another commit is pushed
 concurrency:

.github/workflows/static_analysis.yml

@@ -9,7 +9,6 @@ on:
   pull_request:
     branches:
     - master
-    - release/*
 
 # automatically cancel in-progress builds if another commit is pushed
 concurrency:
@@ -18,7 +17,6 @@ concurrency:
 
 env:
   COMPILER: 'gcc'
-  CONDA_ENV: test-env
   OS_NAME: 'linux'
   PYTHON_VERSION: '3.12'
 

.vsts-ci.yml

@@ -7,12 +7,10 @@ trigger:
     - v*
 pr:
 - master
-- release/*
 variables:
   AZURE: 'true'
   CMAKE_BUILD_PARALLEL_LEVEL: 4
   PYTHON_VERSION: '3.12'
-  CONDA_ENV: test-env
   runCodesignValidationInjection: false
   skipComponentGovernanceDetection: true
   DOTNET_CLI_TELEMETRY_OPTOUT: true
@@ -187,8 +185,8 @@ jobs:
 - job: QEMU_multiarch
 ###########################################
   variables:
+    BUILD_DIRECTORY: /LightGBM
     COMPILER: gcc
-    OS_NAME: 'linux'
     PRODUCES_ARTIFACTS: 'true'
   pool:
     vmImage: ubuntu-22.04
@@ -214,25 +212,12 @@ jobs:
       git clean -d -f -x
     displayName: 'Clean source directory'
   - script: |
-      export ROOT_DOCKER_FOLDER=/LightGBM
-      cat > docker.env <<EOF
-      AZURE=$AZURE
-      OS_NAME=$OS_NAME
-      COMPILER=$COMPILER
-      TASK=$TASK
-      METHOD=$METHOD
-      CONDA_ENV=$CONDA_ENV
-      PYTHON_VERSION=$PYTHON_VERSION
-      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
-      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
-      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
-      EOF
       cat > docker-script.sh <<EOF
       export CONDA=\$HOME/miniforge
       export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
       export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
-      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit 1
-      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit 1
+      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
+      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
       EOF
       IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
       docker pull "${IMAGE_URI}" || exit 1
@@ -241,11 +226,19 @@ jobs:
       docker run \
         --platform "${PLATFORM}" \
         --rm \
-        --env-file docker.env \
-        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
+        --env AZURE=true \
+        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
+        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
+        --env COMPILER=$COMPILER \
+        --env METHOD=$METHOD \
+        --env OS_NAME=linux \
+        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
+        --env PYTHON_VERSION=$PYTHON_VERSION \
+        --env TASK=$TASK \
+        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
         -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
         "${IMAGE_URI}" \
-        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
+        /bin/bash $BUILD_DIRECTORY/docker-script.sh
     displayName: 'Setup and run tests'
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))