Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into pythongh-122445
Browse files Browse the repository at this point in the history
  • Loading branch information
iritkatriel committed Aug 2, 2024
2 parents e55d157 + addbb73 commit fc0c10a
Show file tree
Hide file tree
Showing 108 changed files with 1,989 additions and 1,151 deletions.
131 changes: 13 additions & 118 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,5 @@
name: Tests

# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because
# it prevents to mark a job as mandatory. A PR cannot be merged if a job is
# mandatory but not scheduled because of "paths-ignore".
on:
workflow_dispatch:
push:
Expand All @@ -23,121 +20,19 @@ concurrency:

jobs:
check_source:
name: 'Check for source changes'
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
# Some of the referenced steps set outputs conditionally and there may be
# cases when referencing them evaluates to empty strings. It is nice to
# work with proper booleans so they have to be evaluated through JSON
# conversion in the expressions. However, empty strings used like that
# may trigger all sorts of undefined and hard-to-debug behaviors in
# GitHub Actions CI/CD. To help with this, all of the outputs set here
# that are meant to be used as boolean flags (and not arbitrary strings),
# MUST have fallbacks with default values set. A common pattern would be
# to add ` || false` to all such expressions here, in the output
# definitions. They can then later be safely used through the following
# idiom in job conditionals and other expressions. Here's some examples:
#
# if: fromJSON(needs.check_source.outputs.run-docs)
#
# ${{
# fromJSON(needs.check_source.outputs.run_tests)
# && 'truthy-branch'
# || 'falsy-branch'
# }}
#
run-docs: ${{ steps.docs-changes.outputs.run-docs || false }}
run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi || false }}
run_tests: ${{ steps.check.outputs.run_tests || false }}
run_hypothesis: ${{ steps.check.outputs.run_hypothesis || false }}
run_cifuzz: ${{ steps.check.outputs.run_cifuzz || false }}
config_hash: ${{ steps.config_hash.outputs.hash }} # str
steps:
- uses: actions/checkout@v4
- name: Check for source changes
id: check
run: |
if [ -z "$GITHUB_BASE_REF" ]; then
echo "run_tests=true" >> $GITHUB_OUTPUT
else
git fetch origin $GITHUB_BASE_REF --depth=1
# git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more
# reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots),
# but it requires to download more commits (this job uses
# "git fetch --depth=1").
#
# git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git
# 2.26, but Git 2.28 is stricter and fails with "no merge base".
#
# git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on
# GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF
# into the PR branch anyway.
#
# https://github.com/python/core-workflow/issues/373
git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true
fi
# Check if we should run hypothesis tests
GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
echo $GIT_BRANCH
if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
echo "Branch too old for hypothesis tests"
echo "run_hypothesis=false" >> $GITHUB_OUTPUT
else
echo "Run hypothesis tests"
echo "run_hypothesis=true" >> $GITHUB_OUTPUT
fi
# oss-fuzz maintains a configuration for fuzzing the main branch of
# CPython, so CIFuzz should be run only for code that is likely to be
# merged into the main branch; compatibility with older branches may
# be broken.
FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
# The tests are pretty slow so they are executed only for PRs
# changing relevant files.
echo "Run CIFuzz tests"
echo "run_cifuzz=true" >> $GITHUB_OUTPUT
else
echo "Branch too old for CIFuzz tests; or no C files were changed"
echo "run_cifuzz=false" >> $GITHUB_OUTPUT
fi
- name: Compute hash for config cache key
id: config_hash
run: |
echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT
- name: Get a list of the changed documentation-related files
if: github.event_name == 'pull_request'
id: changed-docs-files
uses: Ana06/[email protected]
with:
filter: |
Doc/**
Misc/**
.github/workflows/reusable-docs.yml
format: csv # works for paths with spaces
- name: Check for docs changes
if: >-
github.event_name == 'pull_request'
&& steps.changed-docs-files.outputs.added_modified_renamed != ''
id: docs-changes
run: |
echo "run-docs=true" >> "${GITHUB_OUTPUT}"
- name: Get a list of the MSI installer-related files
id: changed-win-msi-files
uses: Ana06/[email protected]
with:
filter: |
Tools/msi/**
.github/workflows/reusable-windows-msi.yml
format: csv # works for paths with spaces
- name: Check for changes in MSI installer-related files
if: >-
steps.changed-win-msi-files.outputs.added_modified_renamed != ''
id: win-msi-changes
run: |
echo "run-win-msi=true" >> "${GITHUB_OUTPUT}"
name: Change detection
# To use boolean outputs from this job, parse them as JSON.
# Here's some examples:
#
# if: fromJSON(needs.check_source.outputs.run-docs)
#
# ${{
# fromJSON(needs.check_source.outputs.run_tests)
# && 'truthy-branch'
# || 'falsy-branch'
# }}
#
uses: ./.github/workflows/reusable-change-detection.yml

check-docs:
name: Docs
Expand Down
150 changes: 150 additions & 0 deletions .github/workflows/reusable-change-detection.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
---

name: Change detection

on: # yamllint disable-line rule:truthy
workflow_call:
outputs:
# Some of the referenced steps set outputs conditionally and there may be
# cases when referencing them evaluates to empty strings. It is nice to
# work with proper booleans so they have to be evaluated through JSON
# conversion in the expressions. However, empty strings used like that
# may trigger all sorts of undefined and hard-to-debug behaviors in
# GitHub Actions CI/CD. To help with this, all of the outputs set here
# that are meant to be used as boolean flags (and not arbitrary strings),
# MUST have fallbacks with default values set. A common pattern would be
# to add ` || false` to all such expressions here, in the output
# definitions. They can then later be safely used through the following
# idiom in job conditionals and other expressions. Here's some examples:
#
# if: fromJSON(needs.change-detection.outputs.run-docs)
#
# ${{
# fromJSON(needs.change-detection.outputs.run-tests)
# && 'truthy-branch'
# || 'falsy-branch'
# }}
#
config_hash:
description: Config hash value for use in cache keys
value: ${{ jobs.compute-changes.outputs.config-hash }} # str
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool
run_tests:
description: Whether to run the regular tests
value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool
run-win-msi:
description: Whether to run the MSI installer smoke tests
value: >- # bool
${{ jobs.compute-changes.outputs.run-win-msi || false }}
run_hypothesis:
description: Whether to run the Hypothesis tests
value: >- # bool
${{ jobs.compute-changes.outputs.run-hypothesis || false }}
run_cifuzz:
description: Whether to run the CIFuzz job
value: >- # bool
${{ jobs.compute-changes.outputs.run-cifuzz || false }}
jobs:
compute-changes:
name: Compute changed files
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
config-hash: ${{ steps.config-hash.outputs.hash }}
run-cifuzz: ${{ steps.check.outputs.run-cifuzz }}
run-docs: ${{ steps.docs-changes.outputs.run-docs }}
run-hypothesis: ${{ steps.check.outputs.run-hypothesis }}
run-tests: ${{ steps.check.outputs.run-tests }}
run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }}
steps:
- run: >-
echo '${{ github.event_name }}'
- uses: actions/checkout@v4
- name: Check for source changes
id: check
run: |
if [ -z "$GITHUB_BASE_REF" ]; then
echo "run-tests=true" >> $GITHUB_OUTPUT
else
git fetch origin $GITHUB_BASE_REF --depth=1
# git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more
# reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots),
# but it requires to download more commits (this job uses
# "git fetch --depth=1").
#
# git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git
# 2.26, but Git 2.28 is stricter and fails with "no merge base".
#
# git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on
# GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF
# into the PR branch anyway.
#
# https://github.com/python/core-workflow/issues/373
git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> $GITHUB_OUTPUT || true
fi
# Check if we should run hypothesis tests
GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}}
echo $GIT_BRANCH
if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then
echo "Branch too old for hypothesis tests"
echo "run-hypothesis=false" >> $GITHUB_OUTPUT
else
echo "Run hypothesis tests"
echo "run-hypothesis=true" >> $GITHUB_OUTPUT
fi
# oss-fuzz maintains a configuration for fuzzing the main branch of
# CPython, so CIFuzz should be run only for code that is likely to be
# merged into the main branch; compatibility with older branches may
# be broken.
FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)'
if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then
# The tests are pretty slow so they are executed only for PRs
# changing relevant files.
echo "Run CIFuzz tests"
echo "run-cifuzz=true" >> $GITHUB_OUTPUT
else
echo "Branch too old for CIFuzz tests; or no C files were changed"
echo "run-cifuzz=false" >> $GITHUB_OUTPUT
fi
- name: Compute hash for config cache key
id: config-hash
run: |
echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT
- name: Get a list of the changed documentation-related files
if: github.event_name == 'pull_request'
id: changed-docs-files
uses: Ana06/[email protected]
with:
filter: |
Doc/**
Misc/**
.github/workflows/reusable-docs.yml
format: csv # works for paths with spaces
- name: Check for docs changes
if: >-
github.event_name == 'pull_request'
&& steps.changed-docs-files.outputs.added_modified_renamed != ''
id: docs-changes
run: |
echo "run-docs=true" >> "${GITHUB_OUTPUT}"
- name: Get a list of the MSI installer-related files
id: changed-win-msi-files
uses: Ana06/[email protected]
with:
filter: |
Tools/msi/**
.github/workflows/reusable-windows-msi.yml
format: csv # works for paths with spaces
- name: Check for changes in MSI installer-related files
if: >-
steps.changed-win-msi-files.outputs.added_modified_renamed != ''
id: win-msi-changes
run: |
echo "run-win-msi=true" >> "${GITHUB_OUTPUT}"
...
10 changes: 5 additions & 5 deletions .github/workflows/reusable-tsan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ jobs:
# Install clang-18
wget https://apt.llvm.org/llvm.sh
chmod +x llvm.sh
sudo ./llvm.sh 18
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100
sudo update-alternatives --set clang /usr/bin/clang-18
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100
sudo update-alternatives --set clang++ /usr/bin/clang++-18
sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100
sudo update-alternatives --set clang /usr/bin/clang-17
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100
sudo update-alternatives --set clang++ /usr/bin/clang++-17
# Reduce ASLR to avoid TSAN crashing
sudo sysctl -w vm.mmap_rnd_bits=28
- name: TSAN Option Setup
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reusable-ubuntu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ jobs:
${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
- name: Build CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make -j4 &> compiler_output.txt
run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
- name: Display build info
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: make pythoninfo
Expand Down
2 changes: 1 addition & 1 deletion .readthedocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ sphinx:
configuration: Doc/conf.py

build:
os: ubuntu-22.04
os: ubuntu-24.04
tools:
python: "3"

Expand Down
25 changes: 19 additions & 6 deletions Android/android.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python3

import argparse
from glob import glob
import os
import re
import shutil
Expand All @@ -16,16 +17,21 @@
CROSS_BUILD_DIR = CHECKOUT / "cross-build"


def delete_if_exists(path):
if path.exists():
def delete_glob(pattern):
# Path.glob doesn't accept non-relative patterns.
for path in glob(str(pattern)):
path = Path(path)
print(f"Deleting {path} ...")
shutil.rmtree(path)
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink()


def subdir(name, *, clean=None):
path = CROSS_BUILD_DIR / name
if clean:
delete_if_exists(path)
delete_glob(path)
if not path.exists():
if clean is None:
sys.exit(
Expand Down Expand Up @@ -150,10 +156,17 @@ def configure_host_python(context):


def make_host_python(context):
# The CFLAGS and LDFLAGS set in android-env include the prefix dir, so
# delete any previously-installed Python libs and include files to prevent
# them being used during the build.
host_dir = subdir(context.host)
prefix_dir = host_dir / "prefix"
delete_glob(f"{prefix_dir}/include/python*")
delete_glob(f"{prefix_dir}/lib/libpython*")

os.chdir(host_dir / "build")
run(["make", "-j", str(os.cpu_count())], host=context.host)
run(["make", "install", f"prefix={host_dir}/prefix"], host=context.host)
run(["make", "install", f"prefix={prefix_dir}"], host=context.host)


def build_all(context):
Expand All @@ -164,7 +177,7 @@ def build_all(context):


def clean_all(context):
delete_if_exists(CROSS_BUILD_DIR)
delete_glob(CROSS_BUILD_DIR)


# To avoid distributing compiled artifacts without corresponding source code,
Expand Down
Loading

0 comments on commit fc0c10a

Please sign in to comment.