Merge remote-tracking branch 'origin/main' into chore/span-origin

.circleci/config.templ.yml

@@ -186,6 +186,7 @@ commands:
                 environment:
                   DD_TRACE_AGENT_URL: << parameters.trace_agent_url >>
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
+                  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
                 no_output_timeout: 5m
                 command: |
                   ulimit -c unlimited
@@ -222,6 +223,7 @@ commands:
             - run:
                 environment:
                   RIOT_RUN_RECOMPILE_REQS: "<< pipeline.parameters.riot_run_latest >>"
+                  DD_CIVISIBILITY_AGENTLESS_ENABLED: true
                 no_output_timeout: 5m
                 command: |
                   ulimit -c unlimited

.github/CODEOWNERS

@@ -114,7 +114,7 @@ ddtrace/profiling                   @DataDog/profiling-python
 ddtrace/settings/profiling.py       @DataDog/profiling-python
 ddtrace/internal/datadog/profiling  @DataDog/profiling-python
 tests/profiling                     @DataDog/profiling-python
-tests/profiling-v2                  @DataDog/profiling-python
+tests/profiling_v2                  @DataDog/profiling-python
 .gitlab/tests/profiling.yml         @DataDog/profiling-python
 
 # MLObs

.github/workflows/system-tests.yml

@@ -304,3 +304,15 @@ jobs:
         with:
           name: logs_parametric
           path: artifact.tar.gz
+
+  finished:
+    runs-on: ubuntu-latest
+    needs: [parametric, system-tests]
+    if: success() || failure()
+    steps:
+      - name: True when everything else succeeded
+        if: needs.parametric.result == 'success' && needs.system-tests.result == 'success'
+        run: exit 0
+      - name: Fails if anything else failed
+        if: needs.parametric.result != 'success' || needs.system-tests.result != 'success'
+        run: exit 1

.github/workflows/testrunner.yml

@@ -14,6 +14,9 @@ on:
 jobs:
   build-and-publish:
     uses: ./.github/workflows/build-and-publish-image.yml
+    permissions:
+      contents: read
+      packages: write
     with:
       tags: 'ghcr.io/datadog/dd-trace-py/testrunner:${{ github.sha }},ghcr.io/datadog/dd-trace-py/testrunner:latest'
       platforms: 'linux/amd64,linux/arm64/v8'

.gitlab-ci.yml

@@ -20,6 +20,7 @@ variables:
   timeout: 20m
   before_script:
     - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev
+    - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
 
 include:
   - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/one-pipeline.yml

.gitlab/package.yml

@@ -5,7 +5,7 @@ build_base_venvs:
     matrix:
       - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
   variables:
-    CMAKE_BUILD_PARALLEL_LEVEL: 24
+    CMAKE_BUILD_PARALLEL_LEVEL: 12
     PIP_VERBOSE: 1
   script:
     - pip install riot~=0.19.1

.gitlab/tests.yml

@@ -8,6 +8,8 @@ variables:
   needs: []
   parallel: 4
   script:
+    - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} --ddtrace"
+    - export _DD_CIVISIBILITY_USE_CI_CONTEXT_PROVIDER=true
     - |
       envs=( $(hatch env show --json | jq -r --arg suite_name "$SUITE_NAME" 'keys[] | select(. | contains($suite_name))' | sort | ./.gitlab/ci-split-input.sh) )
       if [[ ${#envs[@]} -eq 0 ]]; then
@@ -53,7 +55,7 @@ variables:
       do
         echo "Running riot hash: ${hash}"
         riot list "${hash}"
-        ${RIOT_RUN_CMD} "${hash}"
+        ${RIOT_RUN_CMD} "${hash}" -- --ddtrace
       done
       ./scripts/check-diff ".riot/requirements/" \
         "Changes detected after running riot. Consider deleting changed files, running scripts/compile-and-prune-test-requirements and committing the result."

.gitlab/tests/appsec.yml

@@ -26,7 +26,7 @@ appsec iast tdd_propagation:
 
 appsec iast memcheck:
   extends: .test_base_riot_snapshot
-  parallel: 5
+  parallel: 4
   variables:
     SUITE_NAME: "appsec_iast_memcheck"
     CI_DEBUG_TRACE: "true" 
@@ -56,7 +56,7 @@ appsec threats fastapi:
 
 appsec aggregated leak testing:
   extends: .test_base_hatch
-  parallel: 5
+  parallel: 4
   variables:
     SUITE_NAME: "appsec_aggregated_leak_testing"
   retry: 2

.riot/requirements/11063bf.txt

@@ -4,37 +4,38 @@
 #
 #    pip-compile --allow-unsafe --no-annotate .riot/requirements/11063bf.in
 #
-ai21==2.9.2
-ai21-tokenizer==0.11.2
-aiohttp==3.9.5
+ai21==2.14.1
+ai21-tokenizer==0.12.0
+aiohappyeyeballs==2.4.0
+aiohttp==3.10.5
 aiosignal==1.3.1
 annotated-types==0.7.0
 anthropic==0.26.0
 anyio==4.4.0
-attrs==23.2.0
+attrs==24.2.0
 boto3==1.34.51
 botocore==1.34.51
-certifi==2024.7.4
+certifi==2024.8.30
 charset-normalizer==3.3.2
 cohere==5.4.0
-coverage[toml]==7.6.0
+coverage[toml]==7.6.1
 dataclasses-json==0.6.7
 defusedxml==0.7.1
 distro==1.9.0
 exceptiongroup==1.2.2
 faiss-cpu==1.8.0
-fastavro==1.9.5
-filelock==3.15.4
+fastavro==1.9.7
+filelock==3.16.0
 frozenlist==1.4.1
-fsspec==2024.6.1
+fsspec==2024.9.0
 greenlet==3.0.3
 h11==0.14.0
 httpcore==1.0.5
-httpx==0.27.0
+httpx==0.27.2
 httpx-sse==0.4.0
-huggingface-hub==0.24.2
+huggingface-hub==0.24.6
 hypothesis==6.45.0
-idna==3.7
+idna==3.8
 iniconfig==2.0.0
 jmespath==1.0.1
 jsonpatch==1.33
@@ -48,45 +49,45 @@ langchain-core==0.1.52
 langchain-openai==0.1.6
 langchain-pinecone==0.1.0
 langchain-text-splitters==0.0.2
-langsmith==0.1.93
-marshmallow==3.21.3
+langsmith==0.1.117
+marshmallow==3.22.0
 mock==5.1.0
-multidict==6.0.5
+multidict==6.1.0
 mypy-extensions==1.0.0
 numexpr==2.8.5
 numpy==1.26.4
 openai==1.30.3
 opentracing==2.4.0
-orjson==3.10.6
+orjson==3.10.7
 packaging==23.2
 pinecone-client==3.2.2
 pluggy==1.5.0
 psutil==6.0.0
-pydantic==2.8.2
-pydantic-core==2.20.1
-pytest==8.3.2
+pydantic==2.9.1
+pydantic-core==2.23.3
+pytest==8.3.3
 pytest-asyncio==0.23.7
 pytest-cov==5.0.0
 pytest-mock==3.14.0
 pytest-randomly==3.10.1
 python-dateutil==2.9.0.post0
-pyyaml==6.0.1
+pyyaml==6.0.2
 regex==2024.7.24
 requests==2.32.3
 s3transfer==0.10.2
 sentencepiece==0.2.0
 six==1.16.0
 sniffio==1.3.1
 sortedcontainers==2.4.0
-sqlalchemy==2.0.31
+sqlalchemy==2.0.34
 tenacity==8.5.0
 tiktoken==0.7.0
 tokenizers==0.19.1
-tqdm==4.66.4
-types-requests==2.32.0.20240712
+tqdm==4.66.5
+types-requests==2.32.0.20240907
 typing-extensions==4.12.2
 typing-inspect==0.9.0
 urllib3==2.0.7
 vcrpy==5.1.0
 wrapt==1.16.0
-yarl==1.9.4
+yarl==1.11.1

.riot/requirements/13a28e1.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate .riot/requirements/13a28e1.in
+#
+attrs==24.2.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+coverage[toml]==7.6.1
+cryptography==43.0.1
+exceptiongroup==1.2.2
+greenlet==3.1.0
+hypothesis==6.45.0
+idna==3.10
+iniconfig==2.0.0
+jinja2==3.1.4
+linkify-it-py==2.0.3
+markdown-it-py[linkify,plugins]==3.0.0
+markupsafe==2.1.5
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+memray==1.14.0
+mock==5.1.0
+opentracing==2.4.0
+packaging==24.1
+platformdirs==4.3.3
+pluggy==1.5.0
+psycopg2-binary==2.9.9
+pycparser==2.22
+pycryptodome==3.20.0
+pygments==2.18.0
+pytest==8.3.3
+pytest-cov==5.0.0
+pytest-memray==1.7.0
+pytest-mock==3.14.0
+requests==2.32.3
+rich==13.8.1
+sortedcontainers==2.4.0
+sqlalchemy==2.0.22
+textual==0.79.1
+tomli==2.0.1
+typing-extensions==4.12.2
+uc-micro-py==1.0.3
+urllib3==2.2.3

.riot/requirements/1548fc3.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --allow-unsafe --no-annotate --resolver=backtracking .riot/requirements/1548fc3.in
+#
+attrs==24.2.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.3.2
+coverage[toml]==7.6.1
+cryptography==43.0.1
+exceptiongroup==1.2.2
+greenlet==3.1.0
+hypothesis==6.45.0
+idna==3.10
+iniconfig==2.0.0
+jinja2==3.1.4
+linkify-it-py==2.0.3
+markdown-it-py[linkify,plugins]==3.0.0
+markupsafe==2.1.5
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+memray==1.14.0
+mock==5.1.0
+opentracing==2.4.0
+packaging==24.1
+platformdirs==4.3.3
+pluggy==1.5.0
+psycopg2-binary==2.9.9
+pycparser==2.22
+pycryptodome==3.20.0
+pygments==2.18.0
+pytest==8.3.3
+pytest-cov==5.0.0
+pytest-memray==1.7.0
+pytest-mock==3.14.0
+requests==2.32.3
+rich==13.8.1
+sortedcontainers==2.4.0
+sqlalchemy==2.0.22
+textual==0.79.1
+tomli==2.0.1
+typing-extensions==4.12.2
+uc-micro-py==1.0.3
+urllib3==2.2.3