From 80ce812411becd0ec013353288b4c7a5b505b2bf Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Wed, 19 Jul 2023 16:58:45 +0200 Subject: [PATCH 01/17] Downgrade PyYaml to 5.3.1 Upgrading to 6.0.1 did not work because there is a dependency conflict with docker-compose. When using the version 5.4.1 the following error message appears when trying to run `pip install -r requirements.txt` ``` AttributeError: cython_source ``` Related issue: https://github.com/yaml/pyyaml/issues/601 --- libbeat/tests/system/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 3f1d5730460..5a4a1bb09ee 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -36,7 +36,7 @@ pyrsistent==0.16.0 pytest==7.3.2 pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 -PyYAML==5.4.1 +PyYAML==5.3.1 redis==2.10.6 requests==2.28.2 semver==2.8.1 From d1237fbca0b3d2fec0f3638dead51e8bca0d6889 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 09:41:39 +0200 Subject: [PATCH 02/17] fix: Ironbank package --- libbeat/tests/system/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 5a4a1bb09ee..3f1d5730460 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -36,7 +36,7 @@ pyrsistent==0.16.0 pytest==7.3.2 pytest-rerunfailures==9.1.1 pytest-timeout==1.4.2 -PyYAML==5.3.1 +PyYAML==5.4.1 redis==2.10.6 requests==2.28.2 semver==2.8.1 From a578e3d51462cbf9304b1d1c3511b98355ac7eaa Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 09:41:55 +0200 Subject: [PATCH 03/17] fix: pip dependency broken --- metricbeat/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index e7ebcc93371..42a20add0a7 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -21,7 +21,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip3 install --upgrade pip==20.1.1 RUN pip3 install --upgrade docker-compose==1.23.2 RUN pip3 install --upgrade setuptools==47.3.2 -RUN pip3 install --upgrade PyYAML==6.0.0 +RUN pip3 install --upgrade PyYAML==5.4.1 # Oracle instant client RUN cd /usr/lib \ From 9d4d08925b68c095e61f4095959eadf22f7f0854 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 10:34:51 +0200 Subject: [PATCH 04/17] test: use Python versions from the .python-version file --- .github/workflows/check-audtibeat.yml | 3 +++ .github/workflows/check-default.yml | 3 +++ .github/workflows/check-dev-tools.yml | 3 +++ .github/workflows/check-docs.yml | 3 +++ .github/workflows/check-filebeat.yml | 3 +++ .github/workflows/check-heartbeat.yml | 3 +++ .github/workflows/check-libbeat.yml | 3 +++ .github/workflows/check-metricbeat.yml | 6 ++++++ .github/workflows/check-packetbeat.yml | 3 +++ .github/workflows/check-winlogbeat.yml | 3 +++ .github/workflows/check-xpack-auditbeat.yml | 3 +++ .github/workflows/check-xpack-dockerlogbeat.yml | 3 +++ .github/workflows/check-xpack-filebeat.yml | 3 +++ .github/workflows/check-xpack-functionbeat.yml | 3 +++ .github/workflows/check-xpack-heartbeat.yml | 3 +++ .github/workflows/check-xpack-libbeat.yml | 3 +++ .github/workflows/check-xpack-metricbeat.yml | 3 +++ .github/workflows/check-xpack-osquerybeat.yml | 3 +++ .github/workflows/check-xpack-packetbeat.yml | 3 +++ .github/workflows/check-xpack-winlogbeat.yml | 3 +++ .github/workflows/macos-auditbeat.yml | 2 +- .github/workflows/macos-build.yml | 3 +++ .github/workflows/macos-filebeat.yml | 2 +- .github/workflows/macos-heartbeat.yml | 2 +- .github/workflows/macos-metricbeat.yml | 2 +- .github/workflows/macos-packetbeat.yml | 2 +- .github/workflows/macos-xpack-auditbeat.yml | 2 +- .github/workflows/macos-xpack-filebeat.yml | 2 +- .github/workflows/macos-xpack-functionbeat.yml | 2 +- .github/workflows/macos-xpack-heartbeat.yml | 2 +- .github/workflows/macos-xpack-metricbeat.yml | 2 +- .github/workflows/macos-xpack-osquerybeat.yml | 2 +- .github/workflows/macos-xpack-packetbeat.yml | 2 +- 33 files changed, 78 insertions(+), 12 deletions(-) diff --git a/.github/workflows/check-audtibeat.yml b/.github/workflows/check-audtibeat.yml index 3941fcdd492..a41519ccc4f 100644 --- a/.github/workflows/check-audtibeat.yml +++ b/.github/workflows/check-audtibeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index 56bf10ab779..5865c74deb2 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -17,6 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-dev-tools.yml b/.github/workflows/check-dev-tools.yml index 4f0ba423466..3b39b128cc2 100644 --- a/.github/workflows/check-dev-tools.yml +++ b/.github/workflows/check-dev-tools.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index 35aa40fb064..0db088e3ace 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -17,6 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml index 0c08232e8af..a3d1634bd8a 100644 --- a/.github/workflows/check-filebeat.yml +++ b/.github/workflows/check-filebeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-heartbeat.yml b/.github/workflows/check-heartbeat.yml index c975398fc2b..6193d79107b 100644 --- a/.github/workflows/check-heartbeat.yml +++ b/.github/workflows/check-heartbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml index 38b04932a86..16e5081abaf 100644 --- a/.github/workflows/check-libbeat.yml +++ b/.github/workflows/check-libbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index 452f0dbedc1..f5c06412fc3 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -15,6 +15,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml index b084e4d962e..f76d43c5290 100644 --- a/.github/workflows/check-packetbeat.yml +++ b/.github/workflows/check-packetbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml index e048d585fa8..1c77af41398 100644 --- a/.github/workflows/check-winlogbeat.yml +++ b/.github/workflows/check-winlogbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-auditbeat.yml b/.github/workflows/check-xpack-auditbeat.yml index d0bf638796b..af406e1068c 100644 --- a/.github/workflows/check-xpack-auditbeat.yml +++ b/.github/workflows/check-xpack-auditbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-dockerlogbeat.yml b/.github/workflows/check-xpack-dockerlogbeat.yml index 44760e6c5e6..1b161dadbc5 100644 --- a/.github/workflows/check-xpack-dockerlogbeat.yml +++ b/.github/workflows/check-xpack-dockerlogbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-filebeat.yml b/.github/workflows/check-xpack-filebeat.yml index 73b5b21d323..7a509374e29 100644 --- a/.github/workflows/check-xpack-filebeat.yml +++ b/.github/workflows/check-xpack-filebeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-functionbeat.yml b/.github/workflows/check-xpack-functionbeat.yml index 089828088d6..dce92656206 100644 --- a/.github/workflows/check-xpack-functionbeat.yml +++ b/.github/workflows/check-xpack-functionbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-heartbeat.yml b/.github/workflows/check-xpack-heartbeat.yml index c9b77cbebb3..222139340f3 100644 --- a/.github/workflows/check-xpack-heartbeat.yml +++ b/.github/workflows/check-xpack-heartbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-libbeat.yml b/.github/workflows/check-xpack-libbeat.yml index 11359887ef0..1f0405134d7 100644 --- a/.github/workflows/check-xpack-libbeat.yml +++ b/.github/workflows/check-xpack-libbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-metricbeat.yml b/.github/workflows/check-xpack-metricbeat.yml index f61967a5eec..04dfcd71e72 100644 --- a/.github/workflows/check-xpack-metricbeat.yml +++ b/.github/workflows/check-xpack-metricbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-osquerybeat.yml b/.github/workflows/check-xpack-osquerybeat.yml index e5c87bcf5bd..ea1fa247cf5 100644 --- a/.github/workflows/check-xpack-osquerybeat.yml +++ b/.github/workflows/check-xpack-osquerybeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-packetbeat.yml b/.github/workflows/check-xpack-packetbeat.yml index 3840d5598aa..b292e2db6d9 100644 --- a/.github/workflows/check-xpack-packetbeat.yml +++ b/.github/workflows/check-xpack-packetbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-winlogbeat.yml b/.github/workflows/check-xpack-winlogbeat.yml index 8656675c3a1..f598d168fe2 100644 --- a/.github/workflows/check-xpack-winlogbeat.yml +++ b/.github/workflows/check-xpack-winlogbeat.yml @@ -15,6 +15,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml index 994ca6dbebc..972a25cada5 100644 --- a/.github/workflows/macos-auditbeat.yml +++ b/.github/workflows/macos-auditbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 55c2d50fa3f..c1e16ffa3f5 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -45,6 +45,9 @@ jobs: uses: actions/setup-go@v3 with: go-version: ${{ github.event.inputs.go_version }} + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - uses: actions/checkout@v3 diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml index 6b43f5bb6c6..f5e9729ddd4 100644 --- a/.github/workflows/macos-filebeat.yml +++ b/.github/workflows/macos-filebeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml index c8e346a4402..35bb7bb3bf4 100644 --- a/.github/workflows/macos-heartbeat.yml +++ b/.github/workflows/macos-heartbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml index 59a225e1601..58a3aaf0538 100644 --- a/.github/workflows/macos-metricbeat.yml +++ b/.github/workflows/macos-metricbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml index be5dc7377e6..efb296f4f7e 100644 --- a/.github/workflows/macos-packetbeat.yml +++ b/.github/workflows/macos-packetbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml index 3adcb46f6da..092538112d4 100644 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ b/.github/workflows/macos-xpack-auditbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml index 936c0913fa4..620193f3018 100644 --- a/.github/workflows/macos-xpack-filebeat.yml +++ b/.github/workflows/macos-xpack-filebeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml index 26a3e311c92..f59445dc9d4 100644 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ b/.github/workflows/macos-xpack-functionbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml index 502d10c1a3e..b1212951898 100644 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ b/.github/workflows/macos-xpack-heartbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml index 38f40b051bc..7257b0b5fcd 100644 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ b/.github/workflows/macos-xpack-metricbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml index 1b3be3e3148..ec9f35e199a 100644 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ b/.github/workflows/macos-xpack-osquerybeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml index 90d9f77e269..00e8a0310eb 100644 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ b/.github/workflows/macos-xpack-packetbeat.yml @@ -23,7 +23,7 @@ jobs: go-version-file: .go-version - uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version-file: '.python-version' - name: Install dependencies run: go install github.com/magefile/mage - name: Run build From 0f474e5af86d702feb79ba8c2c29ec5fe80f2c37 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 14:45:10 +0200 Subject: [PATCH 05/17] fix: buils a Docker image with Python 3.9.13 --- metricbeat/Dockerfile | 67 +++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 42a20add0a7..95d4cf1449e 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,34 +1,71 @@ -FROM golang:1.19.10 +# build a Python virtualenv +FROM golang:1.19.10 as virtualenv +RUN \ + apt update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + wget \ + curl \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libffi-dev \ + liblzma-dev \ + && rm -rf /var/lib/apt/lists/* + +# The project requires Python 3.9, but Debian 12 only has 3.10. We can't use the official Python +# see .python-version file in the root of the project for the version we need. +# https://github.com/pyenv/pyenv-installer +# https://github.com/pyenv/pyenv +ENV PYENV_GIT_TAG=v2.3.22 +ENV PYTHON_VERSION=3.9.13 +RUN curl -sS https://pyenv.run | bash +ENV PYENV_ROOT="/root/.pyenv" +ENV PATH="${PYENV_ROOT}/bin:${PYENV_ROOT}/versions/${PYTHON_VERSION}/bin:${PATH}" +RUN pyenv install ${PYTHON_VERSION} +RUN pyenv global ${PYTHON_VERSION} +RUN [ "$(python3 --version)" = "Python 3.9.13" ] +RUN pip3 install --upgrade pip==20.1.1 +RUN pip3 install --upgrade docker-compose==1.23.2 +RUN pip3 install --upgrade setuptools==47.3.2 +RUN pip3 install --upgrade PyYAML==5.4.1 +# Build the final image +FROM golang:1.19.10 RUN \ apt update \ && DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends \ netcat-openbsd \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ libaio-dev \ unzip \ && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts # with the system Python installation. golang:1.19.10 uses Debian 12 which now enforces PEP668. +ENV PYTHON_VERSION=3.9.13 +ENV PYTHON_HOME=/root/.pyenv/versions/${PYTHON_VERSION} +COPY --from=virtualenv ${PYTHON_HOME} ${PYTHON_HOME} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PYTHON_HOME}/lib +ENV PATH="${PYTHON_HOME}/bin:${PATH}" +# create a virtualenv ENV VIRTUAL_ENV=/opt/venv -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" - -RUN pip3 install --upgrade pip==20.1.1 -RUN pip3 install --upgrade docker-compose==1.23.2 -RUN pip3 install --upgrade setuptools==47.3.2 -RUN pip3 install --upgrade PyYAML==5.4.1 - -# Oracle instant client +RUN python3 -m venv ${VIRTUAL_ENV} +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +# test the Python version is the one we want +RUN [ "$(python3 --version)" = "Python ${PYTHON_VERSION}" ] +# # Oracle instant client RUN cd /usr/lib \ && curl -sLo instantclient-basic-linux.zip https://download.oracle.com/otn_software/linux/instantclient/19600/instantclient-basic-linux.x64-19.6.0.0.0dbru.zip \ && unzip instantclient-basic-linux.zip \ && rm instantclient-basic-linux.zip -ENV LD_LIBRARY_PATH=/usr/lib/instantclient_19_6 +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/lib/instantclient_19_6 # Add healthcheck for the docker/healthcheck metricset to check during testing. HEALTHCHECK CMD exit 0 From 3b9f5ad33b19b54f20eeb3c554073f07ad3f0f3a Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 16:16:29 +0200 Subject: [PATCH 06/17] fix: force to create a virtual env with pytest installed --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 8cddd13bfa8..3f1bf72a7cf 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "mage build unitTest" + mage: "VIRTUAL_ENV=./venv mage build unitTest" stage: mandatory goIntegTest: installK8s: true From debffc075afec206b6b9bb4e3ddb40e64be991c4 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 16:43:19 +0200 Subject: [PATCH 07/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 3f1bf72a7cf..cef26716f74 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "VIRTUAL_ENV=./venv mage build unitTest" + mage: "VIRTUAL_ENV=./venv-clean MAGEFILE_VERBOSE=1 mage build unitTest" stage: mandatory goIntegTest: installK8s: true From 5c036e3811e4341e0627aea3ff4da5856a6afd38 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 17:15:38 +0200 Subject: [PATCH 08/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index cef26716f74..ca9d454b4c3 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "VIRTUAL_ENV=./venv-clean MAGEFILE_VERBOSE=1 mage build unitTest" + mage: "PYTHON_ENV=./venv MAGEFILE_VERBOSE=1 mage build unitTest" stage: mandatory goIntegTest: installK8s: true From 99f7400970156dd6d144a5209e958cf516c4829d Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 17:50:05 +0200 Subject: [PATCH 09/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index ca9d454b4c3..63bd5f51aef 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "PYTHON_ENV=./venv MAGEFILE_VERBOSE=1 mage build unitTest" + mage: "PYTHON_ENV=./venv-clean MAGEFILE_VERBOSE=1 mage build PythonVirtualEnv unitTest" stage: mandatory goIntegTest: installK8s: true From 97c43d1764560d209c911022be7684dacd0dcbb2 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 18:34:45 +0200 Subject: [PATCH 10/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 63bd5f51aef..a6e2f04c8e7 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "PYTHON_ENV=./venv-clean MAGEFILE_VERBOSE=1 mage build PythonVirtualEnv unitTest" + mage: "pip3 install pytest==7.3.2 && mage build unitTest" stage: mandatory goIntegTest: installK8s: true From 3c102b652070ba0eea0716fda849a3a7db02cb00 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 20:52:47 +0200 Subject: [PATCH 11/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index a6e2f04c8e7..25cf6abbc5d 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "pip3 install pytest==7.3.2 && mage build unitTest" + mage: "PYENV_VERSION=$(cat ../.python-version); pyenv install $PYENV_VERSION && eval \"$(pyenv init -)\" && mage build unitTest" stage: mandatory goIntegTest: installK8s: true From 7ab6e461cb3e90f95655e3a2bc593a82482491ea Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 21:23:59 +0200 Subject: [PATCH 12/17] Update metricbeat/Jenkinsfile.yml --- metricbeat/Jenkinsfile.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 25cf6abbc5d..61ab74dac14 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -14,7 +14,7 @@ when: platform: "immutable && ubuntu-22" ## default label for all the stages stages: unitTest: - mage: "PYENV_VERSION=$(cat ../.python-version); pyenv install $PYENV_VERSION && eval \"$(pyenv init -)\" && mage build unitTest" + mage: "PYENV_GIT_TAG=v2.3.22 curl https://pyenv.run | bash; PYENV_VERSION=$(cat ../.python-version); pyenv install $PYENV_VERSION && eval \"$(pyenv init -)\" && mage build unitTest" stage: mandatory goIntegTest: installK8s: true From d51d404a966a7eabc034c815e9808b892101a9fd Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 21:53:50 +0200 Subject: [PATCH 13/17] test: move tests to actions --- .github/workflows/check-metricbeat.yml | 72 ++++++++++++++++++++++++++ metricbeat/Jenkinsfile.yml | 15 ------ 2 files changed, 72 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index f5c06412fc3..de78b83c820 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -29,3 +29,75 @@ jobs: go install github.com/magefile/mage make -C ${{ env.BEAT_MODULE }} check update make check-no-changes + + unitTest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + - name: unitTest + run: | + go install github.com/magefile/mage + mage -d ${{ env.BEAT_MODULE }} build unitTest + + goIntegTest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + - name: unitTest + run: | + go install github.com/magefile/mage + mage -d ${{ env.BEAT_MODULE }} goIntegTest + + pythonIntegTest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + - name: unitTest + run: | + go install github.com/magefile/mage + mage -d ${{ env.BEAT_MODULE }} pythonIntegTest + + crosscompile: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + - name: unitTest + run: | + go install github.com/magefile/mage + make -C ${{ env.BEAT_MODULE }} crosscompile diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 61ab74dac14..0abfc561b36 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -13,21 +13,6 @@ when: tags: true ## for all the tags platform: "immutable && ubuntu-22" ## default label for all the stages stages: - unitTest: - mage: "PYENV_GIT_TAG=v2.3.22 curl https://pyenv.run | bash; PYENV_VERSION=$(cat ../.python-version); pyenv install $PYENV_VERSION && eval \"$(pyenv init -)\" && mage build unitTest" - stage: mandatory - goIntegTest: - installK8s: true - mage: "mage goIntegTest" - withModule: true - stage: mandatory - pythonIntegTest: - mage: "mage pythonIntegTest" - withModule: true - stage: mandatory - crosscompile: - make: "make -C metricbeat crosscompile" - stage: mandatory macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. From c358355859893ab43c234658037618433d47150c Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 22:02:11 +0200 Subject: [PATCH 14/17] test: refactor working dir --- .github/workflows/check-metricbeat.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index de78b83c820..593a5dbe441 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -44,9 +44,10 @@ jobs: with: go-version-file: .go-version - name: unitTest + working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage - mage -d ${{ env.BEAT_MODULE }} build unitTest + mage build unitTest goIntegTest: runs-on: ubuntu-latest @@ -61,10 +62,11 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version - - name: unitTest + - name: goIntegTest + working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage - mage -d ${{ env.BEAT_MODULE }} goIntegTest + mage goIntegTest pythonIntegTest: runs-on: ubuntu-latest @@ -79,10 +81,11 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version - - name: unitTest + - name: pythonIntegTest + working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage - mage -d ${{ env.BEAT_MODULE }} pythonIntegTest + mage pythonIntegTest crosscompile: runs-on: ubuntu-latest @@ -97,7 +100,8 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version - - name: unitTest + - name: crosscompile + working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} crosscompile + make crosscompile From 949eb374ed9cccf263c85e9125b8d1afb01810c4 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 20 Jul 2023 22:24:18 +0200 Subject: [PATCH 15/17] feat: add python cache --- .github/workflows/check-audtibeat.yml | 2 ++ .github/workflows/check-default.yml | 2 ++ .github/workflows/check-dev-tools.yml | 2 ++ .github/workflows/check-docs.yml | 2 ++ .github/workflows/check-filebeat.yml | 2 ++ .github/workflows/check-heartbeat.yml | 2 ++ .github/workflows/check-libbeat.yml | 2 ++ .github/workflows/check-metricbeat.yml | 25 ++++++++----------- .github/workflows/check-packetbeat.yml | 2 ++ .github/workflows/check-winlogbeat.yml | 2 ++ .github/workflows/check-xpack-auditbeat.yml | 2 ++ .../workflows/check-xpack-dockerlogbeat.yml | 2 ++ .github/workflows/check-xpack-filebeat.yml | 2 ++ .../workflows/check-xpack-functionbeat.yml | 2 ++ .github/workflows/check-xpack-heartbeat.yml | 2 ++ .github/workflows/check-xpack-libbeat.yml | 2 ++ .github/workflows/check-xpack-metricbeat.yml | 2 ++ .github/workflows/check-xpack-osquerybeat.yml | 2 ++ .github/workflows/check-xpack-packetbeat.yml | 2 ++ .github/workflows/check-xpack-winlogbeat.yml | 2 ++ .github/workflows/macos-auditbeat.yml | 2 ++ .github/workflows/macos-build.yml | 2 ++ .github/workflows/macos-filebeat.yml | 2 ++ .github/workflows/macos-heartbeat.yml | 2 ++ .github/workflows/macos-metricbeat.yml | 2 ++ .github/workflows/macos-packetbeat.yml | 2 ++ .github/workflows/macos-xpack-auditbeat.yml | 2 ++ .github/workflows/macos-xpack-filebeat.yml | 2 ++ .../workflows/macos-xpack-functionbeat.yml | 2 ++ .github/workflows/macos-xpack-heartbeat.yml | 2 ++ .github/workflows/macos-xpack-metricbeat.yml | 2 ++ .github/workflows/macos-xpack-osquerybeat.yml | 2 ++ .github/workflows/macos-xpack-packetbeat.yml | 2 ++ 33 files changed, 74 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-audtibeat.yml b/.github/workflows/check-audtibeat.yml index a41519ccc4f..91bd31120bd 100644 --- a/.github/workflows/check-audtibeat.yml +++ b/.github/workflows/check-audtibeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index 5865c74deb2..35bab720e65 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -20,6 +20,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-dev-tools.yml b/.github/workflows/check-dev-tools.yml index 3b39b128cc2..beb9529aebe 100644 --- a/.github/workflows/check-dev-tools.yml +++ b/.github/workflows/check-dev-tools.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index 0db088e3ace..f9f9ac021d3 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -20,6 +20,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml index a3d1634bd8a..77490f5d3b6 100644 --- a/.github/workflows/check-filebeat.yml +++ b/.github/workflows/check-filebeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-heartbeat.yml b/.github/workflows/check-heartbeat.yml index 6193d79107b..ae0c67b0d4e 100644 --- a/.github/workflows/check-heartbeat.yml +++ b/.github/workflows/check-heartbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml index 16e5081abaf..777d6c8719c 100644 --- a/.github/workflows/check-libbeat.yml +++ b/.github/workflows/check-libbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index 593a5dbe441..ba1c8176af3 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -18,9 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - - uses: actions/setup-python@v4 - with: - python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version @@ -37,9 +36,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - - uses: actions/setup-python@v4 - with: - python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version @@ -56,9 +54,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - - uses: actions/setup-python@v4 - with: - python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version @@ -75,9 +72,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - - uses: actions/setup-python@v4 - with: - python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version @@ -94,9 +90,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' - - uses: actions/setup-python@v4 - with: - python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml index f76d43c5290..995f435071c 100644 --- a/.github/workflows/check-packetbeat.yml +++ b/.github/workflows/check-packetbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml index 1c77af41398..ee3bfe5ede4 100644 --- a/.github/workflows/check-winlogbeat.yml +++ b/.github/workflows/check-winlogbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-auditbeat.yml b/.github/workflows/check-xpack-auditbeat.yml index af406e1068c..ed048e28fdb 100644 --- a/.github/workflows/check-xpack-auditbeat.yml +++ b/.github/workflows/check-xpack-auditbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-dockerlogbeat.yml b/.github/workflows/check-xpack-dockerlogbeat.yml index 1b161dadbc5..fe75bc1f6f1 100644 --- a/.github/workflows/check-xpack-dockerlogbeat.yml +++ b/.github/workflows/check-xpack-dockerlogbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-filebeat.yml b/.github/workflows/check-xpack-filebeat.yml index 7a509374e29..775a2c31949 100644 --- a/.github/workflows/check-xpack-filebeat.yml +++ b/.github/workflows/check-xpack-filebeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-functionbeat.yml b/.github/workflows/check-xpack-functionbeat.yml index dce92656206..5fb32d637a9 100644 --- a/.github/workflows/check-xpack-functionbeat.yml +++ b/.github/workflows/check-xpack-functionbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-heartbeat.yml b/.github/workflows/check-xpack-heartbeat.yml index 222139340f3..a3e56f3056f 100644 --- a/.github/workflows/check-xpack-heartbeat.yml +++ b/.github/workflows/check-xpack-heartbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-libbeat.yml b/.github/workflows/check-xpack-libbeat.yml index 1f0405134d7..cb7a8ac68c5 100644 --- a/.github/workflows/check-xpack-libbeat.yml +++ b/.github/workflows/check-xpack-libbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-metricbeat.yml b/.github/workflows/check-xpack-metricbeat.yml index 04dfcd71e72..22e64faa33a 100644 --- a/.github/workflows/check-xpack-metricbeat.yml +++ b/.github/workflows/check-xpack-metricbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-osquerybeat.yml b/.github/workflows/check-xpack-osquerybeat.yml index ea1fa247cf5..b4145ace516 100644 --- a/.github/workflows/check-xpack-osquerybeat.yml +++ b/.github/workflows/check-xpack-osquerybeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-packetbeat.yml b/.github/workflows/check-xpack-packetbeat.yml index b292e2db6d9..1bf6afcdd1b 100644 --- a/.github/workflows/check-xpack-packetbeat.yml +++ b/.github/workflows/check-xpack-packetbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/check-xpack-winlogbeat.yml b/.github/workflows/check-xpack-winlogbeat.yml index f598d168fe2..34a83a4c474 100644 --- a/.github/workflows/check-xpack-winlogbeat.yml +++ b/.github/workflows/check-xpack-winlogbeat.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - uses: actions/setup-go@v3 with: go-version-file: .go-version diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml index 972a25cada5..4bc0bc92018 100644 --- a/.github/workflows/macos-auditbeat.yml +++ b/.github/workflows/macos-auditbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index c1e16ffa3f5..ca800988d0f 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -48,6 +48,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - uses: actions/checkout@v3 diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml index f5e9729ddd4..2ad44c0aa22 100644 --- a/.github/workflows/macos-filebeat.yml +++ b/.github/workflows/macos-filebeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml index 35bb7bb3bf4..9bcd8aaebd6 100644 --- a/.github/workflows/macos-heartbeat.yml +++ b/.github/workflows/macos-heartbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml index 58a3aaf0538..a163f79e326 100644 --- a/.github/workflows/macos-metricbeat.yml +++ b/.github/workflows/macos-metricbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml index efb296f4f7e..0d3d1e242fb 100644 --- a/.github/workflows/macos-packetbeat.yml +++ b/.github/workflows/macos-packetbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml index 092538112d4..8f9620c31bb 100644 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ b/.github/workflows/macos-xpack-auditbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml index 620193f3018..5f34e55ceef 100644 --- a/.github/workflows/macos-xpack-filebeat.yml +++ b/.github/workflows/macos-xpack-filebeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml index f59445dc9d4..51a34b55ca9 100644 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ b/.github/workflows/macos-xpack-functionbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml index b1212951898..a5531d59452 100644 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ b/.github/workflows/macos-xpack-heartbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml index 7257b0b5fcd..97f8a4f392f 100644 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ b/.github/workflows/macos-xpack-metricbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml index ec9f35e199a..4bd328381fa 100644 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ b/.github/workflows/macos-xpack-osquerybeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml index 00e8a0310eb..f2e2c76f58c 100644 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ b/.github/workflows/macos-xpack-packetbeat.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt - name: Install dependencies run: go install github.com/magefile/mage - name: Run build From 7211283e703400dd0d687564bd64ea6b9e015908 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 21 Jul 2023 11:17:15 +0200 Subject: [PATCH 16/17] feat: Go cache feat: move windows WF feat: move packaging --- .github/workflows/check-audtibeat.yml | 1 + .github/workflows/check-default.yml | 1 + .github/workflows/check-dev-tools.yml | 1 + .github/workflows/check-docs.yml | 1 + .github/workflows/check-filebeat.yml | 1 + .github/workflows/check-heartbeat.yml | 1 + .github/workflows/check-libbeat.yml | 1 + .github/workflows/check-metricbeat.yml | 71 +++++++++++++++++++ .github/workflows/check-packetbeat.yml | 1 + .github/workflows/check-winlogbeat.yml | 1 + .github/workflows/check-xpack-auditbeat.yml | 1 + .../workflows/check-xpack-dockerlogbeat.yml | 1 + .github/workflows/check-xpack-filebeat.yml | 1 + .../workflows/check-xpack-functionbeat.yml | 1 + .github/workflows/check-xpack-heartbeat.yml | 1 + .github/workflows/check-xpack-libbeat.yml | 1 + .github/workflows/check-xpack-metricbeat.yml | 1 + .github/workflows/check-xpack-osquerybeat.yml | 1 + .github/workflows/check-xpack-packetbeat.yml | 1 + .github/workflows/check-xpack-winlogbeat.yml | 1 + .github/workflows/golangci-lint.yml | 1 + .github/workflows/macos-auditbeat.yml | 1 + .github/workflows/macos-filebeat.yml | 1 + .github/workflows/macos-heartbeat.yml | 1 + .github/workflows/macos-metricbeat.yml | 3 +- .github/workflows/macos-packetbeat.yml | 1 + .github/workflows/macos-xpack-auditbeat.yml | 1 + .github/workflows/macos-xpack-filebeat.yml | 1 + .../workflows/macos-xpack-functionbeat.yml | 1 + .github/workflows/macos-xpack-heartbeat.yml | 1 + .github/workflows/macos-xpack-metricbeat.yml | 1 + .github/workflows/macos-xpack-osquerybeat.yml | 1 + .github/workflows/macos-xpack-packetbeat.yml | 1 + metricbeat/Jenkinsfile.yml | 21 ------ 34 files changed, 104 insertions(+), 22 deletions(-) diff --git a/.github/workflows/check-audtibeat.yml b/.github/workflows/check-audtibeat.yml index 91bd31120bd..1d46c5c34a4 100644 --- a/.github/workflows/check-audtibeat.yml +++ b/.github/workflows/check-audtibeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-default.yml b/.github/workflows/check-default.yml index 35bab720e65..378b5e4b5df 100644 --- a/.github/workflows/check-default.yml +++ b/.github/workflows/check-default.yml @@ -25,6 +25,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check-default run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-dev-tools.yml b/.github/workflows/check-dev-tools.yml index beb9529aebe..9dddb2b7934 100644 --- a/.github/workflows/check-dev-tools.yml +++ b/.github/workflows/check-dev-tools.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-docs.yml b/.github/workflows/check-docs.yml index f9f9ac021d3..aea6606bae7 100644 --- a/.github/workflows/check-docs.yml +++ b/.github/workflows/check-docs.yml @@ -25,6 +25,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libpcap-dev diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml index 77490f5d3b6..7efed65545a 100644 --- a/.github/workflows/check-filebeat.yml +++ b/.github/workflows/check-filebeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libsystemd-dev diff --git a/.github/workflows/check-heartbeat.yml b/.github/workflows/check-heartbeat.yml index ae0c67b0d4e..cd11b87135c 100644 --- a/.github/workflows/check-heartbeat.yml +++ b/.github/workflows/check-heartbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml index 777d6c8719c..e1c916df813 100644 --- a/.github/workflows/check-libbeat.yml +++ b/.github/workflows/check-libbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libpcap-dev diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index ba1c8176af3..44a7035667f 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage @@ -41,6 +42,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: unitTest working-directory: ${{ env.BEAT_MODULE }} run: | @@ -49,6 +51,7 @@ jobs: goIntegTest: runs-on: ubuntu-latest + needs: unitTest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -59,6 +62,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: goIntegTest working-directory: ${{ env.BEAT_MODULE }} run: | @@ -67,6 +71,7 @@ jobs: pythonIntegTest: runs-on: ubuntu-latest + needs: unitTest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -77,6 +82,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: pythonIntegTest working-directory: ${{ env.BEAT_MODULE }} run: | @@ -85,6 +91,7 @@ jobs: crosscompile: runs-on: ubuntu-latest + needs: unitTest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -95,8 +102,72 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: crosscompile working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage make crosscompile + + windows-2022: + runs-on: windows-2022 + needs: unitTest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + cache-dependency-path: ./go.sum + - name: windows-2022 + working-directory: ${{ env.BEAT_MODULE }} + run: | + go install github.com/magefile/mage + mage build unitTest + + windows-2019: + runs-on: windows-2019 + needs: unitTest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + cache-dependency-path: ./go.sum + - name: windows-2019 + working-directory: ${{ env.BEAT_MODULE }} + run: | + go install github.com/magefile/mage + mage build unitTest + + packaging-linux: + # TODO upload packages to GCS Bucket + runs-on: ubuntu-latest + needs: unitTest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version-file: '.python-version' + cache: 'pip' + cache-dependency-path: ./libbeat/tests/system/requirements.txt + - uses: actions/setup-go@v3 + with: + go-version-file: .go-version + cache-dependency-path: ./go.sum + - name: packaging-linux + working-directory: ${{ env.BEAT_MODULE }} + run: | + go install github.com/magefile/mage + make package + env: + PLATFORMS: "+all linux/arm64 linux/amd64 windows/amd64 darwin/amd64 darwin/arm64" diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml index 995f435071c..3e56fcc7a76 100644 --- a/.github/workflows/check-packetbeat.yml +++ b/.github/workflows/check-packetbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libpcap-dev diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml index ee3bfe5ede4..a52bd2ff7c1 100644 --- a/.github/workflows/check-winlogbeat.yml +++ b/.github/workflows/check-winlogbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-auditbeat.yml b/.github/workflows/check-xpack-auditbeat.yml index ed048e28fdb..747a0f223a7 100644 --- a/.github/workflows/check-xpack-auditbeat.yml +++ b/.github/workflows/check-xpack-auditbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install librpm-dev diff --git a/.github/workflows/check-xpack-dockerlogbeat.yml b/.github/workflows/check-xpack-dockerlogbeat.yml index fe75bc1f6f1..a2d161a8fe4 100644 --- a/.github/workflows/check-xpack-dockerlogbeat.yml +++ b/.github/workflows/check-xpack-dockerlogbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-filebeat.yml b/.github/workflows/check-xpack-filebeat.yml index 775a2c31949..0718fd7d605 100644 --- a/.github/workflows/check-xpack-filebeat.yml +++ b/.github/workflows/check-xpack-filebeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libpcap-dev diff --git a/.github/workflows/check-xpack-functionbeat.yml b/.github/workflows/check-xpack-functionbeat.yml index 5fb32d637a9..54773c57d64 100644 --- a/.github/workflows/check-xpack-functionbeat.yml +++ b/.github/workflows/check-xpack-functionbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-heartbeat.yml b/.github/workflows/check-xpack-heartbeat.yml index a3e56f3056f..259311c4c92 100644 --- a/.github/workflows/check-xpack-heartbeat.yml +++ b/.github/workflows/check-xpack-heartbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-libbeat.yml b/.github/workflows/check-xpack-libbeat.yml index cb7a8ac68c5..31cad6eff8f 100644 --- a/.github/workflows/check-xpack-libbeat.yml +++ b/.github/workflows/check-xpack-libbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-metricbeat.yml b/.github/workflows/check-xpack-metricbeat.yml index 22e64faa33a..632518dab26 100644 --- a/.github/workflows/check-xpack-metricbeat.yml +++ b/.github/workflows/check-xpack-metricbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-osquerybeat.yml b/.github/workflows/check-xpack-osquerybeat.yml index b4145ace516..6013f7fe149 100644 --- a/.github/workflows/check-xpack-osquerybeat.yml +++ b/.github/workflows/check-xpack-osquerybeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update run: | go install github.com/magefile/mage diff --git a/.github/workflows/check-xpack-packetbeat.yml b/.github/workflows/check-xpack-packetbeat.yml index 1bf6afcdd1b..a3208becb47 100644 --- a/.github/workflows/check-xpack-packetbeat.yml +++ b/.github/workflows/check-xpack-packetbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Update package lists run: sudo apt-get update - name: Install libpcap-dev diff --git a/.github/workflows/check-xpack-winlogbeat.yml b/.github/workflows/check-xpack-winlogbeat.yml index 34a83a4c474..a52016504b0 100644 --- a/.github/workflows/check-xpack-winlogbeat.yml +++ b/.github/workflows/check-xpack-winlogbeat.yml @@ -23,6 +23,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: Run check/update uses: magefile/mage-action@v2 with: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index c8e871ce772..166325b8797 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -31,6 +31,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - name: golangci-lint env: diff --git a/.github/workflows/macos-auditbeat.yml b/.github/workflows/macos-auditbeat.yml index 4bc0bc92018..25f5f3d2165 100644 --- a/.github/workflows/macos-auditbeat.yml +++ b/.github/workflows/macos-auditbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-filebeat.yml b/.github/workflows/macos-filebeat.yml index 2ad44c0aa22..e787c9c4c26 100644 --- a/.github/workflows/macos-filebeat.yml +++ b/.github/workflows/macos-filebeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-heartbeat.yml b/.github/workflows/macos-heartbeat.yml index 9bcd8aaebd6..9504f3a28b9 100644 --- a/.github/workflows/macos-heartbeat.yml +++ b/.github/workflows/macos-heartbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-metricbeat.yml b/.github/workflows/macos-metricbeat.yml index a163f79e326..b98d041f991 100644 --- a/.github/workflows/macos-metricbeat.yml +++ b/.github/workflows/macos-metricbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' @@ -31,7 +32,7 @@ jobs: - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test - run: cd ${{ env.BEAT_MODULE }} && echo "See https://github.com/elastic/beats/issues/29038" + run: cd ${{ env.BEAT_MODULE }} && mage unitTest - uses: v1v/otel-upload-test-artifact-action@v2 if: always() continue-on-error: true diff --git a/.github/workflows/macos-packetbeat.yml b/.github/workflows/macos-packetbeat.yml index 0d3d1e242fb..8c162a1d09d 100644 --- a/.github/workflows/macos-packetbeat.yml +++ b/.github/workflows/macos-packetbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-auditbeat.yml b/.github/workflows/macos-xpack-auditbeat.yml index 8f9620c31bb..baeab5f8af7 100644 --- a/.github/workflows/macos-xpack-auditbeat.yml +++ b/.github/workflows/macos-xpack-auditbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-filebeat.yml b/.github/workflows/macos-xpack-filebeat.yml index 5f34e55ceef..402c9b37e5a 100644 --- a/.github/workflows/macos-xpack-filebeat.yml +++ b/.github/workflows/macos-xpack-filebeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-functionbeat.yml b/.github/workflows/macos-xpack-functionbeat.yml index 51a34b55ca9..eeeb83b605f 100644 --- a/.github/workflows/macos-xpack-functionbeat.yml +++ b/.github/workflows/macos-xpack-functionbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml index a5531d59452..0dfa3272944 100644 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ b/.github/workflows/macos-xpack-heartbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-metricbeat.yml b/.github/workflows/macos-xpack-metricbeat.yml index 97f8a4f392f..5c7b3cd5f87 100644 --- a/.github/workflows/macos-xpack-metricbeat.yml +++ b/.github/workflows/macos-xpack-metricbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-osquerybeat.yml b/.github/workflows/macos-xpack-osquerybeat.yml index 4bd328381fa..73f4524d020 100644 --- a/.github/workflows/macos-xpack-osquerybeat.yml +++ b/.github/workflows/macos-xpack-osquerybeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/.github/workflows/macos-xpack-packetbeat.yml b/.github/workflows/macos-xpack-packetbeat.yml index f2e2c76f58c..00235059059 100644 --- a/.github/workflows/macos-xpack-packetbeat.yml +++ b/.github/workflows/macos-xpack-packetbeat.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/setup-go@v3 with: go-version-file: .go-version + cache-dependency-path: ./go.sum - uses: actions/setup-python@v4 with: python-version-file: '.python-version' diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 0abfc561b36..a338f182cfe 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -40,16 +40,6 @@ stages: # - "macosM1Test" # tags: false ## for all the tags # stage: extended - windows-2022: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2022" - stage: mandatory - windows-2019: - mage: "mage build unitTest" - platforms: ## override default labels in this specific stage. - - "windows-2019" - stage: extended_win windows-2016: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. @@ -75,17 +65,6 @@ stages: platforms: ## override default labels in this specific stage. - "windows-8" stage: extended_win - packaging-linux: - packaging-linux: "mage package" - e2e: - enabled: false ## e2e is enabled only for x-pack beats - stage: packaging - when: - branches: false ## Only on a PR basis for the time being - tags: false ## packaging on branches/tags is already in place with the downstream build. - changeset: ## when PR contains any of those entries in the changeset - - "^metricbeat/.*" - - "@oss" ## special token regarding the changeset for the oss packaging-arm: packaging-arm: "mage package" e2e: From ae8a39aba45c9747317a6a793b21670793bdd625 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 21 Jul 2023 11:54:19 +0200 Subject: [PATCH 17/17] fix: packaging --- .github/workflows/check-metricbeat.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-metricbeat.yml b/.github/workflows/check-metricbeat.yml index 44a7035667f..cce57b41201 100644 --- a/.github/workflows/check-metricbeat.yml +++ b/.github/workflows/check-metricbeat.yml @@ -49,6 +49,7 @@ jobs: go install github.com/magefile/mage mage build unitTest + # FIXME neeeds more resources some containers does not start goIntegTest: runs-on: ubuntu-latest needs: unitTest @@ -69,6 +70,7 @@ jobs: go install github.com/magefile/mage mage goIntegTest + # FIXME neeeds more resources some containers does not start pythonIntegTest: runs-on: ubuntu-latest needs: unitTest @@ -109,6 +111,7 @@ jobs: go install github.com/magefile/mage make crosscompile + # FIXME Docker linux is not supported (130 tests fail) windows-2022: runs-on: windows-2022 needs: unitTest @@ -129,6 +132,7 @@ jobs: go install github.com/magefile/mage mage build unitTest + # FIXME Docker linux is not supported (130 tests fail) windows-2019: runs-on: windows-2019 needs: unitTest @@ -168,6 +172,6 @@ jobs: working-directory: ${{ env.BEAT_MODULE }} run: | go install github.com/magefile/mage - make package + mage package env: PLATFORMS: "+all linux/arm64 linux/amd64 windows/amd64 darwin/amd64 darwin/arm64"