From 815a42f6490589b9e0bacd186d683dfd7b960f09 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 31 Dec 2024 15:27:24 +0200 Subject: [PATCH] Added missing workflow files Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .github/workflows/assign_milestone.yml | 30 ++ .github/workflows/auto_approve_pr.yml | 29 ++ .github/workflows/check_label.yml | 80 +++++ .../check_make_vtadmin_authz_testgen.yml | 77 +++++ .../check_make_vtadmin_web_proto.yml | 72 +++++ .../workflows/close_stale_pull_requests.yml | 34 ++ .github/workflows/code_freeze.yml | 14 + .github/workflows/codecov.yml | 119 +++++++ .github/workflows/codeql_analysis.yml | 99 ++++++ .github/workflows/create_release.yml | 46 +++ .github/workflows/docker_build_images.yml | 243 ++++++++++++++ .github/workflows/docker_test_cluster_10.yml | 68 ++++ .github/workflows/docker_test_cluster_25.yml | 67 ++++ .github/workflows/e2e_race.yml | 90 ++++++ .github/workflows/endtoend.yml | 91 ++++++ .github/workflows/local_example.yml | 90 ++++++ .github/workflows/region_example.yml | 90 ++++++ .github/workflows/scorecards.yml | 67 ++++ .github/workflows/static_checks_etc.yml | 271 ++++++++++++++++ .github/workflows/unit_race.yml | 153 +++++++++ .github/workflows/unit_race_evalengine.yml | 153 +++++++++ .../workflows/update_golang_dependencies.yml | 59 ++++ .github/workflows/update_golang_version.yml | 94 ++++++ .../upgrade_downgrade_test_backups_e2e.yml | 192 +++++++++++ ...owngrade_test_backups_e2e_next_release.yml | 194 +++++++++++ .../upgrade_downgrade_test_backups_manual.yml | 300 +++++++++++++++++ ...grade_test_backups_manual_next_release.yml | 301 ++++++++++++++++++ .../upgrade_downgrade_test_onlineddl_flow.yml | 255 +++++++++++++++ ...e_downgrade_test_query_serving_queries.yml | 212 ++++++++++++ ...est_query_serving_queries_next_release.yml | 208 ++++++++++++ ...de_downgrade_test_query_serving_schema.yml | 208 ++++++++++++ ...test_query_serving_schema_next_release.yml | 209 ++++++++++++ ...rade_downgrade_test_reparent_new_vtctl.yml | 183 +++++++++++ ...e_downgrade_test_reparent_new_vttablet.yml | 189 +++++++++++ ...rade_downgrade_test_reparent_old_vtctl.yml | 182 +++++++++++ ...e_downgrade_test_reparent_old_vttablet.yml | 181 +++++++++++ .../upgrade_downgrade_test_semi_sync.yml | 164 ++++++++++ .github/workflows/vtadmin_web_build.yml | 58 ++++ .github/workflows/vtadmin_web_lint.yml | 74 +++++ .github/workflows/vtadmin_web_unit_tests.yml | 58 ++++ .github/workflows/vtop_example.yml | 97 ++++++ 41 files changed, 5401 insertions(+) create mode 100644 .github/workflows/assign_milestone.yml create mode 100644 .github/workflows/auto_approve_pr.yml create mode 100644 .github/workflows/check_label.yml create mode 100644 .github/workflows/check_make_vtadmin_authz_testgen.yml create mode 100644 .github/workflows/check_make_vtadmin_web_proto.yml create mode 100644 .github/workflows/close_stale_pull_requests.yml create mode 100644 .github/workflows/code_freeze.yml create mode 100644 .github/workflows/codecov.yml create mode 100644 .github/workflows/codeql_analysis.yml create mode 100644 .github/workflows/create_release.yml create mode 100644 .github/workflows/docker_build_images.yml create mode 100644 .github/workflows/docker_test_cluster_10.yml create mode 100644 .github/workflows/docker_test_cluster_25.yml create mode 100644 .github/workflows/e2e_race.yml create mode 100644 .github/workflows/endtoend.yml create mode 100644 .github/workflows/local_example.yml create mode 100644 .github/workflows/region_example.yml create mode 100644 .github/workflows/scorecards.yml create mode 100644 .github/workflows/static_checks_etc.yml create mode 100644 .github/workflows/unit_race.yml create mode 100644 .github/workflows/unit_race_evalengine.yml create mode 100644 .github/workflows/update_golang_dependencies.yml create mode 100644 .github/workflows/update_golang_version.yml create mode 100644 .github/workflows/upgrade_downgrade_test_backups_e2e.yml create mode 100644 .github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml create mode 100644 .github/workflows/upgrade_downgrade_test_backups_manual.yml create mode 100644 .github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml create mode 100644 .github/workflows/upgrade_downgrade_test_onlineddl_flow.yml create mode 100644 .github/workflows/upgrade_downgrade_test_query_serving_queries.yml create mode 100644 .github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml create mode 100644 .github/workflows/upgrade_downgrade_test_query_serving_schema.yml create mode 100644 .github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml create mode 100644 .github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml create mode 100644 .github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml create mode 100644 .github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml create mode 100644 .github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml create mode 100644 .github/workflows/upgrade_downgrade_test_semi_sync.yml create mode 100644 .github/workflows/vtadmin_web_build.yml create mode 100644 .github/workflows/vtadmin_web_lint.yml create mode 100644 .github/workflows/vtadmin_web_unit_tests.yml create mode 100644 .github/workflows/vtop_example.yml diff --git a/.github/workflows/assign_milestone.yml b/.github/workflows/assign_milestone.yml new file mode 100644 index 00000000000..6687a239b57 --- /dev/null +++ b/.github/workflows/assign_milestone.yml @@ -0,0 +1,30 @@ +name: Assign Milestone + +on: + pull_request_target: + types: [opened] + +permissions: read-all + +env: + GH_TOKEN: ${{ github.token }} + +jobs: + build: + name: Assign Milestone + runs-on: ubuntu-24.04 + permissions: + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Assign Milestone + run: | + gh pr edit ${{ github.event.number }} --milestone "v$(sed -n 's/.*versionName.*\"\([[:digit:]\.]*\).*\"/\1/p' ./go/vt/servenv/version.go)" diff --git a/.github/workflows/auto_approve_pr.yml b/.github/workflows/auto_approve_pr.yml new file mode 100644 index 00000000000..e584337f78b --- /dev/null +++ b/.github/workflows/auto_approve_pr.yml @@ -0,0 +1,29 @@ +name: Auto Approval of Bot Pull Requests +on: + pull_request: + types: [opened, reopened] + +permissions: + contents: read + +jobs: + auto_approve: + name: Auto Approve Pull Request + runs-on: ubuntu-24.04 + + permissions: + pull-requests: write # only given on local PRs, forks run with `read` access + + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Auto Approve Pull Request + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # here we are checking that the PR has been created by the vitess-bot[bot] account and that it is not a draft + # if there is a merge conflict in the backport, the PR will always be created as a draft, meaning we can rely + # on checking whether or not the PR is a draft + if [[ "${{github.event.pull_request.user.login}}" == "vitess-bot[bot]" ]] && [[ "${{github.event.pull_request.draft}}" == "false" ]]; then + gh pr review ${{ github.event.pull_request.number }} --approve + fi diff --git a/.github/workflows/check_label.yml b/.github/workflows/check_label.yml new file mode 100644 index 00000000000..2c6439a29f9 --- /dev/null +++ b/.github/workflows/check_label.yml @@ -0,0 +1,80 @@ +name: Check Pull Request labels +on: + pull_request: + types: [opened, labeled, unlabeled, synchronize] + +permissions: read-all + +jobs: + check_pull_request_labels: + name: Check Pull Request labels + timeout-minutes: 10 + runs-on: ubuntu-24.04 + if: github.repository == 'vitessio/vitess' + steps: + - name: Release Notes label + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'release notes (needs details)')}}" == "true" ]]; then + echo The "release notes (needs details)" label is set. The changes made in this Pull Request need to be documented in the release notes summary "('./changelog/17.0/17.0.0/summary.md')". Once documented, the "release notes (needs details)" label can be removed. + exit 1 + fi + + - name: Check type and component labels + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + LABELS_JSON="/tmp/labels.json" + # Get labels for this pull request + curl -s \ + -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Content-type: application/json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \ + > "$LABELS_JSON" + if ! cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'Component:' ; then + echo "Expecting PR to have label 'Component: ...'" + exit 1 + fi + if ! cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'Type:' ; then + echo "Expecting PR to have label 'Type: ...'" + exit 1 + fi + + - name: Check all Needs labels are off + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + LABELS_JSON="/tmp/labels.json" + # Get labels for this pull request + curl -s \ + -H 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Content-type: application/json" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" \ + > "$LABELS_JSON" + if cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'NeedsDescriptionUpdate' ; then + echo "Expecting PR to not have the NeedsDescriptionUpdate label, please update the PR's description and remove the label." + exit 1 + fi + if cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'NeedsWebsiteDocsUpdate' ; then + echo "Expecting PR to not have the NeedsWebsiteDocsUpdate label, please update the documentation and remove the label." + exit 1 + fi + if cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'NeedsIssue' ; then + echo "Expecting PR to not have the NeedsIssue label; please create a linked issue and remove the label." + exit 1 + fi + if cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'NeedsBackportReason' ; then + if cat ${LABELS_JSON} | jq -r '.[].name ' | grep -q 'Backport to:'; then + echo "Expecting PR to not have the NeedsBackportReason label; please add your justification to the PR description and remove the label." + exit 1 + fi + fi + + + - name: Do Not Merge label + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Do Not Merge')}}" == "true" ]]; then + echo "This PR should not be merged. The 'Do Not Merge' label is set. Please unset it if you wish to merge this PR." + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/check_make_vtadmin_authz_testgen.yml b/.github/workflows/check_make_vtadmin_authz_testgen.yml new file mode 100644 index 00000000000..26543c8a948 --- /dev/null +++ b/.github/workflows/check_make_vtadmin_authz_testgen.yml @@ -0,0 +1,77 @@ +name: check_make_vtadmin_authz_testgen +on: [push, pull_request] + +permissions: read-all + +jobs: + build: + name: Check Make vtadmin_authz_testgen + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + vtadmin_changes: + - 'bootstrap.sh' + - 'tools/**' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'Makefile' + - 'go/vt/vtadmin/**' + - '.github/workflows/check_make_vtadmin_authz_testgen.yml' + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' + run: | + sudo apt-get update + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget + sudo service etcd stop + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' + run: | + make minimaltools + + - name: check_make_vtadmin_authz_testgen + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.vtadmin_changes == 'true' + run: | + tools/check_make_vtadmin_authz_testgen.sh diff --git a/.github/workflows/check_make_vtadmin_web_proto.yml b/.github/workflows/check_make_vtadmin_web_proto.yml new file mode 100644 index 00000000000..1c1bf2cae4f --- /dev/null +++ b/.github/workflows/check_make_vtadmin_web_proto.yml @@ -0,0 +1,72 @@ +name: check_make_vtadmin_web_proto +on: [push, pull_request] + +permissions: read-all + +jobs: + build: + name: Check Make VTAdmin Web Proto + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + proto_changes: + - 'bootstrap.sh' + - 'tools/**' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'Makefile' + - 'go/vt/proto/**' + - 'proto/*.proto' + - 'web/vtadmin/src/proto/**' + - '.github/workflows/check_make_vtadmin_web_proto.yml' + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + with: + go-version-file: go.mod + + - name: Setup Node + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + with: + # node-version should match package.json + node-version: '20.12.2' + + - name: Install npm dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + run: npm ci + working-directory: ./web/vtadmin + + - name: check_make_vtadmin_web_proto + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + run: | + tools/check_make_vtadmin_web_proto.sh diff --git a/.github/workflows/close_stale_pull_requests.yml b/.github/workflows/close_stale_pull_requests.yml new file mode 100644 index 00000000000..dbb71ac79b9 --- /dev/null +++ b/.github/workflows/close_stale_pull_requests.yml @@ -0,0 +1,34 @@ +name: close_stale_pull_requests +on: + schedule: + - cron: '0 1 * * *' + + workflow_dispatch: {} + +permissions: read-all + +jobs: + close_stale_pull_requests: + runs-on: ubuntu-24.04 + permissions: + pull-requests: write + + steps: + - uses: actions/stale@f7176fd3007623b69d27091f9b9d4ab7995f0a06 # v5.2.1 + with: + days-before-stale: 30 + # Do not handle issues at all. We only want to handle PRs. + days-before-issue-stale: -1 + + stale-pr-message: | + This PR is being marked as stale because it has been open for 30 days with no activity. To rectify, you may do any of the following: + + - Push additional commits to the associated branch. + - Remove the stale label. + - Add a comment indicating why it is not stale. + + If no action is taken within 7 days, this PR will be closed. + close-pr-message: "This PR was closed because it has been stale for 7 days with no activity." + + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + diff --git a/.github/workflows/code_freeze.yml b/.github/workflows/code_freeze.yml new file mode 100644 index 00000000000..a66fb6e8b2b --- /dev/null +++ b/.github/workflows/code_freeze.yml @@ -0,0 +1,14 @@ +name: Code Freeze +on: + pull_request: + +permissions: read-all + +jobs: + build: + name: Code Freeze + runs-on: ubuntu-24.04 + steps: + - name: Fail if Code Freeze is enabled + run: | + exit 0 diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 00000000000..c971466b998 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,119 @@ +name: Code Coverage +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Code Coverage') + cancel-in-progress: true + +permissions: read-all + +jobs: + test: + name: Code Coverage + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in files relevant to code coverage + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + changed_files: + - .github/workflows/codecov.yml + - 'go/**' + - go.mod + - go.sum + - Makefile + + - name: Set up Go + if: steps.changes.outputs.changed_files == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.changes.outputs.changed_files == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.changes.outputs.changed_files == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.changes.outputs.changed_files == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + + # mysql80 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + + sudo apt-get install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + mkdir -p dist bin + curl -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + - name: Run make tools + if: steps.changes.outputs.changed_files == 'true' + run: | + make tools + + - name: Run unit tests and generate code coverage reports + if: steps.changes.outputs.changed_files == 'true' + timeout-minutes: 45 + run: | + set -exo pipefail + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + + export NOVTADMINBUILD=1 + + # Exclude endtoend tests from the coverage report. + # TODO: figure out how best to include our endtoend tests in the coverage report. + rm -rf go/test/endtoend go/*/endtoend go/vt/*/endtoend go/cmd/vttestserver + + eatmydata -- make unit_test_cover + + # Restore the files we deleted as codecov tries to fix their paths. + git reset --hard HEAD + + - name: Upload coverage reports to codecov.io + if: steps.changes.outputs.changed_files == 'true' + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7 + with: + fail_ci_if_error: true + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql_analysis.yml b/.github/workflows/codeql_analysis.yml new file mode 100644 index 00000000000..c523e34c697 --- /dev/null +++ b/.github/workflows/codeql_analysis.yml @@ -0,0 +1,99 @@ +name: "CodeQL" + +on: + push: + branches: + - main + - release-**.0 + schedule: + - cron: '0 0 * * 1' + workflow_dispatch: + +permissions: read-all + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + # If you wish to specify cu stom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + - name: Get base dependencies + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Building binaries + timeout-minutes: 30 + run: | + source build.env + make build + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + - name: Slack Workflow Notification + if: ${{ failure() }} + uses: Gamesight/slack-workflow-status@master + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + channel: '#codeql' + name: 'CodeQL Workflows' + + - name: Fail if needed + if: ${{ failure() }} + run: | + exit 1 diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml new file mode 100644 index 00000000000..424cc5bfe15 --- /dev/null +++ b/.github/workflows/create_release.yml @@ -0,0 +1,46 @@ +# This creates a {tar.gz,deb,rpm} file and uploads it to a release. +# To trigger this, create a new release.. but make sure that you publish +# it immediately and do not save it as a DRAFT. + +name: Release +on: + release: + types: [created] + +permissions: read-all + +jobs: + build: + name: Create Release + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + run: | + sudo apt-get update + sudo apt-get install -y make ruby ruby-dev + go mod download + sudo gem install fpm + + - name: Make Packages + run: | + ./tools/make-release-packages.sh + + - name: Upload Files + uses: csexton/release-asset-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: "releases/*.{tar.gz,rpm,deb}" diff --git a/.github/workflows/docker_build_images.yml b/.github/workflows/docker_build_images.yml new file mode 100644 index 00000000000..2024d677205 --- /dev/null +++ b/.github/workflows/docker_build_images.yml @@ -0,0 +1,243 @@ +name: Build Docker Images +on: + push: + branches: + - main + tags: + - 'v[2-9][0-9]*.*' # run only on tags greater or equal to v20.0.0 where this new way of building docker image was changed + workflow_dispatch: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Build Docker Images') + cancel-in-progress: true + +permissions: read-all + +jobs: + build_and_push_vttestserver: + name: Build and push vttestserver + runs-on: gh-hosted-runners-16cores-1-24.04 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ mysql80 ] + + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + echo "DOCKERFILE=./docker/vttestserver/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on main + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/vttestserver:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG=vitess/vttestserver:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_lite: + name: Build and push lite + runs-on: ubuntu-24.04 + if: github.repository == 'vitessio/vitess' + + strategy: + fail-fast: true + matrix: + branch: [ latest, percona80 ] + + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Dockerfile path + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKERFILE=./docker/lite/Dockerfile" >> $GITHUB_ENV + else + echo "DOCKERFILE=./docker/lite/Dockerfile.${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on main + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: vitess/lite:${{ matrix.branch }} + + ###### + # All code below only applies to new tags + ###### + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + if [[ "${{ matrix.branch }}" == "latest" ]]; then + echo "DOCKER_TAG=vitess/lite:${TAG_NAME}" >> $GITHUB_ENV + else + echo "DOCKER_TAG=vitess/lite:${TAG_NAME}-${{ matrix.branch }}" >> $GITHUB_ENV + fi + + - name: Build and push on tags + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: . + file: ${{ env.DOCKERFILE }} + push: true + tags: ${{ env.DOCKER_TAG }} + + build_and_push_components: + name: Build and push + runs-on: gh-hosted-runners-16cores-1-24.04 + if: github.repository == 'vitessio/vitess' && needs.build_and_push_lite.result == 'success' + needs: + - build_and_push_lite + + strategy: + fail-fast: true + matrix: + debian: [ bullseye, bookworm ] + component: [ vtadmin, vtorc, vtgate, vttablet, mysqlctld, mysqlctl, vtctl, vtctlclient, vtctld, vtctldclient, logrotate, logtail, vtbackup, vtexplain ] + + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Login to Docker Hub + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set Docker context path + run: | + echo "DOCKER_CTX=./docker/binaries/${{ matrix.component }}" >> $GITHUB_ENV + + - name: Build and push on main latest tag + if: startsWith(github.ref, 'refs/tags/') == false && matrix.debian == 'bookworm' + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + - name: Build and push on main debian specific tag + if: startsWith(github.ref, 'refs/tags/') == false + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: vitess/${{ matrix.component }}:latest-${{ matrix.debian }} + build-args: | + VT_BASE_VER=latest + DEBIAN_VER=${{ matrix.debian }}-slim + + ###### + # All code below only applies to new tags + ###### + + - name: Get the Git tag + if: startsWith(github.ref, 'refs/tags/') + run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV + + # We push git-tag-based images to three tags, i.e. for 'v19.0.0' we push to: + # + # vitess/${{ matrix.component }}:v19.0.0 (DOCKER_TAG_DEFAULT_DEBIAN) + # vitess/${{ matrix.component }}:v19.0.0-bookworm (DOCKER_TAG) + # vitess/${{ matrix.component }}:v19.0.0-bullseye (DOCKER_TAG) + # + - name: Set Docker tag name + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "DOCKER_TAG_DEFAULT_DEBIAN=vitess/${{ matrix.component }}:${TAG_NAME}" >> $GITHUB_ENV + echo "DOCKER_TAG=vitess/${{ matrix.component }}:${TAG_NAME}-${{ matrix.debian }}" >> $GITHUB_ENV + + # Build and Push component image to DOCKER_TAG, applies to both debian version + - name: Build and push on tags using Debian extension + if: startsWith(github.ref, 'refs/tags/') + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + # Build and Push component image to DOCKER_TAG_DEFAULT_DEBIAN, only applies when building the default Debian version (bookworm) + # It is fine to build a second time here when "matrix.debian == 'bookworm'" as we have cached the first build already + - name: Build and push on tags without Debian extension + if: startsWith(github.ref, 'refs/tags/') && matrix.debian == 'bookworm' + uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 + with: + context: ${{ env.DOCKER_CTX }} + push: true + tags: ${{ env.DOCKER_TAG_DEFAULT_DEBIAN }} + build-args: | + VT_BASE_VER=${{ env.TAG_NAME }} + DEBIAN_VER=${{ matrix.debian }}-slim + + slack_notification: + name: Slack Notification if failed + runs-on: ubuntu-24.04 + needs: + - build_and_push_vttestserver + - build_and_push_lite + - build_and_push_components + if: ${{ failure() }} + steps: + - name: Slack Workflow Notification + uses: Gamesight/slack-workflow-status@68bf00d0dbdbcb206c278399aa1ef6c14f74347a # v1.3.0 + with: + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + channel: '#docker-build-notifications' + name: 'Docker Build Notification' + icon_url: https://avatars.githubusercontent.com/u/33043890?s=96&v=4 \ No newline at end of file diff --git a/.github/workflows/docker_test_cluster_10.yml b/.github/workflows/docker_test_cluster_10.yml new file mode 100644 index 00000000000..10a7c1551c0 --- /dev/null +++ b/.github/workflows/docker_test_cluster_10.yml @@ -0,0 +1,68 @@ +name: docker_test_cluster_10 +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: Docker Test Cluster 10 + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip $skip + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'docker/**' + - '.github/workflows/docker_test_cluster_10.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo "value: " ${{steps.skip-workflow.outputs.skip-workflow}} + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Run tests which require docker - 1 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go run test.go -docker=true --follow -shard 10 diff --git a/.github/workflows/docker_test_cluster_25.yml b/.github/workflows/docker_test_cluster_25.yml new file mode 100644 index 00000000000..34463ce3830 --- /dev/null +++ b/.github/workflows/docker_test_cluster_25.yml @@ -0,0 +1,67 @@ +name: docker_test_cluster_25 +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: Docker Test Cluster 25 + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'docker/**' + - '.github/workflows/docker_test_cluster_25.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Run tests which require docker - 2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go run test.go -docker=true --follow -shard 25 diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml new file mode 100644 index 00000000000..50e1e4b9485 --- /dev/null +++ b/.github/workflows/e2e_race.yml @@ -0,0 +1,90 @@ +name: e2e_race +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: End-to-End Test (Race) + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/e2e_race.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make minimaltools + + - name: e2e_race + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + NOVTADMINBUILD=1 make e2e_test_race diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml new file mode 100644 index 00000000000..f061e8bee68 --- /dev/null +++ b/.github/workflows/endtoend.yml @@ -0,0 +1,91 @@ +name: endtoend +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: End-to-End Test + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/endtoend.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make minimaltools + + - name: Build + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + NOVTADMINBUILD=1 make build + + - name: endtoend + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 30 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + + eatmydata -- tools/e2e_test_runner.sh diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml new file mode 100644 index 00000000000..fc9724aa6ac --- /dev/null +++ b/.github/workflows/local_example.yml @@ -0,0 +1,90 @@ +name: local_example +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: Local example using ${{ matrix.topo }} on Ubuntu + runs-on: gh-hosted-runners-16cores-1-24.04 + strategy: + matrix: + topo: [consul,etcd,zk2] + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + examples: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - '.github/workflows/local_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + make minimaltools + + - name: Build + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + make build + + - name: local_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + timeout-minutes: 30 + run: | + export TOPO=${{matrix.topo}} + if [ ${{matrix.os}} = "macos-latest" ]; then + export PATH="/usr/local/opt/mysql@5.7/bin:$PATH" + fi + eatmydata -- go run test.go -print-log -follow -retry=1 local_example diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml new file mode 100644 index 00000000000..c17f43eec11 --- /dev/null +++ b/.github/workflows/region_example.yml @@ -0,0 +1,90 @@ +name: region_example +on: [push, pull_request] +permissions: read-all +jobs: + + build: + name: Region Sharding example using ${{ matrix.topo }} on Ubuntu + runs-on: gh-hosted-runners-16cores-1-24.04 + strategy: + matrix: + topo: [etcd] + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + examples: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - '.github/workflows/region_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + make minimaltools + + - name: Build + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + run: | + make build + + - name: region_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.examples == 'true' + timeout-minutes: 30 + run: | + export TOPO=${{matrix.topo}} + if [ ${{matrix.os}} = "macos-latest" ]; then + export PATH="/usr/local/opt/mysql@5.7/bin:$PATH" + fi + eatmydata -- go run test.go -print-log -follow -retry=1 region_example diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml new file mode 100644 index 00000000000..de90706da27 --- /dev/null +++ b/.github/workflows/scorecards.yml @@ -0,0 +1,67 @@ +name: Scorecard supply-chain security +on: + # For Branch-Protection check. Only the default branch is supported. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection + branch_protection_rule: + # To guarantee Maintained check is occasionally updated. See + # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained + push: + branches: [ "main" ] + +# Declare default permissions as read only. +permissions: read-all + +jobs: + analysis: + name: Scorecard analysis + runs-on: ubuntu-24.04 + permissions: + # Needed to upload the results to code-scanning dashboard. + security-events: write + # Needed to publish results and get a badge (see publish_results below). + id-token: write + # Uncomment the permissions below if installing in a private repository. + # contents: read + # actions: read + + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: "Run analysis" + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 + with: + results_file: results.sarif + results_format: sarif + # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: + # - you want to enable the Branch-Protection check on a *public* repository, or + # - you are installing Scorecard on a *private* repository + # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. + # repo_token: ${{ secrets.SCORECARD_TOKEN }} + + # Public repositories: + # - Publish results to OpenSSF REST API for easy access by consumers + # - Allows the repository to include the Scorecard badge. + # - See https://github.com/ossf/scorecard-action#publishing-results. + # For private repositories: + # - `publish_results` will always be set to `false`, regardless + # of the value entered here. + publish_results: true + + # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF + # format to the repository Actions tab. + - name: "Upload artifact" + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: SARIF file + path: results.sarif + retention-days: 5 + + # Upload the results to GitHub's code scanning dashboard (optional). + # Commenting out will disable upload of results to your repo's Code Scanning dashboard + - name: "Upload to code-scanning" + uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 + with: + sarif_file: results.sarif diff --git a/.github/workflows/static_checks_etc.yml b/.github/workflows/static_checks_etc.yml new file mode 100644 index 00000000000..cd2cce93827 --- /dev/null +++ b/.github/workflows/static_checks_etc.yml @@ -0,0 +1,271 @@ +name: Static Code Checks, Etc. + +on: + - pull_request + - push + +permissions: read-all + +jobs: + build: + name: Static Code Checks Etc + runs-on: ubuntu-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Checkout code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Run FOSSA scan and upload build data + uses: fossa-contrib/fossa-action@v3 + with: + # This is a push-only API token: https://github.com/fossa-contrib/fossa-action#push-only-api-token + fossa-api-key: f62c11ef0c249fef239947f01279aa0f + github-token: ${{ github.token }} + + - name: Check for changes in Go files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + go_files: + - '**/*.go' + - '*.go' + - 'go.sum' + - 'go.mod' + - '.github/workflows/static_checks_etc.yml' + parser_changes: + - 'go/vt/sqlparser/**' + - 'go.sum' + - 'go.mod' + - 'build.env' + - 'bootstrap.sh' + - 'tools/**' + - '.github/workflows/static_checks_etc.yml' + proto_changes: + - 'bootstrap.sh' + - 'tools/**' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'Makefile' + - 'go/vt/proto/**' + - 'proto/*.proto' + - '.github/workflows/static_checks_etc.yml' + sizegen: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'tools/**' + - 'bootstrap.sh' + - '.github/workflows/static_checks_etc.yml' + visitor: + - 'go/tools/asthelpergen/**' + - 'go/vt/sqlparser/**' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'tools/**' + - 'bootstrap.sh' + - 'misc/git/hooks/asthelpers' + - '.github/workflows/static_checks_etc.yml' + astfmt: + - 'go/tools/astfmtgen/**' + - 'go/vt/sqlparser/**' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'tools/**' + - 'bootstrap.sh' + - '.github/workflows/static_checks_etc.yml' + end_to_end: + - 'docker/**' + - 'test.go' + - 'Makefile' + - 'bootstrap.sh' + - '.github/workflows/static_checks_etc.yml' + ci_config: + - 'test/config.json' + - '.github/workflows/static_checks_etc.yml' + release_notes: + - 'changelog/**' + - './go/tools/releases/**' + - '.github/workflows/static_checks_etc.yml' + workflows: + - '.github/**' + - 'Makefile' + - 'test/ci_workflow_gen.go' + - '.github/workflows/static_checks_etc.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.go_files == 'true' || steps.changes.outputs.parser_changes == 'true' || steps.changes.outputs.proto_changes == 'true') + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Run go fmt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: | + gofmt -l . | grep -vF vendor/ && exit 1 || echo "All files formatted correctly" + + - name: Install goimports + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.go_files == 'true' || steps.changes.outputs.visitor == 'true') + run: | + go install golang.org/x/tools/cmd/goimports@latest + + - name: Run goimports + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: | + out=$(goimports -local vitess.io/vitess -l -w $(find . -name "*.go" | grep -v ".pb.go")) + echo $out | grep go > /dev/null && echo -e "The following files are malformatted:\n$out" && exit 1 || echo "All the files are formatted correctly" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.parser_changes == 'true' || steps.changes.outputs.go_files == 'true') + run: | + sudo apt-get update + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget + sudo service etcd stop + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.parser_changes == 'true' || steps.changes.outputs.go_files == 'true') + run: | + make minimaltools + + - name: check_go_versions + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: + tools/check_go_versions.sh || exit 1 + + - name: check_make_parser + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.parser_changes == 'true' || steps.changes.outputs.go_files == 'true') + run: | + tools/check_make_parser.sh || exit 1 + + - name: check_make_sizegen + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.sizegen == 'true' || steps.changes.outputs.go_files == 'true') + run: | + tools/check_make_sizegen.sh || exit 1 + + - name: check_make_visitor + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.visitor == 'true' || steps.changes.outputs.go_files == 'true') + run: | + misc/git/hooks/asthelpers || exit 1 + + - name: check_ast_format_fast + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.astfmt == 'true' || steps.changes.outputs.go_files == 'true') + run: | + ./tools/check_astfmtgen.sh || exit 1 + + - name: run ensure_bootstrap_version + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make ensure_bootstrap_version + git status + test -z "$(git diff-index --name-only HEAD --)" || exit 1 + + - name: Install golangci-lint + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.2 + + - name: Clean Env + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: $(go env GOPATH)/bin/golangci-lint cache clean + + - name: Print linter version + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: $(go env GOPATH)/bin/golangci-lint --version + + - name: Run golangci-lint + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: $(go env GOPATH)/bin/golangci-lint run go/... --timeout 10m || exit 1 + + - name: Run go mod tidy + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.go_files == 'true' + run: | + set -e + go mod tidy + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + echo 'We wish to maintain a tidy state for go mod. Please run `go mod tidy` on your branch, commit and push again.' + echo 'Running `go mod tidy` on this CI test yields with the following changes:' + echo "$output" + exit 1 + + - name: Setup Node + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3 + with: + # make proto requires newer node than the pre-installed one + node-version: '20.12.2' + + - name: check_make_proto + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.proto_changes == 'true' + run: | + tools/check_make_proto.sh || exit 1 + + - name: Check test/config.json + if: steps.skip-workflow.outputs.skip-workflow == 'false' && (steps.changes.outputs.go_files == 'true' || steps.changes.outputs.ci_config == 'true') + run: | + go run ./go/tools/ci-config/main.go || exit 1 + + - name: Check changelog + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.release_notes == 'true' + run: | + set -e + go run ./go/tools/releases/releases.go + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + echo 'We wish to maintain a consistent changelog directory, please run `go run ./go/tools/releases/releases.go`, commit and push again.' + echo 'Running `go run ./go/tools/releases/releases.go` on CI yields the following changes:' + echo "$output" + echo "" + exit 1 + + - name: Check make generate_ci_workflows + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + set -e + make generate_ci_workflows + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + echo 'Please run `make generate_ci_workflows`, commit and push again.' + echo 'Running `make generate_ci_workflows` on CI yields the following changes:' + echo "$output" + echo "" + exit 1 diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml new file mode 100644 index 00000000000..8ffb74bd2cb --- /dev/null +++ b/.github/workflows/unit_race.yml @@ -0,0 +1,153 @@ +name: unit_race +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'unit_race') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + + build: + name: Unit Test (Race) + runs-on: gh-hosted-runners-16cores-1-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + unit_tests: + - 'go/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/unit_race.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get -qq update + + # mysql80 + sudo apt-get -qq install -y mysql-server mysql-client + + sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + mkdir -p dist bin + curl -s -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run make tools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + make tools + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: unit_race + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + export NOVTADMINBUILD=1 + export VTEVALENGINETEST="0" + + eatmydata -- make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/unit_race_evalengine.yml b/.github/workflows/unit_race_evalengine.yml new file mode 100644 index 00000000000..d6d248f066e --- /dev/null +++ b/.github/workflows/unit_race_evalengine.yml @@ -0,0 +1,153 @@ +name: unit_race_evalengine +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'unit_race_evalengine') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + + build: + name: Unit Test (Evalengine_Race) + runs-on: gh-hosted-runners-16cores-1-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + unit_tests: + - 'go/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/unit_race_evalengine.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + export DEBIAN_FRONTEND="noninteractive" + sudo apt-get -qq update + + # mysql80 + sudo apt-get -qq install -y mysql-server mysql-client + + sudo apt-get -qq install -y make unzip g++ curl git wget ant openjdk-11-jdk eatmydata + sudo service mysql stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + mkdir -p dist bin + curl -s -L https://github.com/coreos/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz | tar -zxC dist + mv dist/etcd-v3.3.10-linux-amd64/{etcd,etcdctl} bin/ + + go mod download + go install golang.org/x/tools/cmd/goimports@latest + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run make tools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + run: | + make tools + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: unit_race_evalengine + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + export NOVTADMINBUILD=1 + export VTEVALENGINETEST="1" + + eatmydata -- make unit_test_race | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always() + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 + with: + paths: "report.xml" + show: "fail" diff --git a/.github/workflows/update_golang_dependencies.yml b/.github/workflows/update_golang_dependencies.yml new file mode 100644 index 00000000000..33c5157097c --- /dev/null +++ b/.github/workflows/update_golang_dependencies.yml @@ -0,0 +1,59 @@ +name: Update Golang Dependencies + +on: + schedule: + - cron: "0 0 1,15 * *" # Runs every month on the 1st and 15th days at midnight UTC + workflow_dispatch: + +permissions: read-all + +jobs: + update_golang_deps: + if: github.repository == 'vitessio/vitess' + permissions: + contents: write + pull-requests: write + name: Update Golang Dependencies + runs-on: ubuntu-24.04 + steps: + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: main + + - name: Upgrade the Golang Dependencies + id: detect-and-update + run: | + go get -u ./... + + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + + go mod tidy + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.CREATE_PR_VITESS_BOT }} + branch: "upgrade-go-deps-on-main" + commit-message: "upgrade go deps" + signoff: true + delete-branch: true + team-reviewers: Release + title: "Upgrade the Golang Dependencies" + body: | + This Pull Request updates all the Golang dependencies to their latest version using `go get -u ./...`. + + cc @vitessio/release + base: main + labels: | + go + Component: General + Type: Dependencies diff --git a/.github/workflows/update_golang_version.yml b/.github/workflows/update_golang_version.yml new file mode 100644 index 00000000000..ad5dceeafa5 --- /dev/null +++ b/.github/workflows/update_golang_version.yml @@ -0,0 +1,94 @@ +name: Update Golang Version + +on: + schedule: + - cron: "0 0 * * *" # Runs every day at midnight UTC + workflow_dispatch: + +permissions: read-all + +jobs: + update_golang_version: + if: github.repository == 'vitessio/vitess' + permissions: + contents: write + pull-requests: write + strategy: + matrix: + branch: [ main, release-21.0, release-20.0, release-19.0 ] + name: Update Golang Version + runs-on: ubuntu-24.04 + steps: + - name: Check out code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ matrix.branch }} + + - name: Set up Go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Detect new version and update codebase + env: + GH_TOKEN: ${{ github.token }} + id: detect-and-update + run: | + old_go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get go-version) + echo "old-go-version=${old_go_version}" >> $GITHUB_OUTPUT + + if [ ${{ matrix.branch }} == "main" ]; then + go run ./go/tools/go-upgrade/go-upgrade.go upgrade --main --allow-major-upgrade + elif [ ${{ matrix.branch }} == "release-21.0" ]; then + go run ./go/tools/go-upgrade/go-upgrade.go upgrade + else + go run ./go/tools/go-upgrade/go-upgrade.go upgrade --workflow-update=false + fi + + output=$(git status -s) + if [ -z "${output}" ]; then + exit 0 + fi + + go_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get go-version) + bootstrap_version=$(go run ./go/tools/go-upgrade/go-upgrade.go get bootstrap-version) + echo "go-version=${go_version}" >> $GITHUB_OUTPUT + echo "bootstrap-version=${bootstrap_version}" >> $GITHUB_OUTPUT + + # Check if the PR already exists, if it does then do not create new PR. + gh pr list -S "is:open [${{ matrix.branch }}] Upgrade the Golang version to go${go_version}" > out.txt 2>&1 | true + if [ -s out.txt ]; then + rm -f out.txt + exit 0 + fi + rm -f out.txt + echo "create-pr=true" >> $GITHUB_OUTPUT + + - name: Create Pull Request + if: steps.detect-and-update.outputs.create-pr == 'true' + uses: peter-evans/create-pull-request@v4 + with: + token: ${{ secrets.CREATE_PR_VITESS_BOT }} + branch: "upgrade-go-to-${{steps.detect-and-update.outputs.go-version}}-on-${{ matrix.branch }}" + commit-message: "bump go version to go${{steps.detect-and-update.outputs.go-version}}" + signoff: true + delete-branch: true + team-reviewers: Release + title: "[${{ matrix.branch }}] Upgrade the Golang version to `go${{steps.detect-and-update.outputs.go-version}}`" + body: | + This Pull Request bumps the Golang version to `go${{steps.detect-and-update.outputs.go-version}}` and the bootstrap version to `${{steps.detect-and-update.outputs.bootstrap-version}}`. + + > Do not trust the bot blindly. A thorough code review must be done to ensure all the files have been correctly modified. + + There are a few manual steps remaining: + - [ ] Make sure you update the Golang version used in the previous and next release branches for the Upgrade/Downgrade tests. + - [ ] Build and Push the bootstrap images to Docker Hub, the bot cannot handle that. + + cc @vitessio/release + base: ${{ matrix.branch }} + labels: | + Skip CI + go + Benchmark me + Component: General + Type: CI/Build diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml new file mode 100644 index 00000000000..a56aad2f523 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml @@ -0,0 +1,192 @@ +name: Backups - E2E - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - E2E') + cancel-in-progress: true + +permissions: read-all + +jobs: + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Upgrade Downgrade Test - Backups - E2E + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_backups_e2e.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap binaries, use last release's VTTablet + - name: Use last release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Run test with VTTablet at version N-1 and VTBackup at version N + - name: Run backups tests (vttablet=N-1, vtbackup=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + eatmydata -- go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups + + # Swap binaries again, use current version's VTTablet, and last release's VTBackup + - name: Use current version VTTablet, and other version VTBackup + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtbackup --version + vttablet --version + + # Run test again with VTTablet at version N, and VTBackup at version N-1 + - name: Run backups tests (vttablet=N, vtbackup=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml new file mode 100644 index 00000000000..7ee6c2db799 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -0,0 +1,194 @@ +name: Backups - E2E - Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - E2E - Next Release') + cancel-in-progress: true + +permissions: read-all + +jobs: + + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Upgrade Downgrade Test - Backups - E2E - Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap binaries, use next release's VTTablet + - name: Use next release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Run test with VTTablet at version N+1 and VTBackup at version N + - name: Run backups tests (vttablet=N+1, vtbackup=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + eatmydata -- go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups + + # Swap binaries again, use current version's VTTablet, and next release's VTBackup + - name: Use current version VTTablet, and other version VTBackup + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup + cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld + vtbackup --version + vttablet --version + + # Run test again with VTTablet at version N, and VTBackup at version N+1 + - name: Run backups tests (vttablet=N, vtbackup=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + eatmydata -- go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml new file mode 100644 index 00000000000..00aab1b78ff --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_backups_manual.yml @@ -0,0 +1,300 @@ +name: Backups - Manual - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual') + cancel-in-progress: true + +permissions: read-all + +jobs: + + # This job usually execute in ± 20 minutes + upgrade_downgrade_test_manual: + timeout-minutes: 40 + name: Run Upgrade Downgrade Test - Backups - Manual + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + # Checkout to this build's commit + - name: Checkout to commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_backups_manual.yml' + - 'examples/**' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Install MySQL 8.0 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Checkout to the other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Checkout to commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make minimaltools + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # We create a sharded Vitess cluster following the local example. + # We also insert a few rows in our three tables. + - name: Create the example Vitess cluster with all components using version N + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./start_cluster.sh + + # Taking a backup + - name: Take a backup of all the shards + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env ; cd examples/backups + ./take_backups.sh + + # We insert more data in every table after the backup. + # When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now. + # The initial number of rows for each table is: + # - customer: 5 + # - product: 2 + # - corder: 5 + # We shall see the same number of rows after restoring the backup. + - name: Insert more data after the backup + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples; source ./common/env.sh + + echo "insert into customer(email) values('new_user_1@domain.com');" | mysql + echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql + echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql + + # Stop all the tablets and remove their data + - name: Stop tablets + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./stop_tablets.sh + + # We downgrade: we use the version N-1 of vttablet + - name: Downgrade - Swap binaries, use VTTablet N-1 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Starting the tablets again, they will automatically start restoring the last backup. + - name: Start new tablets and restore + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./restart_tablets.sh + # give enough time to the tablets to restore the backup + sleep 90 + + # Count the number of rows in each table to make sure the restoration is successful. + - name: Assert the number of rows in every table + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "select count(sku) from product;" | mysql 2>&1| grep 2 + echo "select count(email) from customer;" | mysql 2>&1| grep 5 + echo "select count(sku) from corder;" | mysql 2>&1| grep 5 + + # We insert one more row in every table. + - name: Insert more rows in the tables + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "insert into customer(email) values('new_user_2@domain.com');" | mysql + echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql + echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql + + # Taking a second backup of the cluster. + - name: Take a second backup of all the shards + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./take_backups.sh + + # We upgrade: we swap binaries and use the version N of the tablet. + - name: Upgrade - Swap binaries, use VTTablet N + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Starting the tablets again and restoring the previous backup. + - name: Start new tablets and restore + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./upgrade_cluster.sh + + # We count the number of rows in every table to check that the restore step was successful. + - name: Assert the number of rows in every table + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "select count(sku) from product;" | mysql 2>&1| grep 3 + echo "select count(email) from customer;" | mysql 2>&1| grep 6 + echo "select count(sku) from corder;" | mysql 2>&1| grep 6 + + - name: Stop the Vitess cluster + if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples/local + ./401_teardown.sh || true diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml new file mode 100644 index 00000000000..768268271a1 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml @@ -0,0 +1,301 @@ +name: Backups - Manual - Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual - Next Release') + cancel-in-progress: true + +permissions: read-all + +jobs: + + # This job usually execute in ± 20 minutes + upgrade_downgrade_test_manual: + timeout-minutes: 40 + name: Run Upgrade Downgrade Test - Backups - Manual - Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + # Checkout to this build's commit + - name: Checkout to commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml' + - 'examples/**' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Install MySQL 8.0 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata grep + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the next release of Vitess + - name: Checkout to the other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Checkout to commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Run make minimaltools + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make minimaltools + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # We create a sharded Vitess cluster following the local example. + # We also insert a few rows in our three tables. + - name: Create the example Vitess cluster with all components using version N + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./start_cluster.sh + + # Taking a backup + - name: Take a backup of all the shards + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 5 + run: | + source build.env ; cd examples/backups + ./take_backups.sh + + # We insert more data in every table after the backup. + # When we restore the backup made in the next step, we do not want to see the rows we are about to insert now. + # The initial number of rows for each table is: + # - customer: 5 + # - product: 2 + # - corder: 5 + # We shall see the same number of rows after restoring the backup. + - name: Insert more data after the backup + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "insert into customer(email) values('new_user_1@domain.com');" | mysql + echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql + echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql + + # Stop all the tablets and remove their data + - name: Stop tablets + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./stop_tablets.sh + + # We downgrade: we use the version N+1 of vttablet + - name: Downgrade - Swap binaries, use VTTablet N+1 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Starting the tablets again, they will automatically start restoring the last backup. + - name: Start new tablets and restore + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./restart_tablets.sh + # give enough time to the tablets to restore the backup + sleep 60 + + # Count the number of rows in each table to make sure the restoration is successful. + - name: Assert the number of rows in every table + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "select count(sku) from product;" | mysql 2>&1| grep 2 + echo "select count(email) from customer;" | mysql 2>&1| grep 5 + echo "select count(sku) from corder;" | mysql 2>&1| grep 5 + + # We insert one more row in every table. + - name: Insert more rows in the tables + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "insert into customer(email) values('new_user_2@domain.com');" | mysql + echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql + echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql + + # Taking a second backup of the cluster. + - name: Take a second backup of all the shards + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./take_backups.sh + + # We upgrade: we swap binaries and use the version N of the tablet. + - name: Upgrade - Swap binaries, use VTTablet N + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld + vttablet --version + + # Starting the tablets again and restoring the next backup. + - name: Start new tablets and restore + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env ; cd examples/backups + ./upgrade_cluster.sh + + # We count the number of rows in every table to check that the restore step was successful. + - name: Assert the number of rows in every table + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples ; source ./common/env.sh + + echo "select count(sku) from product;" | mysql 2>&1| grep 3 + echo "select count(email) from customer;" | mysql 2>&1| grep 6 + echo "select count(sku) from corder;" | mysql 2>&1| grep 6 + + - name: Stop the Vitess cluster + if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env ; cd examples/local + ./401_teardown.sh || true diff --git a/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml new file mode 100644 index 00000000000..72426e70a61 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml @@ -0,0 +1,255 @@ +name: Online DDL flow - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Online DDL flow') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our Online DDL + VReplication + throttler components +# work using primary and replica vttablets built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Online DDL flow + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_onlineddl_flow.yml' + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Set output with next release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the last release of Vitess + - name: Check out last version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building last release: $(git rev-parse HEAD)" + source build.env + make build + mkdir -p /tmp/vitess-build-last/ + cp -R bin /tmp/vitess-build-last/ + rm -Rf bin/* + + # Checkout to the next release of Vitess + - name: Check out next version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building next release: $(git rev-parse HEAD)" + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-next/ + cp -R bin /tmp/vitess-build-next/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + echo "building this SHA: $(git rev-parse HEAD)" + source build.env + make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy vttablet and related binaries under new names + - name: Use current version Vtctl, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-last/bin/vttablet $PWD/bin/vttablet-last + cp /tmp/vitess-build-last/bin/mysqlctl $PWD/bin/mysqlctl-last + cp /tmp/vitess-build-last/bin/mysqlctld $PWD/bin/mysqlctld-last + cp /tmp/vitess-build-next/bin/vttablet $PWD/bin/vttablet-next + cp /tmp/vitess-build-next/bin/mysqlctl $PWD/bin/mysqlctl-next + cp /tmp/vitess-build-next/bin/mysqlctld $PWD/bin/mysqlctld-next + $PWD/bin/vttablet-last --version + $PWD/bin/vttablet --version + $PWD/bin/vttablet-next --version + + # Running a test with primary tablet at version n (current SHA) and replica vttablet at version n-1 + - name: Run Online DDL tests (primary=N, replica=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="" + export REPLICA_TABLET_BINARY_SUFFIX="-last" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n-1 and replica vttablet at version n (current SHA) + - name: Run Online DDL tests (primary=N-1, replica=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="-last" + export REPLICA_TABLET_BINARY_SUFFIX="" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n+1 and replica vttablet at version n (current SHA) + - name: Run Online DDL tests (primary=N+1, replica=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="-next" + export REPLICA_TABLET_BINARY_SUFFIX="" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow + + # Running a test with primary tablet at version n (current SHA) and replica vttablet at version n+1 + - name: Run Online DDL tests (primary=N, replica=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + export PRIMARY_TABLET_BINARY_SUFFIX="" + export REPLICA_TABLET_BINARY_SUFFIX="-next" + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_onlineddl_flow diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml new file mode 100644 index 00000000000..5ac1a55334c --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml @@ -0,0 +1,212 @@ +name: Query Serving (Queries) - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries)') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Queries) + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Build current commit's binaries + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + rm -Rf bin/* + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n-1 and keep vttablet at version n + - name: Use last release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp -r /tmp/vitess-build-current/bin/* $PWD/bin/ + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n-1 and vttablet/vtctld at version n + - name: Run query serving tests (vtgate=N-1, vttablet=N, vtctld=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet/vtctld will be at version n-1 + - name: Use current version VTGate, and other version VTTablet/VTctld + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + + cp /tmp/vitess-build-other/bin/vtctld $PWD/bin + cp /tmp/vitess-build-other/bin/vtctldclient $PWD/bin + cp /tmp/vitess-build-other/bin/vtctl $PWD/bin + cp /tmp/vitess-build-other/bin/vtctlclient $PWD/bin + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet/vtctld at version n-1 + - name: Run query serving tests (vtgate=N, vttablet=N-1, vtctld=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml new file mode 100644 index 00000000000..c65fff08c52 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml @@ -0,0 +1,208 @@ +name: Query Serving (Queries) Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Queries) Next Release') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Queries) Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n+1 and keep vttablet at version n + - name: Use next release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n+1 and vttablet at version n + - name: Run query serving tests (vtgate=N+1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 + - name: Use current version VTGate, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet at version n+1 + - name: Run query serving tests (vtgate=N, vttablet=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_queries diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml new file mode 100644 index 00000000000..4b5fad0ab29 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml @@ -0,0 +1,208 @@ +name: Query Serving (Schema) - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Schema)') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Schema) + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_schema.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n-1 and keep vttablet at version n + - name: Use last release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n-1 and vttablet at version n + - name: Run query serving tests (vtgate=N-1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n-1 + - name: Use current version VTGate, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet at version n-1 + - name: Run query serving tests (vtgate=N, vttablet=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml new file mode 100644 index 00000000000..6157079db6f --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml @@ -0,0 +1,209 @@ +name: Query Serving (Schema) Next Release - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Query Serving (Schema) Next Release') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtgate, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Query Serving (Schema) Next Release + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + - name: Convert ErrorContains checks to Error checks + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + find ./go/test/endtoend -name '*.go' -exec sed -i 's/ErrorContains/Error/g' {} + + find ./go/test/endtoend -name '*.go' -exec sed -i 's/EqualError/Error/g' {} + + + # Swap the binaries in the bin. Use vtgate version n+1 and keep vttablet at version n + - name: Use next release's VTGate + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vtgate $PWD/bin/vtgate + vtgate --version + + # Running a test with vtgate at version n+1 and vttablet at version n + - name: Run query serving tests (vtgate=N+1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema + + # Swap the binaries again. This time, vtgate will be at version n, and vttablet will be at version n+1 + - name: Use current version VTGate, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtgate --version + vttablet --version + + # Running a test with vtgate at version n and vttablet at version n+1 + - name: Run query serving tests (vtgate=N, vttablet=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_query_serving_schema diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml new file mode 100644 index 00000000000..931d9174c31 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml @@ -0,0 +1,183 @@ +name: Reparent New Vtctl - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Reparent New Vtctl') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtctl, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Reparent New Vtctl + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap the binaries in the bin. Use vtctl version n+1 and keep vttablet at version n + - name: Use next release's Vtctl + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtctl $PWD/bin/vtctld $PWD/bin/vtctlclient $PWD/bin/vtctldclient + cp /tmp/vitess-build-other/bin/vtctl $PWD/bin/vtctl + cp /tmp/vitess-build-other/bin/vtctld $PWD/bin/vtctld + cp /tmp/vitess-build-other/bin/vtctlclient $PWD/bin/vtctlclient + cp /tmp/vitess-build-other/bin/vtctldclient $PWD/bin/vtctldclient + vtctl --version + vttablet --version + + # Running a test with vtctl at version n+1 and vttablet at version n + - name: Run reparent tests (vtctl=N+1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml new file mode 100644 index 00000000000..d77b7be53b3 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml @@ -0,0 +1,189 @@ +name: Reparent New VTTablet - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Reparent New VTTablet') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtctl, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Reparent New VTTablet + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check out commit's code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-next-release-ref + run: | + next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $next_release_ref + echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + if [[ "${{steps.output-next-release-ref.outputs.next_release_ref}}" == "" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any nextly installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the next release of Vitess + - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} + + - name: Get dependencies for the next release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building next release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap the binaries. Use vtctl version n and keep vttablet at version n+1 + - name: Use current version Vtctl, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtctl --version + vttablet --version + + # Running a test with vtctl at version n and vttablet at version n+1 + - name: Run reparent tests (vtctl=N, vttablet=N+1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml new file mode 100644 index 00000000000..701025d7ecc --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml @@ -0,0 +1,182 @@ +name: Reparent Old Vtctl - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Reparent Old Vtctl') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtctl, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Reparent Old Vtctl + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap the binaries in the bin. Use vtctl version n-1 and keep vttablet at version n + - name: Use last release's Vtctl + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vtctl $PWD/bin/vtctld $PWD/bin/vtctlclient $PWD/bin/vtctldclient + cp /tmp/vitess-build-other/bin/vtctl $PWD/bin/vtctl + cp /tmp/vitess-build-other/bin/vtctld $PWD/bin/vtctld + cp /tmp/vitess-build-other/bin/vtctlclient $PWD/bin/vtctlclient + cp /tmp/vitess-build-other/bin/vtctldclient $PWD/bin/vtctldclient + vtctl --version + vttablet --version + + # Running a test with vtctl at version n-1 and vttablet at version n + - name: Run reparent tests (vtctl=N-1, vttablet=N) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml new file mode 100644 index 00000000000..8b121d4af10 --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml @@ -0,0 +1,181 @@ +name: Reparent Old VTTablet - Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing Reparent Old VTTablet') + cancel-in-progress: true + +permissions: read-all + +# This test ensures that our end-to-end tests work using Vitess components +# (vtctl, vttablet, etc) built on different versions. + +jobs: + + upgrade_downgrade_test: + name: Run Upgrade Downgrade Test - Reparent Old VTTablet + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + id: output-previous-release-ref + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo DEBIAN_FRONTEND="noninteractive" apt-get update + # Uninstall any previously installed MySQL first + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Swap the binaries. Use vtctl version n and keep vttablet at version n-1 + - name: Use current version Vtctl, and other version VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet + cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl + cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld + vtctl --version + vttablet --version + + # Running a test with vtctl at version n and vttablet at version n-1 + - name: Run reparent tests (vtctl=N, vttablet=N-1) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + + source build.env + eatmydata -- go run test.go -skip-build -keep-data=false -docker=false -print-log -follow -tag upgrade_downgrade_reparent diff --git a/.github/workflows/upgrade_downgrade_test_semi_sync.yml b/.github/workflows/upgrade_downgrade_test_semi_sync.yml new file mode 100644 index 00000000000..f12e323654e --- /dev/null +++ b/.github/workflows/upgrade_downgrade_test_semi_sync.yml @@ -0,0 +1,164 @@ +name: Semi Sync Upgrade Downgrade Testing +on: + push: + pull_request: + +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Semi Sync Upgrade Downgrade Testing') + cancel-in-progress: true + +permissions: read-all + +jobs: + upgrade_downgrade_test_e2e: + timeout-minutes: 60 + name: Run Semi Sync Upgrade Downgrade Test + runs-on: gh-hosted-runners-16cores-1-24.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: Set output with latest release branch + if: steps.skip-workflow.outputs.skip-workflow == 'false' + id: output-previous-release-ref + run: | + previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) + echo $previous_release_ref + echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**' + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/upgrade_downgrade_test_semi_sync.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version: 1.22.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + + - name: Get base dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo apt-get install -y gnupg2 + sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release enable-only tools + sudo apt-get update + sudo apt-get install -y percona-xtrabackup-80 + + # Checkout to the last release of Vitess + - name: Check out other version's code (${{ steps.output-previous-release-ref.outputs.previous_release_ref }}) + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} + + - name: Get dependencies for the last release + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building last release's binaries + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-other/ + cp -R bin /tmp/vitess-build-other/ + rm -Rf bin/* + + # Checkout to this build's commit + - name: Check out commit's code + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Get dependencies for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + go mod download + + - name: Building the binaries for this commit + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 10 + run: | + source build.env + NOVTADMINBUILD=1 make build + mkdir -p /tmp/vitess-build-current/ + cp -R bin /tmp/vitess-build-current/ + + # Copy last releases vttablet + - name: Copy last release's VTTablet + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + source build.env + + cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttabletold + vttabletold --version + + - name: Run semi sync tests + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + rm -rf /tmp/vtdataroot + mkdir -p /tmp/vtdataroot + set -x + source build.env + go test -v -count=1 -run="" ./go/test/endtoend/reparent/semisync -alsologtostderr \ No newline at end of file diff --git a/.github/workflows/vtadmin_web_build.yml b/.github/workflows/vtadmin_web_build.yml new file mode 100644 index 00000000000..822fa8a0920 --- /dev/null +++ b/.github/workflows/vtadmin_web_build.yml @@ -0,0 +1,58 @@ +name: vtadmin-web build + +# In specifying the 'paths' property, we need to include the path to this workflow .yml file. +# See https://github.community/t/trigger-a-workflow-on-change-to-the-yml-file-itself/17792/4) +on: + push: + paths: + - '.github/workflows/vtadmin_web_build.yml' + - 'web/vtadmin/**' + pull_request: + paths: + - '.github/workflows/vtadmin_web_build.yml' + - 'web/vtadmin/**' + +permissions: read-all + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + + - uses: actions/setup-node@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + # node-version should match package.json + node-version: '20.12.2' + + - name: Install dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: cd ./web/vtadmin && npm ci + + - name: Build front-end + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: cd ./web/vtadmin && npm run build + + # Cancel pending and in-progress runs of this workflow if a newer ref is pushed to CI. + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true diff --git a/.github/workflows/vtadmin_web_lint.yml b/.github/workflows/vtadmin_web_lint.yml new file mode 100644 index 00000000000..694c24734a6 --- /dev/null +++ b/.github/workflows/vtadmin_web_lint.yml @@ -0,0 +1,74 @@ +name: vtadmin-web linting + formatting + +# In specifying the 'paths' property, we need to include the path to this workflow .yml file. +# See https://github.community/t/trigger-a-workflow-on-change-to-the-yml-file-itself/17792/4) +on: + push: + paths: + - '.github/workflows/vtadmin_web_lint.yml' + - 'web/vtadmin/**' + pull_request: + paths: + - '.github/workflows/vtadmin_web_lint.yml' + - 'web/vtadmin/**' + +permissions: read-all + +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + + - uses: actions/setup-node@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + # node-version should match package.json + node-version: '20.12.2' + + - name: Install dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: cd ./web/vtadmin && npm ci + + # Using "if: always()" means each step will run, even if a previous + # step fails. This is nice because, for example, we want stylelint and + # prettier to run even if eslint fails. + # + # An undesirable secondary effect of this is these steps + # will run even if the install, etc. steps fail, which is... weird. + # A nice enhancement is to parallelize these steps into jobs, with the + # trade-off of more complexity around sharing npm install artifacts. + - name: Run eslint + if: steps.skip-workflow.outputs.skip-workflow == 'false' && always() + run: cd ./web/vtadmin && npm run lint:eslint + + - name: Run stylelint + if: steps.skip-workflow.outputs.skip-workflow == 'false' && always() + run: cd ./web/vtadmin && npm run lint:stylelint -- -f verbose + + - name: Run prettier + if: steps.skip-workflow.outputs.skip-workflow == 'false' && always() + run: cd ./web/vtadmin && npm run lint:prettier + + # Cancel pending and in-progress runs of this workflow if a newer ref is pushed to CI. + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true diff --git a/.github/workflows/vtadmin_web_unit_tests.yml b/.github/workflows/vtadmin_web_unit_tests.yml new file mode 100644 index 00000000000..72a887c1926 --- /dev/null +++ b/.github/workflows/vtadmin_web_unit_tests.yml @@ -0,0 +1,58 @@ +name: vtadmin-web unit tests + +# In specifying the 'paths' property, we need to include the path to this workflow .yml file. +# See https://github.community/t/trigger-a-workflow-on-change-to-the-yml-file-itself/17792/4) +on: + push: + paths: + - '.github/workflows/vtadmin_web_unit_tests.yml' + - 'web/vtadmin/**' + pull_request: + paths: + - '.github/workflows/vtadmin_web_unit_tests.yml' + - 'web/vtadmin/**' + +permissions: read-all + +jobs: + unit-tests: + runs-on: ubuntu-24.04 + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + + - uses: actions/setup-node@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + with: + # node-version should match package.json + node-version: '20.12.2' + + - name: Install dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: cd ./web/vtadmin && npm ci + + - name: Run unit tests + if: steps.skip-workflow.outputs.skip-workflow == 'false' + run: cd ./web/vtadmin && CI=true npm run test + + # Cancel pending and in-progress runs of this workflow if a newer ref is pushed to CI. + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true diff --git a/.github/workflows/vtop_example.yml b/.github/workflows/vtop_example.yml new file mode 100644 index 00000000000..fb5ae87c101 --- /dev/null +++ b/.github/workflows/vtop_example.yml @@ -0,0 +1,97 @@ +name: vtop_example +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example') + cancel-in-progress: true + +jobs: + build: + name: VTop Example + runs-on: self-hosted + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - 'examples/**' + - 'test/**' + - '.github/workflows/vtop_example.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 + with: + go-version-file: go.mod + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Install everything we need, and configure + sudo apt-get install -y eatmydata make + go mod download + + # needed for vtctldclient + - name: Build vitess + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make build + + - name: Install kubectl & kind + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + make install_kubectl_kind + + - name: vtop_example + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 60 + run: | + source build.env + eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example \ No newline at end of file