VStream: Prevent buffering entire transactions (OOM risk), instead se… #33827
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backups - E2E - Next Release - Upgrade Downgrade Testing | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "release-[0-9]+.[0-9]" | |
| tags: '**' | |
| pull_request: | |
| branches: '**' | |
| concurrency: | |
| group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - E2E - Next Release') | |
| cancel-in-progress: true | |
| permissions: read-all | |
| jobs: | |
| upgrade_downgrade_test_e2e: | |
| timeout-minutes: 60 | |
| name: Run Upgrade Downgrade Test - Backups - E2E - Next Release | |
| runs-on: oracle-vm-16cpu-64gb-x86-64 | |
| steps: | |
| - name: Skip CI | |
| run: | | |
| if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then | |
| echo "skipping CI due to the 'Skip CI' label" | |
| exit 1 | |
| fi | |
| - name: Check out commit's code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| fetch-depth: 0 | |
| persist-credentials: 'false' | |
| - name: Set output with latest release branch | |
| id: output-next-release-ref | |
| run: | | |
| next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}}) | |
| echo $next_release_ref | |
| echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT | |
| - name: Check for changes in relevant files | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' | |
| uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 | |
| id: changes | |
| with: | |
| token: '' | |
| filters: | | |
| end_to_end: | |
| - 'test/config.json' | |
| - 'go/**' | |
| - 'go/**/*.go' | |
| - 'test.go' | |
| - 'Makefile' | |
| - 'build.env' | |
| - 'go.sum' | |
| - 'go.mod' | |
| - 'proto/*.proto' | |
| - 'tools/**' | |
| - 'config/**' | |
| - 'bootstrap.sh' | |
| - '.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml' | |
| - name: Tune the OS | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: ./.github/actions/tune-os | |
| - name: Set up python | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 | |
| - name: Setup MySQL | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: ./.github/actions/setup-mysql | |
| with: | |
| flavor: mysql-8.4 | |
| - name: Get base dependencies | |
| timeout-minutes: 10 | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| sudo DEBIAN_FRONTEND="noninteractive" apt-get update | |
| # Install everything else we need, and configure | |
| sudo apt-get install -y make unzip g++ etcd-client etcd-server curl git wget grep | |
| sudo service etcd stop | |
| wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb | |
| sudo apt-get install -y gnupg2 | |
| sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb | |
| sudo percona-release enable-only tools | |
| sudo apt-get update | |
| sudo apt-get install -y percona-xtrabackup-80 | |
| # Checkout to the next release of Vitess | |
| - name: Check out other version's code (${{ steps.output-next-release-ref.outputs.next_release_ref }}) | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: ${{ steps.output-next-release-ref.outputs.next_release_ref }} | |
| persist-credentials: 'false' | |
| - name: Set up Go | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - name: Get dependencies for the next release | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Building next release's binaries | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env | |
| NOVTADMINBUILD=1 make build | |
| mkdir -p /tmp/vitess-build-other/ | |
| cp -R bin /tmp/vitess-build-other/ | |
| rm -Rf bin/* | |
| # Checkout to this build's commit | |
| - name: Check out commit's code | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: 'false' | |
| - name: Set up Go | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 | |
| with: | |
| go-version-file: go.mod | |
| cache: false | |
| - name: Get dependencies for this commit | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Building the binaries for this commit | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 10 | |
| run: | | |
| source build.env | |
| NOVTADMINBUILD=1 make build | |
| mkdir -p /tmp/vitess-build-current/ | |
| cp -R bin /tmp/vitess-build-current/ | |
| # Swap binaries, use next release's VTTablet | |
| - name: Use next release's VTTablet | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld | |
| cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet | |
| cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl | |
| cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld | |
| vttablet --version | |
| # Run test with VTTablet at version N+1 and VTBackup at version N | |
| - name: Run backups tests (vttablet=N+1, vtbackup=N) | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| rm -rf /tmp/vtdataroot | |
| mkdir -p /tmp/vtdataroot | |
| set -x | |
| source build.env | |
| go run test.go -skip-build -docker=false -print-log -follow -tag upgrade_downgrade_backups | |
| # Swap binaries again, use current version's VTTablet, and next release's VTBackup | |
| - name: Use current version VTTablet, and other version VTBackup | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld | |
| cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup | |
| cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet | |
| cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl | |
| cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld | |
| vtbackup --version | |
| vttablet --version | |
| # Run test again with VTTablet at version N, and VTBackup at version N+1 | |
| - name: Run backups tests (vttablet=N, vtbackup=N+1) | |
| if: steps.output-next-release-ref.outputs.next_release_ref != '' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| rm -rf /tmp/vtdataroot | |
| mkdir -p /tmp/vtdataroot | |
| set -x | |
| source build.env | |
| go run test.go -skip-build -keep-data -docker=false -print-log -follow -tag upgrade_downgrade_backups |