diff --git a/delivery.yaml b/delivery.yaml index ad86a843f..adabbf1a6 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -5,7 +5,7 @@ pipeline: - id: push-spilo-ecr env: BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04 - PGVERSION: 15 + PGVERSION: 16 MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid type: script requires_human_approval: true diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 833023794..30664d1b8 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=ubuntu:22.04 -ARG PGVERSION=15 -ARG TIMESCALEDB="1.7.5 2.3.1 2.11.2" +ARG PGVERSION=16 +ARG TIMESCALEDB="2.3.1 2.11.2 2.14.2" ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= @@ -39,7 +39,7 @@ RUN bash /builddeps/prepare.sh COPY --from=ubuntu-18 /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.18 -COPY cron_unprivileged.c dependencies/src /builddeps/ +COPY cron_unprivileged.c /builddeps/ COPY build_scripts/base.sh /builddeps/ COPY --from=dependencies-builder /builddeps/*.deb /builddeps/ @@ -48,7 +48,7 @@ ARG TIMESCALEDB ARG TIMESCALEDB_APACHE_ONLY=false ARG TIMESCALEDB_TOOLKIT=true ARG COMPRESS -ARG PGOLDVERSIONS="10 11 12 13 14" +ARG PGOLDVERSIONS="11 12 13 14 15" ARG WITH_PERL=false ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" @@ -56,17 +56,14 @@ ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" # Install PostgreSQL, extensions and contribs ENV POSTGIS_VERSION=3.4 \ POSTGIS_LEGACY=3.3 \ - POSTGIS_SUPER_LEGACY=3.2 \ - BG_MON_COMMIT=a0e7c93458ee7d96b0c95d9a0a4c5bc7a72ff5f4 \ - PG_AUTH_MON_COMMIT=439697fe2980cf48f1760f45e04c2d69b2748e73 \ - PG_MON_COMMIT=34c35a86e6dea26930bf1f770048cea2c695b80b \ + BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \ + PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \ + PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \ SET_USER=REL4_0_1 \ - PLPROFILER=fix/pg15-shmem \ - PG_PROFILE=4.2 \ + PLPROFILER=REL4_2_4 \ + PG_PROFILE=4.5 \ PAM_OAUTH2=v1.0.1 \ - PLANTUNER_COMMIT=800d81bc85da64ff3ef66e12aed1d4e1e54fc006 \ - PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 \ - PG_TM_AUX_COMMIT=51cf42f6043608919802fcc48458471d0ea3a127 + PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925 WORKDIR /builddeps RUN bash base.sh diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 9462364e2..43ff533f7 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -35,8 +35,7 @@ else make -C pam-oauth2 install # prepare 3rd sources - git clone -b "$PLPROFILER" https://github.com/hughcapet/plprofiler.git - tar -xzf "plantuner-${PLANTUNER_COMMIT}.tar.gz" + git clone -b "$PLPROFILER" https://github.com/bigsql/plprofiler.git curl -sL "https://github.com/zalando-pg/pg_mon/archive/$PG_MON_COMMIT.tar.gz" | tar xz for p in python3-keyring python3-docutils ieee-data; do @@ -55,7 +54,6 @@ fi curl -sL "https://github.com/zalando-pg/bg_mon/archive/$BG_MON_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz" | tar xz -curl -sL "https://github.com/hughcapet/pg_tm_aux/archive/$PG_TM_AUX_COMMIT.tar.gz" | tar xz curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git git clone https://github.com/timescale/timescaledb.git @@ -96,28 +94,15 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}" "postgresql-${version}-postgis-${POSTGIS_VERSION%.*}-scripts" "postgresql-${version}-repack" - "postgresql-${version}-wal2json") - - if [ "$version" != "15" ]; then - # not yet present for pg15 - EXTRAS+=("postgresql-${version}-pllua") - fi + "postgresql-${version}-wal2json" + "postgresql-${version}-decoderbufs" + "postgresql-${version}-pllua" + "postgresql-${version}-pgvector") if [ "$WITH_PERL" = "true" ]; then EXTRAS+=("postgresql-plperl-${version}") fi - if [ "${version%.*}" -ge 10 ]; then - EXTRAS+=("postgresql-${version}-decoderbufs") - fi - - if [ "${version%.*}" -ge 11 ]; then - EXTRAS+=("postgresql-${version}-pgvector") - fi - - if [ "${version%.*}" -lt 11 ]; then - EXTRAS+=("postgresql-${version}-amcheck") - fi fi # Install PostgreSQL binaries, contrib, plproxy and multiple pl's @@ -166,20 +151,15 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do rm /usr/share/keyrings/timescale_E7391C94080429FF.gpg fi + EXTRA_EXTENSIONS=() if [ "$DEMO" != "true" ]; then - EXTRA_EXTENSIONS=("plantuner-${PLANTUNER_COMMIT}" plprofiler) - if [ "${version%.*}" -ge 10 ]; then - EXTRA_EXTENSIONS+=("pg_mon-${PG_MON_COMMIT}") - fi - else - EXTRA_EXTENSIONS=() + EXTRA_EXTENSIONS+=("plprofiler" "pg_mon-${PG_MON_COMMIT}") fi for n in bg_mon-${BG_MON_COMMIT} \ pg_auth_mon-${PG_AUTH_MON_COMMIT} \ set_user \ pg_permissions-${PG_PERMISSIONS_COMMIT} \ - pg_tm_aux-${PG_TM_AUX_COMMIT} \ pg_profile-${PG_PROFILE} \ "${EXTRA_EXTENSIONS[@]}"; do make -C "$n" USE_PGXS=1 clean install-strip @@ -281,17 +261,12 @@ if [ "$DEMO" != "true" ]; then if [ "$v1" = "$v2" ]; then started=1 elif [ $started = 1 ]; then - - used_postgis_version=$POSTGIS_VERSION - if [ "${v1##*/}" = "11" ]; then used_postgis_version=$POSTGIS_LEGACY; fi - - for d1 in extension contrib contrib/postgis-$used_postgis_version; do + for d1 in extension contrib contrib/postgis-$POSTGIS_VERSION; do cd "$v1/$d1" d2="$d1" d1="../../${v1##*/}/$d1" if [ "${d2%-*}" = "contrib/postgis" ]; then - if [ "${v2##*/}" = "11" ]; then d2="${d2%-*}-$POSTGIS_LEGACY" - elif [ "${v2##*/}" = "10" ]; then d2="${d2%-*}-$POSTGIS_SUPER_LEGACY"; fi + if [ "${v2##*/}" = "11" ]; then d2="${d2%-*}-$POSTGIS_LEGACY"; fi d1="../$d1" fi d2="$v2/$d2" diff --git a/postgres-appliance/dependencies/src/plantuner-800d81bc85da64ff3ef66e12aed1d4e1e54fc006.tar.gz b/postgres-appliance/dependencies/src/plantuner-800d81bc85da64ff3ef66e12aed1d4e1e54fc006.tar.gz deleted file mode 100644 index 6c8365d69..000000000 Binary files a/postgres-appliance/dependencies/src/plantuner-800d81bc85da64ff3ef66e12aed1d4e1e54fc006.tar.gz and /dev/null differ diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py index ba56b2156..2a522a301 100644 --- a/postgres-appliance/major_upgrade/pg_upgrade.py +++ b/postgres-appliance/major_upgrade/pg_upgrade.py @@ -11,7 +11,7 @@ class _PostgresqlUpgrade(Postgresql): - _INCOMPATIBLE_EXTENSIONS = ('amcheck_next', 'pg_repack',) + _INCOMPATIBLE_EXTENSIONS = ('pg_repack',) def adjust_shared_preload_libraries(self, version): from spilo_commons import adjust_extensions diff --git a/postgres-appliance/scripts/spilo_commons.py b/postgres-appliance/scripts/spilo_commons.py index 12881cedc..d80a4dae9 100644 --- a/postgres-appliance/scripts/spilo_commons.py +++ b/postgres-appliance/scripts/spilo_commons.py @@ -12,13 +12,13 @@ # (min_version, max_version, shared_preload_libraries, extwlist.extensions) extensions = { - 'timescaledb': (9.6, 15, True, True), - 'pg_cron': (9.5, 15, True, False), - 'pg_stat_kcache': (9.4, 15, True, False), - 'pg_partman': (9.4, 15, False, True) + 'timescaledb': (9.6, 16, True, True), + 'pg_cron': (9.5, 16, True, False), + 'pg_stat_kcache': (9.4, 16, True, False), + 'pg_partman': (9.4, 16, False, True) } if os.environ.get('ENABLE_PG_MON') == 'true': - extensions['pg_mon'] = (11, 15, True, False) + extensions['pg_mon'] = (11, 16, True, False) def adjust_extensions(old, version, extwlist=False): diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index 90a89f9f9..5631d6f31 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -51,7 +51,7 @@ services: postgresql: parameters: shared_buffers: 32MB - PGVERSION: '10' + PGVERSION: '11' # Just to test upgrade with clone. Without CLONE_SCOPE they don't work CLONE_WAL_S3_BUCKET: *bucket CLONE_AWS_ACCESS_KEY_ID: *access_key diff --git a/postgres-appliance/tests/schema.sql b/postgres-appliance/tests/schema.sql index b4ea8224f..d646a8897 100644 --- a/postgres-appliance/tests/schema.sql +++ b/postgres-appliance/tests/schema.sql @@ -1,5 +1,4 @@ -CREATE EXTENSION pg_repack; -CREATE EXTENSION amcheck_next; /* the upgrade script must delete it before running pg_upgrade --check! */ +CREATE EXTENSION pg_repack; /* the upgrade script must delete it before running pg_upgrade --check! */ CREATE DATABASE test_db; \c test_db diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index 7a6c5ca3e..2ac170a25 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -132,15 +132,15 @@ function drop_timescaledb() { } function test_inplace_upgrade_wrong_version() { - docker_exec "$1" "PGVERSION=10 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' + docker_exec "$1" "PGVERSION=11 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' } function test_inplace_upgrade_wrong_capacity() { - docker_exec "$1" "PGVERSION=11 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' + docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' } -function test_successful_inplace_upgrade_to_11() { - docker_exec "$1" "PGVERSION=11 $UPGRADE_SCRIPT 3" +function test_successful_inplace_upgrade_to_12() { + docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 3" } function test_envdir_suffix() { @@ -155,7 +155,7 @@ function test_envdir_updated_to_x() { } function test_failed_inplace_upgrade_big_replication_lag() { - ! test_successful_inplace_upgrade_to_11 "$1" + ! test_successful_inplace_upgrade_to_12 "$1" } function test_successful_inplace_upgrade_to_13() { @@ -174,8 +174,12 @@ function test_successful_inplace_upgrade_to_15() { docker_exec "$1" "PGVERSION=15 $UPGRADE_SCRIPT 3" } -function test_pg_upgrade_to_15_check_failed() { - ! test_successful_inplace_upgrade_to_15 "$1" +function test_successful_inplace_upgrade_to_16() { + docker_exec "$1" "PGVERSION=16 $UPGRADE_SCRIPT 3" +} + +function test_pg_upgrade_to_16_check_failed() { + ! test_successful_inplace_upgrade_to_16 "$1" } function start_clone_with_wale_upgrade_container() { @@ -183,7 +187,7 @@ function start_clone_with_wale_upgrade_container() { docker-compose run \ -e SCOPE=upgrade \ - -e PGVERSION=11 \ + -e PGVERSION=12 \ -e CLONE_SCOPE=demo \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ @@ -196,27 +200,27 @@ function start_clone_with_wale_upgrade_replica_container() { start_clone_with_wale_upgrade_container 2 } -function start_clone_with_wale_upgrade_to_15_container() { +function start_clone_with_wale_upgrade_to_16_container() { docker-compose run \ -e SCOPE=upgrade3 \ - -e PGVERSION=15 \ + -e PGVERSION=16 \ -e CLONE_SCOPE=demo \ - -e CLONE_PGVERSION=10 \ + -e CLONE_PGVERSION=11 \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ --name "${PREFIX}upgrade4" \ -d "spilo3" } -function start_clone_with_wale_15_container() { +function start_clone_with_wale_16_container() { docker-compose run \ - -e SCOPE=clone15 \ - -e PGVERSION=15 \ + -e SCOPE=clone16 \ + -e PGVERSION=16 \ -e CLONE_SCOPE=upgrade3 \ - -e CLONE_PGVERSION=15 \ + -e CLONE_PGVERSION=16 \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_hour)" \ - --name "${PREFIX}clone15" \ + --name "${PREFIX}clone16" \ -d "spilo3" } @@ -224,7 +228,7 @@ function start_clone_with_basebackup_upgrade_container() { local container=$1 docker-compose run \ -e SCOPE=upgrade2 \ - -e PGVERSION=12 \ + -e PGVERSION=13 \ -e CLONE_SCOPE=upgrade \ -e CLONE_METHOD=CLONE_WITH_BASEBACKUP \ -e CLONE_HOST="$(docker_exec "$container" "hostname --ip-address")" \ @@ -250,17 +254,17 @@ function verify_archive_mode_is_on() { } -# TEST SUITE 1 - In-place major upgrade 10->11->...->15 -# TEST SUITE 2 - Major upgrade 10->15 after wal-e clone (with CLONE_PGVERSION set) +# TEST SUITE 1 - In-place major upgrade 11->12->...->16 +# TEST SUITE 2 - Major upgrade 11->16 after wal-e clone (with CLONE_PGVERSION set) # TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+) -# TEST SUITE 4 - Major upgrade 10->11 after wal-e clone (no CLONE_PGVERSION) +# TEST SUITE 4 - Major upgrade 11->12 after wal-e clone (no CLONE_PGVERSION) # TEST SUITE 5 - Replica bootstrap with wal-e # TEST SUITE 6 - Major upgrade 11->12 after clone with basebackup function test_spilo() { # TEST SUITE 1 local container=$1 - run_test test_envdir_suffix "$container" 10 + run_test test_envdir_suffix "$container" 11 log_info "[TS1] Testing wrong upgrade setups" run_test test_inplace_upgrade_wrong_version "$container" @@ -277,68 +281,69 @@ function test_spilo() { # TEST SUITE 2 local upgrade3_container - upgrade3_container=$(start_clone_with_wale_upgrade_to_15_container) # SCOPE=upgrade3 PGVERSION=15 CLONE: _SCOPE=demo _PGVERSION=10 _TARGET_TIME= - log_info "[TS2] Started $upgrade3_container for testing major upgrade 10->15 after clone with wal-e" + upgrade3_container=$(start_clone_with_wale_upgrade_to_16_container) # SCOPE=upgrade3 PGVERSION=16 CLONE: _SCOPE=demo _PGVERSION=11 _TARGET_TIME= + log_info "[TS2] Started $upgrade3_container for testing major upgrade 11->16 after clone with wal-e" # TEST SUITE 4 local upgrade_container - upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=11 CLONE: _SCOPE=demo _TARGET_TIME= - log_info "[TS4] Started $upgrade_container for testing major upgrade 10->11 after clone with wal-e" + upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=12 CLONE: _SCOPE=demo _TARGET_TIME= + log_info "[TS4] Started $upgrade_container for testing major upgrade 11->12 after clone with wal-e" # TEST SUITE 1 - log_info "[TS1] Testing in-place major upgrade 10->11" - run_test test_successful_inplace_upgrade_to_11 "$container" - wait_all_streaming "$container" - run_test test_envdir_updated_to_x 11 - - create_timescaledb "$container" # we don't install it at the beginning, as we do 10->15 in a clone + find_leader "$upgrade_container" # wait clone to finish and prevent timescale installation gets cloned log_info "[TS1] Testing in-place major upgrade 11->13 with failing check" + create_timescaledb "$container" # we don't install it at the beginning, as we do 11->16 in a clone create_table_with_oids "$container" run_test test_pg_upgrade_to_13_check_failed "$container" # pg_upgrade --check complains about OID drop_table_with_oids "$container" + log_info "[TS1] Testing in-place major upgrade 11->12" + wait_zero_lag "$container" + run_test test_successful_inplace_upgrade_to_12 "$container" + wait_all_streaming "$container" + run_test test_envdir_updated_to_x 12 # TEST SUITE 2 - log_info "[TS2] Testing in-place major upgrade 10->15 after wal-e clone" - run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 10 15 + log_info "[TS2] Testing in-place major upgrade 11->16 after wal-e clone" + run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 11 16 run_test verify_archive_mode_is_on "$upgrade3_container" wait_backup "$upgrade3_container" # TEST SUITE 3 - local clone15_container - clone15_container=$(start_clone_with_wale_15_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=15 _TARGET_TIME= - log_info "[TS3] Started $clone15_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" + local clone16_container + clone16_container=$(start_clone_with_wale_16_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=16 _TARGET_TIME= + log_info "[TS3] Started $clone16_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" # TEST SUITE 1 - log_info "[TS1] Testing in-place major upgrade 11->13" - run_test test_successful_inplace_upgrade_to_13 "$container" + log_info "[TS1] Testing in-place major upgrade 12->14" + run_test test_successful_inplace_upgrade_to_14 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 13 + run_test test_envdir_updated_to_x 14 # TEST SUITE 3 - find_leader "$clone15_container" - run_test verify_archive_mode_is_on "$clone15_container" + find_leader "$clone16_container" + run_test verify_archive_mode_is_on "$clone16_container" # TEST SUITE 1 wait_backup "$container" - log_info "[TS1] Testing in-place major upgrade 13->14" - run_test test_successful_inplace_upgrade_to_14 "$container" + log_info "[TS1] Testing in-place major upgrade to 14->15" + run_test test_successful_inplace_upgrade_to_15 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 14 + run_test test_envdir_updated_to_x 15 # TEST SUITE 4 - log_info "[TS4] Testing in-place major upgrade 10->11 after clone with wal-e" - run_test verify_clone_upgrade "$upgrade_container" "wal-e" 10 11 + log_info "[TS4] Testing in-place major upgrade 11->12 after clone with wal-e" + run_test verify_clone_upgrade "$upgrade_container" "wal-e" 11 12 run_test verify_archive_mode_is_on "$upgrade_container" wait_backup "$upgrade_container" @@ -352,30 +357,30 @@ function test_spilo() { # TEST SUITE 6 local basebackup_container - basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=12 CLONE: _SCOPE=upgrade - log_info "[TS6] Started $basebackup_container for testing major upgrade 11->12 after clone with basebackup" + basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=13 CLONE: _SCOPE=upgrade + log_info "[TS6] Started $basebackup_container for testing major upgrade 12->13 after clone with basebackup" # TEST SUITE 1 - # run_test test_pg_upgrade_to_15_check_failed "$container" # pg_upgrade --check complains about timescaledb + # run_test test_pg_upgrade_to_16_check_failed "$container" # pg_upgrade --check complains about timescaledb wait_backup "$container" # drop_timescaledb "$container" - log_info "[TS1] Testing in-place major upgrade 14->15" - run_test test_successful_inplace_upgrade_to_15 "$container" + log_info "[TS1] Testing in-place major upgrade 15->16" + run_test test_successful_inplace_upgrade_to_16 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 15 + run_test test_envdir_updated_to_x 16 # TEST SUITE 5 - log_info "[TS5] Waiting for postgres to start in $upgrade_replica_container and stream from primary..." + log_info "[TS5] Waiting for postgres to start in the $upgrade_replica_container and stream from primary..." wait_all_streaming "$upgrade_container" 1 # TEST SUITE 6 - log_info "[TS6] Testing in-place major upgrade 11->12 after clone with basebackup" - run_test verify_clone_upgrade "$basebackup_container" "basebackup" 11 12 + log_info "[TS6] Testing in-place major upgrade 12->13 after clone with basebackup" + run_test verify_clone_upgrade "$basebackup_container" "basebackup" 12 13 run_test verify_archive_mode_is_on "$basebackup_container" }