From 24743393d12f1609dffb9ddd8bf4885940704373 Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:08:58 +0200 Subject: [PATCH] Remove PG11 (#999) --- postgres-appliance/Dockerfile | 5 +- postgres-appliance/build_scripts/base.sh | 1 - postgres-appliance/scripts/_zmon_schema.dump | 61 +++----------- .../scripts/pg_partman/after-create.sql | 4 +- postgres-appliance/scripts/post_init.sh | 10 +-- postgres-appliance/scripts/wale_restore.sh | 6 +- postgres-appliance/tests/docker-compose.yml | 2 +- postgres-appliance/tests/test_spilo.sh | 81 ++++++++----------- 8 files changed, 53 insertions(+), 117 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 79e4ecb6e..e4651a65c 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=ubuntu:22.04 ARG PGVERSION=16 -ARG TIMESCALEDB="2.3.1 2.11.2 2.14.2" +ARG TIMESCALEDB="2.11.2 2.14.2" ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= @@ -48,14 +48,13 @@ ARG TIMESCALEDB ARG TIMESCALEDB_APACHE_ONLY=true ARG TIMESCALEDB_TOOLKIT=true ARG COMPRESS -ARG PGOLDVERSIONS="11 12 13 14 15" +ARG PGOLDVERSIONS="12 13 14 15" ARG WITH_PERL=false ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION" # Install PostgreSQL, extensions and contribs ENV POSTGIS_VERSION=3.4 \ - POSTGIS_LEGACY=3.3 \ BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \ PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \ PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \ diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 43ff533f7..e0d555d8a 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -266,7 +266,6 @@ if [ "$DEMO" != "true" ]; then d2="$d1" d1="../../${v1##*/}/$d1" if [ "${d2%-*}" = "contrib/postgis" ]; then - if [ "${v2##*/}" = "11" ]; then d2="${d2%-*}-$POSTGIS_LEGACY"; fi d1="../$d1" fi d2="$v2/$d2" diff --git a/postgres-appliance/scripts/_zmon_schema.dump b/postgres-appliance/scripts/_zmon_schema.dump index 9f416d94e..c032d0d86 100644 --- a/postgres-appliance/scripts/_zmon_schema.dump +++ b/postgres-appliance/scripts/_zmon_schema.dump @@ -45,29 +45,16 @@ DECLARE BEGIN server_version_num := current_setting('server_version_num')::integer; SELECT pg_is_in_recovery() INTO in_recovery; - IF server_version_num >= 100000 - THEN - SELECT CASE WHEN in_recovery THEN pg_last_wal_replay_lsn() - ELSE pg_current_wal_lsn() - END INTO xlog_location_string; - SELECT pg_last_wal_receive_lsn() INTO receive_location; - ELSE - SELECT CASE WHEN in_recovery THEN pg_last_xlog_replay_location() - ELSE pg_current_xlog_location() - END INTO xlog_location_string; - SELECT pg_last_xlog_receive_location() INTO receive_location; - END IF; + SELECT CASE WHEN in_recovery THEN pg_last_wal_replay_lsn() + ELSE pg_current_wal_lsn() + END INTO xlog_location_string; + SELECT pg_last_wal_receive_lsn() INTO receive_location; SELECT ('x'||lpad(split_part(xlog_location_string, '/', 1), 16, '0'))::bit(64)::bigint INTO wal_segment; SELECT ('x'||lpad(split_part(xlog_location_string, '/', 2), 16, '0'))::bit(64)::bigint INTO wal_offset; SELECT 1 FROM pg_ls_dir('.') as t(name) WHERE name = 'dontsync' INTO nosync; - IF server_version_num >= 90300 - THEN - wal_multiplier = CAST(x'FFFFFFFF' as bigint); - ELSE - wal_multiplier = CAST(x'FF000000' as bigint); - END IF; + wal_multiplier = CAST(x'FFFFFFFF' as bigint); - IF server_version_num >= 90100 AND in_recovery + IF in_recovery THEN wal_delay_seconds := extract(epoch from now() - pg_last_xact_replay_timestamp())::bigint; END IF; @@ -100,7 +87,7 @@ BEGIN 'synchronous_commit' ) UNION ALL - SELECT 'cluster_name', COALESCE(CASE WHEN server_version_num >= 90500 THEN current_setting('cluster_name') ELSE NULL END, substring(s.setting from E'/pgsql_([^/]+)/[^/]+/data$')) + SELECT 'cluster_name', COALESCE(current_setting('cluster_name'), substring(s.setting from E'/pgsql_([^/]+)/[^/]+/data$')) FROM pg_settings as s WHERE s.name = 'data_directory' UNION ALL @@ -119,11 +106,9 @@ BEGIN SELECT 'is_streaming' as name, CAST(receive_location IS NOT NULL AND in_recovery AS TEXT) as setting UNION ALL SELECT 'archive_nosync' as name, CAST(nosync IS NOT NULL AS TEXT) as setting; - -- pg_stat_activity column names differ depending on server version - IF server_version_num >= 90600 THEN - RETURN QUERY - SELECT a.name, a.setting - FROM unnest((select array[('active_connections'::text, + RETURN QUERY + SELECT a.name, a.setting + FROM unnest((select array[('active_connections'::text, count(CASE WHEN state = 'active' THEN 1 END)::text), ('idle_in_transaction_connections'::text, count(CASE WHEN state = 'idle in transaction' THEN 1 END)::text), @@ -137,28 +122,8 @@ BEGIN -- exclude autovacuum transactions coalesce(extract(epoch from statement_timestamp() - min(CASE WHEN query like 'autovacuum:%' THEN NULL ELSE xact_start END)),0)::text) ] - from pg_stat_activity - )) AS a (name text, setting text); - ELSE - RETURN QUERY - SELECT a.name, a.setting - FROM unnest((select array[('active_connections'::text, - count(CASE WHEN state = 'active' THEN 1 END)::text), - ('idle_in_transaction_connections'::text, - count(CASE WHEN state = 'idle in transaction' THEN 1 END)::text), - ('idle_in_transaction_max_age'::text, - coalesce(max(CASE WHEN state = 'idle in transaction' THEN extract(epoch from statement_timestamp() - state_change) END), 0)::text), - ('locked_connections'::text, - count(CASE WHEN waiting THEN 1 END)::text), - ('current_connections'::text, - count(1)::text), - ('transaction_max_age'::text, - -- exclude autovacuum transactions - coalesce(extract(epoch from statement_timestamp() - min(CASE WHEN query like 'autovacuum:%' THEN NULL ELSE xact_start END)),0)::text) - ] - from pg_stat_activity - )) AS a (name text, setting text); - END IF; + from pg_stat_activity + )) AS a (name text, setting text); END $BODY$ LANGUAGE plpgsql @@ -412,7 +377,7 @@ for r in rv: data_directory = s["data_directory"] log_directory = s["log_directory"] = os.path.join(data_directory, s["log_directory"]) pg_tblspc = os.path.join(data_directory, "pg_tblspc") -pg_xlog = os.path.join(data_directory, "pg_xlog" if int(s["server_version_num"]) < 100000 else "pg_wal") +pg_xlog = os.path.join(data_directory, "pg_wal") result = {} diff --git a/postgres-appliance/scripts/pg_partman/after-create.sql b/postgres-appliance/scripts/pg_partman/after-create.sql index def77bcbb..a7dce8e1d 100644 --- a/postgres-appliance/scripts/pg_partman/after-create.sql +++ b/postgres-appliance/scripts/pg_partman/after-create.sql @@ -8,9 +8,7 @@ BEGIN CREATE ROLE part_man ADMIN admin; END IF; - IF current_setting('server_version_num')::integer >= 110000 THEN - EXECUTE 'GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA @extschema@ TO part_man'; - END IF; + EXECUTE 'GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA @extschema@ TO part_man'; END;$$; GRANT USAGE ON SCHEMA @extschema@ TO part_man; GRANT ALL ON ALL TABLES IN SCHEMA @extschema@ TO part_man; diff --git a/postgres-appliance/scripts/post_init.sh b/postgres-appliance/scripts/post_init.sh index f76bf6585..407d9b982 100755 --- a/postgres-appliance/scripts/post_init.sh +++ b/postgres-appliance/scripts/post_init.sh @@ -5,7 +5,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 export PGOPTIONS="-c synchronous_commit=local -c search_path=pg_catalog" PGVER=$(psql -d "$2" -XtAc "SELECT pg_catalog.current_setting('server_version_num')::int/10000") -if [ "$PGVER" -ge 12 ]; then RESET_ARGS="oid, oid, bigint"; fi +RESET_ARGS="oid, oid, bigint" (echo "\set ON_ERROR_STOP on" echo "DO \$\$ @@ -192,12 +192,8 @@ CREATE EXTENSION IF NOT EXISTS set_user SCHEMA public; ALTER EXTENSION set_user UPDATE; GRANT EXECUTE ON FUNCTION public.set_user(text) TO admin; GRANT EXECUTE ON FUNCTION public.pg_stat_statements_reset($RESET_ARGS) TO admin;" - if [ "$PGVER" -lt 10 ]; then - echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO admin;" - else - echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;" - fi - if [ "$ENABLE_PG_MON" = "true" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi + echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;" + if [ "$ENABLE_PG_MON" = "true" ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi cat metric_helpers.sql done < <(psql -d "$2" -tAc 'select pg_catalog.quote_ident(datname) from pg_catalog.pg_database where datallowconn') ) | psql -Xd "$2" diff --git a/postgres-appliance/scripts/wale_restore.sh b/postgres-appliance/scripts/wale_restore.sh index 6afc0268d..497afe30f 100755 --- a/postgres-appliance/scripts/wale_restore.sh +++ b/postgres-appliance/scripts/wale_restore.sh @@ -61,11 +61,7 @@ if [[ $server_version != "-1" ]]; then readonly lsn_offset=$((16#${wal_segment_backup_start:16:8})) printf -v backup_start_lsn "%X/%X" $lsn_segment $((lsn_offset << 24)) - if [[ $server_version -ge 100000 ]]; then - readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_wal_lsn_diff(COALESCE(pg_last_wal_receive_lsn(), '0/0'), '$backup_start_lsn')::bigint, pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '$backup_start_lsn')::bigint) ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), '$backup_start_lsn')::bigint END" - else - readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(), '0/0'), '$backup_start_lsn')::bigint, pg_xlog_location_diff(pg_last_xlog_replay_location(), '$backup_start_lsn')::bigint) ELSE pg_xlog_location_diff(pg_current_xlog_location(), '$backup_start_lsn')::bigint END" - fi + readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_wal_lsn_diff(COALESCE(pg_last_wal_receive_lsn(), '0/0'), '$backup_start_lsn')::bigint, pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '$backup_start_lsn')::bigint) ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), '$backup_start_lsn')::bigint END" ATTEMPT=0 while true; do diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml index 5631d6f31..31886dba2 100644 --- a/postgres-appliance/tests/docker-compose.yml +++ b/postgres-appliance/tests/docker-compose.yml @@ -51,7 +51,7 @@ services: postgresql: parameters: shared_buffers: 32MB - PGVERSION: '11' + PGVERSION: '12' # Just to test upgrade with clone. Without CLONE_SCOPE they don't work CLONE_WAL_S3_BUCKET: *bucket CLONE_AWS_ACCESS_KEY_ID: *access_key diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index 70dccf711..478c44640 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -115,14 +115,6 @@ function create_schema() { docker_exec -i "$1" "psql -U postgres" < schema.sql } -function create_table_with_oids() { - docker_exec "$1" "psql -U postgres -d test_db -c 'CREATE TABLE with_oids() WITH OIDS'" -} - -function drop_table_with_oids() { - docker_exec "$1" "psql -U postgres -d test_db -c 'DROP TABLE with_oids'" -} - function create_timescaledb() { docker_exec -i "$1" "psql -U postgres" < timescaledb.sql } @@ -132,15 +124,15 @@ function drop_timescaledb() { } function test_inplace_upgrade_wrong_version() { - docker_exec "$1" "PGVERSION=11 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' + docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 3" 2>&1 | grep 'Upgrade is not required' } function test_inplace_upgrade_wrong_capacity() { - docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' + docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 4" 2>&1 | grep 'number of replicas does not match' } -function test_successful_inplace_upgrade_to_12() { - docker_exec "$1" "PGVERSION=12 $UPGRADE_SCRIPT 3" +function test_successful_inplace_upgrade_to_13() { + docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 3" } function test_envdir_suffix() { @@ -155,17 +147,13 @@ function test_envdir_updated_to_x() { } function test_failed_inplace_upgrade_big_replication_lag() { - ! test_successful_inplace_upgrade_to_12 "$1" + ! test_successful_inplace_upgrade_to_13 "$1" } function test_successful_inplace_upgrade_to_13() { docker_exec "$1" "PGVERSION=13 $UPGRADE_SCRIPT 3" } -function test_pg_upgrade_to_13_check_failed() { - ! test_successful_inplace_upgrade_to_13 "$1" -} - function test_successful_inplace_upgrade_to_14() { docker_exec "$1" "PGVERSION=14 $UPGRADE_SCRIPT 3" } @@ -187,7 +175,7 @@ function start_clone_with_wale_upgrade_container() { docker-compose run \ -e SCOPE=upgrade \ - -e PGVERSION=12 \ + -e PGVERSION=13 \ -e CLONE_SCOPE=demo \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ @@ -205,7 +193,7 @@ function start_clone_with_wale_upgrade_to_16_container() { -e SCOPE=upgrade3 \ -e PGVERSION=16 \ -e CLONE_SCOPE=demo \ - -e CLONE_PGVERSION=11 \ + -e CLONE_PGVERSION=12 \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ --name "${PREFIX}upgrade4" \ @@ -228,7 +216,7 @@ function start_clone_with_basebackup_upgrade_container() { local container=$1 docker-compose run \ -e SCOPE=upgrade2 \ - -e PGVERSION=13 \ + -e PGVERSION=14 \ -e CLONE_SCOPE=upgrade \ -e CLONE_METHOD=CLONE_WITH_BASEBACKUP \ -e CLONE_HOST="$(docker_exec "$container" "hostname --ip-address")" \ @@ -254,17 +242,17 @@ function verify_archive_mode_is_on() { } -# TEST SUITE 1 - In-place major upgrade 11->12->...->16 -# TEST SUITE 2 - Major upgrade 11->16 after wal-e clone (with CLONE_PGVERSION set) +# TEST SUITE 1 - In-place major upgrade 12->13->...->16 +# TEST SUITE 2 - Major upgrade 12->16 after wal-e clone (with CLONE_PGVERSION set) # TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+) -# TEST SUITE 4 - Major upgrade 11->12 after wal-e clone (no CLONE_PGVERSION) +# TEST SUITE 4 - Major upgrade 12->13 after wal-e clone (no CLONE_PGVERSION) # TEST SUITE 5 - Replica bootstrap with wal-e -# TEST SUITE 6 - Major upgrade 11->12 after clone with basebackup +# TEST SUITE 6 - Major upgrade 13->14 after clone with basebackup function test_spilo() { # TEST SUITE 1 local container=$1 - run_test test_envdir_suffix "$container" 11 + run_test test_envdir_suffix "$container" 12 log_info "[TS1] Testing wrong upgrade setups" run_test test_inplace_upgrade_wrong_version "$container" @@ -281,36 +269,31 @@ function test_spilo() { # TEST SUITE 2 local upgrade3_container - upgrade3_container=$(start_clone_with_wale_upgrade_to_16_container) # SCOPE=upgrade3 PGVERSION=16 CLONE: _SCOPE=demo _PGVERSION=11 _TARGET_TIME= - log_info "[TS2] Started $upgrade3_container for testing major upgrade 11->16 after clone with wal-e" + upgrade3_container=$(start_clone_with_wale_upgrade_to_16_container) # SCOPE=upgrade3 PGVERSION=16 CLONE: _SCOPE=demo _PGVERSION=12 _TARGET_TIME= + log_info "[TS2] Started $upgrade3_container for testing major upgrade 12->16 after clone with wal-e" # TEST SUITE 4 local upgrade_container - upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=12 CLONE: _SCOPE=demo _TARGET_TIME= - log_info "[TS4] Started $upgrade_container for testing major upgrade 11->12 after clone with wal-e" + upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=13 CLONE: _SCOPE=demo _TARGET_TIME= + log_info "[TS4] Started $upgrade_container for testing major upgrade 12->13 after clone with wal-e" # TEST SUITE 1 # wait clone to finish and prevent timescale installation gets cloned - find_leader "$upgrade_container" find_leader "$upgrade3_container" + find_leader "$upgrade_container" + create_timescaledb "$container" # we don't install it at the beginning, as we do 12->16 in a clone - log_info "[TS1] Testing in-place major upgrade 11->13 with failing check" - create_timescaledb "$container" # we don't install it at the beginning, as we do 11->16 in a clone - create_table_with_oids "$container" - run_test test_pg_upgrade_to_13_check_failed "$container" # pg_upgrade --check complains about OID - drop_table_with_oids "$container" - - log_info "[TS1] Testing in-place major upgrade 11->12" + log_info "[TS1] Testing in-place major upgrade 12->13" wait_zero_lag "$container" - run_test test_successful_inplace_upgrade_to_12 "$container" + run_test test_successful_inplace_upgrade_to_13 "$container" wait_all_streaming "$container" - run_test test_envdir_updated_to_x 12 + run_test test_envdir_updated_to_x 13 # TEST SUITE 2 - log_info "[TS2] Testing in-place major upgrade 11->16 after wal-e clone" - run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 11 16 + log_info "[TS2] Testing in-place major upgrade 12->16 after wal-e clone" + run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 12 16 run_test verify_archive_mode_is_on "$upgrade3_container" wait_backup "$upgrade3_container" @@ -318,12 +301,12 @@ function test_spilo() { # TEST SUITE 3 local clone16_container - clone16_container=$(start_clone_with_wale_16_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=16 _TARGET_TIME= + clone16_container=$(start_clone_with_wale_16_container) # SCOPE=clone16 CLONE: _SCOPE=upgrade3 _PGVERSION=16 _TARGET_TIME= log_info "[TS3] Started $clone16_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" # TEST SUITE 1 - log_info "[TS1] Testing in-place major upgrade 12->14" + log_info "[TS1] Testing in-place major upgrade 13->14" run_test test_successful_inplace_upgrade_to_14 "$container" wait_all_streaming "$container" run_test test_envdir_updated_to_x 14 @@ -344,8 +327,8 @@ function test_spilo() { # TEST SUITE 4 - log_info "[TS4] Testing in-place major upgrade 11->12 after clone with wal-e" - run_test verify_clone_upgrade "$upgrade_container" "wal-e" 11 12 + log_info "[TS4] Testing in-place major upgrade 12->13 after clone with wal-e" + run_test verify_clone_upgrade "$upgrade_container" "wal-e" 12 13 run_test verify_archive_mode_is_on "$upgrade_container" wait_backup "$upgrade_container" @@ -359,8 +342,8 @@ function test_spilo() { # TEST SUITE 6 local basebackup_container - basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=13 CLONE: _SCOPE=upgrade - log_info "[TS6] Started $basebackup_container for testing major upgrade 12->13 after clone with basebackup" + basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=14 CLONE: _SCOPE=upgrade + log_info "[TS6] Started $basebackup_container for testing major upgrade 13->14 after clone with basebackup" # TEST SUITE 1 @@ -381,8 +364,8 @@ function test_spilo() { # TEST SUITE 6 - log_info "[TS6] Testing in-place major upgrade 12->13 after clone with basebackup" - run_test verify_clone_upgrade "$basebackup_container" "basebackup" 12 13 + log_info "[TS6] Testing in-place major upgrade 13->14 after clone with basebackup" + run_test verify_clone_upgrade "$basebackup_container" "basebackup" 13 14 run_test verify_archive_mode_is_on "$basebackup_container" }