Skip to content

Commit

Permalink
Refactor tests (#933)
Browse files Browse the repository at this point in the history 
- wait_all_streaming/wait_zero_lag with configurable replica count
- create table with oids using special func, not in the initial schema
- add pg_repack extension to the initial schema
- separate func for creating timescaledb objects
- set WALE_BACKUP_THRESHOLD_PERCENTAGE to a higher value to ensure wal-e
  replica bootsrap method in test suite 5
- single verify_clone_upgrade func
- adjust comments and formatting
- ensure test suite 4 actually restores latest backup from PG10 and then upgrades
  • Loading branch information
@hughcapet
hughcapet authored Oct 6, 2023
1 parent e576430 commit d554db0
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 69 deletions.
3 changes: 3 additions & 0 deletions postgres-appliance/tests/schema.sql
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
CREATE EXTENSION pg_repack;
CREATE EXTENSION amcheck_next; /* the upgrade script must delete it before running pg_upgrade --check! */

CREATE DATABASE test_db;
\c test_db

Expand Down
133 changes: 68 additions & 65 deletions postgres-appliance/tests/test_spilo.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,27 +100,33 @@ function wait_query() {
}

function wait_all_streaming() {
log_info "Waiting for all replicas to start streaming from the leader..."
wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" 2
local repl_count=${2:-2}
log_info "Waiting for all replicas to start streaming from the leader ($1)..."
wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" "$repl_count"
}

function wait_zero_lag() {
local repl_count=${2:-2}
log_info "Waiting for all replicas to catch up with WAL replay..."
wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" 2
wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" "$repl_count"
}

function create_schema() {
docker_exec -i "$1" "psql -U postgres" < schema.sql
}

function create_schema2() {
docker_exec -i "$1" "psql -U postgres" < schema2.sql
function create_table_with_oids() {
docker_exec "$1" "psql -U postgres -d test_db -c 'CREATE TABLE with_oids() WITH OIDS'"
}

function drop_table_with_oids() {
docker_exec "$1" "psql -U postgres -d test_db -c 'DROP TABLE with_oids'"
}

function create_timescaledb() {
docker_exec -i "$1" "psql -U postgres" < timescaledb.sql
}

function drop_timescaledb() {
docker_exec "$1" "psql -U postgres -d test_db -c 'DROP EXTENSION timescaledb CASCADE'"
}
Expand Down Expand Up @@ -181,6 +187,7 @@ function start_clone_with_wale_upgrade_container() {
-e CLONE_SCOPE=demo \
-e CLONE_METHOD=CLONE_WITH_WALE \
-e CLONE_TARGET_TIME="$(next_minute)" \
-e WALE_BACKUP_THRESHOLD_PERCENTAGE=80 \
--name "${PREFIX}upgrade$ID" \
-d "spilo$ID"
}
Expand Down Expand Up @@ -228,20 +235,13 @@ function start_clone_with_basebackup_upgrade_container() {
-d spilo3
}

function verify_clone_with_wale_upgrade() {
wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 11 2> /dev/null
}

function verify_clone_with_basebackup_upgrade() {
log_info "Waiting for clone with basebackup and upgrade 11->12 to complete..."
function verify_clone_upgrade() {
local type=$2
local from_version=$3
local to_version=$4
log_info "Waiting for clone with $type and upgrade $from_version->$to_version to complete..."
find_leader "$1" 1
wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 12 2> /dev/null
}

function verify_clone_with_wale_upgrade_to_15() {
log_info "Waiting for clone with wal-e and upgrade 10->15 to complete..."
find_leader "$1" 1
wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 15 2> /dev/null
wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" "$to_version" 2> /dev/null
}

function verify_archive_mode_is_on() {
Expand All @@ -251,128 +251,131 @@ function verify_archive_mode_is_on() {


# TEST SUITE 1 - In-place major upgrade 10->11->...->15
# TEST SUITE 2 - Major upgrade 10->15 after wal-e clone
# TEST SUITE 2 - Major upgrade 10->15 after wal-e clone (with CLONE_PGVERSION set)
# TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+)
# TEST SUITE 4 - Major upgrade 10->11 after wal-e clone
# TEST SUITE 4 - Major upgrade 10->11 after wal-e clone (no CLONE_PGVERSION)
# TEST SUITE 5 - Replica bootstrap with wal-e
# TEST SUITE 6 - Major upgrade 11->12 after clone with basebackup
function test_spilo() {
# TEST SUITE 1
local container=$1

run_test test_envdir_suffix "$container" 10

log_info "[TS1] Testing wrong upgrade setups"
run_test test_inplace_upgrade_wrong_version "$container"
run_test test_inplace_upgrade_wrong_capacity "$container"

wait_all_streaming "$container"

create_schema "$container" || exit 1

create_schema "$container" || exit 1 # incompatible upgrade exts, custom tbl with statistics and data
# run_test test_failed_inplace_upgrade_big_replication_lag "$container"

wait_zero_lag "$container"
run_test verify_archive_mode_is_on "$container"
wait_backup "$container"


# TEST SUITE 2
local upgrade3_container
upgrade3_container=$(start_clone_with_wale_upgrade_to_15_container) # SCOPE=upgrade3 PGVERSION=15 CLONE: _SCOPE=demo _PGVERSION=10 _TARGET_TIME=<next_min>
log_info "[TS2] Started $upgrade3_container for testing major upgrade 10->15 after clone with wal-e"


# TEST SUITE 4
local upgrade_container
upgrade_container=$(start_clone_with_wale_upgrade_to_15_container)
log_info "Started $upgrade_container for testing major upgrade 10->15 after clone with wal-e"
upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=11 CLONE: _SCOPE=demo _TARGET_TIME=<next_min>
log_info "[TS4] Started $upgrade_container for testing major upgrade 10->11 after clone with wal-e"

# TEST SUITE 1
wait_backup "$container"

log_info "Testing in-place major upgrade 10->11"
# TEST SUITE 1
log_info "[TS1] Testing in-place major upgrade 10->11"
run_test test_successful_inplace_upgrade_to_11 "$container"

wait_all_streaming "$container"

run_test test_envdir_updated_to_x 11

create_schema2 "$container" || exit 1
create_timescaledb "$container" # we don't install it at the beginning, as we do 10->15 in a clone

log_info "[TS1] Testing in-place major upgrade 11->13 with failing check"
create_table_with_oids "$container"
run_test test_pg_upgrade_to_13_check_failed "$container" # pg_upgrade --check complains about OID
drop_table_with_oids "$container"


# TEST SUITE 2
run_test verify_clone_with_wale_upgrade_to_15 "$upgrade_container"
log_info "[TS2] Testing in-place major upgrade 10->15 after wal-e clone"
run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 10 15

run_test verify_archive_mode_is_on "$upgrade3_container"
wait_backup "$upgrade3_container"

run_test verify_archive_mode_is_on "$upgrade_container"
wait_backup "$upgrade_container"
rm_container "$upgrade_container"

# TEST SUITE 3
local clone15_container
clone15_container=$(start_clone_with_wale_15_container)
log_info "Started $clone15_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+"
clone15_container=$(start_clone_with_wale_15_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=15 _TARGET_TIME=<next_hour>
log_info "[TS3] Started $clone15_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+"

# TEST SUITE 4
wait_backup "$container"
wait_zero_lag "$container"
upgrade_container=$(start_clone_with_wale_upgrade_container)
log_info "Started $upgrade_container for testing major upgrade 10->11 after clone with wal-e"

# TEST SUITE 1
log_info "Testing in-place major upgrade 11->13"
log_info "[TS1] Testing in-place major upgrade 11->13"
run_test test_successful_inplace_upgrade_to_13 "$container"

wait_all_streaming "$container"

run_test test_envdir_updated_to_x 13


# TEST SUITE 3
find_leader "$clone15_container"
run_test verify_archive_mode_is_on "$clone15_container"


# TEST SUITE 1
wait_backup "$container"

log_info "Testing in-place major upgrade to 13->14"
log_info "[TS1] Testing in-place major upgrade 13->14"
run_test test_successful_inplace_upgrade_to_14 "$container"

wait_all_streaming "$container"

run_test test_envdir_updated_to_x 14


# TEST SUITE 4
log_info "Waiting for clone with wal-e and upgrade 10->11 to complete..."
find_leader "$upgrade_container" 1
run_test verify_clone_with_wale_upgrade "$upgrade_container"
log_info "[TS4] Testing in-place major upgrade 10->11 after clone with wal-e"
run_test verify_clone_upgrade "$upgrade_container" "wal-e" 10 11

run_test verify_archive_mode_is_on "$upgrade_container"
wait_backup "$upgrade_container"


# TEST SUITE 5
local upgrade_replica_container
upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container)
log_info "Started $upgrade_replica_container for testing replica bootstrap with wal-e"
upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container) # SCOPE=upgrade
log_info "[TS5] Started $upgrade_replica_container for testing replica bootstrap with wal-e"

# TEST SUITE 4

# TEST SUITE 6
local basebackup_container
basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container")
log_info "Started $basebackup_container for testing major upgrade 11->12 after clone with basebackup"
basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=12 CLONE: _SCOPE=upgrade
log_info "[TS6] Started $basebackup_container for testing major upgrade 11->12 after clone with basebackup"


# TEST SUITE 1
# run_test test_pg_upgrade_to_15_check_failed "$container" # pg_upgrade --check complains about timescaledb

wait_backup "$container"

# drop_timescaledb "$container"

log_info "Testing in-place major upgrade to 14->15"
log_info "[TS1] Testing in-place major upgrade 14->15"
run_test test_successful_inplace_upgrade_to_15 "$container"

wait_all_streaming "$container"

run_test test_envdir_updated_to_x 15

wait_backup "$container"

# TEST SUITE 5
log_info "Waiting for postgres to start in the $upgrade_replica_container..."
run_test verify_clone_with_wale_upgrade "$upgrade_replica_container"
log_info "[TS5] Waiting for postgres to start in $upgrade_replica_container and stream from primary..."
wait_all_streaming "$upgrade_container" 1

# TEST SUITE 4
run_test verify_clone_with_basebackup_upgrade "$basebackup_container"

# TEST SUITE 6
log_info "[TS6] Testing in-place major upgrade 11->12 after clone with basebackup"
run_test verify_clone_upgrade "$basebackup_container" "basebackup" 11 12
run_test verify_archive_mode_is_on "$basebackup_container"
}

Expand Down
4 changes: 0 additions & 4 deletions postgres-appliance/tests/schema2.sql → postgres-appliance/tests/timescaledb.sql
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
CREATE EXTENSION amcheck_next; /* the upgrade script must delete it before running pg_upgrade --check! */

\c test_db

CREATE TABLE with_oids() WITH OIDS;

CREATE EXTENSION timescaledb;

CREATE TABLE "fOo" (id bigint NOT NULL PRIMARY KEY);
Expand Down

0 comments on commit d554db0

Please sign in to comment.