From bd2537659ed8d0f8b853e8ddb76e3365a355ed15 Mon Sep 17 00:00:00 2001 From: Roshan Kanwar Date: Thu, 11 Apr 2024 13:18:59 +0530 Subject: [PATCH] Implement wrapper function for babelfish inconsistency checks (#2465) This commit implements a wrapper function for all the checks to detect inconsistency between babelfish catalogs and pg catalogs. As of now, we are only checking the metadata consistency between these catalogs. Task: BABEL-4139 Signed-off-by: Roshan Kanwar --- .../check-babelfish-inconsistency/action.yml | 11 ++++ .github/workflows/major-version-upgrade.yml | 10 +++- .../workflows/singledb-version-upgrade.yml | 10 +++- contrib/babelfishpg_tsql/sql/ownership.sql | 20 ++++++++ .../babelfishpg_tsql--2.8.0--2.9.0.sql | 21 ++++++++ ...k_for_inconsistent_metadata-vu-cleanup.out | 2 + ...k_for_inconsistent_metadata-vu-prepare.out | 7 +++ ...ck_for_inconsistent_metadata-vu-verify.out | 50 +++++++++++++++++++ ...k_for_inconsistent_metadata-vu-cleanup.mix | 2 + ...k_for_inconsistent_metadata-vu-prepare.mix | 7 +++ ...ck_for_inconsistent_metadata-vu-verify.mix | 21 ++++++++ test/JDBC/upgrade/latest/schedule | 1 + .../expected_dependency.out | 1 + 13 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 .github/composite-actions/check-babelfish-inconsistency/action.yml create mode 100644 test/JDBC/expected/check_for_inconsistent_metadata-vu-cleanup.out create mode 100644 test/JDBC/expected/check_for_inconsistent_metadata-vu-prepare.out create mode 100644 test/JDBC/expected/check_for_inconsistent_metadata-vu-verify.out create mode 100644 test/JDBC/input/check_for_inconsistent_metadata-vu-cleanup.mix create mode 100644 test/JDBC/input/check_for_inconsistent_metadata-vu-prepare.mix create mode 100644 test/JDBC/input/check_for_inconsistent_metadata-vu-verify.mix diff --git a/.github/composite-actions/check-babelfish-inconsistency/action.yml b/.github/composite-actions/check-babelfish-inconsistency/action.yml new file mode 100644 index 0000000000..3d6cc89408 --- /dev/null +++ b/.github/composite-actions/check-babelfish-inconsistency/action.yml @@ -0,0 +1,11 @@ +name: 'Check Babelfish metadata inconsistency' +description: Check for Babelfish metadata inconsistency before Major/Minor Version Upgrade +runs: + using: 'composite' + steps: + - name: Check Babelfish metadata inconsistency + id: check-babelfish-inconsistency + run: | + output=$(sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT sys.check_for_inconsistent_metadata() GO" | tail -n +2) + echo "check_result=$output" >> "$GITHUB_OUTPUT" + shell: bash diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml index 0f9edac894..477c19a7f2 100644 --- a/.github/workflows/major-version-upgrade.yml +++ b/.github/workflows/major-version-upgrade.yml @@ -112,9 +112,17 @@ jobs: sudo echo "host all all $ipaddress/32 trust" >> pg_hba.conf shell: bash + - name: Check Babelfish metadata inconsistency before Major Version Upgrade + id: check-babelfish-inconsistency + if: always() && steps.setup-new-datadir.outcome == 'success' + uses: ./.github/composite-actions/check-babelfish-inconsistency + - name: Run pg_upgrade id: run-pg_upgrade - if: always() && steps.setup-new-datadir.outcome == 'success' + if: | + always() && steps.setup-new-datadir.outcome == 'success' + && steps.check-babelfish-inconsistency.outcome == 'success' + && steps.check-babelfish-inconsistency.outputs.check_result == 0 uses: ./.github/composite-actions/run-pg-upgrade - name: Run JDBC Tests diff --git a/.github/workflows/singledb-version-upgrade.yml b/.github/workflows/singledb-version-upgrade.yml index 8e903220a7..0fd562d430 100644 --- a/.github/workflows/singledb-version-upgrade.yml +++ b/.github/workflows/singledb-version-upgrade.yml @@ -24,9 +24,17 @@ jobs: install_dir: ${{ env.OLD_INSTALL_DIR }} migration_mode: 'single-db' + - name: Check Babelfish metadata inconsistency before Major Version Upgrade + id: check-babelfish-inconsistency + if: always() && steps.setup-base-version.outcome == 'success' + uses: ./.github/composite-actions/check-babelfish-inconsistency + - name: Upgrade to latest version and run verify-cleanup tests id: upgrade-and-verify - if: always() && steps.setup-base-version.outcome == 'success' + if: | + always() && steps.setup-base-version.outcome == 'success' + && steps.check-babelfish-inconsistency.outcome == 'success' + && steps.check-babelfish-inconsistency.outputs.check_result == 0 uses: ./.github/composite-actions/major-version-upgrade-util with: engine_branch: latest diff --git a/contrib/babelfishpg_tsql/sql/ownership.sql b/contrib/babelfishpg_tsql/sql/ownership.sql index 449168b2ba..8986304fef 100644 --- a/contrib/babelfishpg_tsql/sql/ownership.sql +++ b/contrib/babelfishpg_tsql/sql/ownership.sql @@ -495,6 +495,26 @@ RETURNS table ( detail jsonb ) AS 'babelfishpg_tsql', 'babelfish_inconsistent_metadata' LANGUAGE C STABLE; +CREATE OR REPLACE FUNCTION sys.check_for_inconsistent_metadata() +RETURNS BOOLEAN AS $$ +DECLARE + has_inconsistent_metadata BOOLEAN; + num_rows INT; +BEGIN + has_inconsistent_metadata := FALSE; + + -- Count the number of inconsistent metadata rows from Babelfish catalogs + SELECT COUNT(*) INTO num_rows + FROM sys.babelfish_inconsistent_metadata(); + + has_inconsistent_metadata := num_rows > 0; + + -- Additional checks can be added here to update has_inconsistent_metadata accordingly + + RETURN has_inconsistent_metadata; +END; +$$ +LANGUAGE plpgsql STABLE; CREATE OR REPLACE FUNCTION sys.role_id(role_name SYS.SYSNAME) RETURNS INT diff --git a/contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--2.8.0--2.9.0.sql b/contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--2.8.0--2.9.0.sql index 5c52c9d009..586432e27e 100644 --- a/contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--2.8.0--2.9.0.sql +++ b/contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--2.8.0--2.9.0.sql @@ -329,6 +329,27 @@ CREATE OR REPLACE FUNCTION sys.sp_tables_internal( $$ LANGUAGE plpgsql STABLE; +CREATE OR REPLACE FUNCTION sys.check_for_inconsistent_metadata() +RETURNS BOOLEAN AS $$ +DECLARE + has_inconsistent_metadata BOOLEAN; + num_rows INT; +BEGIN + has_inconsistent_metadata := FALSE; + + -- Count the number of inconsistent metadata rows from Babelfish catalogs + SELECT COUNT(*) INTO num_rows + FROM sys.babelfish_inconsistent_metadata(); + + has_inconsistent_metadata := num_rows > 0; + + -- Additional checks can be added here to update has_inconsistent_metadata accordingly + + RETURN has_inconsistent_metadata; +END; +$$ +LANGUAGE plpgsql STABLE; + -- Drops the temporary procedure used by the upgrade script. -- Please have this be one of the last statements executed in this upgrade script. DROP PROCEDURE sys.babelfish_drop_deprecated_object(varchar, varchar, varchar); diff --git a/test/JDBC/expected/check_for_inconsistent_metadata-vu-cleanup.out b/test/JDBC/expected/check_for_inconsistent_metadata-vu-cleanup.out new file mode 100644 index 0000000000..23bb3d3141 --- /dev/null +++ b/test/JDBC/expected/check_for_inconsistent_metadata-vu-cleanup.out @@ -0,0 +1,2 @@ +DROP FUNCTION IF EXISTS check_for_inconsistent_metadata_vu_prepare_func +GO diff --git a/test/JDBC/expected/check_for_inconsistent_metadata-vu-prepare.out b/test/JDBC/expected/check_for_inconsistent_metadata-vu-prepare.out new file mode 100644 index 0000000000..3f745ee0e8 --- /dev/null +++ b/test/JDBC/expected/check_for_inconsistent_metadata-vu-prepare.out @@ -0,0 +1,7 @@ +CREATE FUNCTION check_for_inconsistent_metadata_vu_prepare_func() +RETURNS BOOLEAN +AS +BEGIN +RETURN (SELECT sys.check_for_inconsistent_metadata()) +END +GO diff --git a/test/JDBC/expected/check_for_inconsistent_metadata-vu-verify.out b/test/JDBC/expected/check_for_inconsistent_metadata-vu-verify.out new file mode 100644 index 0000000000..15ab72c2ce --- /dev/null +++ b/test/JDBC/expected/check_for_inconsistent_metadata-vu-verify.out @@ -0,0 +1,50 @@ +-- psql + + + + + +-- Updating one entry from pg_proc pg catalog so that metadata inconsistency fails during check against babelfish_function_ext babelfish catalog +UPDATE pg_catalog.pg_proc SET proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong' WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func'; +SELECT proname, prosrc FROM pg_catalog.pg_proc WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong'; +SELECT nspname, funcname FROM sys.babelfish_function_ext WHERE funcname = 'check_for_inconsistent_metadata_vu_prepare_func'; +-- should return true because of inconsistency +SELECT sys.check_for_inconsistent_metadata(); +-- should return the inconsistent row data +SELECT sys.babelfish_inconsistent_metadata(); +UPDATE pg_catalog.pg_proc SET proname = 'check_for_inconsistent_metadata_vu_prepare_func' WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong'; +GO +~~ROW COUNT: 1~~ + +~~START~~ +name#!#text +check_for_inconsistent_metadata_vu_prepare_func_wrong#!#BEGINRETURN (SELECT sys.check_for_inconsistent_metadata())END +~~END~~ + +~~START~~ +name#!#name +master_dbo#!#check_for_inconsistent_metadata_vu_prepare_func +~~END~~ + +~~START~~ +bool +t +~~END~~ + +~~START~~ +record +(name,pg_catalog,proname,"{""Rule"": "" in babelfish_function_ext must also exist in pg_proc""}") +~~END~~ + +~~ROW COUNT: 1~~ + + +-- tsql +-- since data is consistent now, this should return 0 +SELECT check_for_inconsistent_metadata_vu_prepare_func() +GO +~~START~~ +bit +0 +~~END~~ + diff --git a/test/JDBC/input/check_for_inconsistent_metadata-vu-cleanup.mix b/test/JDBC/input/check_for_inconsistent_metadata-vu-cleanup.mix new file mode 100644 index 0000000000..23bb3d3141 --- /dev/null +++ b/test/JDBC/input/check_for_inconsistent_metadata-vu-cleanup.mix @@ -0,0 +1,2 @@ +DROP FUNCTION IF EXISTS check_for_inconsistent_metadata_vu_prepare_func +GO diff --git a/test/JDBC/input/check_for_inconsistent_metadata-vu-prepare.mix b/test/JDBC/input/check_for_inconsistent_metadata-vu-prepare.mix new file mode 100644 index 0000000000..3f745ee0e8 --- /dev/null +++ b/test/JDBC/input/check_for_inconsistent_metadata-vu-prepare.mix @@ -0,0 +1,7 @@ +CREATE FUNCTION check_for_inconsistent_metadata_vu_prepare_func() +RETURNS BOOLEAN +AS +BEGIN +RETURN (SELECT sys.check_for_inconsistent_metadata()) +END +GO diff --git a/test/JDBC/input/check_for_inconsistent_metadata-vu-verify.mix b/test/JDBC/input/check_for_inconsistent_metadata-vu-verify.mix new file mode 100644 index 0000000000..c5754f479e --- /dev/null +++ b/test/JDBC/input/check_for_inconsistent_metadata-vu-verify.mix @@ -0,0 +1,21 @@ +-- Updating one entry from pg_proc pg catalog so that metadata inconsistency fails during check against babelfish_function_ext babelfish catalog +-- psql +UPDATE pg_catalog.pg_proc SET proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong' WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func'; + +SELECT proname, prosrc FROM pg_catalog.pg_proc WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong'; + +SELECT nspname, funcname FROM sys.babelfish_function_ext WHERE funcname = 'check_for_inconsistent_metadata_vu_prepare_func'; + +-- should return true because of inconsistency +SELECT sys.check_for_inconsistent_metadata(); + +-- should return the inconsistent row data +SELECT sys.babelfish_inconsistent_metadata(); + +UPDATE pg_catalog.pg_proc SET proname = 'check_for_inconsistent_metadata_vu_prepare_func' WHERE proname = 'check_for_inconsistent_metadata_vu_prepare_func_wrong'; +GO + +-- tsql +-- since data is consistent now, this should return 0 +SELECT check_for_inconsistent_metadata_vu_prepare_func() +GO diff --git a/test/JDBC/upgrade/latest/schedule b/test/JDBC/upgrade/latest/schedule index fca297aad6..2f150565d9 100644 --- a/test/JDBC/upgrade/latest/schedule +++ b/test/JDBC/upgrade/latest/schedule @@ -10,6 +10,7 @@ BABEL-3613 babelfish_cast_floor +check_for_inconsistent_metadata babel_try_parse TestBigInt TestBinary diff --git a/test/python/expected/upgrade_validation/expected_dependency.out b/test/python/expected/upgrade_validation/expected_dependency.out index 574377242b..aecfc08497 100644 --- a/test/python/expected/upgrade_validation/expected_dependency.out +++ b/test/python/expected/upgrade_validation/expected_dependency.out @@ -281,6 +281,7 @@ Function sys.char_to_fixeddecimal(sys."varchar") Function sys.char_to_fixeddecimal(sys.bpchar) Function sys.char_to_fixeddecimal(text) Function sys.charindex(text,text,integer) +Function sys.check_for_inconsistent_metadata() Function sys.checksum(text[]) Function sys.collationproperty(text,text) Function sys.columnproperty(oid,name,text)