From 4c4211627fce919b855e9cc73ece4edd0afa7a47 Mon Sep 17 00:00:00 2001 From: Tim Chang Date: Thu, 12 Oct 2023 17:47:53 +0000 Subject: [PATCH] Support session-local OID buffer for temp tables in TSQL Signed-off-by: Tim Chang Remove stress test due to timeout Support full OID range Make GUC unalterable for now blank commit empty Add null check in logical schema Remove unnecessary volatile blank Adding some simple unit testing Signed-off-by: Tim Chang Added tests to upgrade Signed-off-by: Tim Chang Added new JDBC tests for temp tables Moved new OID generation functions into extension hooks Signed-off-by: Tim Chang Added extra tests for buffer Signed-off-by: Tim Chang Minor fixes Signed-off-by: Tim Chang Rerun tests Bump SLA Signed-off-by: Tim Chang Add invariant checks in functions, added unit tests, expanded JDBC tests Signed-off-by: Tim Chang Minor test changes, comments --- contrib/babelfishpg_tsql/src/catalog.c | 2 +- contrib/babelfishpg_tsql/src/guc.c | 25 ++ contrib/babelfishpg_tsql/src/hooks.c | 292 +++++++++++++++ contrib/babelfishpg_unit/Makefile | 2 +- contrib/babelfishpg_unit/babelfishpg_unit.c | 7 + contrib/babelfishpg_unit/babelfishpg_unit.h | 7 + contrib/babelfishpg_unit/test_temp_tables.c | 328 +++++++++++++++++ test/JDBC/expected/temp-tables-vu-cleanup.out | 6 + test/JDBC/expected/temp-tables-vu-prepare.out | 6 + test/JDBC/expected/temp-tables-vu-verify.out | 85 +++++ test/JDBC/expected/temp_table_jdbc.out | 0 .../temp_tables_concurrent-vu-cleanup.out | 9 + .../temp_tables_concurrent-vu-prepare.out | 9 + .../temp_tables_concurrent-vu-verify.out | 45 +++ .../JDBC/input/ddl/temp-tables-vu-cleanup.sql | 6 + .../JDBC/input/ddl/temp-tables-vu-prepare.sql | 6 + test/JDBC/input/ddl/temp-tables-vu-verify.sql | 47 ++- .../input/temp_tables/temp_table_jdbc.txt | 1 + .../temp_tables_concurrent-vu-cleanup.mix | 9 + .../temp_tables_concurrent-vu-prepare.mix | 9 + .../temp_tables_concurrent-vu-verify.mix | 35 ++ .../java/com/sqlsamples/JDBCTempTable.java | 348 ++++++++++++++++++ .../java/com/sqlsamples/TestQueryFile.java | 8 +- 23 files changed, 1288 insertions(+), 4 deletions(-) create mode 100644 contrib/babelfishpg_unit/test_temp_tables.c create mode 100644 test/JDBC/expected/temp_table_jdbc.out create mode 100644 test/JDBC/expected/temp_tables_concurrent-vu-cleanup.out create mode 100644 test/JDBC/expected/temp_tables_concurrent-vu-prepare.out create mode 100644 test/JDBC/expected/temp_tables_concurrent-vu-verify.out create mode 100644 test/JDBC/input/temp_tables/temp_table_jdbc.txt create mode 100644 test/JDBC/input/temp_tables/temp_tables_concurrent-vu-cleanup.mix create mode 100644 test/JDBC/input/temp_tables/temp_tables_concurrent-vu-prepare.mix create mode 100644 test/JDBC/input/temp_tables/temp_tables_concurrent-vu-verify.mix create mode 100644 test/JDBC/src/main/java/com/sqlsamples/JDBCTempTable.java diff --git a/contrib/babelfishpg_tsql/src/catalog.c b/contrib/babelfishpg_tsql/src/catalog.c index c9b91d8af0..458af485e6 100644 --- a/contrib/babelfishpg_tsql/src/catalog.c +++ b/contrib/babelfishpg_tsql/src/catalog.c @@ -514,7 +514,7 @@ get_logical_schema_name(const char *physical_schema_name, bool missingOk) TupleDesc dsc; bool isnull; - if (get_namespace_oid(physical_schema_name, missingOk) == InvalidOid) + if (!physical_schema_name || get_namespace_oid(physical_schema_name, missingOk) == InvalidOid) return NULL; rel = table_open(namespace_ext_oid, AccessShareLock); diff --git a/contrib/babelfishpg_tsql/src/guc.c b/contrib/babelfishpg_tsql/src/guc.c index 7dc52f2cc0..60fc125ec3 100644 --- a/contrib/babelfishpg_tsql/src/guc.c +++ b/contrib/babelfishpg_tsql/src/guc.c @@ -1143,6 +1143,31 @@ define_custom_variables(void) GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_DISALLOW_IN_AUTO_FILE, check_babelfish_dump_restore_min_oid, NULL, NULL); + /* + * This is an int here but actually represents an Oid (uint32). It's accessed only in hooks.c using the + * macros defined there. + */ + DefineCustomIntVariable("babelfishpg_tsql.temp_oid_buffer_start", + gettext_noop("Internal. Specifies the start range of the buffer for temp oids"), + NULL, + &temp_oid_buffer_start, + INT_MIN, INT_MIN, INT_MAX, + PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_DISALLOW_IN_AUTO_FILE, + NULL, NULL, NULL); + + /* + * For now, Oid buffer size will not be adjustable by users. A value of 0 will disable temp oid generation. + */ + DefineCustomIntVariable("babelfishpg_tsql.temp_oid_buffer_size", + gettext_noop("Temp oid buffer size"), + NULL, + &temp_oid_buffer_size, + 65536, 0, 131072, + PGC_SUSET, + GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE | GUC_DISALLOW_IN_AUTO_FILE, + NULL, NULL, NULL); + /* T-SQL Hint Mapping */ DefineCustomBoolVariable("babelfishpg_tsql.enable_hint_mapping", gettext_noop("Enables T-SQL hint mapping in ANTLR parser"), diff --git a/contrib/babelfishpg_tsql/src/hooks.c b/contrib/babelfishpg_tsql/src/hooks.c index ee6f8b1a69..956eaa4af6 100644 --- a/contrib/babelfishpg_tsql/src/hooks.c +++ b/contrib/babelfishpg_tsql/src/hooks.c @@ -1,8 +1,11 @@ #include "postgres.h" +#include + #include "access/genam.h" #include "access/htup.h" #include "access/table.h" +#include "access/transam.h" #include "catalog/heap.h" #include "utils/pg_locale.h" #include "access/xact.h" @@ -12,13 +15,16 @@ #include "catalog/pg_aggregate.h" #include "catalog/pg_attrdef_d.h" #include "catalog/pg_authid.h" +#include "catalog/pg_db_role_setting.h" #include "catalog/pg_namespace.h" #include "catalog/pg_proc.h" #include "catalog/pg_trigger.h" #include "catalog/pg_trigger_d.h" #include "catalog/pg_type.h" #include "catalog/pg_operator.h" +#include "catalog/pg_tablespace.h" #include "commands/copy.h" +#include "commands/dbcommands.h" #include "commands/explain.h" #include "commands/tablecmds.h" #include "commands/view.h" @@ -52,6 +58,7 @@ #include "utils/rel.h" #include "utils/relcache.h" #include "utils/ruleutils.h" +#include "utils/snapmgr.h" #include "utils/syscache.h" #include "utils/numeric.h" #include @@ -75,6 +82,11 @@ extern char *babelfish_dump_restore_min_oid; extern bool pltsql_quoted_identifier; extern bool pltsql_ansi_nulls; +#define OID_TO_BUFFER_START(oid) ((oid) + INT_MIN) +#define BUFFER_START_TO_OID ((Oid) (temp_oid_buffer_start) - INT_MIN) + +/* For unit testing, to avoid concurrent heap update issues. */ +bool TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; /***************************************** * Catalog Hooks @@ -143,6 +155,9 @@ static bool pltsql_detect_numeric_overflow(int weight, int dscale, int first_blo static void insert_pltsql_function_defaults(HeapTuple func_tuple, List *defaults, Node **argarray); static int print_pltsql_function_arguments(StringInfo buf, HeapTuple proctup, bool print_table_args, bool print_defaults); static void pltsql_GetNewObjectId(VariableCache variableCache); +static Oid pltsql_GetNewTempObjectId(void); +static Oid pltsql_GetNewTempOidWithIndex(Relation relation, Oid indexId, AttrNumber oidcolumn); +static bool set_and_persist_temp_oid_buffer_start(Oid new_oid); static void pltsql_validate_var_datatype_scale(const TypeName *typeName, Type typ); static bool pltsql_bbfCustomProcessUtility(ParseState *pstate, PlannedStmt *pstmt, @@ -210,6 +225,8 @@ static ExecutorRun_hook_type prev_ExecutorRun = NULL; static ExecutorFinish_hook_type prev_ExecutorFinish = NULL; static ExecutorEnd_hook_type prev_ExecutorEnd = NULL; static GetNewObjectId_hook_type prev_GetNewObjectId_hook = NULL; +static GetNewTempObjectId_hook_type prev_GetNewTempObjectId_hook = NULL; +static GetNewTempOidWithIndex_hook_type prev_GetNewTempOidWithIndex_hook = NULL; static inherit_view_constraints_from_table_hook_type prev_inherit_view_constraints_from_table = NULL; static bbfViewHasInsteadofTrigger_hook_type prev_bbfViewHasInsteadofTrigger_hook = NULL; static detect_numeric_overflow_hook_type prev_detect_numeric_overflow_hook = NULL; @@ -323,6 +340,12 @@ InstallExtendedHooks(void) prev_GetNewObjectId_hook = GetNewObjectId_hook; GetNewObjectId_hook = pltsql_GetNewObjectId; + prev_GetNewTempObjectId_hook = GetNewTempObjectId_hook; + GetNewTempObjectId_hook = pltsql_GetNewTempObjectId; + + prev_GetNewTempOidWithIndex_hook = GetNewTempOidWithIndex_hook; + GetNewTempOidWithIndex_hook = pltsql_GetNewTempOidWithIndex; + prev_inherit_view_constraints_from_table = inherit_view_constraints_from_table_hook; inherit_view_constraints_from_table_hook = preserve_view_constraints_from_base_table; TriggerRecuresiveCheck_hook = plsql_TriggerRecursiveCheck; @@ -438,6 +461,8 @@ UninstallExtendedHooks(void) ExecutorFinish_hook = prev_ExecutorFinish; ExecutorEnd_hook = prev_ExecutorEnd; GetNewObjectId_hook = prev_GetNewObjectId_hook; + GetNewTempObjectId_hook = prev_GetNewTempObjectId_hook; + GetNewTempOidWithIndex_hook = prev_GetNewTempOidWithIndex_hook; inherit_view_constraints_from_table_hook = prev_inherit_view_constraints_from_table; bbfViewHasInsteadofTrigger_hook = prev_bbfViewHasInsteadofTrigger_hook; detect_numeric_overflow_hook = prev_detect_numeric_overflow_hook; @@ -557,6 +582,191 @@ pltsql_GetNewObjectId(VariableCache variableCache) ShmemVariableCache->oidCount = 0; } +static Oid +pltsql_GetNewTempObjectId() +{ + Oid result; + Oid tempOidStart; + static Oid nextTempOid = InvalidOid; + + /* safety check, we should never get this far in a HS standby */ + if (RecoveryInProgress()) + elog(ERROR, "cannot assign OIDs during recovery"); + + /* + * temp_oid_buffer_size = 0 would indicate that the feature is + * disabled, so we shouldn't even reach this code. + */ + if (temp_oid_buffer_size <= 0) + elog(ERROR, "temp_oid_buffer use is disabled"); + + if (!OidIsValid(BUFFER_START_TO_OID)) /* InvalidOid means it needs assigning */ + { + /* First check to see if another connection has already picked a start, then update. */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + if (OidIsValid(ShmemVariableCache->tempOidStart)) + { + /* + * Persist the newfound value of temp_oid_buffer_start to disk (via pg_settings). + * + * If for whatever reason this fails, we will fallback to manually updating the GUC, + * which won't be crash-resilient, but won't cause loss of functionality. + */ + if (!set_and_persist_temp_oid_buffer_start(ShmemVariableCache->tempOidStart)) + { + elog(WARNING, "unable to persist temp_oid_buffer_start"); + temp_oid_buffer_start = OID_TO_BUFFER_START(ShmemVariableCache->tempOidStart); + } + nextTempOid = ShmemVariableCache->tempOidStart; + } + else + { + /* We need to pick a new start for the buffer range. */ + tempOidStart = ShmemVariableCache->nextOid; + + /* + * Decrement ShmemVariableCache->oidCount to take into account the new buffer we're allocating + */ + if (ShmemVariableCache->oidCount < temp_oid_buffer_size) + ShmemVariableCache->oidCount = 0; + else + ShmemVariableCache->oidCount -= temp_oid_buffer_size; + + /* + * If ShmemVariableCache->nextOid is below FirstNormalObjectId then we can start at FirstNormalObjectId here and + * GetNewObjectId will return the right value on the next call. + */ + if (tempOidStart < FirstNormalObjectId) + tempOidStart = FirstNormalObjectId; + + /* If the OID range would wraparound, start from beginning instead. */ + if (tempOidStart + temp_oid_buffer_size < tempOidStart) + { + tempOidStart = FirstNormalObjectId; + + /* As in GetNewObjectId - wraparound in standalone mode (unlikely but possible) */ + ShmemVariableCache->oidCount = 0; + } + + if (!set_and_persist_temp_oid_buffer_start(tempOidStart)) + { + elog(WARNING, "unable to persist temp_oid_buffer_start"); + temp_oid_buffer_start = OID_TO_BUFFER_START(ShmemVariableCache->tempOidStart); + } + ShmemVariableCache->tempOidStart = tempOidStart; + + nextTempOid = (Oid) tempOidStart; + + /* Skip nextOid ahead to end of range here as well. */ + ShmemVariableCache->nextOid = (Oid) (tempOidStart + temp_oid_buffer_size); + } + + /* + * Invariant checks: + * - We should not be in the restricted OID range + * - We should be continuous (IE the end of the buffer shouldn't wrap around to restricted OID range) + * - We should be separate from the normal OID range + */ + if ((BUFFER_START_TO_OID < FirstNormalObjectId) + || (((Oid) (BUFFER_START_TO_OID + temp_oid_buffer_size)) < BUFFER_START_TO_OID)) + elog(ERROR, "OID buffer start is invalid"); + if (ShmemVariableCache->nextOid < ((Oid) (BUFFER_START_TO_OID + temp_oid_buffer_size)) && ShmemVariableCache->nextOid > BUFFER_START_TO_OID) + elog(ERROR, "Normal OID range and Temp OID buffer intersect"); + + /* If we run out of logged for use oids then we must log more */ + if (ShmemVariableCache->oidCount == 0) + { + XLogPutNextOid(ShmemVariableCache->nextOid + GetVarOidPrefetch()); + ShmemVariableCache->oidCount = GetVarOidPrefetch(); + } + + LWLockRelease(OidGenLock); + } + + /* + * Check for wraparound of the temp OID buffer. + */ + if (nextTempOid >= (Oid) (BUFFER_START_TO_OID + temp_oid_buffer_size) + || nextTempOid < BUFFER_START_TO_OID) + { + /* + * Invariant checks: + * - We should not be in the restricted OID range + * - We should be continuous (IE the end of the buffer shouldn't wrap around to restricted OID range) + * - We should be separate from the normal OID range + */ + Assert(BUFFER_START_TO_OID >= FirstNormalObjectId); + Assert(((Oid) (BUFFER_START_TO_OID + temp_oid_buffer_size)) > BUFFER_START_TO_OID); + Assert(BUFFER_START_TO_OID != ShmemVariableCache->nextOid); + + nextTempOid = BUFFER_START_TO_OID; + } + + result = nextTempOid; + nextTempOid++; + + return result; +} + +Oid +pltsql_GetNewTempOidWithIndex(Relation relation, Oid indexId, AttrNumber oidcolumn) +{ + Oid newOid; + SysScanDesc scan; + ScanKeyData key; + bool collides; + uint64 retries = 0; + + /* Only system relations are supported */ + Assert(IsSystemRelation(relation)); + + /* + * We should never be asked to generate a new pg_type OID during + * pg_upgrade; doing so would risk collisions with the OIDs it wants to + * assign. Hitting this assert means there's some path where we failed to + * ensure that a type OID is determined by commands in the dump script. + */ + Assert(!IsBinaryUpgrade); + + /* + * temp_oid_buffer_size = 0 would indicate that the feature is + * disabled, so we shouldn't even reach this code. + */ + Assert(temp_oid_buffer_size > 0); + + /* Generate new OIDs until we find one not in the table */ + do + { + CHECK_FOR_INTERRUPTS(); + + newOid = pltsql_GetNewTempObjectId(); + + ScanKeyInit(&key, + oidcolumn, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(newOid)); + + /* see notes above about using SnapshotAny */ + scan = systable_beginscan(relation, indexId, true, + SnapshotAny, 1, &key); + + collides = HeapTupleIsValid(systable_getnext(scan)); + + systable_endscan(scan); + + /* + * Provide a useful error message about temp table OID usage if the entire buffer is used. + */ + if (retries >= temp_oid_buffer_size) + ereport(ERROR, + (errmsg("Unable to allocate oid for temp table. Drop some temporary tables or start a new session."))); + + retries++; + } while (collides); + + return newOid; +} + static void pltsql_ExecutorStart(QueryDesc *queryDesc, int eflags) { @@ -4153,6 +4363,88 @@ pltsql_validate_var_datatype_scale(const TypeName *typeName, Type typ) } } +/* + * To properly persist a new value of temp_oid_buffer_start, we must set it + * in pg_settings, as it would in an ALTER DATABASE ... SET ... command. + * + * Returns true on success. + */ +static bool set_and_persist_temp_oid_buffer_start(Oid new_oid) +{ + HeapTuple tuple, newtuple; + Relation rel; + ScanKeyData scankey[2]; + SysScanDesc scan; + const char *babelfish_db_name = NULL; + char *new_oid_str = NULL; + Oid babelfish_db_id = InvalidOid; + int translated_oid = OID_TO_BUFFER_START(new_oid); + Datum repl_val[Natts_pg_db_role_setting]; + bool repl_null[Natts_pg_db_role_setting]; + bool repl_repl[Natts_pg_db_role_setting]; + Datum datum; + ArrayType *a; + + babelfish_db_name = GetConfigOption("babelfishpg_tsql.database_name", true, false); + if (!babelfish_db_name) + return false; + + babelfish_db_id = get_database_oid(babelfish_db_name, true); + + if (!OidIsValid(babelfish_db_id)) + return false; + + new_oid_str = psprintf("%d", translated_oid); + + rel = table_open(DbRoleSettingRelationId, RowExclusiveLock); + ScanKeyInit(&scankey[0], + Anum_pg_db_role_setting_setdatabase, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(babelfish_db_id)); + ScanKeyInit(&scankey[1], + Anum_pg_db_role_setting_setrole, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(InvalidOid)); + scan = systable_beginscan(rel, DbRoleSettingDatidRolidIndexId, true, + NULL, 2, scankey); + tuple = systable_getnext(scan); + + /* temp_oid_buffer_start has a default setting, so it should be there already. */ + if (!HeapTupleIsValid(tuple)) + return false; + + memset(repl_repl, false, sizeof(repl_repl)); + repl_repl[Anum_pg_db_role_setting_setconfig - 1] = true; + repl_null[Anum_pg_db_role_setting_setconfig - 1] = false; + + Assert(strlen(new_oid_str) > 0); + datum = CStringGetTextDatum(psprintf("%s=%s", "babelfishpg_tsql.temp_oid_buffer_start", new_oid_str)); + a = construct_array(&datum, 1, + TEXTOID, + -1, false, TYPALIGN_INT); + + repl_val[Anum_pg_db_role_setting_setconfig - 1] = + PointerGetDatum(a); + + newtuple = heap_modify_tuple(tuple, RelationGetDescr(rel), + repl_val, repl_null, repl_repl); + + /* + * During unit tests, this will crash if it's executed multiple times in the same transaction since + * concurrent tuple updates are not allowed. + */ + if (!TEST_persist_temp_oid_buffer_start_disable_catalog_update) + CatalogTupleUpdate(rel, &tuple->t_self, newtuple); + + systable_endscan(scan); + + table_close(rel, RowExclusiveLock); + + temp_oid_buffer_start = translated_oid; + + return true; +} + /* * Modify the Tuple Descriptor to match the expected * result set. Currently used only for T-SQL OPENQUERY. diff --git a/contrib/babelfishpg_unit/Makefile b/contrib/babelfishpg_unit/Makefile index 214f50ed4d..9528ca2b2b 100644 --- a/contrib/babelfishpg_unit/Makefile +++ b/contrib/babelfishpg_unit/Makefile @@ -4,7 +4,7 @@ DATA = babelfishpg_unit--1.0.0.sql # script file to install OBJS = $(SRCS:.c=.o) # object files # source code files -SRCS = babelfishpg_unit.c test_money.c \ +SRCS = babelfishpg_unit.c test_money.c test_temp_tables.c \ # for posgres build PG_CONFIG = pg_config diff --git a/contrib/babelfishpg_unit/babelfishpg_unit.c b/contrib/babelfishpg_unit/babelfishpg_unit.c index 6c70e83ad8..93b73ede5a 100644 --- a/contrib/babelfishpg_unit/babelfishpg_unit.c +++ b/contrib/babelfishpg_unit/babelfishpg_unit.c @@ -77,6 +77,13 @@ TestInfo tests[]= {&test_fixeddecimal_int2_gt, true, "GreaterThanCheck_FIXEDDECIMAL_INT2", "babelfish_money_datatype"}, {&test_fixeddecimal_int2_ne, true, "NotEqualToCheck_FIXEDDECIMAL_INT2", "babelfish_money_datatype"}, {&test_fixeddecimal_int2_cmp, true, "Comparison_FIXEDDECIMAL_INT2", "babelfish_money_datatype"}, + {&test_pltsql_GetNewTempObjectId, true, "Test pltsql_GetNewTempObjectId", "temp_table"}, + {&test_pltsql_GetNewTempObjectId_tempOidStartSet, true, "Test test_pltsql_GetNewTempObjectId_tempOidStartSet", "temp_table"}, + {&test_pltsql_GetNewTempObjectId_oidCount, true, "Test test_pltsql_GetNewTempObjectId_oidCount", "temp_table"}, + {&test_pltsql_GetNewTempObjectId_endWraparound, true, "Test test_pltsql_GetNewTempObjectId_endWraparound", "temp_table"}, + {&test_pltsql_GetNewTempObjectId_endBuffer, true, "Test test_pltsql_GetNewTempObjectId_endBuffer", "temp_table"}, + {&test_pltsql_GetNewTempOidWithIndex, true, "Test pltsql_GetNewTempOidWithIndex", "temp_table"}, + {&test_pltsql_GetNewTempOidWithIndex_endBuffer, true, "Test test_pltsql_GetNewTempOidWithIndex_endBuffer", "temp_table"}, }; diff --git a/contrib/babelfishpg_unit/babelfishpg_unit.h b/contrib/babelfishpg_unit/babelfishpg_unit.h index 2bcf272fdb..565a7c8b7c 100644 --- a/contrib/babelfishpg_unit/babelfishpg_unit.h +++ b/contrib/babelfishpg_unit/babelfishpg_unit.h @@ -57,3 +57,10 @@ extern TestResult *test_fixeddecimal_int2_le(void); extern TestResult *test_fixeddecimal_int2_gt(void); extern TestResult *test_fixeddecimal_int2_ne(void); extern TestResult *test_fixeddecimal_int2_cmp(void); +extern TestResult *test_pltsql_GetNewTempObjectId(void); +extern TestResult *test_pltsql_GetNewTempObjectId_tempOidStartSet(void); +extern TestResult *test_pltsql_GetNewTempObjectId_oidCount(void); +extern TestResult *test_pltsql_GetNewTempObjectId_endWraparound(void); +extern TestResult *test_pltsql_GetNewTempObjectId_endBuffer(void); +extern TestResult *test_pltsql_GetNewTempOidWithIndex(void); +extern TestResult *test_pltsql_GetNewTempOidWithIndex_endBuffer(void); diff --git a/contrib/babelfishpg_unit/test_temp_tables.c b/contrib/babelfishpg_unit/test_temp_tables.c new file mode 100644 index 0000000000..2421d97740 --- /dev/null +++ b/contrib/babelfishpg_unit/test_temp_tables.c @@ -0,0 +1,328 @@ +#include "babelfishpg_unit.h" +#include "../babelfishpg_tsql/src/hooks.c" + +/* + * Unit testing for GetNewTempObjectId. + * + * It's best to not run this in parallel with other sessions connected to the same database, + * since we are directly manipulating ShmemVariableCache for testing purposes. + * + * This also means that there's a small chance that tests may fail if there are other sessions + * consuming OIDs at the same time, between us releasing/acquiring locks in this function. + */ +TestResult *test_pltsql_GetNewTempObjectId(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + testResult->result = true; + + /* First assignment. Note we also test the case for nextOid in wraparound, so we start at FirstNormalObjectId. */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = 0; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 1000; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 100; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + TEST_ASSERT_TESTCASE(result == FirstNormalObjectId, testResult); + TEST_ASSERT(result == FirstNormalObjectId, testResult); + + /* Check nextOid and oidCount as well. */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + TEST_ASSERT_TESTCASE(ShmemVariableCache->nextOid == FirstNormalObjectId + 100, testResult); + TEST_ASSERT(ShmemVariableCache->nextOid == FirstNormalObjectId + 100, testResult); + + TEST_ASSERT_TESTCASE(ShmemVariableCache->oidCount == 900, testResult); + TEST_ASSERT(ShmemVariableCache->oidCount == 900, testResult); + LWLockRelease(OidGenLock); + + /* + * Next value should be incremented by 1. + */ + result = pltsql_GetNewTempObjectId(); + TEST_ASSERT_TESTCASE(result == (FirstNormalObjectId + 1), testResult); + TEST_ASSERT(result == (FirstNormalObjectId + 1), testResult); + + return testResult; +} + +TestResult *test_pltsql_GetNewTempObjectId_tempOidStartSet(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + testResult->result = true; + + /* + * If tempOidStart is set properly in ShmemVariableCache, we will start there instead. + */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->tempOidStart = 222222; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + TEST_ASSERT_TESTCASE(result == 222222, testResult); + TEST_ASSERT(result == 222222, testResult); + + TEST_ASSERT_TESTCASE(BUFFER_START_TO_OID == 222222, testResult); + TEST_ASSERT(BUFFER_START_TO_OID == 222222, testResult); + + return testResult; +} + +TestResult *test_pltsql_GetNewTempObjectId_oidCount(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + testResult->result = true; + + /* + * Check oidCount properly set to VAR_OID_PREFETCH. + */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = 0; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 100; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 100; + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + TEST_ASSERT_TESTCASE(ShmemVariableCache->oidCount == GetVarOidPrefetch(), testResult); + TEST_ASSERT(ShmemVariableCache->oidCount == GetVarOidPrefetch(), testResult); + + TEST_ASSERT_TESTCASE(ShmemVariableCache->nextOid == FirstNormalObjectId + 100, testResult); + TEST_ASSERT(ShmemVariableCache->nextOid == FirstNormalObjectId + 100, testResult); + LWLockRelease(OidGenLock); + + return testResult; +} + +TestResult *test_pltsql_GetNewTempObjectId_endWraparound(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + testResult->result = true; + + /* + * Assignment when end of buffer would wrap into reserved OIDs. + */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = OID_MAX - 100; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 1000; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 101; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + TEST_ASSERT_TESTCASE(result == FirstNormalObjectId, testResult); + TEST_ASSERT(result == FirstNormalObjectId, testResult); + + TEST_ASSERT_TESTCASE(ShmemVariableCache->oidCount == GetVarOidPrefetch(), testResult); + TEST_ASSERT(ShmemVariableCache->oidCount == GetVarOidPrefetch(), testResult); + + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + TEST_ASSERT_TESTCASE(ShmemVariableCache->nextOid == FirstNormalObjectId + 101, testResult); + TEST_ASSERT(ShmemVariableCache->nextOid == FirstNormalObjectId + 101, testResult); + LWLockRelease(OidGenLock); + + TEST_ASSERT_TESTCASE(BUFFER_START_TO_OID == FirstNormalObjectId, testResult); + TEST_ASSERT(BUFFER_START_TO_OID == FirstNormalObjectId, testResult); + + return testResult; +} + +TestResult *test_pltsql_GetNewTempObjectId_endBuffer(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + testResult->result = true; + + /* + * Assignment when end of buffer is MAX_OID. + */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = OID_MAX - 10; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 1000; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 10; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + /* + * Check for proper assignment and proper wraparound. + */ + TEST_ASSERT_TESTCASE(result == (OID_MAX - 10), testResult); + TEST_ASSERT(result == (OID_MAX - 10), testResult); + + for (int i = 0; i < 10; i++) + { + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempObjectId(); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + } + + TEST_ASSERT_TESTCASE(result == (OID_MAX - 10), testResult); + TEST_ASSERT(result == (OID_MAX - 10), testResult); + + return testResult; +} + +/* + * Unit testing for GetNewTempOidWithIndex. + * + * Testing is similar to the above, since this is mostly a wrapper around + * GetNewTempObjectId. Cases that can't be covered by unit tests are + * covered in JDBC tests. + */ +TestResult *test_pltsql_GetNewTempOidWithIndex(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + Relation rel = table_open(RelationRelationId, RowExclusiveLock); + testResult->result = true; + + /* First assignment. Note we also test the case for nextOid in wraparound, so we start at FirstNormalObjectId. */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = 0; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 1000; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 100; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempOidWithIndex(rel, ClassOidIndexId, Anum_pg_class_oid); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + TEST_ASSERT_TESTCASE(result == FirstNormalObjectId, testResult); + TEST_ASSERT(result == FirstNormalObjectId, testResult); + + table_close(rel, RowExclusiveLock); + + return testResult; +} + +TestResult *test_pltsql_GetNewTempOidWithIndex_endBuffer(void) +{ + TestResult* testResult = palloc0(sizeof(TestResult)); + Oid result = 0; + Relation rel = table_open(RelationRelationId, RowExclusiveLock); + testResult->result = true; + + /* + * Assignment when end of buffer is MAX_OID. + */ + LWLockAcquire(OidGenLock, LW_EXCLUSIVE); + ShmemVariableCache->nextOid = OID_MAX - 10; + ShmemVariableCache->tempOidStart = InvalidOid; + ShmemVariableCache->oidCount = 1000; + LWLockRelease(OidGenLock); + temp_oid_buffer_start = INT_MIN; + temp_oid_buffer_size = 10; + + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempOidWithIndex(rel, ClassOidIndexId, Anum_pg_class_oid); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + + /* + * Check for proper assignment and proper wraparound. + */ + TEST_ASSERT_TESTCASE(result == (OID_MAX - 10), testResult); + TEST_ASSERT(result == (OID_MAX - 10), testResult); + + for (int i = 0; i < 10; i++) + { + PG_TRY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = true; + result = pltsql_GetNewTempOidWithIndex(rel, ClassOidIndexId, Anum_pg_class_oid); + } + PG_FINALLY(); + { + TEST_persist_temp_oid_buffer_start_disable_catalog_update = false; + } + PG_END_TRY(); + } + + TEST_ASSERT_TESTCASE(result == (OID_MAX - 10), testResult); + TEST_ASSERT(result == (OID_MAX - 10), testResult); + + table_close(rel, RowExclusiveLock); + + return testResult; +} diff --git a/test/JDBC/expected/temp-tables-vu-cleanup.out b/test/JDBC/expected/temp-tables-vu-cleanup.out index 4e9150b1e7..7d20f0cdcc 100644 --- a/test/JDBC/expected/temp-tables-vu-cleanup.out +++ b/test/JDBC/expected/temp-tables-vu-cleanup.out @@ -12,3 +12,9 @@ GO DROP TABLE temp_table_vu_prepare_t1; GO + +DROP TYPE temp_table_type_int +GO + +DROP TYPE temp_table_type_char +GO diff --git a/test/JDBC/expected/temp-tables-vu-prepare.out b/test/JDBC/expected/temp-tables-vu-prepare.out index bd671afb86..068fb77083 100644 --- a/test/JDBC/expected/temp-tables-vu-prepare.out +++ b/test/JDBC/expected/temp-tables-vu-prepare.out @@ -35,3 +35,9 @@ BEGIN CREATE TABLE #tt (a int); -- throws error END; GO + +CREATE TYPE temp_table_type_int FROM int +GO + +CREATE TYPE temp_table_type_char FROM nvarchar(200) +GO diff --git a/test/JDBC/expected/temp-tables-vu-verify.out b/test/JDBC/expected/temp-tables-vu-verify.out index 2227727212..ec5ee84ccc 100644 --- a/test/JDBC/expected/temp-tables-vu-verify.out +++ b/test/JDBC/expected/temp-tables-vu-verify.out @@ -15,3 +15,88 @@ GO ~~ERROR (Message: relation "#tt" already exists)~~ + +-- Test temp table creation with toast is cleaned up properly. +CREATE TABLE #temp_table_create(a int, b nvarchar(200)) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO +~~START~~ +int +2 +~~END~~ + + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO +~~START~~ +int +1 +~~END~~ + + +DROP TABLE #temp_table_create +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO +~~START~~ +int +0 +~~END~~ + + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO +~~START~~ +int +0 +~~END~~ + + +-- Test temp table alter with toast +CREATE TABLE #temp_table_alter1(col1 int) +GO +ALTER TABLE #temp_table_alter1 ADD col2 varchar(20) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO +~~START~~ +int +2 +~~END~~ + + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO +~~START~~ +int +1 +~~END~~ + + +DROP TABLE #temp_table_alter1 +GO + +CREATE TABLE #temp_table_alter2(col1 int, col2 varchar(20)) +GO +ALTER TABLE #temp_table_alter2 ADD col3 int IDENTITY(1, 1) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO +~~START~~ +int +2 +~~END~~ + + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO +~~START~~ +int +1 +~~END~~ + diff --git a/test/JDBC/expected/temp_table_jdbc.out b/test/JDBC/expected/temp_table_jdbc.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/JDBC/expected/temp_tables_concurrent-vu-cleanup.out b/test/JDBC/expected/temp_tables_concurrent-vu-cleanup.out new file mode 100644 index 0000000000..52a09a371b --- /dev/null +++ b/test/JDBC/expected/temp_tables_concurrent-vu-cleanup.out @@ -0,0 +1,9 @@ + +--tsql user=u1 password=123 +DROP TABLE non_temp_table_1 +GO + + +--tsql user=u2 password=123 +DROP TABLE non_temp_table_2 +GO diff --git a/test/JDBC/expected/temp_tables_concurrent-vu-prepare.out b/test/JDBC/expected/temp_tables_concurrent-vu-prepare.out new file mode 100644 index 0000000000..6cfc3397c0 --- /dev/null +++ b/test/JDBC/expected/temp_tables_concurrent-vu-prepare.out @@ -0,0 +1,9 @@ + +--tsql user=u1 password=123 +CREATE TABLE non_temp_table_1(a int) +GO + + +--tsql user=u2 password=123 +CREATE TABLE non_temp_table_2(a int) +GO diff --git a/test/JDBC/expected/temp_tables_concurrent-vu-verify.out b/test/JDBC/expected/temp_tables_concurrent-vu-verify.out new file mode 100644 index 0000000000..3cb67f26a5 --- /dev/null +++ b/test/JDBC/expected/temp_tables_concurrent-vu-verify.out @@ -0,0 +1,45 @@ +--tsql user=u1 password=123 +CREATE TABLE #temp_table_t1(a int) +GO + +CREATE TABLE #temp_table_t2(a int) +GO + +declare @r1 int +declare @r2 int +select @r1 = oid FROM pg_class WHERE relname='non_temp_table_1'; +select @r2 = reloid FROM sys.babelfish_get_enr_list() WHERE relname='#temp_table_t1'; +select 1 where @r1 - @r2 > 65000 +GO +~~START~~ +int +1 +~~END~~ + + +DROP TABLE #temp_table_t1 +DROP TABLE #temp_table_t2 +GO + +--tsql user=u2 password=123 +CREATE TABLE #temp_table_t1(a int) +GO + +CREATE TABLE #temp_table_t2(a int) +GO + +declare @r1 int +declare @r2 int +select @r1 = oid FROM pg_class WHERE relname='non_temp_table_2'; +select @r2 = reloid FROM sys.babelfish_get_enr_list() WHERE relname='#temp_table_t1'; +select 1 where @r1 - @r2 > 65000 +GO +~~START~~ +int +1 +~~END~~ + + +DROP TABLE #temp_table_t1 +DROP TABLE #temp_table_t2 +GO diff --git a/test/JDBC/input/ddl/temp-tables-vu-cleanup.sql b/test/JDBC/input/ddl/temp-tables-vu-cleanup.sql index 2a26a44c9e..e70feaae5e 100644 --- a/test/JDBC/input/ddl/temp-tables-vu-cleanup.sql +++ b/test/JDBC/input/ddl/temp-tables-vu-cleanup.sql @@ -11,4 +11,10 @@ DROP PROCEDURE temp_table_vu_prepare_sp_exception; GO DROP TABLE temp_table_vu_prepare_t1; +GO + +DROP TYPE temp_table_type_int +GO + +DROP TYPE temp_table_type_char GO \ No newline at end of file diff --git a/test/JDBC/input/ddl/temp-tables-vu-prepare.sql b/test/JDBC/input/ddl/temp-tables-vu-prepare.sql index 4912f17483..090761fb99 100644 --- a/test/JDBC/input/ddl/temp-tables-vu-prepare.sql +++ b/test/JDBC/input/ddl/temp-tables-vu-prepare.sql @@ -30,4 +30,10 @@ BEGIN CREATE TABLE #tt (a int); CREATE TABLE #tt (a int); -- throws error END; +GO + +CREATE TYPE temp_table_type_int FROM int +GO + +CREATE TYPE temp_table_type_char FROM nvarchar(200) GO \ No newline at end of file diff --git a/test/JDBC/input/ddl/temp-tables-vu-verify.sql b/test/JDBC/input/ddl/temp-tables-vu-verify.sql index dbb16d60e2..01540f60e5 100644 --- a/test/JDBC/input/ddl/temp-tables-vu-verify.sql +++ b/test/JDBC/input/ddl/temp-tables-vu-verify.sql @@ -8,4 +8,49 @@ EXEC temp_table_vu_prepare_sp_drop; GO EXEC temp_table_vu_prepare_sp_exception; -GO \ No newline at end of file +GO + +-- Test temp table creation with toast is cleaned up properly. +CREATE TABLE #temp_table_create(a int, b nvarchar(200)) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO + +DROP TABLE #temp_table_create +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO + +-- Test temp table alter with toast +CREATE TABLE #temp_table_alter1(col1 int) +GO +ALTER TABLE #temp_table_alter1 ADD col2 varchar(20) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO + +DROP TABLE #temp_table_alter1 +GO + +CREATE TABLE #temp_table_alter2(col1 int, col2 varchar(20)) +GO +ALTER TABLE #temp_table_alter2 ADD col3 int IDENTITY(1, 1) +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%' +GO + +select count(*) FROM sys.babelfish_get_enr_list() WHERE relname LIKE '#pg_toast_%_index' +GO diff --git a/test/JDBC/input/temp_tables/temp_table_jdbc.txt b/test/JDBC/input/temp_tables/temp_table_jdbc.txt new file mode 100644 index 0000000000..3b3ea471e8 --- /dev/null +++ b/test/JDBC/input/temp_tables/temp_table_jdbc.txt @@ -0,0 +1 @@ +The test implementation can be found in JDBCTempTable.java. This file is just here so that the test can be toggled like any other test in jdbc_schedule. \ No newline at end of file diff --git a/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-cleanup.mix b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-cleanup.mix new file mode 100644 index 0000000000..4ee9652933 --- /dev/null +++ b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-cleanup.mix @@ -0,0 +1,9 @@ +--tsql user=u1 password=123 + +DROP TABLE non_temp_table_1 +GO + +--tsql user=u2 password=123 + +DROP TABLE non_temp_table_2 +GO \ No newline at end of file diff --git a/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-prepare.mix b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-prepare.mix new file mode 100644 index 0000000000..438143e2e2 --- /dev/null +++ b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-prepare.mix @@ -0,0 +1,9 @@ +--tsql user=u1 password=123 + +CREATE TABLE non_temp_table_1(a int) +GO + +--tsql user=u2 password=123 + +CREATE TABLE non_temp_table_2(a int) +GO \ No newline at end of file diff --git a/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-verify.mix b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-verify.mix new file mode 100644 index 0000000000..163c64fd41 --- /dev/null +++ b/test/JDBC/input/temp_tables/temp_tables_concurrent-vu-verify.mix @@ -0,0 +1,35 @@ +--tsql user=u1 password=123 +CREATE TABLE #temp_table_t1(a int) +GO + +CREATE TABLE #temp_table_t2(a int) +GO + +declare @r1 int +declare @r2 int +select @r1 = oid FROM pg_class WHERE relname='non_temp_table_1'; +select @r2 = reloid FROM sys.babelfish_get_enr_list() WHERE relname='#temp_table_t1'; +select 1 where @r1 - @r2 > 65000 +GO + +DROP TABLE #temp_table_t1 +DROP TABLE #temp_table_t2 +GO + +--tsql user=u2 password=123 +CREATE TABLE #temp_table_t1(a int) +GO + +CREATE TABLE #temp_table_t2(a int) +GO + +declare @r1 int +declare @r2 int +select @r1 = oid FROM pg_class WHERE relname='non_temp_table_2'; +select @r2 = reloid FROM sys.babelfish_get_enr_list() WHERE relname='#temp_table_t1'; +select 1 where @r1 - @r2 > 65000 +GO + +DROP TABLE #temp_table_t1 +DROP TABLE #temp_table_t2 +GO \ No newline at end of file diff --git a/test/JDBC/src/main/java/com/sqlsamples/JDBCTempTable.java b/test/JDBC/src/main/java/com/sqlsamples/JDBCTempTable.java new file mode 100644 index 0000000000..bb8b7dac26 --- /dev/null +++ b/test/JDBC/src/main/java/com/sqlsamples/JDBCTempTable.java @@ -0,0 +1,348 @@ +package com.sqlsamples; + +import org.apache.logging.log4j.Logger; +import java.io.*; +import java.util.*; +import java.sql.*; + +import org.junit.jupiter.api.*; +import java.util.concurrent.ThreadLocalRandom; + +import static com.sqlsamples.Config.*; +import static com.sqlsamples.Statistics.curr_exec_time; + +public class JDBCTempTable { + public static boolean toRun = false; + + private static String initializeConnectionString() { + String url = properties.getProperty("URL"); + String port = properties.getProperty("tsql_port"); + String databaseName = properties.getProperty("databaseName"); + String user = properties.getProperty("user"); + String password = properties.getProperty("password"); + + return createSQLServerConnectionString(url, port, databaseName, user, password); + } + + public static void runTest(BufferedWriter bw, Logger logger) { + long startTime = System.nanoTime(); + + try { + check_oids_equal(bw); + test_oid_buffer(bw, logger); + concurrency_test(bw); + psql_test(bw, logger); + } catch (Exception e) { + try { + bw.write(e.getMessage()); + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + + long endTime = System.nanoTime(); + curr_exec_time = endTime - startTime; + } + + /* + * Helper function that creates the specified number of connections, creates a temp table on each connection, and returns whether all the OIDs are equal or not. + */ + private static boolean check_oids_equal_helper(int num_connections) throws Exception { + ArrayList connections = new ArrayList(); + String connectionString = initializeConnectionString(); + + /* Create connections */ + for (int i = 0; i < num_connections; i++) { + Connection connection = DriverManager.getConnection(connectionString); + connections.add(connection); + } + + /* Run test on each connection */ + ArrayList oids = new ArrayList<>(); + String queryString = ""; + String tableName = ""; + int count = 0; + for (Connection c : connections) + { + tableName = "#t" + count; + queryString = "CREATE TABLE " + tableName + " (a int)"; + int result = create_table_and_report_oid(c, queryString, tableName); + oids.add(result); + count++; + c.close(); + } + + // The oids should all be equal here. + for (Integer i : oids) + { + if (!i.equals(oids.get(0))) { + return false; + } + } + return true; + } + + /* + * This is a straightforward test to assert that temp tables created across different connections will have the same OID start. + */ + private static void check_oids_equal(BufferedWriter bw) throws Exception { + int num_connections = 2; + + if (!check_oids_equal_helper(num_connections)) + { + bw.write("OID check failed! Not all oids were equal:"); + bw.newLine(); + } + } + + /* + * Create a table (via provided query string), report back the OID of the table that was just created. + */ + private static int create_table_and_report_oid(Connection c, String queryString, String tablename) throws Exception { + Statement s = c.createStatement(); + s.execute(queryString); + + ResultSet rs = s.executeQuery("SELECT * FROM sys.babelfish_get_enr_list() WHERE RELNAME = '" + tablename + "'"); + + if (!rs.next()) { + throw new Exception("Tablename not found in sys.babelfish_get_enr_list"); + } + String reloid = rs.getString("reloid"); + + return Integer.parseInt(reloid); + } + + /* + * This will be a short stress test to create multiple tables in parallel to ensure no issues or crashes are encountered. + */ + private static void concurrency_test(BufferedWriter bw) throws Exception { + int num_connections = 10; + int num_tables = 5000; + + /* Create a UDT so that we can test non-ENR temp tables */ + String connectionString = initializeConnectionString(); + ArrayList cxns = new ArrayList<>(); + Connection c = DriverManager.getConnection(connectionString); + cxns.add(c); + c.createStatement().execute("CREATE TYPE my_temp_type FROM int"); + + ArrayList threads = new ArrayList<>(); + + /* Create connections */ + for (int i = 0; i < num_connections; i++) { + Connection connection = DriverManager.getConnection(connectionString); + cxns.add(connection); + Thread t = new Thread(new Worker(connection, i + 1, num_tables / num_connections, bw)); + threads.add(t); + t.start(); + } + for (Thread t : threads) + { + t.join(); + } + c.createStatement().execute("DROP TYPE my_temp_type"); + + for (Connection cxn : cxns) { + cxn.close(); + } + } + + private static void test_oid_buffer(BufferedWriter bw, Logger logger) throws Exception { + String connectionString = initializeConnectionString(); + Connection c = DriverManager.getConnection(connectionString); + JDBCCrossDialect cx = new JDBCCrossDialect(c); + + Connection psql = cx.getPsqlConnection("-- psql", bw, logger); + int num_connections = 2; + + /* + * TEST: After disabling GUC, ensure that the OIDs are not equal, meaning we aren't using + * the OID buffer. + */ + Statement alter_guc = psql.createStatement(); + alter_guc.execute("ALTER DATABASE jdbc_testdb SET babelfishpg_tsql.temp_oid_buffer_size = 0"); + + if (check_oids_equal_helper(num_connections)) { + bw.write("OID check failed! Oids were equal after disabling guc."); + bw.newLine(); + } + + /* + * TEST: Ensure that we can create up to (and no more) than the oid buffer size. + */ + alter_guc.execute("ALTER DATABASE jdbc_testdb SET babelfishpg_tsql.temp_oid_buffer_size = 10"); + + /* We need a new connection here to pick up the updated guc. */ + Connection c2 = DriverManager.getConnection(connectionString); + Statement s = c2.createStatement(); + + try { + for (int i = 0; i < 11; i++) { + String queryString = "CREATE TABLE #tab" + i + " (a int)"; + s.execute(queryString); + } + /* If we reach this point, we created more tables than we should have been able to. */ + bw.write("Created more tables than buffer should have allowed for"); + bw.newLine(); + } catch (Exception e) { + if (!e.getMessage().startsWith("Unable to allocate oid for temp table.")) { + bw.write(e.getMessage()); + bw.newLine(); + } + } + + /* If the table was created, throw an error. */ + ResultSet rs = s.executeQuery("SELECT * FROM babelfish_get_enr_list() WHERE relname = \'#table_cant_be_created\'"); + if (rs.next()) { + bw.write("A table was created that should have reached buffer size limit."); + bw.newLine(); + } + + /* + * TEST: Ensure that we can wraparound properly. + */ + rs = s.executeQuery("SELECT * FROM babelfish_get_enr_list() WHERE relname = \'#tab0\'"); + if (!rs.next()) { + bw.write("Table is missing."); + bw.newLine(); + } + int old_oid = Integer.parseInt(rs.getString("reloid")); + s.execute("DROP TABLE #tab0"); + int new_oid = create_table_and_report_oid(c2, "CREATE TABLE #new_table(a int)", "#new_table"); + + if (old_oid != new_oid) { + bw.write("Wraparound did not handle new OIDs properly."); + bw.newLine(); + } + + c2.close(); + + /* Restore GUC after tests. */ + alter_guc.execute("ALTER DATABASE jdbc_testdb SET babelfishpg_tsql.temp_oid_buffer_size = 65536"); + psql.close(); + c.close(); + } + + /* + * Sanity checks that must be done from psql endpoint. + */ + private static void psql_test(BufferedWriter bw, Logger logger) throws Exception { + /* Test GUC crash resiliency - Create a few normal tables (to advance OID counter). Ensure that the GUC remains the same from a different session. */ + Connection connection = DriverManager.getConnection(connectionString); + Statement s = connection.createStatement(); + + /* Create a few tables so that the OID counter is not a predictable value. */ + for (int i = 0; i < 100; i++) + { + s.execute("CREATE TABLE crash_res_dummy_table(a int)"); + s.execute("DROP TABLE crash_res_dummy_table"); + } + + s.execute("CREATE TABLE #t1(a int)"); + + /* Create two new connections to check the GUC value from postgres. */ + Connection c = DriverManager.getConnection(connectionString); + JDBCCrossDialect cx = new JDBCCrossDialect(c); + Connection psql = cx.getPsqlConnection("-- psql", bw, logger); + + Statement check_guc = psql.createStatement(); + ResultSet rs = check_guc.executeQuery("SHOW babelfishpg_tsql.temp_oid_buffer_start"); + if (!rs.next()) { + bw.write("Table is missing."); + bw.newLine(); + } + int buffer_start = Integer.parseInt(rs.getString("babelfishpg_tsql.temp_oid_buffer_start")); + + if (buffer_start == Integer.MIN_VALUE) + { + bw.write("Oid buffer guc was not set properly!"); + bw.newLine(); + } + + Connection c2 = DriverManager.getConnection(connectionString); + JDBCCrossDialect cx2 = new JDBCCrossDialect(c2); + Connection psql2 = cx2.getPsqlConnection("-- psql", bw, logger); + + check_guc = psql2.createStatement(); + rs = check_guc.executeQuery("SHOW babelfishpg_tsql.temp_oid_buffer_start"); + if (!rs.next()) { + bw.write("Table is missing."); + bw.newLine(); + } + int buffer_start_2 = Integer.parseInt(rs.getString("babelfishpg_tsql.temp_oid_buffer_start")); + + if (buffer_start_2 == Integer.MIN_VALUE) + { + bw.write("2nd Oid buffer guc was not set properly!"); + bw.newLine(); + } + + if (buffer_start_2 != buffer_start) { + bw.write("Oid buffer guc was different between two connections!"); + bw.newLine(); + } + + /* Ensure that psql still can't create tables with '#'. */ + try { + check_guc.execute("CREATE TABLE #psql_table_with_hash(a int)"); + } catch (Exception e) { + if (!e.getMessage().startsWith("ERROR: syntax error at or near \"#\"")) { + bw.write(e.getMessage()); + bw.newLine(); + } + } + + psql2.close(); + c2.close(); + psql.close(); + c.close(); + connection.close(); + } +} + +class Worker implements Runnable { + public static int table_count = 0; + private int num_to_create; + private BufferedWriter bw; + + Connection c; + String prefix; + String[] column_descriptions = new String[]{ + "(a int)", /* Plain table */ + "(a my_temp_type)", /* Non-ENR */ + "(a nvarchar(200))", /* Toasted */ + "(a int primary key identity)", /* Primary key, identity */ + "(a int primary key identity, b nvarchar(200))" /* Primary key, identity, toasted */ + }; + + Worker(Connection c, int i, int num_to_create, BufferedWriter bw) { + this.c = c; + this.prefix = "thr" + i + "_"; + this.num_to_create = num_to_create; + this.bw = bw; + } + + public void run() { + try { + try { + Statement s = c.createStatement(); + for (int i = 0; i < num_to_create; i++) { + /* Pick a random table type to create */ + int r = ThreadLocalRandom.current().nextInt(0, column_descriptions.length); + + /* Name format: thr[thread number]_#tab[table number]_[column_description index] */ + String tablename = prefix + "#tab" + i + "_" + r; + + s.execute("CREATE TABLE " + tablename + column_descriptions[r]); + s.execute("DROP TABLE " + tablename); + table_count++; + } + } catch (Exception e) { + bw.write(e.getMessage()); + bw.newLine(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/test/JDBC/src/test/java/com/sqlsamples/TestQueryFile.java b/test/JDBC/src/test/java/com/sqlsamples/TestQueryFile.java index 337d026352..6b94912ffa 100644 --- a/test/JDBC/src/test/java/com/sqlsamples/TestQueryFile.java +++ b/test/JDBC/src/test/java/com/sqlsamples/TestQueryFile.java @@ -14,6 +14,7 @@ import java.util.stream.Stream; import static com.sqlsamples.Config.*; +import static com.sqlsamples.JDBCTempTable.*; import static com.sqlsamples.Statistics.exec_times; import static com.sqlsamples.Statistics.curr_exec_time; @@ -422,7 +423,12 @@ public void TestQueryBatch(String inputFileName) throws SQLException, ClassNotFo BufferedWriter bw = new BufferedWriter(fw); curr_exec_time = 0L; checkParallelQueryExpected = false; - batch_run.batch_run_sql(connection_bbl, bw, testFilePath, logger); + if (inputFileName.equals("temp_table_jdbc")) { + JDBCTempTable.runTest(bw, logger); + sla = defaultSLA*1000000L * 2; /* Increase SLA to avoid flakiness */ + } else { + batch_run.batch_run_sql(connection_bbl, bw, testFilePath, logger); + } bw.close(); if(sla == 0){ sla = defaultSLA*1000000L;