From fed2faf19c94a7f16e369a312fc661dc89b6bed8 Mon Sep 17 00:00:00 2001 From: melkij Date: Wed, 24 Apr 2019 14:46:29 +0300 Subject: [PATCH 01/14] allow build with PostgreSQL 12 Conflict changes was: new argument for RenameRelationInternal (,,,is_index) heap_open/heap_closewas moved to access/table.h (well, this is macro now, but I did not change callers code) --- lib/repack.c | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/repack.c b/lib/repack.c index 77bcac59..a8658c34 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -17,6 +17,14 @@ #include "catalog/indexing.h" #include "catalog/namespace.h" +/* + * heap_open/heap_close was moved to table_open/table_close in 12.0 + * table.h has macros mapping the old names to the new ones + */ +#if PG_VERSION_NUM >= 120000 +#include "access/table.h" +#endif + /* * utils/rel.h no longer includes pg_am.h as of 9.6, so need to include * it explicitly. @@ -112,8 +120,18 @@ must_be_superuser(const char *func) #define RENAME_REL(relid, newrelname) RenameRelationInternal(relid, newrelname, PG_TOAST_NAMESPACE); #elif PG_VERSION_NUM < 90300 #define RENAME_REL(relid, newrelname) RenameRelationInternal(relid, newrelname); -#else +#elif PG_VERSION_NUM < 120000 #define RENAME_REL(relid, newrelname) RenameRelationInternal(relid, newrelname, true); +#else +#define RENAME_REL(relid, newrelname) RenameRelationInternal(relid, newrelname, true, false); +#endif +/* + * is_index flag was added in 12.0, prefer separate macro for relation and index + */ +#if PG_VERSION_NUM < 120000 +#define RENAME_INDEX(relid, newrelname) RENAME_REL(relid, newrelname); +#else +#define RENAME_INDEX(relid, newrelname) RenameRelationInternal(relid, newrelname, true, false); #endif #ifdef REPACK_VERSION @@ -931,7 +949,7 @@ repack_swap(PG_FUNCTION_ARGS) snprintf(name, NAMEDATALEN, "pg_toast_%u", oid2); RENAME_REL(reltoastrelid1, name); snprintf(name, NAMEDATALEN, "pg_toast_%u_index", oid2); - RENAME_REL(reltoastidxid1, name); + RENAME_INDEX(reltoastidxid1, name); CommandCounterIncrement(); } else if (reltoastrelid1 != InvalidOid) @@ -943,21 +961,21 @@ repack_swap(PG_FUNCTION_ARGS) snprintf(name, NAMEDATALEN, "pg_toast_pid%d", pid); RENAME_REL(reltoastrelid1, name); snprintf(name, NAMEDATALEN, "pg_toast_pid%d_index", pid); - RENAME_REL(reltoastidxid1, name); + RENAME_INDEX(reltoastidxid1, name); CommandCounterIncrement(); /* rename Y to X */ snprintf(name, NAMEDATALEN, "pg_toast_%u", oid); RENAME_REL(reltoastrelid2, name); snprintf(name, NAMEDATALEN, "pg_toast_%u_index", oid); - RENAME_REL(reltoastidxid2, name); + RENAME_INDEX(reltoastidxid2, name); CommandCounterIncrement(); /* rename TEMP to Y */ snprintf(name, NAMEDATALEN, "pg_toast_%u", oid2); RENAME_REL(reltoastrelid1, name); snprintf(name, NAMEDATALEN, "pg_toast_%u_index", oid2); - RENAME_REL(reltoastidxid1, name); + RENAME_INDEX(reltoastidxid1, name); CommandCounterIncrement(); } From 32d32e5c4574cb294b391af4e1341b756ef157bf Mon Sep 17 00:00:00 2001 From: melkij Date: Wed, 24 Apr 2019 15:06:18 +0300 Subject: [PATCH 02/14] PostgreSQL 12 has no more OIDS support for user tables. relhasoids was removed from pg_class, so create separate get_relhasoids C function --- lib/exports.txt | 22 ++++++++++++---------- lib/pg_repack.sql.in | 8 +++++--- lib/repack.c | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/lib/exports.txt b/lib/exports.txt index b3bd9534..8927bb2e 100644 --- a/lib/exports.txt +++ b/lib/exports.txt @@ -9,13 +9,15 @@ pg_finfo_repack_trigger 8 pg_finfo_repack_version 9 pg_finfo_repack_index_swap 10 pg_finfo_repack_get_table_and_inheritors 11 -repack_apply 12 -repack_disable_autovacuum 13 -repack_drop 14 -repack_get_order_by 15 -repack_indexdef 16 -repack_swap 17 -repack_trigger 18 -repack_version 19 -repack_index_swap 20 -repack_get_table_and_inheritors 21 +pg_finfo_repack_get_relhasoids 12 +repack_apply 13 +repack_disable_autovacuum 14 +repack_drop 15 +repack_get_order_by 16 +repack_indexdef 17 +repack_swap 18 +repack_trigger 19 +repack_version 20 +repack_index_swap 21 +repack_get_table_and_inheritors 22 +repack_get_relhasoids 23 diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index 99003b6e..6235cbb3 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -154,6 +154,10 @@ WHERE $$ LANGUAGE sql STABLE STRICT; +CREATE FUNCTION repack.get_relhasoids(oid) + RETURNS BOOL AS 'MODULE_PATHNAME', 'repack_get_relhasoids' +LANGUAGE C STABLE STRICT; + -- Get a comma-separated storage paramter for the table including -- paramters for the corresponding TOAST table. -- Note that since oid setting is always not NULL, this function @@ -178,12 +182,10 @@ FROM ( UNION ALL -- table oid SELECT 'oids = ' || - CASE WHEN relhasoids + CASE WHEN repack.get_relhasoids($1) THEN 'true' ELSE 'false' END - FROM pg_class - WHERE oid = $1 ) as t $$ diff --git a/lib/repack.c b/lib/repack.c index a8658c34..05229c83 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -80,6 +80,7 @@ extern Datum PGUT_EXPORT repack_drop(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_disable_autovacuum(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_index_swap(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_get_table_and_inheritors(PG_FUNCTION_ARGS); +extern Datum PGUT_EXPORT repack_get_relhasoids(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(repack_version); PG_FUNCTION_INFO_V1(repack_trigger); @@ -91,6 +92,7 @@ PG_FUNCTION_INFO_V1(repack_drop); PG_FUNCTION_INFO_V1(repack_disable_autovacuum); PG_FUNCTION_INFO_V1(repack_index_swap); PG_FUNCTION_INFO_V1(repack_get_table_and_inheritors); +PG_FUNCTION_INFO_V1(repack_get_relhasoids); static void repack_init(void); static SPIPlanPtr repack_prepare(const char *src, int nargs, Oid *argtypes); @@ -1429,3 +1431,34 @@ repack_get_table_and_inheritors(PG_FUNCTION_ARGS) PG_RETURN_ARRAYTYPE_P(result); } + +/* + * PostgreSQL 12 has no more support for "with oids" option and does + * not have pg_class.relhasoids. Extract pg_class.relhasoid lookup to + * seperate C routine + */ +Datum +repack_get_relhasoids(PG_FUNCTION_ARGS) +{ +#if PG_VERSION_NUM >= 120000 + PG_RETURN_BOOL(false); +#else + Relation class_rel; + HeapTuple tuple; + Form_pg_class tuple_class; + bool relhasoid; + + class_rel = heap_open(RelationRelationId, RowShareLock); + + tuple = SearchSysCacheCopy1(RELOID, PG_GETARG_DATUM(0)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for relation %u", + PG_GETARG_OID(0)); + tuple_class = (Form_pg_class) GETSTRUCT(tuple); + + relhasoid = tuple_class->relhasoids; + + heap_close(class_rel, RowShareLock); + PG_RETURN_BOOL(relhasoid); +#endif +} From 6732338a5aaea6f246bc119ec1d4634b661cf67f Mon Sep 17 00:00:00 2001 From: melkij Date: Sun, 25 Aug 2019 13:17:01 +0300 Subject: [PATCH 03/14] simplify relhasoids with replace from Makefile --- lib/Makefile | 11 ++++++++++- lib/exports.txt | 22 ++++++++++------------ lib/pg_repack.sql.in | 8 +++----- lib/repack.c | 35 +---------------------------------- 4 files changed, 24 insertions(+), 52 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 57307650..9d403935 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -19,6 +19,14 @@ OBJS = repack.o pgut/pgut-spi.o SHLIB_EXPORTS = exports.txt + +# It is not possible to create tables with OIDs on PostgreSQL 12 or later +ifeq ($(shell echo $$(($(INTVERSION) < 1200))),1) + RELHASOIDS := relhasoids +else + RELHASOIDS := false +endif + # The version number of the program. It should be the same of the library. REPACK_VERSION = $(shell grep '"version":' ../META.json | head -1 \ | sed -e 's/[ ]*"version":[ ]*"\(.*\)",/\1/') @@ -43,7 +51,8 @@ pg_repack.sql: pg_repack.sql.in echo "COMMIT;" >> $@; pg_repack--$(REPACK_VERSION).sql: pg_repack.sql.in - sed 's,REPACK_VERSION,$(REPACK_VERSION),g' $< > $@; + sed 's,REPACK_VERSION,$(REPACK_VERSION),g' $< \ + | sed 's,relhasoids,$(RELHASOIDS),g'> $@; pg_repack.control: pg_repack.control.in sed 's,REPACK_VERSION,$(REPACK_VERSION),g' $< > $@ diff --git a/lib/exports.txt b/lib/exports.txt index 8927bb2e..b3bd9534 100644 --- a/lib/exports.txt +++ b/lib/exports.txt @@ -9,15 +9,13 @@ pg_finfo_repack_trigger 8 pg_finfo_repack_version 9 pg_finfo_repack_index_swap 10 pg_finfo_repack_get_table_and_inheritors 11 -pg_finfo_repack_get_relhasoids 12 -repack_apply 13 -repack_disable_autovacuum 14 -repack_drop 15 -repack_get_order_by 16 -repack_indexdef 17 -repack_swap 18 -repack_trigger 19 -repack_version 20 -repack_index_swap 21 -repack_get_table_and_inheritors 22 -repack_get_relhasoids 23 +repack_apply 12 +repack_disable_autovacuum 13 +repack_drop 14 +repack_get_order_by 15 +repack_indexdef 16 +repack_swap 17 +repack_trigger 18 +repack_version 19 +repack_index_swap 20 +repack_get_table_and_inheritors 21 diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index 6235cbb3..99003b6e 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -154,10 +154,6 @@ WHERE $$ LANGUAGE sql STABLE STRICT; -CREATE FUNCTION repack.get_relhasoids(oid) - RETURNS BOOL AS 'MODULE_PATHNAME', 'repack_get_relhasoids' -LANGUAGE C STABLE STRICT; - -- Get a comma-separated storage paramter for the table including -- paramters for the corresponding TOAST table. -- Note that since oid setting is always not NULL, this function @@ -182,10 +178,12 @@ FROM ( UNION ALL -- table oid SELECT 'oids = ' || - CASE WHEN repack.get_relhasoids($1) + CASE WHEN relhasoids THEN 'true' ELSE 'false' END + FROM pg_class + WHERE oid = $1 ) as t $$ diff --git a/lib/repack.c b/lib/repack.c index 05229c83..d75f2562 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -80,7 +80,6 @@ extern Datum PGUT_EXPORT repack_drop(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_disable_autovacuum(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_index_swap(PG_FUNCTION_ARGS); extern Datum PGUT_EXPORT repack_get_table_and_inheritors(PG_FUNCTION_ARGS); -extern Datum PGUT_EXPORT repack_get_relhasoids(PG_FUNCTION_ARGS); PG_FUNCTION_INFO_V1(repack_version); PG_FUNCTION_INFO_V1(repack_trigger); @@ -92,7 +91,6 @@ PG_FUNCTION_INFO_V1(repack_drop); PG_FUNCTION_INFO_V1(repack_disable_autovacuum); PG_FUNCTION_INFO_V1(repack_index_swap); PG_FUNCTION_INFO_V1(repack_get_table_and_inheritors); -PG_FUNCTION_INFO_V1(repack_get_relhasoids); static void repack_init(void); static SPIPlanPtr repack_prepare(const char *src, int nargs, Oid *argtypes); @@ -133,7 +131,7 @@ must_be_superuser(const char *func) #if PG_VERSION_NUM < 120000 #define RENAME_INDEX(relid, newrelname) RENAME_REL(relid, newrelname); #else -#define RENAME_INDEX(relid, newrelname) RenameRelationInternal(relid, newrelname, true, false); +#define RENAME_INDEX(relid, newrelname) RenameRelationInternal(relid, newrelname, true, true); #endif #ifdef REPACK_VERSION @@ -1431,34 +1429,3 @@ repack_get_table_and_inheritors(PG_FUNCTION_ARGS) PG_RETURN_ARRAYTYPE_P(result); } - -/* - * PostgreSQL 12 has no more support for "with oids" option and does - * not have pg_class.relhasoids. Extract pg_class.relhasoid lookup to - * seperate C routine - */ -Datum -repack_get_relhasoids(PG_FUNCTION_ARGS) -{ -#if PG_VERSION_NUM >= 120000 - PG_RETURN_BOOL(false); -#else - Relation class_rel; - HeapTuple tuple; - Form_pg_class tuple_class; - bool relhasoid; - - class_rel = heap_open(RelationRelationId, RowShareLock); - - tuple = SearchSysCacheCopy1(RELOID, PG_GETARG_DATUM(0)); - if (!HeapTupleIsValid(tuple)) - elog(ERROR, "cache lookup failed for relation %u", - PG_GETARG_OID(0)); - tuple_class = (Form_pg_class) GETSTRUCT(tuple); - - relhasoid = tuple_class->relhasoids; - - heap_close(class_rel, RowShareLock); - PG_RETURN_BOOL(relhasoid); -#endif -} From 91fe63e272ca6e2e9d24278286e7e00999dd6848 Mon Sep 17 00:00:00 2001 From: melkij Date: Sun, 25 Aug 2019 14:58:53 +0300 Subject: [PATCH 04/14] tablespace test varian with qualified table name and without "TABLESPACE pg_default" sentence Due late april postgresql commit 87259588d0ab0b8e742e30596afa7ae25caadb18 --- regress/expected/tablespace_4.out | 234 ++++++++++++++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 regress/expected/tablespace_4.out diff --git a/regress/expected/tablespace_4.out b/regress/expected/tablespace_4.out new file mode 100644 index 00000000..6427185a --- /dev/null +++ b/regress/expected/tablespace_4.out @@ -0,0 +1,234 @@ +SET client_min_messages = warning; +-- +-- Tablespace features tests +-- +-- Note: in order to pass this test you must create a tablespace called 'testts' +-- +SELECT spcname FROM pg_tablespace WHERE spcname = 'testts'; + spcname +--------- + testts +(1 row) + +-- If the query above failed you must create the 'testts' tablespace; +CREATE TABLE testts1 (id serial primary key, data text); +CREATE INDEX testts1_partial_idx on testts1 (id) where (id > 0); +CREATE INDEX testts1_with_idx on testts1 (id) with (fillfactor=80); +INSERT INTO testts1 (data) values ('a'); +INSERT INTO testts1 (data) values ('b'); +INSERT INTO testts1 (data) values ('c'); +-- check the indexes definitions +SELECT regexp_replace( + repack.repack_indexdef(indexrelid, 'testts1'::regclass, NULL, false), + '_[0-9]+', '_OID', 'g') +FROM pg_index i join pg_class c ON c.oid = indexrelid +WHERE indrelid = 'testts1'::regclass ORDER BY relname; + regexp_replace +------------------------------------------------------------------------------------ + CREATE INDEX index_OID ON repack.table_OID USING btree (id) WHERE (id > 0) + CREATE UNIQUE INDEX index_OID ON repack.table_OID USING btree (id) + CREATE INDEX index_OID ON repack.table_OID USING btree (id) WITH (fillfactor='80') +(3 rows) + +SELECT regexp_replace( + repack.repack_indexdef(indexrelid, 'testts1'::regclass, 'foo', false), + '_[0-9]+', '_OID', 'g') +FROM pg_index i join pg_class c ON c.oid = indexrelid +WHERE indrelid = 'testts1'::regclass ORDER BY relname; + regexp_replace +--------------------------------------------------------------------------------------------------- + CREATE INDEX index_OID ON repack.table_OID USING btree (id) TABLESPACE foo WHERE (id > 0) + CREATE UNIQUE INDEX index_OID ON repack.table_OID USING btree (id) TABLESPACE foo + CREATE INDEX index_OID ON repack.table_OID USING btree (id) WITH (fillfactor='80') TABLESPACE foo +(3 rows) + +SELECT regexp_replace( + repack.repack_indexdef(indexrelid, 'testts1'::regclass, NULL, true), + '_[0-9]+', '_OID', 'g') +FROM pg_index i join pg_class c ON c.oid = indexrelid +WHERE indrelid = 'testts1'::regclass ORDER BY relname; + regexp_replace +----------------------------------------------------------------------------------------------- + CREATE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) WHERE (id > 0) + CREATE UNIQUE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) + CREATE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) WITH (fillfactor='80') +(3 rows) + +SELECT regexp_replace( + repack.repack_indexdef(indexrelid, 'testts1'::regclass, 'foo', true), + '_[0-9]+', '_OID', 'g') +FROM pg_index i join pg_class c ON c.oid = indexrelid +WHERE indrelid = 'testts1'::regclass ORDER BY relname; + regexp_replace +-------------------------------------------------------------------------------------------------------------- + CREATE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) TABLESPACE foo WHERE (id > 0) + CREATE UNIQUE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) TABLESPACE foo + CREATE INDEX CONCURRENTLY index_OID ON public.testts1 USING btree (id) WITH (fillfactor='80') TABLESPACE foo +(3 rows) + +-- can move the tablespace from default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts +INFO: repacking table "public.testts1" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- + testts1 | testts +(1 row) + +SELECT * from testts1 order by id; + id | data +----+------ + 1 | a + 2 | b + 3 | c +(3 rows) + +-- tablespace stays where it is +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 +INFO: repacking table "public.testts1" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- + testts1 | testts +(1 row) + +-- can move the ts back to default +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -s pg_default +INFO: repacking table "public.testts1" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- +(0 rows) + +-- can move the table together with the indexes +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --tablespace testts --moveidx +INFO: repacking table "public.testts1" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------------------+--------- + testts1 | testts + testts1_partial_idx | testts + testts1_pkey | testts + testts1_with_idx | testts +(4 rows) + +-- can't specify --moveidx without --tablespace +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 --moveidx +ERROR: cannot specify --moveidx (-S) without --tablespace (-s) +\! pg_repack --dbname=contrib_regression --no-order --table=testts1 -S +ERROR: cannot specify --moveidx (-S) without --tablespace (-s) +-- not broken with order +\! pg_repack --dbname=contrib_regression -o id --table=testts1 --tablespace pg_default --moveidx +INFO: repacking table "public.testts1" +--move all indexes of the table to a tablespace +\! pg_repack --dbname=contrib_regression --table=testts1 --only-indexes --tablespace=testts +INFO: repacking indexes of "testts1" +INFO: repacking index "public.testts1_partial_idx" +INFO: repacking index "public.testts1_pkey" +INFO: repacking index "public.testts1_with_idx" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------------------+--------- + testts1_partial_idx | testts + testts1_pkey | testts + testts1_with_idx | testts +(3 rows) + +--all indexes of tablespace remain in same tablespace +\! pg_repack --dbname=contrib_regression --table=testts1 --only-indexes +INFO: repacking indexes of "testts1" +INFO: repacking index "public.testts1_partial_idx" +INFO: repacking index "public.testts1_pkey" +INFO: repacking index "public.testts1_with_idx" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------------------+--------- + testts1_partial_idx | testts + testts1_pkey | testts + testts1_with_idx | testts +(3 rows) + +--move all indexes of the table to pg_default +\! pg_repack --dbname=contrib_regression --table=testts1 --only-indexes --tablespace=pg_default +INFO: repacking indexes of "testts1" +INFO: repacking index "public.testts1_partial_idx" +INFO: repacking index "public.testts1_pkey" +INFO: repacking index "public.testts1_with_idx" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- +(0 rows) + +--move one index to a tablespace +\! pg_repack --dbname=contrib_regression --index=testts1_pkey --tablespace=testts +INFO: repacking index "public.testts1_pkey" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +--------------+--------- + testts1_pkey | testts +(1 row) + +--index tablespace stays as is +\! pg_repack --dbname=contrib_regression --index=testts1_pkey +INFO: repacking index "public.testts1_pkey" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +--------------+--------- + testts1_pkey | testts +(1 row) + +--move index to pg_default +\! pg_repack --dbname=contrib_regression --index=testts1_pkey --tablespace=pg_default +INFO: repacking index "public.testts1_pkey" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +---------+--------- +(0 rows) + +--using multiple --index option +\! pg_repack --dbname=contrib_regression --index=testts1_pkey --index=testts1_with_idx --tablespace=testts +INFO: repacking index "public.testts1_pkey" +INFO: repacking index "public.testts1_with_idx" +SELECT relname, spcname +FROM pg_class JOIN pg_tablespace ts ON ts.oid = reltablespace +WHERE relname ~ '^testts1' +ORDER BY relname; + relname | spcname +------------------+--------- + testts1_pkey | testts + testts1_with_idx | testts +(2 rows) + +--using --indexes-only and --index option together +\! pg_repack --dbname=contrib_regression --table=testts1 --only-indexes --index=testts1_pkey +ERROR: cannot specify --index (-i) and --table (-t) From 9e68b063cb4e0038a545fb12bb47417a0b47f807 Mon Sep 17 00:00:00 2001 From: melkij Date: Fri, 4 Oct 2019 14:41:14 +0300 Subject: [PATCH 05/14] support for PostgreSQL 12: documentation changes, adding PG 12 to travis testing grid Prepare to releasee 1.4.5 --- .travis.yml | 2 ++ META.json | 4 ++-- doc/pg_repack.rst | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a60037d5..d354fa19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ dist: trusty sudo: required env: + - PGVER=12 + PGTESTING=1 - PGVER=11 PGTESTING=1 - PGVER=10 diff --git a/META.json b/META.json index 62dd4049..a53ce0d8 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.4.4", + "version": "1.4.5", "maintainer": [ "Beena Emerson ", "Josh Kupershmidt ", @@ -15,7 +15,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.4.4", + "version": "1.4.5", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index e5ffced6..efa60bc5 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -40,7 +40,7 @@ Requirements ------------ PostgreSQL versions - PostgreSQL 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 10, 11 + PostgreSQL 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 10, 11, 12 Disks Performing a full-table repack requires free disk space about twice as @@ -466,6 +466,11 @@ Creating indexes concurrently comes with a few caveats, please see `the document Releases -------- +* pg_repack 1.4.5 + + * Added support for PostgreSQL 12 + * Fixed parallel processing for indexes with operators from public schema + * pg_repack 1.4.4 * Added support for PostgreSQL 11 (issue #181) From 1cb05a157ffecd788266f8360b06a0437d88dad9 Mon Sep 17 00:00:00 2001 From: melkij Date: Fri, 4 Oct 2019 14:51:52 +0300 Subject: [PATCH 06/14] Ubuntu trusty is EOL, try move to bionic 18.04 release --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d354fa19..56b6c33c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # Travis CI configuration file for psycopg2 -dist: trusty +dist: bionic sudo: required env: From c3e3a79350c142d4922ae15739fd0ef82072f903 Mon Sep 17 00:00:00 2001 From: melkij Date: Fri, 4 Oct 2019 15:21:37 +0300 Subject: [PATCH 07/14] try travis on xenial with all supported versions --- .travis.yml | 2 +- regress/travis_prepare.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 56b6c33c..17b59835 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ # Travis CI configuration file for psycopg2 -dist: bionic +dist: xenial sudo: required env: diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index dbb1b407..020d8674 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -12,11 +12,11 @@ export PATH="$PGBIN:$PATH" # This also stops the server currently running on port 5432 sudo apt-get remove -y libpq5 -if [ "$PGVER" = "9.1" ]; then +if [[ "$PGVER" = "9.1" || "$PGVER" = "9.2" || "$PGVER" = "9.3" ]]; then # Dinosaur package kindly provided by psycopg sudo mkdir -p /usr/lib/postgresql - wget -O - http://initd.org/psycopg/upload/postgresql/postgresql-${PGVER}.tar.bz2 \ + wget -O - http://initd.org/psycopg/upload/postgresql/postgresql-${PGVER}-xenial.tar.bz2 \ | sudo tar xjf - -C /usr/lib/postgresql cd / @@ -32,7 +32,7 @@ else /etc/apt/sources.list.d/pgdg.list if [ "$PGTESTING" != "" ]; then - sudo sed -i "s/trusty-pgdg/trusty-pgdg-testing/" \ + sudo sed -i "s/xenial-pgdg/xenial-pgdg-testing/" \ /etc/apt/sources.list.d/pgdg.list fi From d52c633675c9c7e5e970f25ef7e228b366a8add3 Mon Sep 17 00:00:00 2001 From: melkij Date: Fri, 4 Oct 2019 16:39:45 +0300 Subject: [PATCH 08/14] yet another try to make travis works --- .travis.yml | 3 --- regress/travis_prepare.sh | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17b59835..47d0101c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,8 @@ sudo: required env: - PGVER=12 - PGTESTING=1 - PGVER=11 - PGTESTING=1 - PGVER=10 - PGTESTING=1 - PGVER=9.6 - PGVER=9.5 - PGVER=9.4 diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 020d8674..c84ee02d 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -40,6 +40,19 @@ else sudo apt-get install -y "libpq5=${PGVER}*" "libpq-dev=${PGVER}*" sudo apt-mark hold libpq5 sudo apt-get install -y postgresql-server-dev-$PGVER postgresql-$PGVER + + # ensure PostgreSQL is running on 5432 port with proper auth + sudo sed -i \ + 's/\(^local[[:space:]]\+all[[:space:]]\+all[[:space:]]\+\).*/\1trust/' \ + "$CONFDIR/pg_hba.conf" + sudo bash -c "echo 'port=5432' >> $CONFDIR/postgresql.conf" + sudo service postgresql restart $PGVER + + # ensure travis user exists. May be missed if the database was not provided by Travis + userexists=`"$PGBIN/psql" -tc "select count(*) from pg_catalog.pg_user where usename='travis';"` + if [ ${userexists} -eq 0 ]; then + sudo -u postgres "$PGBIN/psql" -c "create user travis superuser" + fi fi # Go somewhere else or sudo will fail @@ -51,17 +64,5 @@ sudo -u postgres mkdir -p /var/lib/postgresql/testts sudo -u postgres "$PGBIN/psql" \ -c "create tablespace testts location '/var/lib/postgresql/testts'" -# If the database was not provided by Travis it needs some customization -if [ "$PGTESTING" != "" ]; then - - # Allow local connections with no password - sudo sed -i \ - 's/\(^local[[:space:]]\+all[[:space:]]\+all[[:space:]]\+\).*/\1trust/' \ - "$CONFDIR/pg_hba.conf" - sudo -u postgres "$PGBIN/pg_ctl" -D "$DATADIR" reload - - sudo -u postgres "$PGBIN/psql" -c "create user travis superuser" -fi - # Go back to the build dir cd - From 58614f14bdef792385aad0edf8c7b0bf88b8a622 Mon Sep 17 00:00:00 2001 From: melkij Date: Fri, 4 Oct 2019 16:43:02 +0300 Subject: [PATCH 09/14] one missed sudo, yet another try --- regress/travis_prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index c84ee02d..80db9823 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -49,7 +49,7 @@ else sudo service postgresql restart $PGVER # ensure travis user exists. May be missed if the database was not provided by Travis - userexists=`"$PGBIN/psql" -tc "select count(*) from pg_catalog.pg_user where usename='travis';"` + userexists=`sudo -u postgres "$PGBIN/psql" -tc "select count(*) from pg_catalog.pg_user where usename='travis';"` if [ ${userexists} -eq 0 ]; then sudo -u postgres "$PGBIN/psql" -c "create user travis superuser" fi From 7513317c1108f949997a50169308c564ddcc0a25 Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Wed, 18 Aug 2021 21:03:51 +0900 Subject: [PATCH 10/14] Take ACCESS EXCLUSIVE LOCK carefully (#8) Try to get the lock in loop with timeouts using `lock_exclusive()`. This allows to not stuck lock queue for readers if pg_repack cannot take the lock fast enough. --- bin/pg_repack.c | 25 +++++++++++++++++++++++-- bin/pgut/pgut.h | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 38d40e22..d721a33f 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -1164,7 +1164,7 @@ repack_one_table(repack_table *table, const char *orderby) return; /* push repack_cleanup_callback() on stack to clean temporary objects */ - pgut_atexit_push(repack_cleanup_callback, &table->target_oid); + pgut_atexit_push(repack_cleanup_callback, table); /* * 1. Setup advisory lock and trigger on main table. @@ -1854,7 +1854,8 @@ lock_exclusive(PGconn *conn, const char *relid, const char *lock_query, bool sta void repack_cleanup_callback(bool fatal, void *userdata) { - Oid target_table = *(Oid *) userdata; + repack_table *table = (repack_table *) userdata; + Oid target_table = table->target_oid; const char *params[2]; char buffer[12]; char num_buff[12]; @@ -1869,7 +1870,17 @@ repack_cleanup_callback(bool fatal, void *userdata) * so just use an unconditional reconnect(). */ reconnect(ERROR); + + command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL); + if (!(lock_exclusive(connection, params[0], table->lock_table, false))) + { + elog(WARNING, "lock_exclusive() failed in connection for %s during cleanup callback", + table->target_name); + return; + } + command("SELECT repack.repack_drop($1, $2)", 2, params); + command("COMMIT", 0, NULL); temp_obj_num = 0; /* reset temporary object counter after cleanup */ } } @@ -1899,7 +1910,17 @@ repack_cleanup(bool fatal, const repack_table *table) /* do cleanup */ params[0] = utoa(table->target_oid, buffer); params[1] = utoa(temp_obj_num, num_buff); + + command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL); + if (!(lock_exclusive(connection, params[0], table->lock_table, false))) + { + elog(WARNING, "lock_exclusive() failed in connection for %s during cleanup", + table->target_name); + return; + } + command("SELECT repack.repack_drop($1, $2)", 2, params); + command("COMMIT", 0, NULL); temp_obj_num = 0; /* reset temporary object counter after cleanup */ } } diff --git a/bin/pgut/pgut.h b/bin/pgut/pgut.h index e347bea7..9253855b 100644 --- a/bin/pgut/pgut.h +++ b/bin/pgut/pgut.h @@ -51,7 +51,7 @@ extern const char *PROGRAM_ISSUES; extern bool interrupted; extern int pgut_log_level; extern int pgut_abort_level; -extern bool pgut_echo; +extern bool pgut_echo; extern void pgut_init(int argc, char **argv); extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); From 95dd7fce12baa06191618ec320fdbdf37bc973f1 Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Fri, 20 Aug 2021 18:17:25 +0900 Subject: [PATCH 11/14] Raise ERROR message during repack_drop() (#9) It is better to raise ERROR message if lock_exclusive() failed during calling repack.repack_drop() otherwise a user might not notice that pg_repack didn't clean up leftovers. --- bin/pg_repack.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index d721a33f..019db9ea 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -1874,9 +1874,8 @@ repack_cleanup_callback(bool fatal, void *userdata) command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL); if (!(lock_exclusive(connection, params[0], table->lock_table, false))) { - elog(WARNING, "lock_exclusive() failed in connection for %s during cleanup callback", + elog(ERROR, "lock_exclusive() failed in connection for %s during cleanup callback", table->target_name); - return; } command("SELECT repack.repack_drop($1, $2)", 2, params); @@ -1914,9 +1913,8 @@ repack_cleanup(bool fatal, const repack_table *table) command("BEGIN ISOLATION LEVEL READ COMMITTED", 0, NULL); if (!(lock_exclusive(connection, params[0], table->lock_table, false))) { - elog(WARNING, "lock_exclusive() failed in connection for %s during cleanup", + elog(ERROR, "lock_exclusive() failed in connection for %s during cleanup", table->target_name); - return; } command("SELECT repack.repack_drop($1, $2)", 2, params); From 0b2ecf7122b792d3de4f380dbedd19ba9ac54f6c Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Tue, 1 Feb 2022 20:31:42 +0100 Subject: [PATCH 12/14] Merge upstream (#10) * simple_prompt signature was changed * Use SearchSysCacheCopy1 macro instead of SearchSysCacheCopy direct call as more future-proof Per syscache.h: > The use of the macros below rather than direct calls to the corresponding > functions is encouraged, as it insulates the caller from changes in the > maximum number of keys. Also this fixes segfault on pg14, but still not sure why exactly. * Get rid the custom array_accum aggregate in favor of postgresql's built-in array_agg and string_agg. In postgresql 14, the signature of array_append was changed, and our create aggregate throws the error "function array_append(anyarray, anyelement) does not exist". Postgresql's built-in string_agg was introduced in postgresql 9.0, array_agg was from 8.4 release. Both are too old and no longer supported by pg_repack. So, instead of fixing the repack.array_accum, I want to drop it. One notable behavior difference is handling empty sets: array_agg will produce NULL. So I put several coalesce to avoid altering the query results. * Check for the existence of the tables specified by --table or --parent-table PostgreSQL 14 will produce something like ERROR: pg_repack failed with error: ERROR: relation "dummy_table" does not exist CONTEXT: unnamed portal parameter $2 = '...' The second line looks weird and breaks tests. The second word "ERROR" also looks strange. So an explicit check for the existence of a table has been added. * Prepare release 1.4.7 First sketch to see what buildfarm thinks. Also recheck PGVER=9.6 build * ubuntu xenial is EOL, try focal 20.04 LTS * third party apt-repositories removed from the Bionic build image. Next try https://docs.travis-ci.com/user/reference/bionic/#third-party-apt-repositories-removed * amd64 packages only, please * apt repository section was changed? * postgresql-14 beta1 should be already in repo, retry * try a workaround for pg9.5, 9.6 Possible we decide to drop these versions, but just check * missing build dependency by postgresql-server-dev lz4 * update docs broken links * Remove connection info from error log * remove unnecessary plus operator * fix typo in pgut.c * Fix typo Fix typo * Reassure the user that it's ok to drop the extension See #281 Co-authored-by: melkij Co-authored-by: Valeriya Popova Co-authored-by: Daniel Merken Co-authored-by: lincuiping <57204139+lincuiping@users.noreply.github.com> Co-authored-by: sunhm89 <93200340+sunhm89@users.noreply.github.com> Co-authored-by: zhuqx-fnst <94515051+zhuqx-fnst@users.noreply.github.com> Co-authored-by: Daniele Varrazzo --- .travis.yml | 5 +- META.json | 8 +-- README.rst | 4 +- bin/pg_repack.c | 123 ++++++++++++++++++++++++++++++++++++-- bin/pgut/pgut.c | 23 +++++-- doc/pg_repack.rst | 11 ++-- doc/pg_repack_jp.rst | 3 +- lib/pg_repack.sql.in | 35 +++++------ lib/repack.c | 8 +-- regress/travis_prepare.sh | 14 +++-- 10 files changed, 178 insertions(+), 56 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcd27df8..06b38614 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,17 @@ # Travis CI configuration file for psycopg2 -dist: xenial +dist: focal sudo: required env: + - PGVER=14 - PGVER=13 - PGVER=12 - PGVER=11 - PGVER=10 # Disabled because packages broken at least on xenial # https://www.postgresql.org/message-id/CA%2Bmi_8a1oEnCzkt0CvqysgY4MQ6jEefjmS%3Dq_K-AvOx%3DF7m2%2BQ%40mail.gmail.com - # - PGVER=9.6 + - PGVER=9.6 - PGVER=9.5 - PGVER=9.4 diff --git a/META.json b/META.json index d12358b1..54ed45c0 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_repack", "abstract": "PostgreSQL module for data reorganization", "description": "Reorganize tables in PostgreSQL databases with minimal locks", - "version": "1.4.6", + "version": "1.4.7", "maintainer": [ "Beena Emerson ", "Josh Kupershmidt ", @@ -15,7 +15,7 @@ "provides": { "pg_repack": { "file": "lib/pg_repack.sql", - "version": "1.4.6", + "version": "1.4.7", "abstract": "Reorganize tables in PostgreSQL databases with minimal locks" } }, @@ -27,7 +27,7 @@ } }, "resources": { - "homepage": "http://reorg.github.com/pg_repack", + "homepage": "https://reorg.github.io/pg_repack", "bugtracker": { "web": "https://github.com/reorg/pg_repack/issues" }, @@ -39,6 +39,6 @@ }, "meta-spec": { "version": "1.0.0", - "url": "http://pgxn.org/meta/spec.txt" + "url": "https://pgxn.org/meta/spec.txt" } } diff --git a/README.rst b/README.rst index ef10b317..247be93d 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ pg_repack -- Reorganize tables in PostgreSQL databases with minimal locks ========================================================================= -- Homepage: https://reorg.github.com/pg_repack +- Homepage: https://reorg.github.io/pg_repack - Download: https://pgxn.org/dist/pg_repack/ - Development: https://github.com/reorg/pg_repack - Bug Report: https://github.com/reorg/pg_repack/issues @@ -22,7 +22,7 @@ CLUSTER directly. Please check the documentation (in the ``doc`` directory or online_) for installation and usage instructions. -.. _pg_repack: https://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: https://www.postgresql.org/docs/current/static/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: https://www.postgresql.org/docs/current/static/sql-vacuum.html diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 019db9ea..377d3e49 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -75,7 +75,7 @@ const char *PROGRAM_VERSION = "unknown"; * pg_regress. */ #define SQL_XID_SNAPSHOT_90200 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l " \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.pid " \ @@ -90,7 +90,7 @@ const char *PROGRAM_VERSION = "unknown"; " AND ((d.datname IS NULL OR d.datname = current_database()) OR l.database = 0)" #define SQL_XID_SNAPSHOT_90000 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l " \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.procpid " \ @@ -108,7 +108,7 @@ const char *PROGRAM_VERSION = "unknown"; * the WHERE clause is just to eat the $2 parameter (application name). */ #define SQL_XID_SNAPSHOT_80300 \ - "SELECT repack.array_accum(l.virtualtransaction) " \ + "SELECT coalesce(array_agg(l.virtualtransaction), '{}') " \ " FROM pg_locks AS l" \ " LEFT JOIN pg_stat_activity AS a " \ " ON l.pid = a.procpid " \ @@ -212,6 +212,7 @@ typedef struct repack_table static bool is_superuser(void); static void check_tablespace(void); static bool preliminary_checks(char *errbuf, size_t errsize); +static bool is_requested_relation_exists(char *errbuf, size_t errsize); static void repack_all_databases(const char *order_by); static bool repack_one_database(const char *order_by, char *errbuf, size_t errsize); static void repack_one_table(repack_table *table, const char *order_by); @@ -514,7 +515,8 @@ preliminary_checks(char *errbuf, size_t errsize){ { if (errbuf) snprintf(errbuf, errsize, - "extension '%s' required, found extension '%s'", + "extension '%s' required, found '%s';" + " please drop and re-create the extension", buf, libver); goto cleanup; } @@ -558,6 +560,113 @@ preliminary_checks(char *errbuf, size_t errsize){ return ret; } +/* + * Check the presence of tables specified by --parent-table and --table + * otherwise format user-friendly message + */ +static bool +is_requested_relation_exists(char *errbuf, size_t errsize){ + bool ret = false; + PGresult *res = NULL; + const char **params = NULL; + int iparam = 0; + StringInfoData sql; + int num_relations; + SimpleStringListCell *cell; + + num_relations = simple_string_list_size(parent_table_list) + + simple_string_list_size(table_list); + + /* nothing was implicitly requested, so nothing to do here */ + if (num_relations == 0) + return true; + + /* has no suitable to_regclass(text) */ + if (PQserverVersion(connection)<90600) + return true; + + params = pgut_malloc(num_relations * sizeof(char *)); + initStringInfo(&sql); + appendStringInfoString(&sql, "SELECT r FROM (VALUES "); + + for (cell = table_list.head; cell; cell = cell->next) + { + appendStringInfo(&sql, "($%d)", iparam + 1); + params[iparam++] = cell->val; + if (iparam < num_relations) + appendStringInfoChar(&sql, ','); + } + for (cell = parent_table_list.head; cell; cell = cell->next) + { + appendStringInfo(&sql, "($%d)", iparam + 1); + params[iparam++] = cell->val; + if (iparam < num_relations) + appendStringInfoChar(&sql, ','); + } + appendStringInfoString(&sql, + ") AS given_t(r)" + " WHERE NOT EXISTS(" + " SELECT FROM repack.tables WHERE relid=to_regclass(given_t.r) )" + ); + + /* double check the parameters array is sane */ + if (iparam != num_relations) + { + if (errbuf) + snprintf(errbuf, errsize, + "internal error: bad parameters count: %i instead of %i", + iparam, num_relations); + goto cleanup; + } + + res = execute_elevel(sql.data, iparam, params, DEBUG2); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + int num; + + num = PQntuples(res); + + if (num != 0) + { + int i; + StringInfoData rel_names; + initStringInfo(&rel_names); + + for (i = 0; i < num; i++) + { + appendStringInfo(&rel_names, "\"%s\"", getstr(res, i, 0)); + if ((i + 1) != num) + appendStringInfoString(&rel_names, ", "); + } + + if (errbuf) + { + if (num > 1) + snprintf(errbuf, errsize, + "relations do not exist: %s", rel_names.data); + else + snprintf(errbuf, errsize, + "ERROR: relation %s does not exist", rel_names.data); + } + termStringInfo(&rel_names); + } + else + ret = true; + } + else + { + if (errbuf) + snprintf(errbuf, errsize, "%s", PQerrorMessage(connection)); + } + CLEARPGRES(res); + +cleanup: + CLEARPGRES(res); + termStringInfo(&sql); + free(params); + return ret; +} + /* * Call repack_one_database for each database. */ @@ -657,6 +766,9 @@ repack_one_database(const char *orderby, char *errbuf, size_t errsize) if (!preliminary_checks(errbuf, errsize)) goto cleanup; + if (!is_requested_relation_exists(errbuf, errsize)) + goto cleanup; + /* acquire target tables */ appendStringInfoString(&sql, "SELECT t.*," @@ -2134,6 +2246,9 @@ repack_all_indexes(char *errbuf, size_t errsize) if (!preliminary_checks(errbuf, errsize)) goto cleanup; + if (!is_requested_relation_exists(errbuf, errsize)) + goto cleanup; + if (r_index.head) { appendStringInfoString(&sql, diff --git a/bin/pgut/pgut.c b/bin/pgut/pgut.c index 8804195f..2cd796d1 100644 --- a/bin/pgut/pgut.c +++ b/bin/pgut/pgut.c @@ -10,6 +10,10 @@ #include "postgres_fe.h" #include "libpq/pqsignal.h" +#if PG_VERSION_NUM >= 140000 +#include "common/string.h" /* for simple_prompt */ +#endif + #include #include #include @@ -342,7 +346,7 @@ parse_time(const char *value, time_t *time) char junk[2]; /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + + 1); + tmp = pgut_malloc(strlen(value) + 1); len = 0; for (i = 0; value[i]; i++) tmp[len++] = (IsAlnum(value[i]) ? value[i] : ' '); @@ -450,7 +454,7 @@ prompt_for_password(void) passwdbuf = pgut_malloc(BUFSIZE); memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); } -#else +#elif PG_VERSION_NUM < 140000 buf = pgut_malloc(BUFSIZE); if (have_passwd) { memcpy(buf, passwdbuf, sizeof(char)*BUFSIZE); @@ -461,6 +465,16 @@ prompt_for_password(void) passwdbuf = pgut_malloc(BUFSIZE); memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); } +#else + if (have_passwd) { + buf = pgut_malloc(BUFSIZE); + memcpy(buf, passwdbuf, sizeof(char)*BUFSIZE); + } else { + buf = simple_prompt("Password: ", false); + have_passwd = true; + passwdbuf = pgut_malloc(BUFSIZE); + memcpy(passwdbuf, buf, sizeof(char)*BUFSIZE); + } #endif if (buf == NULL) @@ -548,8 +562,7 @@ pgut_connect(const char *info, YesNo prompt, int elevel) ereport(elevel, (errcode(E_PG_CONNECT), - errmsg("could not connect to database with \"%s\": %s", - info, PQerrorMessage(conn)))); + errmsg("could not connect to database: %s", PQerrorMessage(conn)))); PQfinish(conn); return NULL; } @@ -1151,7 +1164,7 @@ on_interrupt(void) pgutConn *c; int save_errno = errno; - /* Set interruped flag */ + /* Set interrupted flag */ interrupted = true; if (in_cleanup) diff --git a/doc/pg_repack.rst b/doc/pg_repack.rst index 7d2ff174..a5efc22d 100644 --- a/doc/pg_repack.rst +++ b/doc/pg_repack.rst @@ -28,7 +28,7 @@ NOTICE: * Target table must have a PRIMARY KEY, or at least a UNIQUE total index on a NOT NULL column. -.. _pg_repack: http://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: http://www.postgresql.org/docs/current/static/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: http://www.postgresql.org/docs/current/static/sql-vacuum.html @@ -40,7 +40,7 @@ Requirements ------------ PostgreSQL versions - PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13 + PostgreSQL 9.4, 9.5, 9.6, 10, 11, 12, 13, 14 Disks Performing a full-table repack requires free disk space about twice as @@ -356,7 +356,7 @@ ERROR: program 'pg_repack V1' does not match database library 'pg_repack V2' database; if they are what expected you may need to repeat pg_repack installation. -ERROR: extension 'pg_repack V1' required, found extension 'pg_repack V2' +ERROR: extension 'pg_repack V1' required, found 'pg_repack V2' The SQL extension found in the database does not match the version required by the pg_repack program. @@ -466,6 +466,10 @@ Creating indexes concurrently comes with a few caveats, please see `the document Releases -------- +* pg_repack 1.4.7 + + * Added support for PostgreSQL 14 + * pg_repack 1.4.6 * Added support for PostgreSQL 13 @@ -587,4 +591,3 @@ See Also * `clusterdb `__ * `vacuumdb `__ - diff --git a/doc/pg_repack_jp.rst b/doc/pg_repack_jp.rst index 8f314992..b4ac5c5c 100644 --- a/doc/pg_repack_jp.rst +++ b/doc/pg_repack_jp.rst @@ -50,7 +50,7 @@ pg_repackでは再編成する方法として次のものが選択できます * DBのスーパーユーザだけがpg_repackを実行できます * 対象となるテーブルは主キー、もしくはNOT NULL制約を持つカラムへのユニーク制約をもつインデックスが存在している必要があります -.. _pg_repack: http://reorg.github.com/pg_repack +.. _pg_repack: https://reorg.github.io/pg_repack .. _CLUSTER: http://www.postgresql.jp/document/current/html/sql-cluster.html .. _VACUUM FULL: VACUUM_ .. _VACUUM: http://www.postgresql.jp/document/current/html/sql-vacuum.html @@ -1050,4 +1050,3 @@ ACCESS EXCLUSIVEロックを取得します。その他のステップでは、A * `clusterdb `__ * `vacuumdb `__ - diff --git a/lib/pg_repack.sql.in b/lib/pg_repack.sql.in index ebbf9f2c..ea45b4f7 100644 --- a/lib/pg_repack.sql.in +++ b/lib/pg_repack.sql.in @@ -16,13 +16,6 @@ CREATE FUNCTION repack.version_sql() RETURNS text AS $$SELECT 'pg_repack REPACK_VERSION'::text$$ LANGUAGE SQL IMMUTABLE STRICT; -CREATE AGGREGATE repack.array_accum ( - sfunc = array_append, - basetype = anyelement, - stype = anyarray, - initcond = '{}' -); - -- Always specify search_path to 'pg_catalog' so that we -- always can get schema-qualified relation name CREATE FUNCTION repack.oid2text(oid) RETURNS text AS @@ -33,7 +26,7 @@ LANGUAGE sql STABLE STRICT SET search_path to 'pg_catalog'; CREATE FUNCTION repack.get_index_columns(oid, text) RETURNS text AS $$ - SELECT array_to_string(repack.array_accum(quote_ident(attname)), $2) + SELECT coalesce(string_agg(quote_ident(attname), $2), '') FROM pg_attribute, (SELECT indrelid, indkey, @@ -53,8 +46,8 @@ LANGUAGE C STABLE STRICT; CREATE FUNCTION repack.get_create_index_type(oid, name) RETURNS text AS $$ SELECT 'CREATE TYPE ' || $2 || ' AS (' || - array_to_string(repack.array_accum(quote_ident(attname) || ' ' || - pg_catalog.format_type(atttypid, atttypmod)), ', ') || ')' + coalesce(string_agg(quote_ident(attname) || ' ' || + pg_catalog.format_type(atttypid, atttypmod), ', '), '') || ')' FROM pg_attribute, (SELECT indrelid, indkey, @@ -90,9 +83,9 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_assign(oid, text) RETURNS text AS $$ - SELECT '(' || array_to_string(repack.array_accum(quote_ident(attname)), ', ') || + SELECT '(' || coalesce(string_agg(quote_ident(attname), ', '), '') || ') = (' || $2 || '.' || - array_to_string(repack.array_accum(quote_ident(attname)), ', ' || $2 || '.') || ')' + coalesce(string_agg(quote_ident(attname), ', ' || $2 || '.'), '') || ')' FROM (SELECT attname FROM pg_attribute WHERE attrelid = $1 AND attnum > 0 AND NOT attisdropped ORDER BY attnum) tmp; @@ -102,9 +95,9 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_compare_pkey(oid, text) RETURNS text AS $$ - SELECT '(' || array_to_string(repack.array_accum(quote_ident(attname)), ', ') || + SELECT '(' || coalesce(string_agg(quote_ident(attname), ', '), '') || ') = (' || $2 || '.' || - array_to_string(repack.array_accum(quote_ident(attname)), ', ' || $2 || '.') || ')' + coalesce(string_agg(quote_ident(attname), ', ' || $2 || '.'), '') || ')' FROM pg_attribute, (SELECT indrelid, indkey, @@ -122,7 +115,7 @@ LANGUAGE sql STABLE STRICT; CREATE FUNCTION repack.get_columns_for_create_as(oid) RETURNS text AS $$ -SELECT array_to_string(repack.array_accum(c), ',') FROM (SELECT +SELECT coalesce(string_agg(c, ','), '') FROM (SELECT CASE WHEN attisdropped THEN 'NULL::integer AS ' || quote_ident(attname) ELSE quote_ident(attname) @@ -142,7 +135,7 @@ SELECT 'ALTER TABLE ' || $2 || ' ' || array_to_string(dropped_columns, ', ') FROM ( SELECT - repack.array_accum('DROP COLUMN ' || quote_ident(attname)) AS dropped_columns + array_agg('DROP COLUMN ' || quote_ident(attname)) AS dropped_columns FROM ( SELECT * FROM pg_attribute WHERE attrelid = $1 AND attnum > 0 AND attisdropped @@ -154,14 +147,14 @@ WHERE $$ LANGUAGE sql STABLE STRICT; --- Get a comma-separated storage paramter for the table including --- paramters for the corresponding TOAST table. +-- Get a comma-separated storage parameter for the table including +-- parameters for the corresponding TOAST table. -- Note that since oid setting is always not NULL, this function -- never returns NULL CREATE FUNCTION repack.get_storage_param(oid) RETURNS TEXT AS $$ -SELECT array_to_string(array_agg(param), ', ') +SELECT string_agg(param, ', ') FROM ( -- table storage parameter SELECT unnest(reloptions) as param @@ -196,7 +189,7 @@ $$ SELECT 'ALTER TABLE repack.table_' || $1 || array_to_string(column_storage, ',') FROM ( SELECT - repack.array_accum(' ALTER ' || quote_ident(attname) || + array_agg(' ALTER ' || quote_ident(attname) || CASE attstorage WHEN 'p' THEN ' SET STORAGE PLAIN' WHEN 'm' THEN ' SET STORAGE MAIN' @@ -222,7 +215,7 @@ LANGUAGE sql STABLE STRICT; -- includes not only PRIMARY KEYS but also UNIQUE NOT NULL keys CREATE VIEW repack.primary_keys AS - SELECT indrelid, (repack.array_accum(indexrelid))[1] AS indexrelid + SELECT indrelid, min(indexrelid) AS indexrelid FROM (SELECT indrelid, indexrelid FROM pg_index WHERE indisunique AND indisvalid diff --git a/lib/repack.c b/lib/repack.c index 2690a9ce..96660c0d 100644 --- a/lib/repack.c +++ b/lib/repack.c @@ -1167,16 +1167,12 @@ swap_heap_or_index_files(Oid r1, Oid r2) relRelation = heap_open(RelationRelationId, RowExclusiveLock); #endif - reltup1 = SearchSysCacheCopy(RELOID, - ObjectIdGetDatum(r1), - 0, 0, 0); + reltup1 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r1)); if (!HeapTupleIsValid(reltup1)) elog(ERROR, "cache lookup failed for relation %u", r1); relform1 = (Form_pg_class) GETSTRUCT(reltup1); - reltup2 = SearchSysCacheCopy(RELOID, - ObjectIdGetDatum(r2), - 0, 0, 0); + reltup2 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r2)); if (!HeapTupleIsValid(reltup2)) elog(ERROR, "cache lookup failed for relation %u", r2); relform2 = (Form_pg_class) GETSTRUCT(reltup2); diff --git a/regress/travis_prepare.sh b/regress/travis_prepare.sh index 8e105a4a..cee73cc5 100755 --- a/regress/travis_prepare.sh +++ b/regress/travis_prepare.sh @@ -14,13 +14,10 @@ sudo apt-get remove -y libpq5 # Match libpq and server-dev packages # See https://github.com/reorg/pg_repack/issues/63 -sudo sed -i "s/main[[:space:]]*$/main ${PGVER}/" \ - /etc/apt/sources.list.d/pgdg.list +sudo sh -c 'echo "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main ${PGVER}" > /etc/apt/sources.list.d/pgdg.list' -if [ "$PGTESTING" != "" ]; then - sudo sed -i "s/xenial-pgdg/xenial-pgdg-testing/" \ - /etc/apt/sources.list.d/pgdg.list -fi +# Import the repository signing key: +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update @@ -32,6 +29,11 @@ if [[ "$PGVER" = "9.4" ]]; then sudo apt-mark hold libpq5 fi +# missing build dependency by postgresql-server-dev +if [[ "$PGVER" -ge "14" ]]; then + sudo apt-get install -y liblz4-dev +fi + if ! sudo apt-get install -y \ postgresql-$PGVER \ postgresql-client-$PGVER \ From e463b81d488af91a4ef58625a8ada415a4e0e7c8 Mon Sep 17 00:00:00 2001 From: Dhatta Sai Chanuka Sista <113626707+chanukya571@users.noreply.github.com> Date: Tue, 20 Dec 2022 14:51:02 +0100 Subject: [PATCH 13/14] error messages for system tables to use VACUUM FULL. --- bin/pg_repack.c | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 377d3e49..74289b7a 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -211,6 +211,7 @@ typedef struct repack_table static bool is_superuser(void); static void check_tablespace(void); +static bool check_systemtables(void); static bool preliminary_checks(char *errbuf, size_t errsize); static bool is_requested_relation_exists(char *errbuf, size_t errsize); static void repack_all_databases(const char *order_by); @@ -421,6 +422,51 @@ is_superuser(void) return false; } +bool +check_systemtables() + +{ + PGresult *query_result = NULL; + int num; + SimpleStringListCell *cell; + StringInfoData sql; + int iparam = 0; + + + num = simple_string_list_size(table_list); + + initStringInfo(&sql); + + appendStringInfoString(&sql, "select 1 from pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace where n.nspname IN ('pg_catalog','information_schema') AND c.oid IN ("); + + + for (cell = table_list.head; cell; cell = cell->next) + { + appendStringInfo(&sql, "'%s'::regclass::oid", cell->val); + iparam++; + if (iparam < num) + appendStringInfoChar(&sql, ','); + } + + appendStringInfoString(&sql,")"); + + + query_result = execute_elevel(sql.data,0,NULL, DEBUG2); + + if (PQresultStatus(query_result) == PGRES_TUPLES_OK) + { + + if (PQntuples(query_result) >= 1) + { + return true; + } + } + + CLEARPGRES(query_result); + + return false; +} + /* * Check if the tablespace requested exists. * @@ -487,6 +533,12 @@ preliminary_checks(char *errbuf, size_t errsize){ goto cleanup; } + if (check_systemtables()) { + if (errbuf) + snprintf(errbuf, errsize, "For System Tables Use VACUUM FULL."); + goto cleanup; + } + /* Query the extension version. Exit if no match */ res = execute_elevel("select repack.version(), repack.version_sql()", 0, NULL, DEBUG2); From 2fb32615efd2df5a256a7a2cfb54db2f899949c9 Mon Sep 17 00:00:00 2001 From: Dhatta Sai Chanuka Sista <113626707+chanukya571@users.noreply.github.com> Date: Tue, 20 Dec 2022 14:56:47 +0100 Subject: [PATCH 14/14] code alignment --- bin/pg_repack.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/bin/pg_repack.c b/bin/pg_repack.c index 74289b7a..bed8427d 100644 --- a/bin/pg_repack.c +++ b/bin/pg_repack.c @@ -426,12 +426,11 @@ bool check_systemtables() { - PGresult *query_result = NULL; - int num; - SimpleStringListCell *cell; - StringInfoData sql; - int iparam = 0; - + PGresult *query_result = NULL; + int num; + SimpleStringListCell *cell; + StringInfoData sql; + int iparam = 0; num = simple_string_list_size(table_list); @@ -439,7 +438,6 @@ check_systemtables() appendStringInfoString(&sql, "select 1 from pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace where n.nspname IN ('pg_catalog','information_schema') AND c.oid IN ("); - for (cell = table_list.head; cell; cell = cell->next) { appendStringInfo(&sql, "'%s'::regclass::oid", cell->val); @@ -450,17 +448,15 @@ check_systemtables() appendStringInfoString(&sql,")"); - query_result = execute_elevel(sql.data,0,NULL, DEBUG2); if (PQresultStatus(query_result) == PGRES_TUPLES_OK) + { + if (PQntuples(query_result) >= 1) { - - if (PQntuples(query_result) >= 1) - { - return true; - } + return true; } + } CLEARPGRES(query_result);