From 9fa73a60874efaadb8acd9f958ff7ce0e4a298bd Mon Sep 17 00:00:00 2001 From: Muhammad Usama Date: Thu, 24 Aug 2023 18:25:22 +0500 Subject: [PATCH] Code re-arrangement, Makefile fixes and implementing '.tde' relation fork (#5) - Fixing the Makefile to link with ssl and crypto libs required by the extension. - An entry file pg_tde.c for all extension-related initialization functions - A new ".tde" relation fork to store the encryption key and related data. Currently, the relation fork gets created with new relations and a dummy encryption key gets stored in the fork. - Re-arranging the directory structure to separate source and header files --- Makefile | 21 ++++++- meson.build | 4 +- src/Makefile | 19 ------ src/access/pg_tde_io.c | 6 +- src/access/pg_tde_prune.c | 4 +- src/access/pg_tde_rewrite.c | 8 +-- src/access/pg_tde_tdemap.c | 58 +++++++++++++++++++ src/access/pg_tde_vacuumlazy.c | 6 +- src/access/pg_tde_visibilitymap.c | 4 +- src/access/pg_tdeam.c | 10 ++-- src/access/pg_tdeam_handler.c | 16 +++-- src/access/pg_tdeam_visibility.c | 2 +- src/access/pg_tdetoast.c | 4 +- src/encryption/enc_tuple.c | 2 +- src/{ => include}/access/pg_tde_io.h | 0 src/{ => include}/access/pg_tde_rewrite.h | 0 src/include/access/pg_tde_tdemap.h | 16 +++++ .../access/pg_tde_visibilitymap.h | 0 src/{ => include}/access/pg_tdeam.h | 0 src/{ => include}/access/pg_tdeam_xlog.h | 0 src/{ => include}/access/pg_tdetoast.h | 0 src/{ => include}/encryption/enc_aes.h | 0 src/{ => include}/encryption/enc_tuple.h | 0 src/{access => include}/pg_tde_defines.h | 0 src/pg_tde.c | 23 ++++++++ 25 files changed, 152 insertions(+), 51 deletions(-) delete mode 100644 src/Makefile create mode 100644 src/access/pg_tde_tdemap.c rename src/{ => include}/access/pg_tde_io.h (100%) rename src/{ => include}/access/pg_tde_rewrite.h (100%) create mode 100644 src/include/access/pg_tde_tdemap.h rename src/{ => include}/access/pg_tde_visibilitymap.h (100%) rename src/{ => include}/access/pg_tdeam.h (100%) rename src/{ => include}/access/pg_tdeam_xlog.h (100%) rename src/{ => include}/access/pg_tdetoast.h (100%) rename src/{ => include}/encryption/enc_aes.h (100%) rename src/{ => include}/encryption/enc_tuple.h (100%) rename src/{access => include}/pg_tde_defines.h (100%) create mode 100644 src/pg_tde.c diff --git a/Makefile b/Makefile index 6a23f4e4..9d54a85e 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,38 @@ # contrib/pg_tde/Makefile PGFILEDESC = "pg_tde access method" - +MODULE_big = pg_tde EXTENSION = pg_tde DATA = pg_tde--1.0.sql REGRESS = pg_tde TAP_TESTS = 0 -SUBDIRS = src +OBJS = src/encryption/enc_tuple.o \ +src/encryption/enc_aes.o \ +src/access/pg_tde_io.o \ +src/access/pg_tdeam_visibility.o \ +src/access/pg_tde_tdemap.o \ +src/access/pg_tdeam.o \ +src/access/pg_tdetoast.o \ +src/access/pg_tde_prune.o \ +src/access/pg_tde_vacuumlazy.o \ +src/access/pg_tde_visibilitymap.o \ +src/access/pg_tde_rewrite.o \ +src/access/pg_tdeam_handler.o \ +src/pg_tde.o + ifdef USE_PGXS PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) +override PG_CPPFLAGS += -I$(CURDIR)/src/include include $(PGXS) else subdir = contrib/postgres-tde-ext top_builddir = ../.. +override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include include $(top_builddir)/src/Makefile.global include $(top_srcdir)/contrib/contrib-global.mk endif -$(recurse) +SHLIB_LINK += $(filter -lcrypto -lssl, $(LIBS)) diff --git a/meson.build b/meson.build index b65b602c..4a5a4650 100644 --- a/meson.build +++ b/meson.build @@ -1,4 +1,6 @@ pg_tde_sources = files( + 'src/pg_tde.c', + 'src/access/pg_tde_tdemap.c', 'src/access/pg_tdeam.c', 'src/access/pg_tdeam_handler.c', 'src/access/pg_tdeam_visibility.c', @@ -12,7 +14,7 @@ pg_tde_sources = files( 'src/encryption/enc_aes.c', ) -incdir = include_directories('src') +incdir = include_directories('src/include') pg_tde = shared_module('pg_tde', pg_tde_sources, diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index 99ad7784..00000000 --- a/src/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -# contrib/pg_tde/Makefile - -MODULE_big = pg_tde - -tde_subdirs = access encryption - -OBJS = \ - $(patsubst %.c,%.o,$(foreach dir,$(tde_subdirs), $(wildcard $(dir)/*.c))) - -ifdef USE_PGXS -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) -else -subdir = contrib/postgres-tde-ext/src -top_builddir = ../../.. -include $(top_builddir)/src/Makefile.global -include $(top_srcdir)/contrib/contrib-global.mk -endif diff --git a/src/access/pg_tde_io.c b/src/access/pg_tde_io.c index ae7ed0d2..5be12926 100644 --- a/src/access/pg_tde_io.c +++ b/src/access/pg_tde_io.c @@ -17,9 +17,9 @@ #include "postgres.h" -#include "pg_tdeam.h" -#include "pg_tde_io.h" -#include "pg_tde_visibilitymap.h" +#include "access/pg_tdeam.h" +#include "access/pg_tde_io.h" +#include "access/pg_tde_visibilitymap.h" #include "encryption/enc_tuple.h" #include "access/htup_details.h" diff --git a/src/access/pg_tde_prune.c b/src/access/pg_tde_prune.c index 25494145..28f36663 100644 --- a/src/access/pg_tde_prune.c +++ b/src/access/pg_tde_prune.c @@ -16,8 +16,8 @@ #include "postgres.h" -#include "pg_tdeam.h" -#include "pg_tdeam_xlog.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdeam_xlog.h" #include "access/htup_details.h" #include "access/transam.h" diff --git a/src/access/pg_tde_rewrite.c b/src/access/pg_tde_rewrite.c index cc6952ce..19a15095 100644 --- a/src/access/pg_tde_rewrite.c +++ b/src/access/pg_tde_rewrite.c @@ -106,10 +106,10 @@ #include -#include "pg_tdeam.h" -#include "pg_tdeam_xlog.h" -#include "pg_tdetoast.h" -#include "pg_tde_rewrite.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdeam_xlog.h" +#include "access/pg_tdetoast.h" +#include "access/pg_tde_rewrite.h" #include "encryption/enc_tuple.h" #include "access/transam.h" diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c new file mode 100644 index 00000000..958b12cf --- /dev/null +++ b/src/access/pg_tde_tdemap.c @@ -0,0 +1,58 @@ +/*------------------------------------------------------------------------- + * + * pg_tde_tdemap.c + * tde relation fork manager code + * + * + * IDENTIFICATION + * src/access/pg_tde_tdemap.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/pg_tde_tdemap.h" +#include "storage/fd.h" +#include "utils/wait_event.h" + +/* + * Creates a relation fork file relfilenode.tde that contains the + * encryption key for the relation. + */ +void +pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) +{ + char *rel_file_path; + char *key_file_path; + File file = -1; + char enc_key[256]; /* Dummy key */ + + /* We get a relation name for MAIN fork and manually append the + * .tde postfix to the file name + */ + rel_file_path = relpathperm(*newrlocator, MAIN_FORKNUM); + key_file_path = psprintf("%s.tde", rel_file_path); + pfree(rel_file_path); + + file = PathNameOpenFile(key_file_path, O_RDWR | O_CREAT | PG_BINARY); + if (file < 0) + { + ereport(FATAL, + (errcode_for_file_access(), + errmsg("could not open tde key file %s", key_file_path))); + } + /* TODO: + * For now just write a dummy data to the file. We will write the actual + * key later. + */ + snprintf(enc_key, sizeof(enc_key), "Percona TDE Dummy key for relation:%s", RelationGetRelationName(rel)); + if (FileWrite(file, enc_key, sizeof(enc_key), + 0, WAIT_EVENT_DATA_FILE_WRITE) != sizeof(enc_key)) + ereport(FATAL, (errcode_for_file_access(), + errmsg("Could not write key data to file: %s", + key_file_path))); + + /* For now just clode the key file.*/ + pfree(key_file_path); + FileClose(file); +} \ No newline at end of file diff --git a/src/access/pg_tde_vacuumlazy.c b/src/access/pg_tde_vacuumlazy.c index 2164e6a5..b40ec95d 100644 --- a/src/access/pg_tde_vacuumlazy.c +++ b/src/access/pg_tde_vacuumlazy.c @@ -36,9 +36,9 @@ #include -#include "pg_tdeam.h" -#include "pg_tdeam_xlog.h" -#include "pg_tde_visibilitymap.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdeam_xlog.h" +#include "access/pg_tde_visibilitymap.h" #include "encryption/enc_tuple.h" #include "access/amapi.h" diff --git a/src/access/pg_tde_visibilitymap.c b/src/access/pg_tde_visibilitymap.c index 296927e4..4d25e633 100644 --- a/src/access/pg_tde_visibilitymap.c +++ b/src/access/pg_tde_visibilitymap.c @@ -88,8 +88,8 @@ #include "postgres.h" -#include "pg_tdeam_xlog.h" -#include "pg_tde_visibilitymap.h" +#include "access/pg_tdeam_xlog.h" +#include "access/pg_tde_visibilitymap.h" #include "access/xloginsert.h" #include "access/xlogutils.h" diff --git a/src/access/pg_tdeam.c b/src/access/pg_tdeam.c index 1e3b1e39..e1a264dc 100644 --- a/src/access/pg_tdeam.c +++ b/src/access/pg_tdeam.c @@ -34,11 +34,11 @@ #include "postgres.h" -#include "pg_tdeam.h" -#include "pg_tdeam_xlog.h" -#include "pg_tdetoast.h" -#include "pg_tde_io.h" -#include "pg_tde_visibilitymap.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdeam_xlog.h" +#include "access/pg_tdetoast.h" +#include "access/pg_tde_io.h" +#include "access/pg_tde_visibilitymap.h" #include "encryption/enc_tuple.h" #include "access/bufmask.h" diff --git a/src/access/pg_tdeam_handler.c b/src/access/pg_tdeam_handler.c index 6eb01920..58d795db 100644 --- a/src/access/pg_tdeam_handler.c +++ b/src/access/pg_tdeam_handler.c @@ -22,9 +22,10 @@ #include "postgres.h" -#include "pg_tdeam.h" -#include "pg_tdetoast.h" -#include "pg_tde_rewrite.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdetoast.h" +#include "access/pg_tde_rewrite.h" +#include "access/pg_tde_tdemap.h" #include "encryption/enc_tuple.h" @@ -51,8 +52,6 @@ #include "utils/builtins.h" #include "utils/rel.h" -PG_MODULE_MAGIC; - PG_FUNCTION_INFO_V1(pg_tdeam_handler); @@ -635,6 +634,13 @@ pg_tdeam_relation_set_new_filelocator(Relation rel, } smgrclose(srel); + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_MATVIEW ) + { + ereport(DEBUG2, + (errmsg("creating key file for relation %s", RelationGetRelationName(rel)))); + pg_tde_create_key_fork(newrlocator, rel); + } } static void diff --git a/src/access/pg_tdeam_visibility.c b/src/access/pg_tdeam_visibility.c index e9fdbe23..c037e30c 100644 --- a/src/access/pg_tdeam_visibility.c +++ b/src/access/pg_tdeam_visibility.c @@ -68,7 +68,7 @@ #include "postgres.h" -#include "pg_tdeam.h" +#include "access/pg_tdeam.h" #include "access/htup_details.h" #include "access/multixact.h" diff --git a/src/access/pg_tdetoast.c b/src/access/pg_tdetoast.c index 21ed0d58..117dc207 100644 --- a/src/access/pg_tdetoast.c +++ b/src/access/pg_tdetoast.c @@ -25,8 +25,8 @@ #include "postgres.h" -#include "pg_tdeam.h" -#include "pg_tdetoast.h" +#include "access/pg_tdeam.h" +#include "access/pg_tdetoast.h" #include "access/detoast.h" #include "access/genam.h" diff --git a/src/encryption/enc_tuple.c b/src/encryption/enc_tuple.c index 84d429e4..d13c06a7 100644 --- a/src/encryption/enc_tuple.c +++ b/src/encryption/enc_tuple.c @@ -1,4 +1,4 @@ -#include "access/pg_tde_defines.h" +#include "pg_tde_defines.h" #define ENCRYPTION_DEBUG 1 #include "postgres.h" diff --git a/src/access/pg_tde_io.h b/src/include/access/pg_tde_io.h similarity index 100% rename from src/access/pg_tde_io.h rename to src/include/access/pg_tde_io.h diff --git a/src/access/pg_tde_rewrite.h b/src/include/access/pg_tde_rewrite.h similarity index 100% rename from src/access/pg_tde_rewrite.h rename to src/include/access/pg_tde_rewrite.h diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h new file mode 100644 index 00000000..f8f53b97 --- /dev/null +++ b/src/include/access/pg_tde_tdemap.h @@ -0,0 +1,16 @@ +/*------------------------------------------------------------------------- + * + * pg_tde_tdemap.h + * TDE relation fork manapulation. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_TDE_MAP_H +#define PG_TDE_MAP_H + +#include "utils/rel.h" +#include "storage/relfilelocator.h" + +extern void pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel); + +#endif /* PG_TDE_MAP_H */ \ No newline at end of file diff --git a/src/access/pg_tde_visibilitymap.h b/src/include/access/pg_tde_visibilitymap.h similarity index 100% rename from src/access/pg_tde_visibilitymap.h rename to src/include/access/pg_tde_visibilitymap.h diff --git a/src/access/pg_tdeam.h b/src/include/access/pg_tdeam.h similarity index 100% rename from src/access/pg_tdeam.h rename to src/include/access/pg_tdeam.h diff --git a/src/access/pg_tdeam_xlog.h b/src/include/access/pg_tdeam_xlog.h similarity index 100% rename from src/access/pg_tdeam_xlog.h rename to src/include/access/pg_tdeam_xlog.h diff --git a/src/access/pg_tdetoast.h b/src/include/access/pg_tdetoast.h similarity index 100% rename from src/access/pg_tdetoast.h rename to src/include/access/pg_tdetoast.h diff --git a/src/encryption/enc_aes.h b/src/include/encryption/enc_aes.h similarity index 100% rename from src/encryption/enc_aes.h rename to src/include/encryption/enc_aes.h diff --git a/src/encryption/enc_tuple.h b/src/include/encryption/enc_tuple.h similarity index 100% rename from src/encryption/enc_tuple.h rename to src/include/encryption/enc_tuple.h diff --git a/src/access/pg_tde_defines.h b/src/include/pg_tde_defines.h similarity index 100% rename from src/access/pg_tde_defines.h rename to src/include/pg_tde_defines.h diff --git a/src/pg_tde.c b/src/pg_tde.c new file mode 100644 index 00000000..7b113d57 --- /dev/null +++ b/src/pg_tde.c @@ -0,0 +1,23 @@ +/*------------------------------------------------------------------------- + * + * pg_tde.c + * Main file: setup GUCs, shared memory, hooks and other general-purpose + * routines. + * + * IDENTIFICATION + * contrib/pg_tde/src/pg_tde.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "funcapi.h" + +PG_MODULE_MAGIC; +void _PG_init(void); + +void + _PG_init(void) + { + + }