From 397221ab512e3c8b3bc4a5e78c613cfc942a54b0 Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Wed, 7 Feb 2024 20:24:55 +0500 Subject: [PATCH 1/5] The patch implements on disk "key map and data" structure. It replaces the old "tde" fork architecture. This new architecture implements a two file pair with: (1) Map File (2) Key Data File Both files contain a header that contains the name of the master key that was to encrypt the data keys and a file version. The file version is set to PG_TDE_FILEMAGIC at the moment and it can be used to differiate between different file format versions in case we change the structure later on. The map file is a list of relNumber, flags and key index. - relNumber is the Oid of the associated relation. - Flags define if the map entry is free or in use. - Key index points to the starting position of the key in the key data file. The flags play a pivotal role in avoiding the file to grow infinitely. When a relation is either deleted or a transaction is aborted, the entry map entry is marked as MAP_ENTRY_FREE. Any next transaction requiring to store its relation key will pick the first entry with flag set to MAP_ENTRY_FREE. The key data file is simply a list of keys. No flags are needed as the validity is identified by the map file. Writing to the file is performed using FileWrite function. This avoids any locking in the key data file. Pending: - Implementation of key rotation - Locking of file during key rotation or map entry - Review of fflush calls - Review of the WAL --- pg_tde--1.0.sql | 5 + src/access/pg_tde_ddl.c | 2 +- src/access/pg_tde_tdemap.c | 944 ++++++++++++++++------ src/access/pg_tdeam_handler.c | 5 +- src/encryption/enc_aes.c | 53 ++ src/include/access/pg_tde_tdemap.h | 9 +- src/include/encryption/enc_aes.h | 5 + src/include/keyring/keyring_api.h | 4 +- src/include/transam/pg_tde_xact_handler.h | 2 +- src/keyring/keyring_api.c | 28 + src/transam/pg_tde_xact_handler.c | 90 +-- 11 files changed, 852 insertions(+), 295 deletions(-) diff --git a/pg_tde--1.0.sql b/pg_tde--1.0.sql index a8cfa0ca..7b12abcf 100644 --- a/pg_tde--1.0.sql +++ b/pg_tde--1.0.sql @@ -13,6 +13,11 @@ RETURNS boolean AS $$ SELECT amname = 'pg_tde' FROM pg_class INNER JOIN pg_am ON pg_am.oid = pg_class.relam WHERE relname = table_name $$ LANGUAGE SQL; +CREATE FUNCTION pg_tde_rotate_key(key_name VARCHAR) +RETURNS boolean +AS 'MODULE_PATHNAME' +LANGUAGE C; + -- Access method CREATE ACCESS METHOD pg_tde TYPE TABLE HANDLER pg_tdeam_handler; COMMENT ON ACCESS METHOD pg_tde IS 'pg_tde table access method'; diff --git a/src/access/pg_tde_ddl.c b/src/access/pg_tde_ddl.c index 94b9e1a0..c61fd15f 100644 --- a/src/access/pg_tde_ddl.c +++ b/src/access/pg_tde_ddl.c @@ -45,7 +45,7 @@ static void rel->rd_rel->relkind == RELKIND_MATVIEW) && (subId == 0) && is_pg_tde_rel(rel)) { - pg_tde_delete_key_fork(rel); + pg_tde_delete_key_map_entry(&rel->rd_locator); } relation_close(rel, AccessShareLock); } diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 53535324..29363497 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -19,6 +19,8 @@ #include "access/xlog.h" #include "access/xlog_internal.h" #include "access/xloginsert.h" +#include "utils/builtins.h" +#include "miscadmin.h" #include "access/pg_tde_tdemap.h" #include "encryption/enc_aes.h" @@ -26,47 +28,88 @@ #include #include +#include #include "pg_tde_defines.h" -/* TODO: should be a user defined */ +/* A useful macro when debugging key encryption/decryption */ +#ifdef DEBUG +#define ELOG_KEY(_msg, _key) \ +{ \ + int i; \ + char buf[1024]; \ + for (i = 0; i < sizeof(_key->internal_key[0].key); i++) \ + sprintf(buf+i, "%02X", _key->internal_key[0].key[i]); \ + buf[i] = '\0'; \ + elog(INFO, "[%s] INTERNAL KEY => %s", _msg, buf); \ +} +#endif + +#define PG_TDE_MAP_FILENAME "pg_tde.map" +#define PG_TDE_KEYDATA_FILENAME "pg_tde.dat" + +#define PG_TDE_FILEMAGIC 0x01454454 /* version ID value = TDE 01 */ + +#define MAP_ENTRY_FREE 0x00 +#define MAP_ENTRY_VALID 0x01 + +#define MAP_ENTRY_SIZE sizeof(TDEMapEntry) +#define TDE_FILE_HEADER_SIZE sizeof(TDEFileHeader) + +typedef struct TDEFileHeader +{ + int32 file_version; + char master_key_name[MASTER_KEY_NAME_LEN]; +} TDEFileHeader; + +typedef struct TDEMapEntry +{ + RelFileNumber relNumber; + int32 flags; + int32 key_index; +} TDEMapEntry; + +/* Global variables */ +static char db_path[MAXPGPATH] = {0}; +static char db_map_path[MAXPGPATH] = {0}; +static char db_keydata_path[MAXPGPATH] = {0}; + +/* TODO: should be a user defined */ static const char *MasterKeyName = "master-key"; -static inline char* pg_tde_get_key_file_path(const RelFileLocator *newrlocator); static void put_keys_into_map(Oid rel_id, RelKeysData *keys); -static void pg_tde_add_rel_keys(const RelFileLocator *rlocator, InternalKey *key, InternalKey *keyEnc, const char *masterkeyname); static void pg_tde_xlog_create_fork(XLogReaderState *record); -static void pg_tde_decrypt_internal_key(const InternalKey *in, InternalKey *out, const char *masterkeyname); -void -pg_tde_delete_key_fork(Relation rel) -{ - /* TODO: delete related internal keys from cache */ - char *key_file_path = pg_tde_get_key_file_path(&rel->rd_locator); - if (!key_file_path) - { - ereport(ERROR, - (errmsg("failed to get key file path"))); - } - RegisterFileForDeletion(key_file_path, true); - pfree(key_file_path); -} +static RelKeysData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info); +static RelKeysData* tde_encrypt_rel_key(const keyInfo *master_key_info, RelKeysData *rel_key_data); +static RelKeysData* tde_decrypt_rel_key(const keyInfo *master_key_info, RelKeysData *enc_rel_key_data); +static bool pg_tde_perform_rotate_key(const char *new_master_key_name); + +static void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *str_append); +static File pg_tde_open_file(char *tde_filename, const char *master_key_name, int fileFlags, bool *is_new_file, off_t *offset); +static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *key, const char *master_key_name); + +static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, const char *master_key_name); +static int32 pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); +static int32 pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete); +static bool pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset); + +static void pg_tde_write_keydata(char *db_keydata_path, const char *master_key_name, int32 key_index, RelKeysData *enc_rel_key_data); +static void pg_tde_write_one_keydata(File keydata_file, off_t header_size, int32 key_index, RelKeysData *enc_rel_key_data); +static RelKeysData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, const char *master_key_name); +static RelKeysData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, const char *master_key_name); +static RelKeysData* pg_tde_read_one_keydata(File keydata_file, off_t header_size, int32 key_index, const char *master_key_name); /* * Creates a relation fork file relfilenode.tde that contains the * encryption key for the relation. */ void -pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) +pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) { - InternalKey int_key = {0}; - InternalKey int_key_enc = {0}; - uint8 mkey_len; - const keyInfo *master_key_info; - int encsz; + InternalKey int_key; - // TODO: use proper iv stored in the file! - unsigned char iv[INTERNAL_KEY_LEN] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + memset(&int_key, 0, sizeof(InternalKey)); if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) { @@ -76,161 +119,19 @@ pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) RelationGetRelationName(rel), ERR_error_string(ERR_get_error(), NULL)))); } - master_key_info = keyringGetLatestKey(MasterKeyName); - if(master_key_info == NULL) - { - master_key_info = keyringGenerateKey(MasterKeyName, INTERNAL_KEY_LEN); - } - if(master_key_info == NULL) - { - ereport(ERROR, - (errmsg("failed to retrieve master key"))); - } - - AesEncrypt(master_key_info->data.data, iv, (unsigned char*) &int_key, INTERNAL_KEY_LEN, (unsigned char*) &int_key_enc, &encsz); - - mkey_len = (uint8) strlen(master_key_info->name.name); /* XLOG internal keys */ XLogBeginInsert(); XLogRegisterData((char *) newrlocator, sizeof(RelFileLocator)); - XLogRegisterData((char *) &mkey_len, sizeof(uint8)); - XLogRegisterData((char *) master_key_info->name.name, strlen(master_key_info->name.name)+1); - XLogRegisterData((char *) &int_key_enc, sizeof(InternalKey)); + XLogRegisterData((char *) &int_key, sizeof(InternalKey)); XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CREATE_FORK); - /* TODO: should DB crash after sending XLog, secondaries would create a fork + /* + * TODO: should DB crash after sending XLog, secondaries would create a fork * file but the relation won't be created either on primary or secondaries. * Hence, the *.tde file will remain as garbage on secondaries. */ - pg_tde_add_rel_keys(newrlocator, &int_key, &int_key_enc, master_key_info->name.name); -} - -/* - * pg_tde_add_rel_keys - * - * Creates a relation key and writers it to the fork file w/ the encrypted internal key - * and to the inmemory cache (just a perf optimisation) w/ the unencrypted internal key. -*/ -void -pg_tde_add_rel_keys(const RelFileLocator *rlocator, InternalKey *key, InternalKey *keyEnc, const char *masterkeyname) -{ - char *key_file_path; - File file = -1; - RelKeysData *data; - unsigned char dataEnc[1024]; - size_t sz; - - key_file_path = pg_tde_get_key_file_path(rlocator); - if (!key_file_path) - ereport(ERROR, - (errmsg("failed to get key file path"))); - - file = PathNameOpenFile(key_file_path, O_RDWR | O_CREAT | PG_BINARY); - if (file < 0) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not open tde key file \"%s\": %m", - key_file_path))); - - /* Allocate in TopMemoryContext and don't pfree sice we add it to - * the cache as well */ - data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, SizeOfRelKeysData(1)); - - strncpy(data->master_key_name, masterkeyname, MASTER_KEY_NAME_LEN); - data->internal_key[0] = *key; - data->internal_keys_len = 1; - - sz = SizeOfRelKeysData(data->internal_keys_len); - - memcpy(dataEnc, data, sz); - memcpy(dataEnc + SizeOfRelKeysDataHeader, keyEnc, INTERNAL_KEY_LEN); - - if (FileWrite(file, dataEnc, sz, 0, WAIT_EVENT_DATA_FILE_WRITE) != sz) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not write key data to file \"%s\": %m", - key_file_path))); - - /* Register the file for delete in case transaction Aborts */ - RegisterFileForDeletion(key_file_path, false); - - /* Add to the cache */ - put_keys_into_map(rlocator->relNumber, data); - - pfree(key_file_path); - FileClose(file); -} - -/* - * Reads tde keys for the relation fork file. - */ -RelKeysData * -pg_tde_get_keys_from_fork(const RelFileLocator *rlocator) -{ - char *key_file_path; - File file = -1; - Size sz; - int nbytes; - RelKeysData *keys; - - key_file_path = pg_tde_get_key_file_path(rlocator); - if (!key_file_path) - { - ereport(ERROR, - (errmsg("failed to get key file path"))); - } - - file = PathNameOpenFile(key_file_path, O_RDONLY | PG_BINARY); - if (file < 0) - { - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not open tde key file \"%s\": %m", - key_file_path))); - } - - - sz = (Size) FileSize(file); - keys = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, sz); - - /* - * TODO: internal key(s) should be encrypted - */ - nbytes = FileRead(file, keys, sz, 0, WAIT_EVENT_DATA_FILE_READ); - if (nbytes < 0) - ereport(FATAL, - (errcode_for_file_access(), - errmsg("could not read key data file \"%s\": %m", - key_file_path))); - else if (nbytes < SizeOfRelKeysData(1) || - (nbytes - SizeOfRelKeysDataHeader) % sizeof(InternalKey) != 0) - { - ereport(FATAL, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("corrupted key data in file \"%s\"", - key_file_path))); - } - -#if TDE_FORK_DEBUG - for (Size i = 0; i < keys->internal_keys_len; i++) - ereport(DEBUG2, - (errmsg("encrypted fork file keys: [%lu] %s: %s", i+1, keys->master_key_name, tde_sprint_key(&keys->internal_key[i])))); -#endif - - pg_tde_decrypt_internal_key(&keys->internal_key[0], &keys->internal_key[0], keys->master_key_name); - -#if TDE_FORK_DEBUG - for (Size i = 0; i < keys->internal_keys_len; i++) - ereport(DEBUG2, - (errmsg("fork file keys: [%lu] %s: %s", i+1, keys->master_key_name, tde_sprint_key(&keys->internal_key[i])))); -#endif - - pfree(key_file_path); - /* For now just close the key file.*/ - FileClose(file); - - return keys; + pg_tde_write_key_map_entry(newrlocator, &int_key, MasterKeyName); } /* Head of the keys cache (linked list) */ @@ -250,19 +151,13 @@ GetRelationKeys(RelFileLocator rel) Oid rel_id = rel.relNumber; for (curr = tde_rel_keys_map; curr != NULL; curr = curr->next) { - if (curr->rel_id == rel_id) { -#if TDE_FORK_DEBUG - ereport(DEBUG2, - (errmsg("TDE: cache hit, \"%s\" %s | (%d)", - curr->keys->master_key_name, - tde_sprint_key(&curr->keys->internal_key[0]), - rel_id))); -#endif + if (curr->rel_id == rel_id) + { return curr->keys; } } - keys = pg_tde_get_keys_from_fork(&rel); + keys = pg_tde_get_key_from_file(&rel, MasterKeyName); put_keys_into_map(rel.relNumber, keys); @@ -277,7 +172,7 @@ put_keys_into_map(Oid rel_id, RelKeysData *keys) { new = (RelKeys *) MemoryContextAlloc(TopMemoryContext, sizeof(RelKeys)); new->rel_id = rel_id; new->keys = keys; - new->next = NULL; + new->next = NULL; if (prev == NULL) tde_rel_keys_map = new; @@ -311,48 +206,652 @@ tde_sprint_masterkey(const keyData *k) return buf; } -/* returns the palloc'd key (TDE relation fork) file path */ -static inline char* -pg_tde_get_key_file_path(const RelFileLocator *newrlocator) +/* + * Creates a key for a relation identified by rlocator. Returns the newly + * created key. + */ +RelKeysData * +tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info) { - char *rel_file_path; - char *key_file_path = NULL; - - /* We get a relation name for MAIN fork and manually append the - * .tde postfix to the file name - */ - rel_file_path = relpathperm(*newrlocator, MAIN_FORKNUM); - if (rel_file_path) - { - key_file_path = psprintf("%s."TDE_FORK_EXT, rel_file_path); - pfree(rel_file_path); - } - return key_file_path; + RelKeysData *rel_key_data; + + rel_key_data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, SizeOfRelKeysData(1)); + + strncpy(rel_key_data->master_key_name, master_key_info->name.name, MASTER_KEY_NAME_LEN); + rel_key_data->internal_key[0] = *key; + rel_key_data->internal_keys_len = 1; + + /* Add to the cache */ + put_keys_into_map(rlocator->relNumber, rel_key_data); + + return rel_key_data; } -void -pg_tde_decrypt_internal_key(const InternalKey *in, InternalKey *out, const char *masterkeyname) +/* + * Encrypts a given key and returns the encrypted one. + */ +RelKeysData * +tde_encrypt_rel_key(const keyInfo *master_key_info, RelKeysData *rel_key_data) +{ + RelKeysData *enc_rel_key_data; + size_t enc_key_bytes; + + AesEncryptKey(master_key_info, rel_key_data, &enc_rel_key_data, &enc_key_bytes); + + return enc_rel_key_data; +} + +/* + * Decrypts a given key and returns the decrypted one. + */ +RelKeysData * +tde_decrypt_rel_key(const keyInfo *master_key_info, RelKeysData *enc_rel_key_data) +{ + RelKeysData *rel_key_data = NULL; + size_t key_bytes; + + AesDecryptKey(master_key_info, &rel_key_data, enc_rel_key_data, &key_bytes); + + return rel_key_data; +} + +/* + * Sets the global variables so that we don't have to do this again for this + * backend lifetime. + */ +void +pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *str_append) +{ + /* Return if the values are already set */ + if (*db_path && *db_map_path && *db_keydata_path) + return; + + /* Fill in the values */ + snprintf(db_path, MAXPGPATH, "%s", GetDatabasePath(rlocator->dbOid, rlocator->spcOid)); + + /* Set the file nanes for map and keydata */ + snprintf(db_map_path, MAXPGPATH, "%s/%s%s", db_path, PG_TDE_MAP_FILENAME, ((str_append) ? str_append : "")); + snprintf(db_keydata_path, MAXPGPATH, "%s/%s%s", db_path, PG_TDE_KEYDATA_FILENAME, ((str_append) ? str_append : "")); +} + +/* + * Path data clean up once the transaction is done. + */ +void +pg_tde_cleanup_path_vars(void) +{ + *db_path = *db_map_path = *db_keydata_path = 0; +} + +/* + * Open and Validate File Header [pg_tde.*]: + * header: {Format Version, Master Key Name} + * + * Returns the file descriptor in case of a success. Otherwise, fatal error + * is raised. + * + * Also, it sets the is_new_file to true if the file is just created. This is + * useful to know when reading a file so that we can skip further processing. + * + * Plus, there is nothing wrong with a create even if we are going to read + * data. This will save the creation overhead the next time. Ideally, this + * should never happen for a read operation as it indicates a missing file. + * + * The caller can pass the required flags to ensure that file is created + * or an error is thrown if the file does not exist. + */ +File +pg_tde_open_file(char *tde_filename, const char *master_key_name, int fileFlags, bool *is_new_file, off_t *curr_pos) { - const keyInfo *master_key_info; - unsigned char dataDec[1024]; - int encsz; - // TODO: use proper iv stored in the file! - unsigned char iv[INTERNAL_KEY_LEN] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - - keyName master_key_name; - strncpy(master_key_name.name, masterkeyname, MASTER_KEY_NAME_LEN); - master_key_info = keyringGetKey(master_key_name); - if(master_key_info == NULL) + File tde_file = -1; + TDEFileHeader fheader; + off_t bytes_read = 0; + off_t bytes_written = 0; + + Assert(is_new_file); + + /* + * Ensuring that we always open the file in binary mode. The caller must + * specify other flags for reading, writing or creating the file. + */ + tde_file = PathNameOpenFile(tde_filename, fileFlags | PG_BINARY); + if (tde_file < 0) { ereport(ERROR, - (errmsg("failed to retrieve master key"))); + (errcode_for_file_access(), + errmsg("Could not open tde file \"%s\": %m", + tde_filename))); + } + + bytes_read = FileRead(tde_file, &fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_READ); + *is_new_file = (bytes_read == 0); + + /* File doesn't exist */ + if (bytes_read == 0) + { + int len = strlen(master_key_name); + len = (len > MASTER_KEY_NAME_LEN) ? MASTER_KEY_NAME_LEN : len; + + /* Create the header for this file. */ + fheader.file_version = PG_TDE_FILEMAGIC; + + memset(fheader.master_key_name, 0, MASTER_KEY_NAME_LEN); + memcpy(fheader.master_key_name, master_key_name, len); + + bytes_written = FileWrite(tde_file, &fheader, TDE_FILE_HEADER_SIZE, 0, WAIT_EVENT_DATA_FILE_WRITE); + + if (bytes_written != TDE_FILE_HEADER_SIZE) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("Could not write tde file \"%s\": %m", + tde_filename))); + } + } + else if (bytes_read != TDE_FILE_HEADER_SIZE + || fheader.file_version != PG_TDE_FILEMAGIC) + { + /* Corrupt file */ + ereport(FATAL, + (errcode_for_file_access(), + errmsg("TDE map file \"%s\" is corrupted: %m", + tde_filename))); + } + + *curr_pos = bytes_read + bytes_written; + return tde_file; +} + +/* + * Key Map Table [pg_tde.map]: + * header: {Format Version, Master Key Name} + * data: {OID, Flag, index of key in pg_tde.dat}... + * + * Returns the index of the key to be written in the key data file. + * The caller must hold an exclusive lock on the map file to avoid + * concurrent in place updates leading to data conflicts. + */ +int32 +pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, const char *master_key_name) +{ + File map_file = -1; + int32 key_index = 0; + TDEMapEntry map_entry; + bool is_new_file; + off_t curr_pos = 0; + off_t prev_pos = 0; + bool found = false; + + /* Open and vaidate file for basic correctness. */ + map_file = pg_tde_open_file(db_map_path, master_key_name, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + + /* + * Read until we find an empty slot. Otherwise, read until end. This seems + * to be less frequent than vacuum. So let's keep this function here rather + * than overloading the vacuum process. + */ + while(1) + { + prev_pos = curr_pos; + found = pg_tde_read_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, &map_entry, &curr_pos); + + /* We either reach EOF or found an empty slot in the middle of the file */ + if (prev_pos == curr_pos || found) + break; + + /* Increment the offset and the key index */ + key_index++; + } + + /* Write the given entry at the location pointed by prev_pos */ + pg_tde_write_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, key_index, &map_entry, &prev_pos); + + /* Let's close the file. */ + FileClose(map_file); + + /* Register the entry to be freed in case the transaction aborts */ + RegisterEntryForDeletion(rlocator, curr_pos, false); + + return key_index; +} + +/* + * Based on the given arguments, creates and write the entry into the key + * map file. + */ +int32 +pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset) +{ + Assert(map_entry); + + /* Fill in the map entry structure */ + map_entry->relNumber = (rlocator == NULL) ? 0 : rlocator->relNumber; + map_entry->flags = flags; + map_entry->key_index = key_index; + + /* Add the entry to the file */ + if (FileWrite(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_WRITE) != MAP_ENTRY_SIZE) + { + ereport(FATAL, + (errcode_for_file_access(), + errmsg("Could not write tde map file \"%s\": %m", + db_map_path))); + } + + return key_index; +} + +/* + * Returns the index of the read map if we find a valid match; i.e. + * - flags is set to MAP_ENTRY_VALID and the relNumber matches the one + * provided in rlocator. + * - If should_delete is true, we delete the entry. An offset value may + * be passed to speed up the file reading operation. + * + * The function expects that the offset points to a valid map start location. + */ +int32 +pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_t *offset, bool should_delete) +{ + File map_file = -1; + int32 key_index = 0; + TDEMapEntry map_entry; + bool is_new_file; + bool found = false; + off_t prev_pos = 0; + off_t curr_pos = 0; + + Assert(offset); + + /* + * Open and vaidate file for basic correctness. DO NOT create it. + * The file should pre-exist otherwise we should never be here. + */ + map_file = pg_tde_open_file(db_map_path, NULL, O_RDWR, &is_new_file, &curr_pos); + + /* + * If we need to delete an entry, we expect an offset value to the start + * of the entry to speed up the operation. Otherwise, we'd be sequntially + * scanning the entire map file. + */ + if (should_delete == true && *offset > 0) + { + curr_pos = lseek(map_file, *offset, SEEK_SET); + + if (curr_pos == -1) + { + ereport(FATAL, + (errcode_for_file_access(), + errmsg("Could not seek in tde map file \"%s\": %m", + db_map_path))); + } + } + else + { + /* Otherwise, let's just offset to zero */ + *offset = 0; } - AesDecrypt(master_key_info->data.data, iv, (unsigned char*) in, INTERNAL_KEY_LEN, dataDec, &encsz); - memcpy(out, dataDec, INTERNAL_KEY_LEN); + + /* + * Read until we find an empty slot. Otherwise, read until end. This seems + * to be less frequent than vacuum. So let's keep this function here rather + * than overloading the vacuum process. + */ + while(1) + { + prev_pos = curr_pos; + found = pg_tde_read_one_map_entry(map_file, rlocator, MAP_ENTRY_VALID, &map_entry, &curr_pos); + + /* We've reached EOF */ + if (curr_pos == prev_pos) + break; + + /* We found a valid entry for the relNumber */ + if (found) + { + /* Mark the entry pointed by prev_pos as free */ + if (should_delete) + { + pg_tde_write_one_map_entry(map_file, NULL, MAP_ENTRY_FREE, 0, &map_entry, &prev_pos); + } + + break; + } + + /* Increment the offset and the key index */ + key_index++; + } + + /* Let's close the file. */ + FileClose(map_file); + + /* Return -1 indicating that no entry was removed */ + return ((found) ? key_index : -1); } -/* - * TDE fork XLog +/* + * Returns true if a valid map entry if found. Otherwise, it only increments + * the offset and returns false. If the same offset value is set, it indicates + * to the caller that nothing was read. + * + * If a non-NULL rlocator is provided, the function compares the read value + * against the relNumber of rlocator. It sets found accordingly. + * + * The caller is reponsible for identifying that we have reached EOF by + * comparing old and new value of the offset. + */ +bool +pg_tde_read_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, TDEMapEntry *map_entry, off_t *offset) +{ + bool found; + off_t bytes_read = 0; + + Assert(map_entry); + Assert(offset); + + /* Read the entry at the given offset */ + bytes_read = FileRead(map_file, map_entry, MAP_ENTRY_SIZE, *offset, WAIT_EVENT_DATA_FILE_READ); + *offset += bytes_read; + + /* We found a valid entry for the relNumber */ + found = (bytes_read > 0 && map_entry->flags == flags); + + /* If a valid rlocator is provided, let's compare and set found value */ + found &= (rlocator == NULL) ? true : (map_entry->relNumber == rlocator->relNumber); + + return found; +} + +/* + * Key Data [pg_tde.dat]: + * header: {Format Version: x} + * data: {Encrypted Key} + * + * Requires a valid index of the key to be written. The function with seek to + * the required location in the file. Any holes will be filled when another + * job finds an empty index. + */ +void +pg_tde_write_keydata(char *db_keydata_path, const char *master_key_name, int32 key_index, RelKeysData *enc_rel_key_data) +{ + File keydata_file = -1; + bool is_new_file; + off_t curr_pos = 0; + + /* Open and vaidate file for basic correctness. */ + keydata_file = pg_tde_open_file(db_keydata_path, master_key_name, O_RDWR | O_CREAT, &is_new_file, &curr_pos); + + /* Write a single key data */ + pg_tde_write_one_keydata(keydata_file, curr_pos, key_index, enc_rel_key_data); + + /* Let's close the file. */ + FileClose(keydata_file); +} + +/* + * Function writes a single RelKeysData into the file at the given index. + */ +void +pg_tde_write_one_keydata(File keydata_file, off_t header_size, int32 key_index, RelKeysData *enc_rel_key_data) +{ + size_t key_size; + off_t curr_pos = header_size; + + Assert(keydata_file != -1); + + /* Calculate the writing position in the file. */ + curr_pos += (key_index * SizeOfRelKeysData(1)); + + key_size = SizeOfRelKeysData(1); + + if (FileWrite(keydata_file, enc_rel_key_data, key_size, curr_pos, WAIT_EVENT_DATA_FILE_WRITE) != key_size) + { + ereport(FATAL, + (errcode_for_file_access(), + errmsg("Could not write tde key data file \"%s\": %m", + db_keydata_path))); + } +} + +/* + * Open the file and read the required key data from file and return encrypted key. + */ +RelKeysData * +pg_tde_read_keydata(char *db_keydata_path, int32 key_index, const char *master_key_name) +{ + File keydata_file = -1; + RelKeysData *enc_rel_key_data; + off_t read_pos = 0; + bool is_new_file; + + /* Open and vaidate file for basic correctness. */ + keydata_file = pg_tde_open_file(db_keydata_path, master_key_name, O_RDONLY, &is_new_file, &read_pos); + + /* Read the encrypted key from file */ + enc_rel_key_data = pg_tde_read_one_keydata(keydata_file, read_pos, key_index, master_key_name); + + /* Let's close the file. */ + FileClose(keydata_file); + + return enc_rel_key_data; +} + +/* + * Reads a single keydata from the file. + */ +RelKeysData * +pg_tde_read_one_keydata(File keydata_file, off_t header_size, int32 key_index, const char *master_key_name) +{ + RelKeysData *enc_rel_key_data; + off_t read_pos = 0; + size_t key_size; + int key_count = 1; + + /* Set the sizes for key and key data */ + key_size = SizeOfRelKeysData(key_count); + + /* Allocate and fill in the structure */ + enc_rel_key_data = (RelKeysData *) palloc(key_size); + + strncpy(enc_rel_key_data->master_key_name, master_key_name, MASTER_KEY_NAME_LEN); + enc_rel_key_data->internal_keys_len = key_count; + + /* Calculate the reading position in the file. */ + read_pos += (key_index * SizeOfRelKeysData(key_count)) + TDE_FILE_HEADER_SIZE; + + /* Check if the file has a valid key */ + if ((read_pos + key_size) > FileSize(keydata_file)) + { + ereport(FATAL, + (errcode(ERRCODE_NO_DATA_FOUND), + errmsg("Could not find the required key at index %d in tde data file \"%s\": %m", + key_index, + db_keydata_path))); + } + + /* Read the encrypted key */ + if (FileRead(keydata_file, enc_rel_key_data, key_size, read_pos, WAIT_EVENT_DATA_FILE_READ) != key_size) + { + ereport(FATAL, + (errcode_for_file_access(), + errmsg("Could not read key at index %d in tde key data file \"%s\": %m", + key_index, + db_keydata_path))); + } + + return enc_rel_key_data; +} + +/* + * Calls the create map entry function to get an index into the keydata. This + * The keydata function will then write the encrypted key on the desired + * location. + * + * The map file must be updated while holding an exclusive lock. + */ +void +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *key, const char *master_key_name) +{ + int32 key_index = 0; + const keyInfo *master_key_info; + RelKeysData *rel_key_data; + RelKeysData *enc_rel_key_data; + + /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ + master_key_info = getMasterKey(master_key_name, true, true); + + rel_key_data = tde_create_rel_key(rlocator, key, master_key_info); + enc_rel_key_data = tde_encrypt_rel_key(master_key_info, rel_key_data); + + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, NULL); + + /* Create the map entry and then add the encrypted key to the data file */ + key_index = pg_tde_write_map_entry(rlocator, db_map_path, master_key_name); + + /* Add the encrypted key to the data file. */ + pg_tde_write_keydata(db_keydata_path, master_key_name, key_index, enc_rel_key_data); +} + +/* + * Deletes a map entry by setting marking it as unused. We don't have to delete + * the actual key data as valid key data entries are identify by valid map entries. + */ +void +pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) +{ + int32 key_index = 0; + off_t offset = 0; + + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, NULL); + + /* Remove the map entry if found */ + key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); + + if (key_index == -1) + { + ereport(WARNING, + (errcode(ERRCODE_NO_DATA_FOUND), + errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + rlocator->relNumber, + db_map_path))); + + return; + } + + /* Register the entry to be freed when transaction commits */ + RegisterEntryForDeletion(rlocator, offset, true); +} + +/* + * Called when transaction is being completed; either committed or aborted. + * By default, when a transaction creates an entry, we mark it as MAP_ENTRY_VALID. + * Only during the abort phase of the transaction that we are proceed on with + * marking the entry as MAP_ENTRY_FREE. This optimistic strategy that assumes + * that transaction will commit more often then getting aborted avoids + * unnecessary locking. + * + * The offset allows us to simply seek to the desired location and mark the entry + * as MAP_ENTRY_FREE without needing any further processing. + */ +void +pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) +{ + int32 key_index = 0; + + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, NULL); + + /* Remove the map entry if found */ + key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, true); + + if (key_index == -1) + { + ereport(WARNING, + (errcode(ERRCODE_NO_DATA_FOUND), + errmsg("Could not find the required map entry for deletion of relation %d in tde map file \"%s\": %m", + rlocator->relNumber, + db_map_path))); + + return; + } +} + +/* + * Reads the key of the required relation. It identifies its map entry and then simply + * reads the key data from the keydata file. + */ +RelKeysData * +pg_tde_get_key_from_file(const RelFileLocator *rlocator, const char *master_key_name) +{ + int32 key_index = 0; + const keyInfo *master_key_info; + RelKeysData *rel_key_data; + RelKeysData *enc_rel_key_data; + off_t offset = 0; + + /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ + master_key_info = getMasterKey(master_key_name, false, true); + + /* Get the file paths */ + pg_tde_set_db_file_paths(rlocator, NULL); + + /* Read the map entry and get the index of the relation key */ + key_index = pg_tde_process_map_entry(rlocator, db_map_path, &offset, false); + + /* Add the encrypted key to the data file. */ + enc_rel_key_data = pg_tde_read_keydata(db_keydata_path, key_index, master_key_info->name.name); + rel_key_data = tde_decrypt_rel_key(master_key_info, enc_rel_key_data); + + return rel_key_data; +} + +PG_FUNCTION_INFO_V1(pg_tde_rotate_key); +Datum +pg_tde_rotate_key(PG_FUNCTION_ARGS) +{ + const char *new_key; + bool ret; + + if (PG_ARGISNULL(0)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("new master key name cannot be NULL"))); + + new_key = TextDatumGetCString(PG_GETARG_DATUM(0)); + ret = pg_tde_perform_rotate_key(new_key); + PG_RETURN_BOOL(ret); +} + +/* + * TODO: + * - How do we get the old key name and the key itself? + * - We need to specify this for a current or all databases? + */ +bool +pg_tde_perform_rotate_key(const char *new_master_key_name) +{ + /* + * Implementation: + * - Get names of the new and old master keys; either via arguments or function calls + * - Open old map file + * - Open old keydata file + * - Open new map file + * - Open new keydata file + * - Read old map entry using its index. + * - Write to new map file using its index. + * - Read keydata from old file + * - Decrypt it using the old master key + * - Encrypt it using the new master key + * - Write to new keydata file + */ + + return true; +} + +/* + * TDE fork XLog */ void pg_tde_rmgr_redo(XLogReaderState *record) @@ -397,10 +896,9 @@ pg_tde_xlog_create_fork(XLogReaderState *record) { char *rec = XLogRecGetData(record); RelFileLocator rlocator; - InternalKey int_key = {0}; - InternalKey int_key_enc = {0}; - uint8 mkey_len; - char mkey_name[MASTER_KEY_NAME_LEN]; + InternalKey int_key; + + memset(&int_key, 0, sizeof(InternalKey)); if (XLogRecGetDataLen(record) < sizeof(InternalKey)+sizeof(RelFileLocator)) { @@ -409,18 +907,14 @@ pg_tde_xlog_create_fork(XLogReaderState *record) errmsg("corrupted XLOG_TDE_CREATE_FORK data"))); } - /* Format [RelFileLocator][MasterKeyNameLen][MasterKeyName][InternalKey] */ + /* Format [RelFileLocator][InternalKey] */ memcpy(&rlocator, rec, sizeof(RelFileLocator)); - memcpy(&mkey_len, rec+sizeof(RelFileLocator), sizeof(mkey_len)); - memcpy(&mkey_name, rec+sizeof(RelFileLocator)+sizeof(mkey_len), mkey_len+1); - memcpy(&int_key_enc, rec+sizeof(RelFileLocator)+sizeof(mkey_len)+1+mkey_len, sizeof(InternalKey)); - - pg_tde_decrypt_internal_key(&int_key_enc, &int_key, mkey_name); + memcpy(&int_key, rec+sizeof(RelFileLocator), sizeof(InternalKey)); #if TDE_FORK_DEBUG ereport(DEBUG2, (errmsg("xlog internal_key: %s", tde_sprint_key(&int_key)))); #endif - pg_tde_add_rel_keys(&rlocator, &int_key, &int_key_enc, mkey_name); + pg_tde_write_key_map_entry(&rlocator, &int_key, MasterKeyName); } diff --git a/src/access/pg_tdeam_handler.c b/src/access/pg_tdeam_handler.c index 3cd7e5ab..018234f8 100644 --- a/src/access/pg_tdeam_handler.c +++ b/src/access/pg_tdeam_handler.c @@ -634,13 +634,16 @@ pg_tdeam_relation_set_new_filelocator(Relation rel, } smgrclose(srel); + + /* Update TDE filemap */ if (rel->rd_rel->relkind == RELKIND_RELATION || rel->rd_rel->relkind == RELKIND_MATVIEW || rel->rd_rel->relkind == RELKIND_TOASTVALUE) { ereport(DEBUG1, (errmsg("creating key file for relation %s", RelationGetRelationName(rel)))); - pg_tde_create_key_fork(newrlocator, rel); + + pg_tde_create_key_map_entry(newrlocator, rel); } } diff --git a/src/encryption/enc_aes.c b/src/encryption/enc_aes.c index 3f26acb7..251f5e64 100644 --- a/src/encryption/enc_aes.c +++ b/src/encryption/enc_aes.c @@ -6,7 +6,10 @@ #define Assert(p) assert(p) #endif +#include "access/pg_tde_tdemap.h" #include "encryption/enc_aes.h" +#include "keyring/keyring_api.h" +#include "utils/memutils.h" #include #include @@ -159,11 +162,61 @@ void AesEncrypt(const unsigned char* key, const unsigned char* iv, const unsigne AesRunCbc(1, key, iv, in, in_len, out, out_len); } +/* + * Provide a simple interface to encrypt a given key. + * + * The function pallocs and updates the p_enc_rel_key_data along with key bytes. The memory + * is allocated in the current memory context as this key should be ephemeral with a very + * short lifespan until it is written to disk. + */ +void +AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes) +{ + size_t sz; + unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + + /* Ensure we are getting a valid pointer here */ + Assert(master_key_info); + + sz = SizeOfRelKeysData(1); + + *p_enc_rel_key_data = (RelKeysData *) palloc(sz); + memcpy(*p_enc_rel_key_data, rel_key_data, sz); + + AesEncrypt(master_key_info->data.data, iv, ((unsigned char*)rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_enc_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)enc_key_bytes); +} + void AesDecrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len) { AesRunCbc(0, key, iv, in, in_len, out, out_len); } +/* + * Provide a simple interface to decrypt a given key. + * + * The function pallocs and updates the p_rel_key_data along with key bytes. It's important + * to note that memory is allocated in the TopMemoryContext so we expect this to be added + * to our key cache. + */ +void +AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes) +{ + size_t sz; + unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + + /* Ensure we are getting a valid pointer here */ + Assert(master_key_info); + + sz = SizeOfRelKeysData(enc_rel_key_data->internal_keys_len); + + *p_rel_key_data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, sz); + + /* Fill in the structure */ + memcpy(*p_rel_key_data, enc_rel_key_data, sz); + + AesDecrypt(master_key_info->data.data, iv, ((unsigned char*) enc_rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)key_bytes); +} + /* * We want to avoid dynamic memory allocation, so the function only allows * to process NUM_AES_BLOCKS_IN_BATCH number of blocks at a time. diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 9071c93e..a044c001 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -52,10 +52,13 @@ typedef struct RelKeys struct RelKeys *next; } RelKeys; -extern void pg_tde_delete_key_fork(Relation rel); -extern void pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel); +extern void pg_tde_delete_key_map_entry(const RelFileLocator *rlocator); +extern void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset); +extern void pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel); extern RelKeysData *pg_tde_get_keys_from_fork(const RelFileLocator *rlocator); extern RelKeysData *GetRelationKeys(RelFileLocator rel); +extern void pg_tde_cleanup_path_vars(void); + const char * tde_sprint_key(InternalKey *k); /* TDE XLOG resource manager */ @@ -68,6 +71,8 @@ extern void pg_tde_rmgr_redo(XLogReaderState *record); extern void pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record); extern const char * pg_tde_rmgr_identify(uint8 info); + +/* Move this to pg_tde.c file */ static const RmgrData pg_tde_rmgr = { .rm_name = RM_TDERMGR_NAME, .rm_redo = pg_tde_rmgr_redo, diff --git a/src/include/encryption/enc_aes.h b/src/include/encryption/enc_aes.h index 7cca1e36..a59bb5c9 100644 --- a/src/include/encryption/enc_aes.h +++ b/src/include/encryption/enc_aes.h @@ -11,6 +11,8 @@ #define ENC_AES_H #include +#include "access/pg_tde_tdemap.h" +#include "keyring/keyring_api.h" #define AES_BLOCK_SIZE 16 #define NUM_AES_BLOCKS_IN_BATCH 100 @@ -23,4 +25,7 @@ extern void Aes128EncryptedZeroBlocks(void* ctxPtr, const unsigned char* key, co extern void AesEncrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len); extern void AesDecrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len); +extern void AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes); +extern void AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes); + #endif /*ENC_AES_H*/ \ No newline at end of file diff --git a/src/include/keyring/keyring_api.h b/src/include/keyring/keyring_api.h index 28d1e4f3..90e64b86 100644 --- a/src/include/keyring/keyring_api.h +++ b/src/include/keyring/keyring_api.h @@ -1,8 +1,6 @@ - #ifndef KEYRING_API_H #define KEYRING_API_H - typedef struct keyName { char name[256]; // enough for now @@ -51,4 +49,6 @@ void keyringInitCache(void); const keyInfo* keyringCacheStoreKey(keyName name, keyData data); const char * tde_sprint_masterkey(const keyData *k); +const keyInfo* getMasterKey(const char* internalName, int doGenerateKey, int doRaiseError); + #endif // KEYRING_API_H diff --git a/src/include/transam/pg_tde_xact_handler.h b/src/include/transam/pg_tde_xact_handler.h index 5bdafcb0..231dff67 100644 --- a/src/include/transam/pg_tde_xact_handler.h +++ b/src/include/transam/pg_tde_xact_handler.h @@ -15,7 +15,7 @@ extern void pg_tde_xact_callback(XactEvent event, void *arg); extern void pg_tde_subxact_callback(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg); -extern void RegisterFileForDeletion(const char *filePath, bool atCommit); +extern void RegisterEntryForDeletion(const RelFileLocator *rlocator, off_t map_entry_offset, bool atCommit); #endif /* PG_TDE_XACT_HANDLER_H */ \ No newline at end of file diff --git a/src/keyring/keyring_api.c b/src/keyring/keyring_api.c index 44f1b1f5..b75ac0e0 100644 --- a/src/keyring/keyring_api.c +++ b/src/keyring/keyring_api.c @@ -171,3 +171,31 @@ const keyInfo* keyringGenerateKey(const char* internalName, unsigned keyLen) return keyringStoreKey(keyringConstructKeyName(internalName, i), kd); } +/* + * Simplifying the interface to get the master key without having to worry + * generating a new one. If master key does not exist, and doGenerateKey is + * set, a new key is generated. This is useful during write operations. + * + * However, when performing a read operation and a master is expected to exist, + * doGenerateKey should be false and doRaiseError should be set to indicate + * that master key is expected but could not be accessed. + */ +const keyInfo* getMasterKey(const char* internalName, int doGenerateKey, int doRaiseError) +{ + const keyInfo* key = NULL; + + key = keyringGetLatestKey(internalName); + + if (key == NULL && doGenerateKey) + { + key = keyringGenerateKey(internalName, 16); + } + + if (key == NULL && doRaiseError) + { + ereport(ERROR, + (errmsg("failed to retrieve master key"))); + } + + return key; +} diff --git a/src/transam/pg_tde_xact_handler.c b/src/transam/pg_tde_xact_handler.c index 9f5e41e3..ff84c3d9 100644 --- a/src/transam/pg_tde_xact_handler.c +++ b/src/transam/pg_tde_xact_handler.c @@ -16,18 +16,19 @@ #include "utils/elog.h" #include "storage/fd.h" #include "transam/pg_tde_xact_handler.h" +#include "access/pg_tde_tdemap.h" -typedef struct PendingFileDelete +typedef struct PendingMapEntryDelete { - char *path; /* file that may need to be deleted */ - bool atCommit; /* T=delete at commit; F=delete at abort */ - int nestLevel; /* xact nesting level of request */ - struct PendingFileDelete *next; /* linked-list link */ -} PendingFileDelete; + off_t map_entry_offset; /* map entry offset */ + RelFileLocator rlocator; /* main for use as relation OID */ + bool atCommit; /* T=delete at commit; F=delete at abort */ + int nestLevel; /* xact nesting level of request */ + struct PendingMapEntryDelete *next; /* linked-list link */ +} PendingMapEntryDelete; -static PendingFileDelete *pendingDeletes = NULL; /* head of linked list */ +static PendingMapEntryDelete *pendingDeletes = NULL; /* head of linked list */ -static void cleanup_pending_deletes(bool atCommit); static void do_pending_deletes(bool isCommit); static void pending_delete_cleanup(void); @@ -41,7 +42,6 @@ pg_tde_xact_callback(XactEvent event, void *arg) ereport(DEBUG2, (errmsg("pg_tde_xact_callback: aborting transaction"))); do_pending_deletes(false); - } else if (event == XACT_EVENT_COMMIT) { @@ -52,6 +52,8 @@ pg_tde_xact_callback(XactEvent event, void *arg) { pending_delete_cleanup(); } + + pg_tde_cleanup_path_vars(); } void @@ -68,52 +70,18 @@ pg_tde_subxact_callback(SubXactEvent event, SubTransactionId mySubid, } void -RegisterFileForDeletion(const char *filePath, bool atCommit) +RegisterEntryForDeletion(const RelFileLocator *rlocator, off_t map_entry_offset, bool atCommit) { - PendingFileDelete *pending; - pending = (PendingFileDelete *) MemoryContextAlloc(TopMemoryContext, sizeof(PendingFileDelete)); - pending->path = MemoryContextStrdup(TopMemoryContext, filePath); + PendingMapEntryDelete *pending; + pending = (PendingMapEntryDelete *) MemoryContextAlloc(TopMemoryContext, sizeof(PendingMapEntryDelete)); + pending->map_entry_offset = map_entry_offset; + memcpy(&pending->rlocator, rlocator, sizeof(RelFileLocator)); pending->atCommit = atCommit; /* delete if abort */ pending->nestLevel = GetCurrentTransactionNestLevel(); pending->next = pendingDeletes; pendingDeletes = pending; } -/* - * cleanup_pending_deletes - * Mark a relation as not to be deleted after all. - */ -static void -cleanup_pending_deletes(bool atCommit) -{ - PendingFileDelete *pending; - PendingFileDelete *prev; - PendingFileDelete *next; - - prev = NULL; - for (pending = pendingDeletes; pending != NULL; pending = next) - { - next = pending->next; - if (pending->atCommit == atCommit) - { - /* unlink and delete list entry */ - if (prev) - prev->next = next; - else - pendingDeletes = next; - if (pending->path) - pfree(pending->path); - pfree(pending); - /* prev does not change */ - } - else - { - /* unrelated entry, don't touch it */ - prev = pending; - } - } -} - /* * do_pending_deletes() -- Take care of file deletes at end of xact. * @@ -124,10 +92,10 @@ cleanup_pending_deletes(bool atCommit) static void do_pending_deletes(bool isCommit) { - int nestLevel = GetCurrentTransactionNestLevel(); - PendingFileDelete *pending; - PendingFileDelete *prev; - PendingFileDelete *next; + int nestLevel = GetCurrentTransactionNestLevel(); + PendingMapEntryDelete *pending; + PendingMapEntryDelete *prev; + PendingMapEntryDelete *next; prev = NULL; for (pending = pendingDeletes; pending != NULL; pending = next) @@ -149,13 +117,12 @@ do_pending_deletes(bool isCommit) if (pending->atCommit == isCommit) { ereport(LOG, - (errmsg("pg_tde_xact_callback: deletingx file %s", - pending->path))); - durable_unlink(pending->path, WARNING); /* TODO: should it be ERROR? */ + (errmsg("pg_tde_xact_callback: deleting entry at offset %d", + (int)(pending->map_entry_offset)))); + + pg_tde_free_key_map_entry(&pending->rlocator, pending->map_entry_offset); } - /* must explicitly free the list entry */ - if(pending->path) - pfree(pending->path); + pfree(pending); /* prev does not change */ } @@ -172,16 +139,13 @@ do_pending_deletes(bool isCommit) static void pending_delete_cleanup(void) { - PendingFileDelete *pending; - PendingFileDelete *next; + PendingMapEntryDelete *pending; + PendingMapEntryDelete *next; for (pending = pendingDeletes; pending != NULL; pending = next) { next = pending->next; pendingDeletes = next; - /* must explicitly free the list entry */ - if(pending->path) - pfree(pending->path); pfree(pending); } } From b83a4f046405e20152e17945bc7a6bc98c778c95 Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Wed, 14 Feb 2024 11:56:48 +0500 Subject: [PATCH 2/5] Refactoring based on the Zsolt's comments on the PR. Moving the key encryption/decryption functions to the enc_tuple file and renaming the files according to the functionality. --- Makefile.in | 2 +- meson.build | 2 +- src/access/pg_tde_io.c | 2 +- src/access/pg_tde_prune.c | 2 +- src/access/pg_tde_rewrite.c | 2 +- src/access/pg_tde_vacuumlazy.c | 2 +- src/access/pg_tdeam.c | 2 +- src/access/pg_tdeam_handler.c | 2 +- src/access/pg_tdetoast.c | 2 +- src/encryption/enc_aes.c | 53 ------------------- src/encryption/{enc_tuple.c => enc_tde.c} | 52 +++++++++++++++++- src/include/encryption/enc_aes.h | 5 -- .../encryption/{enc_tuple.h => enc_tde.h} | 16 +++--- 13 files changed, 70 insertions(+), 74 deletions(-) rename src/encryption/{enc_tuple.c => enc_tde.c} (77%) rename src/include/encryption/{enc_tuple.h => enc_tde.h} (81%) diff --git a/Makefile.in b/Makefile.in index 8ddbb9a6..a0d985fa 100644 --- a/Makefile.in +++ b/Makefile.in @@ -16,7 +16,7 @@ multi_insert \ trigger_on_view TAP_TESTS = 1 -OBJS = src/encryption/enc_tuple.o \ +OBJS = src/encryption/enc_tde.o \ src/encryption/enc_aes.o \ src/access/pg_tde_io.o \ src/access/pg_tdeam_visibility.o \ diff --git a/meson.build b/meson.build index 236e9ddc..630dc76c 100644 --- a/meson.build +++ b/meson.build @@ -18,7 +18,7 @@ pg_tde_sources = files( 'src/access/pg_tde_visibilitymap.c', 'src/access/pg_tde_ddl.c', - 'src/encryption/enc_tuple.c', + 'src/encryption/enc_tde.c', 'src/encryption/enc_aes.c', 'src/keyring/keyring_config.c', diff --git a/src/access/pg_tde_io.c b/src/access/pg_tde_io.c index 78c8dfc3..4499a51c 100644 --- a/src/access/pg_tde_io.c +++ b/src/access/pg_tde_io.c @@ -20,7 +20,7 @@ #include "access/pg_tdeam.h" #include "access/pg_tde_io.h" #include "access/pg_tde_visibilitymap.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/htup_details.h" #include "storage/bufmgr.h" diff --git a/src/access/pg_tde_prune.c b/src/access/pg_tde_prune.c index 35c36f7d..0047498d 100644 --- a/src/access/pg_tde_prune.c +++ b/src/access/pg_tde_prune.c @@ -16,7 +16,7 @@ #include "postgres.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/pg_tdeam.h" #include "access/pg_tdeam_xlog.h" diff --git a/src/access/pg_tde_rewrite.c b/src/access/pg_tde_rewrite.c index 7a672775..dc7f9bd2 100644 --- a/src/access/pg_tde_rewrite.c +++ b/src/access/pg_tde_rewrite.c @@ -110,7 +110,7 @@ #include "access/pg_tdeam_xlog.h" #include "access/pg_tdetoast.h" #include "access/pg_tde_rewrite.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/transam.h" #include "access/xact.h" diff --git a/src/access/pg_tde_vacuumlazy.c b/src/access/pg_tde_vacuumlazy.c index b40ec95d..bd7eca4e 100644 --- a/src/access/pg_tde_vacuumlazy.c +++ b/src/access/pg_tde_vacuumlazy.c @@ -39,7 +39,7 @@ #include "access/pg_tdeam.h" #include "access/pg_tdeam_xlog.h" #include "access/pg_tde_visibilitymap.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/amapi.h" #include "access/genam.h" diff --git a/src/access/pg_tdeam.c b/src/access/pg_tdeam.c index 308e7a4d..dfa0e149 100644 --- a/src/access/pg_tdeam.c +++ b/src/access/pg_tdeam.c @@ -39,7 +39,7 @@ #include "access/pg_tdetoast.h" #include "access/pg_tde_io.h" #include "access/pg_tde_visibilitymap.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/bufmask.h" #include "access/genam.h" diff --git a/src/access/pg_tdeam_handler.c b/src/access/pg_tdeam_handler.c index 018234f8..a944c4e2 100644 --- a/src/access/pg_tdeam_handler.c +++ b/src/access/pg_tdeam_handler.c @@ -27,7 +27,7 @@ #include "access/pg_tde_rewrite.h" #include "access/pg_tde_tdemap.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "access/genam.h" #include "access/multixact.h" diff --git a/src/access/pg_tdetoast.c b/src/access/pg_tdetoast.c index 7b18ccb2..eccf45ac 100644 --- a/src/access/pg_tdetoast.c +++ b/src/access/pg_tdetoast.c @@ -35,7 +35,7 @@ #include "miscadmin.h" #include "utils/fmgroids.h" #include "utils/snapmgr.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #define TDE_TOAST_COMPRESS_HEADER_SIZE (VARHDRSZ_COMPRESSED - VARHDRSZ) diff --git a/src/encryption/enc_aes.c b/src/encryption/enc_aes.c index 251f5e64..3f26acb7 100644 --- a/src/encryption/enc_aes.c +++ b/src/encryption/enc_aes.c @@ -6,10 +6,7 @@ #define Assert(p) assert(p) #endif -#include "access/pg_tde_tdemap.h" #include "encryption/enc_aes.h" -#include "keyring/keyring_api.h" -#include "utils/memutils.h" #include #include @@ -162,61 +159,11 @@ void AesEncrypt(const unsigned char* key, const unsigned char* iv, const unsigne AesRunCbc(1, key, iv, in, in_len, out, out_len); } -/* - * Provide a simple interface to encrypt a given key. - * - * The function pallocs and updates the p_enc_rel_key_data along with key bytes. The memory - * is allocated in the current memory context as this key should be ephemeral with a very - * short lifespan until it is written to disk. - */ -void -AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes) -{ - size_t sz; - unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - - /* Ensure we are getting a valid pointer here */ - Assert(master_key_info); - - sz = SizeOfRelKeysData(1); - - *p_enc_rel_key_data = (RelKeysData *) palloc(sz); - memcpy(*p_enc_rel_key_data, rel_key_data, sz); - - AesEncrypt(master_key_info->data.data, iv, ((unsigned char*)rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_enc_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)enc_key_bytes); -} - void AesDecrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len) { AesRunCbc(0, key, iv, in, in_len, out, out_len); } -/* - * Provide a simple interface to decrypt a given key. - * - * The function pallocs and updates the p_rel_key_data along with key bytes. It's important - * to note that memory is allocated in the TopMemoryContext so we expect this to be added - * to our key cache. - */ -void -AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes) -{ - size_t sz; - unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - - /* Ensure we are getting a valid pointer here */ - Assert(master_key_info); - - sz = SizeOfRelKeysData(enc_rel_key_data->internal_keys_len); - - *p_rel_key_data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, sz); - - /* Fill in the structure */ - memcpy(*p_rel_key_data, enc_rel_key_data, sz); - - AesDecrypt(master_key_info->data.data, iv, ((unsigned char*) enc_rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)key_bytes); -} - /* * We want to avoid dynamic memory allocation, so the function only allows * to process NUM_AES_BLOCKS_IN_BATCH number of blocks at a time. diff --git a/src/encryption/enc_tuple.c b/src/encryption/enc_tde.c similarity index 77% rename from src/encryption/enc_tuple.c rename to src/encryption/enc_tde.c index 4c2d5a1a..20a1d50d 100644 --- a/src/encryption/enc_tuple.c +++ b/src/encryption/enc_tde.c @@ -4,7 +4,7 @@ #include "utils/memutils.h" #include "access/pg_tde_tdemap.h" -#include "encryption/enc_tuple.h" +#include "encryption/enc_tde.h" #include "encryption/enc_aes.h" #include "storage/bufmgr.h" #include "keyring/keyring_api.h" @@ -208,3 +208,53 @@ PGTdeExecStorePinnedBufferHeapTuple(Relation rel, HeapTuple tuple, TupleTableSlo } return ExecStorePinnedBufferHeapTuple(tuple, slot, buffer); } + +/* + * Provide a simple interface to encrypt a given key. + * + * The function pallocs and updates the p_enc_rel_key_data along with key bytes. The memory + * is allocated in the current memory context as this key should be ephemeral with a very + * short lifespan until it is written to disk. + */ +void +AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes) +{ + size_t sz; + unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + + /* Ensure we are getting a valid pointer here */ + Assert(master_key_info); + + sz = SizeOfRelKeysData(1); + + *p_enc_rel_key_data = (RelKeysData *) palloc(sz); + memcpy(*p_enc_rel_key_data, rel_key_data, sz); + + AesEncrypt(master_key_info->data.data, iv, ((unsigned char*)rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_enc_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)enc_key_bytes); +} + +/* + * Provide a simple interface to decrypt a given key. + * + * The function pallocs and updates the p_rel_key_data along with key bytes. It's important + * to note that memory is allocated in the TopMemoryContext so we expect this to be added + * to our key cache. + */ +void +AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes) +{ + size_t sz; + unsigned char iv[16] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + + /* Ensure we are getting a valid pointer here */ + Assert(master_key_info); + + sz = SizeOfRelKeysData(enc_rel_key_data->internal_keys_len); + + *p_rel_key_data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, sz); + + /* Fill in the structure */ + memcpy(*p_rel_key_data, enc_rel_key_data, sz); + + AesDecrypt(master_key_info->data.data, iv, ((unsigned char*) enc_rel_key_data) + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, ((unsigned char *)(*p_rel_key_data)) + SizeOfRelKeysDataHeader, (int *)key_bytes); +} diff --git a/src/include/encryption/enc_aes.h b/src/include/encryption/enc_aes.h index a59bb5c9..7cca1e36 100644 --- a/src/include/encryption/enc_aes.h +++ b/src/include/encryption/enc_aes.h @@ -11,8 +11,6 @@ #define ENC_AES_H #include -#include "access/pg_tde_tdemap.h" -#include "keyring/keyring_api.h" #define AES_BLOCK_SIZE 16 #define NUM_AES_BLOCKS_IN_BATCH 100 @@ -25,7 +23,4 @@ extern void Aes128EncryptedZeroBlocks(void* ctxPtr, const unsigned char* key, co extern void AesEncrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len); extern void AesDecrypt(const unsigned char* key, const unsigned char* iv, const unsigned char* in, int in_len, unsigned char* out, int* out_len); -extern void AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes); -extern void AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes); - #endif /*ENC_AES_H*/ \ No newline at end of file diff --git a/src/include/encryption/enc_tuple.h b/src/include/encryption/enc_tde.h similarity index 81% rename from src/include/encryption/enc_tuple.h rename to src/include/encryption/enc_tde.h index f0173373..84a9a6bc 100644 --- a/src/include/encryption/enc_tuple.h +++ b/src/include/encryption/enc_tde.h @@ -1,20 +1,21 @@ /*------------------------------------------------------------------------- * - * enc_tuple.h - * Encryption / Decryption of tuples and item data + * enc_tde.h + * Encryption / Decryption of functions for TDE * - * src/include/encryption/enc_tuple.h + * src/include/encryption/enc_tde.h * *------------------------------------------------------------------------- */ -#ifndef ENC_TUPLE_H -#define ENC_TUPLE_H +#ifndef ENC_TDE_H +#define ENC_TDE_H #include "utils/rel.h" #include "storage/bufpage.h" #include "executor/tuptable.h" #include "executor/tuptable.h" #include "access/pg_tde_tdemap.h" +#include "keyring/keyring_api.h" extern void pg_tde_crypt(const char* iv_prefix, uint32 start_offset, const char* data, uint32 data_len, char* out, RelKeysData* keys, const char* context); @@ -57,4 +58,7 @@ PGTdeExecStorePinnedBufferHeapTuple(Relation rel, HeapTuple tuple, TupleTableSlo pg_tde_crypt(_iv_prefix, _iv_prefix_len, _data, _data_len, _out, _keys, "ENCRYPT-PAGE-ITEM"); \ } while(0) -#endif /*ENC_TUPLE_H*/ +extern void AesEncryptKey(const keyInfo *master_key_info, RelKeysData *rel_key_data, RelKeysData **p_enc_rel_key_data, size_t *enc_key_bytes); +extern void AesDecryptKey(const keyInfo *master_key_info, RelKeysData **p_rel_key_data, RelKeysData *enc_rel_key_data, size_t *key_bytes); + +#endif /*ENC_TDE_H*/ From 05d6b4427f4653b0d8a3680a34f2a512f9f6ab3f Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Wed, 14 Feb 2024 16:24:38 +0500 Subject: [PATCH 3/5] Adding the XLOG handling for internal key during relation creation and when redo-ing the log. Also, updated the handling of master key to accomodate versioning. --- src/access/pg_tde_tdemap.c | 61 +++++++++++++++++------------- src/include/access/pg_tde_tdemap.h | 2 +- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 29363497..826b88cd 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -24,6 +24,7 @@ #include "access/pg_tde_tdemap.h" #include "encryption/enc_aes.h" +#include "encryption/enc_tde.h" #include "keyring/keyring_api.h" #include @@ -78,7 +79,7 @@ static char db_keydata_path[MAXPGPATH] = {0}; static const char *MasterKeyName = "master-key"; static void put_keys_into_map(Oid rel_id, RelKeysData *keys); -static void pg_tde_xlog_create_fork(XLogReaderState *record); +static void pg_tde_xlog_create_relation(XLogReaderState *record); static RelKeysData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info); static RelKeysData* tde_encrypt_rel_key(const keyInfo *master_key_info, RelKeysData *rel_key_data); @@ -87,7 +88,7 @@ static bool pg_tde_perform_rotate_key(const char *new_master_key_name); static void pg_tde_set_db_file_paths(const RelFileLocator *rlocator, char *str_append); static File pg_tde_open_file(char *tde_filename, const char *master_key_name, int fileFlags, bool *is_new_file, off_t *offset); -static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *key, const char *master_key_name); +static void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeysData *enc_rel_key_data, const keyInfo *master_key_info); static int32 pg_tde_write_map_entry(const RelFileLocator *rlocator, char *db_map_path, const char *master_key_name); static int32 pg_tde_write_one_map_entry(File map_file, const RelFileLocator *rlocator, int flags, int32 key_index, TDEMapEntry *map_entry, off_t *offset); @@ -108,6 +109,12 @@ void pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) { InternalKey int_key; + RelKeysData *rel_key_data; + RelKeysData *enc_rel_key_data; + const keyInfo *master_key_info; + + /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ + master_key_info = getMasterKey(MasterKeyName, true, true); memset(&int_key, 0, sizeof(InternalKey)); @@ -119,19 +126,22 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) RelationGetRelationName(rel), ERR_error_string(ERR_get_error(), NULL)))); } + /* Encrypt the key */ + rel_key_data = tde_create_rel_key(newrlocator, &int_key, master_key_info); + enc_rel_key_data = tde_encrypt_rel_key(master_key_info, rel_key_data); + /* XLOG internal keys */ XLogBeginInsert(); XLogRegisterData((char *) newrlocator, sizeof(RelFileLocator)); - XLogRegisterData((char *) &int_key, sizeof(InternalKey)); - XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CREATE_FORK); + XLogRegisterData((char *) enc_rel_key_data->internal_key, sizeof(InternalKey)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_RELATION_KEY); /* * TODO: should DB crash after sending XLog, secondaries would create a fork * file but the relation won't be created either on primary or secondaries. * Hence, the *.tde file will remain as garbage on secondaries. */ - - pg_tde_write_key_map_entry(newrlocator, &int_key, MasterKeyName); + pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, master_key_info); } /* Head of the keys cache (linked list) */ @@ -690,27 +700,18 @@ pg_tde_read_one_keydata(File keydata_file, off_t header_size, int32 key_index, c * The map file must be updated while holding an exclusive lock. */ void -pg_tde_write_key_map_entry(const RelFileLocator *rlocator, InternalKey *key, const char *master_key_name) +pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeysData *enc_rel_key_data, const keyInfo *master_key_info) { int32 key_index = 0; - const keyInfo *master_key_info; - RelKeysData *rel_key_data; - RelKeysData *enc_rel_key_data; - - /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ - master_key_info = getMasterKey(master_key_name, true, true); - - rel_key_data = tde_create_rel_key(rlocator, key, master_key_info); - enc_rel_key_data = tde_encrypt_rel_key(master_key_info, rel_key_data); /* Get the file paths */ pg_tde_set_db_file_paths(rlocator, NULL); /* Create the map entry and then add the encrypted key to the data file */ - key_index = pg_tde_write_map_entry(rlocator, db_map_path, master_key_name); + key_index = pg_tde_write_map_entry(rlocator, db_map_path, master_key_info->name.name); /* Add the encrypted key to the data file. */ - pg_tde_write_keydata(db_keydata_path, master_key_name, key_index, enc_rel_key_data); + pg_tde_write_keydata(db_keydata_path, master_key_info->name.name, key_index, enc_rel_key_data); } /* @@ -860,8 +861,8 @@ pg_tde_rmgr_redo(XLogReaderState *record) switch (info) { - case XLOG_TDE_CREATE_FORK: - pg_tde_xlog_create_fork(record); + case XLOG_TDE_RELATION_KEY: + pg_tde_xlog_create_relation(record); break; default: elog(PANIC, "pg_tde_redo: unknown op code %u", info); @@ -875,7 +876,7 @@ pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record) char *rec = XLogRecGetData(record); RelFileLocator rlocator; - if (info == XLOG_TDE_CREATE_FORK) + if (info == XLOG_TDE_RELATION_KEY) { memcpy(&rlocator, rec, sizeof(RelFileLocator)); appendStringInfo(buf, "create tde fork for relation %u/%u", rlocator.dbOid, rlocator.relNumber); @@ -885,18 +886,20 @@ pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record) const char * pg_tde_rmgr_identify(uint8 info) { - if ((info & ~XLR_INFO_MASK) == XLOG_TDE_CREATE_FORK) - return "TDE_CREATE_FORK"; + if ((info & ~XLR_INFO_MASK) == XLOG_TDE_RELATION_KEY) + return "XLOG_TDE_RELATION_KEY"; return NULL; } static void -pg_tde_xlog_create_fork(XLogReaderState *record) +pg_tde_xlog_create_relation(XLogReaderState *record) { char *rec = XLogRecGetData(record); RelFileLocator rlocator; InternalKey int_key; + const keyInfo *master_key_info; + RelKeysData *enc_rel_key_data; memset(&int_key, 0, sizeof(InternalKey)); @@ -904,7 +907,7 @@ pg_tde_xlog_create_fork(XLogReaderState *record) { ereport(FATAL, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("corrupted XLOG_TDE_CREATE_FORK data"))); + errmsg("corrupted XLOG_TDE_RELATION_KEY data"))); } /* Format [RelFileLocator][InternalKey] */ @@ -916,5 +919,11 @@ pg_tde_xlog_create_fork(XLogReaderState *record) (errmsg("xlog internal_key: %s", tde_sprint_key(&int_key)))); #endif - pg_tde_write_key_map_entry(&rlocator, &int_key, MasterKeyName); + /* Get/generate a master, create the key for relation and get the encrypted key with bytes to write */ + master_key_info = getMasterKey(MasterKeyName, true, true); + + /* Get the the keydata structure for the encrypted key */ + enc_rel_key_data = tde_create_rel_key(&rlocator, &int_key, master_key_info); + + pg_tde_write_key_map_entry(&rlocator, enc_rel_key_data, master_key_info); } diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index a044c001..3180a491 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -62,7 +62,7 @@ extern void pg_tde_cleanup_path_vars(void); const char * tde_sprint_key(InternalKey *k); /* TDE XLOG resource manager */ -#define XLOG_TDE_CREATE_FORK 0x00 +#define XLOG_TDE_RELATION_KEY 0x00 /* TODO: ID has to be registedred and changed: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ #define RM_TDERMGR_ID RM_EXPERIMENTAL_ID #define RM_TDERMGR_NAME "test_pg_tde_custom_rmgr" From 0ba4a5958bcf80c38e23b747e6e559c8bea042ef Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Wed, 14 Feb 2024 16:30:21 +0500 Subject: [PATCH 4/5] Updating the comment as it is no longer valid. --- src/access/pg_tde_tdemap.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 826b88cd..468e4223 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -137,9 +137,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) XLogInsert(RM_TDERMGR_ID, XLOG_TDE_RELATION_KEY); /* - * TODO: should DB crash after sending XLog, secondaries would create a fork - * file but the relation won't be created either on primary or secondaries. - * Hence, the *.tde file will remain as garbage on secondaries. + * Add the encyrpted key to the key map data file structure. */ pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, master_key_info); } From 262e8c706fcc0e946a0e315f9ea1b2105c121025 Mon Sep 17 00:00:00 2001 From: Hamid Akhtar Date: Thu, 15 Feb 2024 16:54:42 +0500 Subject: [PATCH 5/5] Updated: - getMasterKey function argument types to bool from int - Before xlog redo, the decrypted key is added to the key cache. --- src/access/pg_tde_tdemap.c | 25 ++++++++++++++++--------- src/include/access/pg_tde_tdemap.h | 2 -- src/include/keyring/keyring_api.h | 4 +++- src/keyring/keyring_api.c | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 468e4223..be1b8be2 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -81,7 +81,7 @@ static const char *MasterKeyName = "master-key"; static void put_keys_into_map(Oid rel_id, RelKeysData *keys); static void pg_tde_xlog_create_relation(XLogReaderState *record); -static RelKeysData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info); +static RelKeysData* tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info, bool is_key_decrypted); static RelKeysData* tde_encrypt_rel_key(const keyInfo *master_key_info, RelKeysData *rel_key_data); static RelKeysData* tde_decrypt_rel_key(const keyInfo *master_key_info, RelKeysData *enc_rel_key_data); static bool pg_tde_perform_rotate_key(const char *new_master_key_name); @@ -127,7 +127,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator, Relation rel) } /* Encrypt the key */ - rel_key_data = tde_create_rel_key(newrlocator, &int_key, master_key_info); + rel_key_data = tde_create_rel_key(newrlocator, &int_key, master_key_info, true); enc_rel_key_data = tde_encrypt_rel_key(master_key_info, rel_key_data); /* XLOG internal keys */ @@ -219,18 +219,22 @@ tde_sprint_masterkey(const keyData *k) * created key. */ RelKeysData * -tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info) +tde_create_rel_key(const RelFileLocator *rlocator, InternalKey *key, const keyInfo *master_key_info, bool is_key_decrypted) { RelKeysData *rel_key_data; + MemoryContext context = (is_key_decrypted ? TopMemoryContext : CurrentMemoryContext); - rel_key_data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, SizeOfRelKeysData(1)); + Assert(context); + + rel_key_data = (RelKeysData *) MemoryContextAlloc(context, SizeOfRelKeysData(1)); strncpy(rel_key_data->master_key_name, master_key_info->name.name, MASTER_KEY_NAME_LEN); - rel_key_data->internal_key[0] = *key; + memcpy(&rel_key_data->internal_key[0], key, sizeof(InternalKey)); rel_key_data->internal_keys_len = 1; - /* Add to the cache */ - put_keys_into_map(rlocator->relNumber, rel_key_data); + /* Add to the decrypted key to cache */ + if (is_key_decrypted) + put_keys_into_map(rlocator->relNumber, rel_key_data); return rel_key_data; } @@ -702,7 +706,7 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeysData *enc_rel_ { int32 key_index = 0; - /* Get the file paths */ + /* Set the file paths */ pg_tde_set_db_file_paths(rlocator, NULL); /* Create the map entry and then add the encrypted key to the data file */ @@ -921,7 +925,10 @@ pg_tde_xlog_create_relation(XLogReaderState *record) master_key_info = getMasterKey(MasterKeyName, true, true); /* Get the the keydata structure for the encrypted key */ - enc_rel_key_data = tde_create_rel_key(&rlocator, &int_key, master_key_info); + enc_rel_key_data = tde_create_rel_key(&rlocator, &int_key, master_key_info, true); + + /* Add the key to key cache */ + tde_create_rel_key(&rlocator, &enc_rel_key_data->internal_key[0], master_key_info, false); pg_tde_write_key_map_entry(&rlocator, enc_rel_key_data, master_key_info); } diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 3180a491..e110f802 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -12,8 +12,6 @@ #include "storage/relfilelocator.h" #include "access/xlog_internal.h" -#define TDE_FORK_EXT "tde" - #define INTERNAL_KEY_LEN 16 typedef struct InternalKey { diff --git a/src/include/keyring/keyring_api.h b/src/include/keyring/keyring_api.h index 90e64b86..921e4325 100644 --- a/src/include/keyring/keyring_api.h +++ b/src/include/keyring/keyring_api.h @@ -1,6 +1,8 @@ #ifndef KEYRING_API_H #define KEYRING_API_H +#include "postgres.h" + typedef struct keyName { char name[256]; // enough for now @@ -49,6 +51,6 @@ void keyringInitCache(void); const keyInfo* keyringCacheStoreKey(keyName name, keyData data); const char * tde_sprint_masterkey(const keyData *k); -const keyInfo* getMasterKey(const char* internalName, int doGenerateKey, int doRaiseError); +const keyInfo* getMasterKey(const char* internalName, bool doGenerateKey, bool doRaiseError); #endif // KEYRING_API_H diff --git a/src/keyring/keyring_api.c b/src/keyring/keyring_api.c index b75ac0e0..40b4a926 100644 --- a/src/keyring/keyring_api.c +++ b/src/keyring/keyring_api.c @@ -180,7 +180,7 @@ const keyInfo* keyringGenerateKey(const char* internalName, unsigned keyLen) * doGenerateKey should be false and doRaiseError should be set to indicate * that master key is expected but could not be accessed. */ -const keyInfo* getMasterKey(const char* internalName, int doGenerateKey, int doRaiseError) +const keyInfo* getMasterKey(const char* internalName, bool doGenerateKey, bool doRaiseError) { const keyInfo* key = NULL;