From 5a38a6cdcf161b7a1dc1ca3693674de7c9507f76 Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoy Date: Wed, 25 Sep 2024 18:20:22 +0300 Subject: [PATCH] Refactor internal/principal key LWLocks Before this commit, several flaws existed in the lock protection of the Internal and Principal keys. Such as double locking, insufficient locking while rotating the principal key and internal key retrieval from the disk. We also had two locks guarding the same actions, which in 90% of cases were locked and unlocked together. This commit tries to address all these issues. For: https://perconadev.atlassian.net/browse/PG-858 --- src/access/pg_tde_tdemap.c | 65 +++++----- src/access/pg_tde_xlog.c | 6 + src/catalog/tde_principal_key.c | 162 ++++++++++-------------- src/include/access/pg_tde_tdemap.h | 2 +- src/include/catalog/tde_principal_key.h | 8 +- src/include/common/pg_tde_shmem.h | 3 +- src/include/pg_tde_fe.h | 5 + src/smgr/pg_tde_smgr.c | 6 +- 8 files changed, 121 insertions(+), 136 deletions(-) diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 67bfc84e..82d3f032 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -144,10 +144,13 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) RelKeyData *enc_rel_key_data; TDEPrincipalKey *principal_key; XLogRelKey xlrec; + LWLock *lock_pk = tde_lwlock_enc_keys(); - principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid); + LWLockAcquire(lock_pk, LW_EXCLUSIVE); + principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid, LW_EXCLUSIVE); if (principal_key == NULL) { + LWLockRelease(lock_pk); ereport(ERROR, (errmsg("failed to retrieve principal key"))); @@ -158,6 +161,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) { + LWLockRelease(lock_pk); ereport(FATAL, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not generate internal key for relation \"%s\": %s", @@ -184,6 +188,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator) * Add the encyrpted key to the key map data file structure. */ pg_tde_write_key_map_entry(newrlocator, enc_rel_key_data, &principal_key->keyInfo); + LWLockRelease(lock_pk); pfree(enc_rel_key_data); return rel_key_data; } @@ -486,13 +491,12 @@ pg_tde_write_one_keydata(int fd, int32 key_index, RelKeyData *enc_rel_key_data) * The keydata function will then write the encrypted key on the desired * location. * - * The map file must be updated while holding an exclusive lock. + * The caller must hold an exclusive lock tde_lwlock_enc_keys. */ void pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_key_data, TDEPrincipalKeyInfo *principal_key_info) { int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; @@ -502,12 +506,10 @@ pg_tde_write_key_map_entry(const RelFileLocator *rlocator, RelKeyData *enc_rel_k pg_tde_set_db_file_paths(rlocator, db_map_path, db_keydata_path); /* Create the map entry and then add the encrypted key to the data file */ - LWLockAcquire(lock_files, LW_EXCLUSIVE); key_index = pg_tde_write_map_entry(rlocator, db_map_path, principal_key_info); /* Add the encrypted key to the data file. */ pg_tde_write_keydata(db_keydata_path, principal_key_info, key_index, enc_rel_key_data); - LWLockRelease(lock_files); } /* @@ -519,7 +521,7 @@ pg_tde_delete_key_map_entry(const RelFileLocator *rlocator) { int32 key_index = 0; off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_files = tde_lwlock_enc_keys(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; @@ -563,7 +565,7 @@ void pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset) { int32 key_index = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_files = tde_lwlock_enc_keys(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; @@ -649,8 +651,6 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p off_t keydata_size; XLogPrincipalKeyRotate *xlrec; off_t xlrec_size; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; @@ -665,9 +665,6 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p strncpy(m_path[OLD_PRINCIPAL_KEY], db_map_path, MAXPGPATH); strncpy(k_path[OLD_PRINCIPAL_KEY], db_keydata_path, MAXPGPATH); - LWLockAcquire(lock_files, LW_EXCLUSIVE); - LWLockAcquire(lock_cache, LW_EXCLUSIVE); - /* Open both files in read only mode. We don't need to track the current position of the keydata file. We always use the key index */ m_fd[OLD_PRINCIPAL_KEY] = pg_tde_open_file(m_path[OLD_PRINCIPAL_KEY], &principal_key->keyInfo, false, O_RDONLY, &is_new_file, &curr_pos[OLD_PRINCIPAL_KEY]); k_fd[OLD_PRINCIPAL_KEY] = pg_tde_open_file(k_path[OLD_PRINCIPAL_KEY], &principal_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos_tmp); @@ -744,9 +741,6 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p finalize_key_rotation(m_path[OLD_PRINCIPAL_KEY], k_path[OLD_PRINCIPAL_KEY], m_path[NEW_PRINCIPAL_KEY], k_path[NEW_PRINCIPAL_KEY]); - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - /* Free up the palloc'ed data */ pfree(xlrec); @@ -771,8 +765,6 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ bool is_new_file; off_t curr_pos = 0; off_t read_pos_tmp = 0; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; bool is_err = false; @@ -787,9 +779,6 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ 0}, db_map_path, db_keydata_path); - LWLockAcquire(lock_files, LW_EXCLUSIVE); - LWLockAcquire(lock_cache, LW_EXCLUSIVE); - /* Initialize the new files and set the names */ m_fd_new = keyrotation_init_file(&fheader->principal_key_info, m_path_new, db_map_path, &is_new_file, &curr_pos); k_fd_new = keyrotation_init_file(&fheader->principal_key_info, k_path_new, db_keydata_path, &is_new_file, &read_pos_tmp); @@ -839,9 +828,6 @@ pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_ if (!is_err) finalize_key_rotation(db_map_path, db_keydata_path, m_path_new, k_path_new); - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - return !is_err; } @@ -859,19 +845,30 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) RelKeyData *rel_key_data; RelKeyData *enc_rel_key_data; off_t offset = 0; - LWLock *lock_files = tde_lwlock_mk_files(); + LWLock *lock_pk = tde_lwlock_enc_keys(); char db_map_path[MAXPGPATH] = {0}; char db_keydata_path[MAXPGPATH] = {0}; Assert(rlocator); - LWLockAcquire(lock_files, LW_SHARED); - - /* Get/generate a principal key, create the key for relation and get the encrypted key with bytes to write */ - principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid); + /* + * Get/generate a principal key, create the key for relation and get the + * encrypted key with bytes to write + * + * We should hold the lock until the internal key is loaded to be sure the + * retrieved key was encrypted with the obtained principal key. Otherwise, + * the next may happen: + * - GetPrincipalKey returns key "PKey_1". + * - Some other process rotates the Principal key and re-encrypt an + * Internal key with "PKey_2". + * - We read the Internal key and decrypt it with "PKey_1" (that's what + * we've got). As the result we return an invalid Internal key. + */ + LWLockAcquire(lock_pk, LW_SHARED); + principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid, LW_SHARED); if (principal_key == NULL) { - LWLockRelease(lock_files); + LWLockRelease(lock_pk); ereport(ERROR, (errmsg("failed to retrieve principal key"))); } @@ -884,12 +881,12 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator) if (key_index == -1) { - LWLockRelease(lock_files); + LWLockRelease(lock_pk); return NULL; } enc_rel_key_data = pg_tde_read_keydata(db_keydata_path, key_index, principal_key); - LWLockRelease(lock_files); + LWLockRelease(lock_pk); rel_key_data = tde_decrypt_rel_key(principal_key, enc_rel_key_data, rlocator); @@ -1000,6 +997,7 @@ pg_tde_process_map_entry(const RelFileLocator *rlocator, char *db_map_path, off_ /* * Open the file and read the required key data from file and return encrypted key. + * The caller should hold */ static RelKeyData * pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *principal_key) @@ -1008,10 +1006,8 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *pri RelKeyData *enc_rel_key_data; off_t read_pos = 0; bool is_new_file; - LWLock *lock_files = tde_lwlock_mk_files(); /* Open and validate file for basic correctness. */ - LWLockAcquire(lock_files, LW_SHARED); fd = pg_tde_open_file(db_keydata_path, &principal_key->keyInfo, false, O_RDONLY, &is_new_file, &read_pos); /* Read the encrypted key from file */ @@ -1019,7 +1015,6 @@ pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *pri /* Let's close the file. */ close(fd); - LWLockRelease(lock_files); return enc_rel_key_data; } @@ -1241,7 +1236,7 @@ pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *princi * a LW_SHARED or higher lock on files before calling this function. */ TDEPrincipalKeyInfo * -pg_tde_get_principal_key(Oid dbOid, Oid spcOid) +pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid) { int fd = -1; TDEFileHeader fheader; diff --git a/src/access/pg_tde_xlog.c b/src/access/pg_tde_xlog.c index 05b8cae2..e3039305 100644 --- a/src/access/pg_tde_xlog.c +++ b/src/access/pg_tde_xlog.c @@ -39,13 +39,17 @@ tdeheap_rmgr_redo(XLogReaderState *record) { XLogRelKey *xlrec = (XLogRelKey *) XLogRecGetData(record); + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); pg_tde_write_key_map_entry(&xlrec->rlocator, &xlrec->relKey, NULL); + LWLockRelease(tde_lwlock_enc_keys()); } else if (info == XLOG_TDE_ADD_PRINCIPAL_KEY) { TDEPrincipalKeyInfo *mkey = (TDEPrincipalKeyInfo *) XLogRecGetData(record); + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); save_principal_key_info(mkey); + LWLockRelease(tde_lwlock_enc_keys()); } else if (info == XLOG_TDE_EXTENSION_INSTALL_KEY) { @@ -64,7 +68,9 @@ tdeheap_rmgr_redo(XLogReaderState *record) { XLogPrincipalKeyRotate *xlrec = (XLogPrincipalKeyRotate *) XLogRecGetData(record); + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); xl_tde_perform_rotate_key(xlrec); + LWLockRelease(tde_lwlock_enc_keys()); } else { diff --git a/src/catalog/tde_principal_key.c b/src/catalog/tde_principal_key.c index cde95af8..e7ba8ed9 100644 --- a/src/catalog/tde_principal_key.c +++ b/src/catalog/tde_principal_key.c @@ -101,20 +101,16 @@ void InitializePrincipalKeyInfo(void) on_ext_install(principal_key_startup_cleanup, NULL); } +/* + * Lock to guard internal/principal key. Usually, this lock has to be held until + * the caller fetches an internal_key or rotates the principal. + */ LWLock * -tde_lwlock_mk_files(void) -{ - Assert(principalKeyLocalState.sharedPrincipalKeyState); - - return &principalKeyLocalState.sharedPrincipalKeyState->Locks[TDE_LWLOCK_MK_FILES]; -} - -LWLock * -tde_lwlock_mk_cache(void) +tde_lwlock_enc_keys(void) { Assert(principalKeyLocalState.sharedPrincipalKeyState); - return &principalKeyLocalState.sharedPrincipalKeyState->Locks[TDE_LWLOCK_MK_CACHE]; + return &principalKeyLocalState.sharedPrincipalKeyState->Locks[TDE_LWLOCK_ENC_KEY]; } static Size @@ -218,36 +214,28 @@ save_principal_key_info(TDEPrincipalKeyInfo *principal_key_info) /* * SetPrincipalkey: * We need to ensure that only one principal key is set for a database. - * To do that we take a little help from cache. Before setting the - * principal key we take an exclusive lock on the cache entry for the - * database. - * After acquiring the exclusive lock we check for the entry again - * to make sure if some other caller has not added a principal key for - * same database while we were waiting for the lock. */ TDEPrincipalKey * set_principal_key_with_keyring(const char *key_name, GenericKeyring *keyring, Oid dbOid, Oid spcOid, bool ensure_new_key) { TDEPrincipalKey *principalKey = NULL; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + LWLock *lock_files = tde_lwlock_enc_keys(); bool is_dup_key = false; /* * Try to get principal key from cache. */ LWLockAcquire(lock_files, LW_EXCLUSIVE); - LWLockAcquire(lock_cache, LW_EXCLUSIVE); principalKey = get_principal_key_from_cache(dbOid); is_dup_key = (principalKey != NULL); /* TODO: Add the key in the cache? */ - if (is_dup_key == false) - is_dup_key = (pg_tde_get_principal_key(dbOid, spcOid) != NULL); + if (!is_dup_key) + is_dup_key = (pg_tde_get_principal_key_info(dbOid, spcOid) != NULL); - if (is_dup_key == false) + if (!is_dup_key) { const keyInfo *keyInfo = NULL; @@ -266,7 +254,6 @@ set_principal_key_with_keyring(const char *key_name, GenericKeyring *keyring, if (keyInfo == NULL) { - LWLockRelease(lock_cache); LWLockRelease(lock_files); ereport(ERROR, @@ -286,7 +273,6 @@ set_principal_key_with_keyring(const char *key_name, GenericKeyring *keyring, push_principal_key_to_cache(principalKey); } - LWLockRelease(lock_cache); LWLockRelease(lock_files); if (is_dup_key) @@ -395,7 +381,7 @@ xl_tde_perform_rotate_key(XLogPrincipalKeyRotate *xlrec) bool ret; ret = pg_tde_write_map_keydata_files(xlrec->map_size, xlrec->buff, xlrec->keydata_size, &xlrec->buff[xlrec->map_size]); - clear_principal_key_cache(MyDatabaseId); + clear_principal_key_cache(xlrec->databaseId); return ret; } @@ -476,11 +462,9 @@ GetPrincipalKeyProviderId(void) TDEPrincipalKeyInfo *principalKeyInfo = NULL; Oid keyringId = InvalidOid; Oid dbOid = MyDatabaseId; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); + LWLock *lock_files = tde_lwlock_enc_keys(); LWLockAcquire(lock_files, LW_SHARED); - LWLockAcquire(lock_cache, LW_SHARED); principalKey = get_principal_key_from_cache(dbOid); if (principalKey) @@ -489,7 +473,7 @@ GetPrincipalKeyProviderId(void) } { /* Principal key not present in cache. Try Loading it from the info file */ - principalKeyInfo = pg_tde_get_principal_key(dbOid, MyDatabaseTableSpace); + principalKeyInfo = pg_tde_get_principal_key_info(dbOid, MyDatabaseTableSpace); if (principalKeyInfo) { keyringId = principalKeyInfo->keyringId; @@ -497,7 +481,6 @@ GetPrincipalKeyProviderId(void) } } - LWLockRelease(lock_cache); LWLockRelease(lock_files); return keyringId; @@ -661,8 +644,12 @@ pg_tde_rotate_principal_key_internal(PG_FUNCTION_ARGS) new_principal_key_name, new_provider_name, is_global ? "cluster" : "database"))); - current_key = GetPrincipalKey(dbOid, spcOid); + + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + current_key = GetPrincipalKey(dbOid, spcOid, LW_EXCLUSIVE); ret = RotatePrincipalKey(current_key, new_principal_key_name, new_provider_name, ensure_new_key); + LWLockRelease(tde_lwlock_enc_keys()); + PG_RETURN_BOOL(ret); } @@ -700,7 +687,9 @@ pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid, Oid spcOid) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context that cannot accept type record"))); - principal_key = GetPrincipalKey(dbOid, spcOid); + LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); + principal_key = GetPrincipalKey(dbOid, spcOid, LW_SHARED); + LWLockRelease(tde_lwlock_enc_keys()); if (principal_key == NULL) { ereport(ERROR, @@ -751,86 +740,36 @@ pg_tde_get_key_info(PG_FUNCTION_ARGS, Oid dbOid, Oid spcOid) } #endif /* FRONTEND */ -/* - * Public interface to get the principal key for the current database - * If the principal key is not present in the cache, it is loaded from - * the keyring and stored in the cache. - * When the principal key is not set for the database. The function returns - * throws an error. +/* + * Gets principal key form the keyring and pops it into cache if key exists + * Caller should hold an exclusive tde_lwlock_enc_keys lock */ TDEPrincipalKey * -GetPrincipalKey(Oid dbOid, Oid spcOid) +get_principal_key_from_keyring(Oid dbOid, Oid spcOid) { GenericKeyring *keyring; TDEPrincipalKey *principalKey = NULL; TDEPrincipalKeyInfo *principalKeyInfo = NULL; const keyInfo *keyInfo = NULL; KeyringReturnCodes keyring_ret; - LWLock *lock_files = tde_lwlock_mk_files(); - LWLock *lock_cache = tde_lwlock_mk_cache(); - -#ifndef FRONTEND - /* We don't store global space key in cache */ - if (spcOid != GLOBALTABLESPACE_OID) - { - LWLockAcquire(lock_cache, LW_SHARED); - principalKey = get_principal_key_from_cache(dbOid); - LWLockRelease(lock_cache); - } - - if (principalKey) - { - return principalKey; - } -#endif - /* - * We should hold an exclusive lock here to ensure that a valid principal key, if found, is added - * to the cache without any interference. - */ - LWLockAcquire(lock_files, LW_SHARED); - LWLockAcquire(lock_cache, LW_EXCLUSIVE); - -#ifndef FRONTEND - /* We don't store global space key in cache */ - if (spcOid != GLOBALTABLESPACE_OID) - { - principalKey = get_principal_key_from_cache(dbOid); - } + Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), LW_EXCLUSIVE)); - if (principalKey) - { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - return principalKey; - } -#endif - - /* Principal key not present in cache. Load from the keyring */ - principalKeyInfo = pg_tde_get_principal_key(dbOid, spcOid); + principalKeyInfo = pg_tde_get_principal_key_info(dbOid, spcOid); if (principalKeyInfo == NULL) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - return NULL; } keyring = GetKeyProviderByID(principalKeyInfo->keyringId, dbOid, spcOid); if (keyring == NULL) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - return NULL; } keyInfo = KeyringGetKey(keyring, principalKeyInfo->keyId.versioned_name, false, &keyring_ret); if (keyInfo == NULL) { - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - return NULL; } @@ -850,12 +789,51 @@ GetPrincipalKey(Oid dbOid, Oid spcOid) } #endif - /* Release the exclusive locks here */ - LWLockRelease(lock_cache); - LWLockRelease(lock_files); - if (principalKeyInfo) pfree(principalKeyInfo); return principalKey; } + +/* + * A public interface to get the principal key for the database. + * If the principal key is not present in the cache, it is loaded from + * the keyring and stored in the cache. + * When the principal key is not set for the database. The function returns + * throws an error. + * + * The caller must hold a `tde_lwlock_enc_keys` lock and pass its obtained mode + * via the `lockMode` param (LW_SHARED or LW_EXCLUSIVE). We expect the key to be + * most likely in the cache. So the caller should use LW_SHARED if there are no + * principal key changes planned as this is faster and creates less contention. + * But if there is no key in the cache, we have to switch the lock + * (LWLockRelease + LWLockAcquire) to LW_EXCLUSIVE mode to write the key to the + * cache. + */ +TDEPrincipalKey * +GetPrincipalKey(Oid dbOid, Oid spcOid, LWLockMode lockMode) +{ +#ifndef FRONTEND + TDEPrincipalKey *principalKey = NULL; + + Assert(LWLockHeldByMeInMode(tde_lwlock_enc_keys(), lockMode)); + /* We don't store global space key in cache */ + if (spcOid != GLOBALTABLESPACE_OID) + { + principalKey = get_principal_key_from_cache(dbOid); + } + + if (likely(principalKey)) + { + return principalKey; + } + + if (lockMode != LW_EXCLUSIVE) + { + LWLockRelease(tde_lwlock_enc_keys()); + LWLockAcquire(tde_lwlock_enc_keys(), LW_EXCLUSIVE); + } +#endif + + return get_principal_key_from_keyring(dbOid, spcOid); +} \ No newline at end of file diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index 43e218ef..b07223ff 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -42,7 +42,7 @@ extern RelKeyData *GetRelationKey(RelFileLocator rel); extern void pg_tde_delete_tde_files(Oid dbOid, Oid spcOid); -extern TDEPrincipalKeyInfo *pg_tde_get_principal_key(Oid dbOid, Oid spcOid); +extern TDEPrincipalKeyInfo *pg_tde_get_principal_key_info(Oid dbOid, Oid spcOid); extern bool pg_tde_save_principal_key(TDEPrincipalKeyInfo *principal_key_info); extern bool pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_principal_key); extern bool pg_tde_write_map_keydata_files(off_t map_size, char *m_file_data, off_t keydata_size, char *k_file_data); diff --git a/src/include/catalog/tde_principal_key.h b/src/include/catalog/tde_principal_key.h index 66be92f6..3904fb66 100644 --- a/src/include/catalog/tde_principal_key.h +++ b/src/include/catalog/tde_principal_key.h @@ -61,17 +61,15 @@ extern void InitializePrincipalKeyInfo(void); extern void cleanup_principal_key_info(Oid databaseId, Oid tablespaceId); #ifndef FRONTEND -extern LWLock *tde_lwlock_mk_files(void); -extern LWLock *tde_lwlock_mk_cache(void); +extern LWLock *tde_lwlock_enc_keys(void); +extern TDEPrincipalKey* GetPrincipalKey(Oid dbOid, Oid spcOid, LWLockMode lockMode); #else -#define tde_lwlock_mk_files() NULL -#define tde_lwlock_mk_cache() NULL +extern TDEPrincipalKey* GetPrincipalKey(Oid dbOid, Oid spcOid, void *lockMode); #endif extern bool save_principal_key_info(TDEPrincipalKeyInfo *principalKeyInfo); extern Oid GetPrincipalKeyProviderId(void); -extern TDEPrincipalKey* GetPrincipalKey(Oid dbOid, Oid spcOid); extern bool SetPrincipalKey(const char *key_name, const char *provider_name, bool ensure_new_key); extern bool RotatePrincipalKey(TDEPrincipalKey *current_key, const char *new_key_name, const char *new_provider_name, bool ensure_new_key); extern bool xl_tde_perform_rotate_key(XLogPrincipalKeyRotate *xlrec); diff --git a/src/include/common/pg_tde_shmem.h b/src/include/common/pg_tde_shmem.h index 7fa0c3de..70a30df1 100644 --- a/src/include/common/pg_tde_shmem.h +++ b/src/include/common/pg_tde_shmem.h @@ -18,8 +18,7 @@ typedef enum { - TDE_LWLOCK_MK_CACHE, - TDE_LWLOCK_MK_FILES, + TDE_LWLOCK_ENC_KEY, TDE_LWLOCK_PI_FILES, /* Must be the last entry in the enum */ diff --git a/src/include/pg_tde_fe.h b/src/include/pg_tde_fe.h index 44f5c6a6..c49a9370 100644 --- a/src/include/pg_tde_fe.h +++ b/src/include/pg_tde_fe.h @@ -76,7 +76,12 @@ static int tde_fe_error_level = 0; #define LWLockAcquire(lock, mode) NULL #define LWLockRelease(lock_files) NULL +#define LWLockHeldByMeInMode(lock, mode) NULL #define LWLock void +#define LWLockMode void* +#define LW_SHARED NULL +#define LW_EXCLUSIVE NULL +#define tde_lwlock_enc_keys() NULL #define BasicOpenFile(fileName, fileFlags) open(fileName, fileFlags, PG_FILE_MODE_OWNER) diff --git a/src/smgr/pg_tde_smgr.c b/src/smgr/pg_tde_smgr.c index 4f084e19..ca36ae10 100644 --- a/src/smgr/pg_tde_smgr.c +++ b/src/smgr/pg_tde_smgr.c @@ -15,6 +15,7 @@ tde_smgr_get_key(SMgrRelation reln) { TdeCreateEvent *event; RelKeyData *rkd; + TDEPrincipalKey *pk; if(IsCatalogRelationOid(reln->smgr_rlocator.locator.relNumber)) { @@ -22,7 +23,10 @@ tde_smgr_get_key(SMgrRelation reln) return NULL; } - if(GetPrincipalKey(reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.spcOid)==NULL) + LWLockAcquire(tde_lwlock_enc_keys(), LW_SHARED); + pk = GetPrincipalKey(reln->smgr_rlocator.locator.dbOid, reln->smgr_rlocator.locator.spcOid, LW_SHARED); + LWLockRelease(tde_lwlock_enc_keys()); + if(pk == NULL) { return NULL; }