Skip to content

Commit

Permalink
Implementing remote options for keyring configuration
Browse files Browse the repository at this point in the history
This commit adds the possibility to extract parameters from the
main keyring configuration file into a separate file, or remote HTTP
objects. Basic tests using perl are also added.

Two simple configuration files are 'keyring-w-file.json' and
'keyring-w-http.json'. Additionally, the HTTP version also support
a 'ca' parameter for SSL, similar to the main keyring vault configuration.

With this commit, all string parameters support these options,
except the main 'provider' option.
  • Loading branch information
dutow committed Dec 18, 2023
1 parent d7c552c commit ea98264
Show file tree
Hide file tree
Showing 22 changed files with 471 additions and 82 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/postgresql-16-pgdg-package-pgxs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
run: |
sudo apt-get install -y libreadline6-dev systemtap-sdt-dev wget \
zlib1g-dev libssl-dev libpam0g-dev bison flex libipc-run-perl \
libjson-c-dev libcurl4-openssl-dev
libjson-c-dev libcurl4-openssl-dev libhttp-server-simple-perl
sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
Expand Down Expand Up @@ -62,7 +62,7 @@ jobs:
sudo tee -a /etc/postgresql/16/main/postgresql.conf
echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" |
sudo tee -a /etc/postgresql/16/main/postgresql.conf
cp keyring.json /tmp/keyring.json
cp keyring* /tmp/
sudo service postgresql start
sudo psql -V
sudo -u postgres bash -c 'make installcheck USE_PGXS=1'
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/postgresql-16-src-make.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
llvm-11 llvm-11-dev libselinux1-dev python3-dev \
uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev
uuid-dev liblz4-dev libjson-c-dev libcurl4-openssl-dev libhttp-server-simple-perl
sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
Expand Down Expand Up @@ -68,7 +68,7 @@ jobs:
/opt/pgsql/data/postgresql.conf
echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> \
/opt/pgsql/data/postgresql.conf
cp src/contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
cp src/contrib/postgres-tde-ext/keyring* /tmp/
pg_ctl -D /opt/pgsql/data -l logfile start
- name: Test postgres-tde-ext
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/postgresql-16-src-meson-perf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
llvm-11 llvm-11-dev libselinux1-dev python3-dev \
uuid-dev liblz4-dev meson ninja-build libjson-c-dev \
sysbench libcurl4-openssl-dev
sysbench libcurl4-openssl-dev libhttp-server-simple-perl
sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:

- name: Test postgres-tde-ext
run: |
cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
cp ../contrib/postgres-tde-ext/keyring* /tmp/
meson test --suite setup -v
meson test --suite postgres-tde-ext -v --num-processes 1
working-directory: src/build
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/postgresql-16-src-meson.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
llvm-11 llvm-11-dev libselinux1-dev python3-dev \
uuid-dev liblz4-dev meson ninja-build libjson-c-dev \
gpg wget libcurl4-openssl-dev
gpg wget libcurl4-openssl-dev libhttp-server-simple-perl
sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:

- name: Test postgres-tde-ext with keyring_file
run: |
cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
cp ../contrib/postgres-tde-ext/keyring* /tmp/
meson test --suite setup -v
meson test --suite postgres-tde-ext -v --num-processes 1
working-directory: src/build
Expand Down Expand Up @@ -92,6 +92,6 @@ jobs:
with:
name: Regressions diff and postgresql log
path: |
src/build/testrun/postgres-tde-ext/regress/
src/build/testrun/postgres-tde-ext/
retention-days: 3

1 change: 1 addition & 0 deletions Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ src/access/pg_tdeam_handler.o \
src/access/pg_tde_ddl.o \
src/transam/pg_tde_xact_handler.o \
src/keyring/keyring_config.o \
src/keyring/keyring_curl.o \
src/keyring/keyring_file.o \
src/keyring/keyring_vault.o \
src/keyring/keyring_api.o \
Expand Down
6 changes: 5 additions & 1 deletion keyring-vault.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,9 @@
'provider': 'vault-v2',
'token': 'ROOT_TOKEN',
'url': 'http://127.0.0.1:8200',
'mountPath': 'secret'
'mountPath': {
'type': 'file',
'path': '/tmp/vault-mount'

}
}
7 changes: 7 additions & 0 deletions keyring-w-file.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
'provider': 'file',
'datafile': {
'type': 'file',
'path': '/tmp/datafile-location'
}
}
7 changes: 7 additions & 0 deletions keyring-w-http.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
'provider': 'file',
'datafile': {
'type': 'remote',
'url': 'http://localhost:8888/hello'
}
}
3 changes: 3 additions & 0 deletions meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pg_tde_sources = files(
'src/keyring/keyring_config.c',
'src/keyring/keyring_file.c',
'src/keyring/keyring_vault.c',
'src/keyring/keyring_curl.c',
'src/keyring/keyring_api.c',

'src/pg_tde.c',
Expand Down Expand Up @@ -84,6 +85,8 @@ tests += {
'tap': {
'tests': [
't/001_basic.pl',
't/002_remote_config.pl',
't/003_file_config.pl',
],
},
}
2 changes: 1 addition & 1 deletion src/include/keyring/keyring_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ bool keyringLoadConfiguration(const char* configFileName);
// { type: 'remote'. url: 'http://...' }
// If it doesn't have a type key / not remote / ... returns NULL
// Otherwise it retuns the JSON value interpreted as a string
const char* keyringParseStringParam(json_object* object);
bool keyringParseStringParam(const char* name, json_object* object, char* out, long outLen);

#endif // KEYRING_CONFIG_H
19 changes: 19 additions & 0 deletions src/include/keyring/keyring_curl.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@

#ifndef KEYRING_CURL_H
#define KEYRING_CURL_H

#include "pg_tde_defines.h"

#include <stdbool.h>
#include <curl/curl.h>

typedef struct curlString {
char *ptr;
size_t len;
} curlString;

extern CURL* keyringCurl;

bool curlSetupSession(const char* url, const char* caFile, curlString* outStr);

#endif //KEYRING_CURL_H
111 changes: 108 additions & 3 deletions src/keyring/keyring_config.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include "keyring/keyring_config.h"
#include "keyring/keyring_file.h"
#include "keyring/keyring_vault.h"
#include "keyring/keyring_curl.h"

#include <stdio.h>
#include <assert.h>
Expand Down Expand Up @@ -154,12 +155,116 @@ bool keyringLoadConfiguration(const char* configFileName)
return ret;
}

const char* keyringParseStringParam(json_object* object)
bool keyringParseStringParam(const char* name, json_object* object, char* out, long outLen)
{
if(json_object_get_type(object) == json_type_object)
{
elog(WARNING, "Remote parameters are not yet implemented");
json_object* typeO;
const char* type;

if(!json_object_object_get_ex(object, "type", &typeO))
{
elog(ERROR, "Invalid object value for %s: Missing 'type'.", name);
return false;
}

type = json_object_get_string(typeO);

if(strncmp("remote", type, 7) == 0)
{
json_object* caO;
json_object* urlO;
const char* ca = NULL;
const char* url = NULL;

long httpCode;
curlString outStr;

if(json_object_object_get_ex(object, "ca", &caO))
{
ca = json_object_get_string(caO);
}
if(json_object_object_get_ex(object, "url", &urlO))
{
url = json_object_get_string(urlO);
} else
{
elog(ERROR, "Invalid remote object for %s: Missing 'url'.", name);
return false;
}

outStr.ptr = palloc0(1);
outStr.len = 0;
if(!curlSetupSession(url, ca, &outStr))
{
elog(ERROR, "CURL error for remote object %s", name);
return false;
}
if(curl_easy_perform(keyringCurl) != CURLE_OK)
{
elog(ERROR, "HTTP request error for remote object %s", name);
return false;
}
if(curl_easy_getinfo(keyringCurl, CURLINFO_RESPONSE_CODE, &httpCode) != CURLE_OK)
{
elog(ERROR, "HTTP error for remote object %s, HTTP code %li", name, httpCode);
return false;
}
#if KEYRING_DEBUG
elog(DEBUG2, "HTTP response for config [%s] '%s'", name, outStr->ptr != NULL ? outStr->ptr : "");
#endif
strncpy(out, outStr.ptr, outLen);
out[strcspn(out, "\r\n")] = 0;

return true;
}

if(strncmp("file", type, 5) == 0)
{
json_object* pathO;
const char* path = NULL;
FILE* f;

if(json_object_object_get_ex(object, "path", &pathO))
{
path = json_object_get_string(pathO);
} else
{
elog(ERROR, "Invalid file object for %s: Missing 'path'.", name);
return false;
}

if(access(path, R_OK) != 0)
{
elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", name, path);
return false;
}

f = fopen(path, "r");

if(!f)
{
elog(ERROR, "The file referenced by %s doesn't exists, or is not readable to postgres: %s", name, path);
return false;
}

fgets(out, outLen, f);
out[strcspn(out, "\r\n")] = 0;

fclose(f);

return true;
}

elog(ERROR, "Unknown type for %s: %s", name, type);
return false;
}

return json_object_get_string(object);
{
const char* string;
string = json_object_get_string(object);

strncpy(out, string, outLen);
}
return true;
}
56 changes: 56 additions & 0 deletions src/keyring/keyring_curl.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@

#include "keyring/keyring_curl.h"
#include "keyring/keyring_config.h"
#include "pg_tde_defines.h"

CURL* keyringCurl = NULL;

static size_t writefunc(void *ptr, size_t size, size_t nmemb, struct curlString *s)
{
size_t new_len = s->len + size*nmemb;
s->ptr = repalloc(s->ptr, new_len+1);
if (s->ptr == NULL) {
exit(EXIT_FAILURE);
}
memcpy(s->ptr+s->len, ptr, size*nmemb);
s->ptr[new_len] = '\0';
s->len = new_len;

return size*nmemb;
}

bool curlSetupSession(const char* url, const char* caFile, curlString* outStr)
{
if(keyringCurl == NULL)
{
keyringCurl = curl_easy_init();

if(keyringCurl == NULL) return 0;
}

if(curl_easy_setopt(keyringCurl, CURLOPT_SSL_VERIFYPEER, 1) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_SSL_VERIFYHOST, 1) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_USE_SSL, CURLUSESSL_ALL) != CURLE_OK) return 0;
if(caFile == NULL || strlen(caFile) == 0)
{
//if(curl_easy_setopt(keyringCurl, CURLOPT_CAINFO, "") != CURLE_OK) return 0;
//if(curl_easy_setopt(keyringCurl, CURLSSLOPT_NATIVE_CA, 1) != CURLE_OK) return 0;
} else
{
if(curl_easy_setopt(keyringCurl, CURLSSLOPT_NATIVE_CA, 0) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_CAINFO, caFile) != CURLE_OK) return 0;
}
if(curl_easy_setopt(keyringCurl, CURLOPT_FOLLOWLOCATION, 1L) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_CONNECTTIMEOUT, 3) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_TIMEOUT, 10) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_HTTP_VERSION,(long)CURL_HTTP_VERSION_1_1) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_WRITEFUNCTION,writefunc) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_WRITEDATA,outStr) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_URL, url) != CURLE_OK) return 0;

if(curl_easy_setopt(keyringCurl, CURLOPT_POSTFIELDS, NULL) != CURLE_OK) return 0;
if(curl_easy_setopt(keyringCurl, CURLOPT_POST, 0) != CURLE_OK) return 0;

return 1;
}

8 changes: 2 additions & 6 deletions src/keyring/keyring_file.c
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,20 @@ extern keyringCache* cache; // TODO: do not depend on cache internals
int keyringFileParseConfiguration(json_object* configRoot)
{
json_object* dataO;
const char* datafile;

if(!json_object_object_get_ex(configRoot, "datafile", &dataO))
{
elog(ERROR, "Missing 'datafile' attribute.");
return 0;
}

datafile = keyringParseStringParam(dataO);

if(datafile == NULL)
keyringFileDataFileName[0] = 0;
if(!keyringParseStringParam("datafile", dataO, keyringFileDataFileName, 512))
{
elog(ERROR, "Couldn't parse 'datafile' attribute.");
return 0;
}

strcpy(keyringFileDataFileName, datafile);

return 1;
}

Expand Down
Loading

0 comments on commit ea98264

Please sign in to comment.