From a4711f682b03fe1cb120c4637059a1cab313a7da Mon Sep 17 00:00:00 2001 From: Andrew Pogrebnoi Date: Wed, 8 Nov 2023 11:25:28 +0200 Subject: [PATCH] XLog: replicate fork file and encrypt tuples (#59) 1. Inserts and Updates now are encrypted in WAL. We encrypt new tuples directly in Buffer after they were insrerted there. To pass it to XLog we could memcpy Buffer data into into the tuple. But later tuple has to be unencrypted for index instertions etc. So we pass directly data from the Buffer into XLog. 2. Log into WAL and replicate *.tde forks creation. 3. Added docker-compose for the streaming replication test setup. (not perfect - needs two `up -d` in a row to start the secondary) 4. Added tests for multi inserts. Need tests for replications though. --- docker/Dockerfile | 5 +- docker/docker-compose.yaml | 26 + docker/pg-tde-create-ext.sh | 2 + docker/pg-tde-streaming-repl.sh | 19 + expected/multi_insert.out | 91 ++ meson.build | 1 + sql/multi_insert.sql | 1393 ++++++++++++++++++++++++++++ src/access/pg_tde_tdemap.c | 139 ++- src/access/pg_tdeam.c | 16 +- src/include/access/pg_tde_tdemap.h | 19 + src/pg_tde.c | 3 + 11 files changed, 1676 insertions(+), 38 deletions(-) create mode 100644 docker/docker-compose.yaml create mode 100644 docker/pg-tde-streaming-repl.sh create mode 100644 expected/multi_insert.out create mode 100644 sql/multi_insert.sql diff --git a/docker/Dockerfile b/docker/Dockerfile index 29ffa524..f50c04cc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,11 +17,14 @@ RUN make USE_PGXS=1 && \ make USE_PGXS=1 install RUN cp /usr/share/postgresql/postgresql.conf.sample /etc/postgresql/postgresql.conf; \ echo "shared_preload_libraries = 'pg_tde'" >> /etc/postgresql/postgresql.conf; \ + # echo "log_min_messages = debug3" >> /etc/postgresql/postgresql.conf; \ + # echo "log_min_error_statement = debug3" >> /etc/postgresql/postgresql.conf; \ echo "pg_tde.keyringConfigFile = '/etc/postgresql/tde_conf.json'" >> /etc/postgresql/postgresql.conf; \ - echo "{'provider': 'file','datafile': '/tmp/pgkeyring',}" > /etc/postgresql/tde_conf.json; \ + echo "{'provider': 'file','datafile': '/etc/postgresql/pgkeyring',}" > /etc/postgresql/tde_conf.json; \ chown postgres /etc/postgresql/tde_conf.json; \ mkdir -p /docker-entrypoint-initdb.d COPY ./docker/pg-tde-create-ext.sh /docker-entrypoint-initdb.d/pg-tde-create-ext.sh +COPY ./docker/pg-tde-streaming-repl.sh /docker-entrypoint-initdb.d/pg-tde-streaming-repl.sh VOLUME /etc/postgresql/ diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..65bf3158 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,26 @@ +# TODO: needs improvements as currentlly ` docker-compose up -d --build` has to be run twice +# as replication init on secodary doen't work 100% properly +version: "3.4" +services: + pg-primary: + build: + dockerfile: ./docker/Dockerfile + context: .. + environment: + - "POSTGRES_PASSWORD=testpass" + - "PG_PRIMARY=true" + - "POSTGRES_HOST_AUTH_METHOD=trust" + - "PG_REPLICATION=true" + ports: + - "5433:5432" + pg-secondary: + build: + dockerfile: ./docker/Dockerfile + context: .. + depends_on: + - pg-primary + environment: + - "POSTGRES_PASSWORD=testpass" + - "PG_REPLICATION=true" + ports: + - "5434:5432" diff --git a/docker/pg-tde-create-ext.sh b/docker/pg-tde-create-ext.sh index 09868e96..98e0f0ab 100644 --- a/docker/pg-tde-create-ext.sh +++ b/docker/pg-tde-create-ext.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -e + psql -c 'CREATE EXTENSION pg_tde;' psql -d template1 -c 'CREATE EXTENSION pg_tde;' diff --git a/docker/pg-tde-streaming-repl.sh b/docker/pg-tde-streaming-repl.sh new file mode 100644 index 00000000..70e08c27 --- /dev/null +++ b/docker/pg-tde-streaming-repl.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +PG_PRIMARY=${PG_PRIMARY:-"false"} +PG_REPLICATION=${PG_REPLICATION:-"false"} +REPL_PASS=${REPL_PASS:-"replpass"} + +if [ !PG_REPLICATION = "true "] ; then + exit 0 +fi + +if [ $PG_PRIMARY == "true" ] ; then + psql -c "CREATE ROLE repl WITH REPLICATION PASSWORD '${REPL_PASS}' LOGIN;" + echo "host replication repl 0.0.0.0/0 trust" >> ${PGDATA}/pg_hba.conf +else + rm -rf ${PGDATA}/* + pg_basebackup -h pg-primary -p 5432 -U repl -D ${PGDATA} -Fp -Xs -R +fi \ No newline at end of file diff --git a/expected/multi_insert.out b/expected/multi_insert.out new file mode 100644 index 00000000..6bb893cc --- /dev/null +++ b/expected/multi_insert.out @@ -0,0 +1,91 @@ +-- trigger multi_insert path +-- +CREATE EXTENSION pg_tde; +CREATE TABLE albums ( + album_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + artist_id INTEGER, + title TEXT NOT NULL, + released DATE NOT NULL +) USING pg_tde; +COPY albums FROM stdin CSV HEADER; +SELECT * FROM albums; + album_id | artist_id | title | released +----------+-----------+--------------------+------------ + 1 | 1 | Mirror | 06-24-2009 + 2 | 2 | Pretzel Logic | 02-20-1974 + 3 | 3 | Under Construction | 11-12-2002 + 4 | 4 | Return to Wherever | 07-11-2019 + 5 | 5 | The Nightfly | 10-01-1982 + 6 | 6 | It's Alive | 10-15-2013 + 7 | 7 | Pure Ella | 02-15-1994 +(7 rows) + +SELECT * FROM albums where album_id > 5; + album_id | artist_id | title | released +----------+-----------+------------+------------ + 6 | 6 | It's Alive | 10-15-2013 + 7 | 7 | Pure Ella | 02-15-1994 +(2 rows) + +-- On replica: +-- SELECT * FROM albums; +-- album_id | artist_id | title | released +-- ----------+-----------+--------------------+------------ +-- 1 | 1 | Mirror | 2009-06-24 +-- 2 | 2 | Pretzel Logic | 1974-02-20 +-- 3 | 3 | Under Construction | 2002-11-12 +-- 4 | 4 | Return to Wherever | 2019-07-11 +-- 5 | 5 | The Nightfly | 1982-10-01 +-- 6 | 6 | It's Alive | 2013-10-15 +-- 7 | 7 | Pure Ella | 1994-02-15 +-- (7 rows) +-- +-- SELECT * FROM albums where album_id > 5; +-- album_id | artist_id | title | released +-- ----------+-----------+------------+------------ +-- 6 | 6 | It's Alive | 2013-10-15 +-- 7 | 7 | Pure Ella | 1994-02-15 +-- (2 rows) +-- +DROP TABLE albums; +-- multi_insert2 +-- more data to take multiple pages +CREATE TABLE Towns ( + id SERIAL UNIQUE NOT NULL, + code VARCHAR(10) NOT NULL, + article TEXT, + name TEXT NOT NULL, + department VARCHAR(4) NOT NULL, + UNIQUE (code, department) +) USING pg_tde; +COPY towns (id, code, article, name, department) FROM stdin; +SELECT count(*) FROM towns; + count +------- + 1313 +(1 row) + +SELECT * FROM towns where id in (13, 666); + id | code | article | name | department +-----+------+-----------+----------------+------------ + 13 | 014 | some_text | Arbent | 01 + 666 | 252 | some_text | Cuissy-et-Geny | 02 +(2 rows) + +-- ON REPLICA +-- +-- select count(*) from towns; +-- count +-- ------- +-- 1313 +-- (1 row) +-- +-- select * from towns where id in (13, 666); +-- id | code | article | name | department +-- -----+------+-----------+----------------+------------ +-- 13 | 014 | some_text | Arbent | 01 +-- 666 | 252 | some_text | Cuissy-et-Geny | 02 +-- (2 rows) +-- +DROP TABLE towns; +DROP EXTENSION pg_tde; diff --git a/meson.build b/meson.build index 321016d2..0f0796ec 100644 --- a/meson.build +++ b/meson.build @@ -70,6 +70,7 @@ tests += { 'non_sorted_off_compact', 'update_compare_indexes', 'pgtde_is_encrypted', + 'multi_insert', ], 'regress_args': ['--temp-config', files('postgres-tde-ext.conf')], 'runningcheck': false, diff --git a/sql/multi_insert.sql b/sql/multi_insert.sql new file mode 100644 index 00000000..03c283a3 --- /dev/null +++ b/sql/multi_insert.sql @@ -0,0 +1,1393 @@ +-- trigger multi_insert path +-- +CREATE EXTENSION pg_tde; + +CREATE TABLE albums ( + album_id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + artist_id INTEGER, + title TEXT NOT NULL, + released DATE NOT NULL +) USING pg_tde; + +COPY albums FROM stdin CSV HEADER; +album_id,artist_id,title,released +1,1,"Mirror",2009-06-24 +2,2,"Pretzel Logic",1974-02-20 +3,3,"Under Construction",2002-11-12 +4,4,"Return to Wherever",2019-07-11 +5,5,"The Nightfly",1982-10-01 +6,6,"It's Alive",2013-10-15 +7,7,"Pure Ella",1994-02-15 +\. + +SELECT * FROM albums; +SELECT * FROM albums where album_id > 5; +-- On replica: +-- SELECT * FROM albums; +-- album_id | artist_id | title | released +-- ----------+-----------+--------------------+------------ +-- 1 | 1 | Mirror | 2009-06-24 +-- 2 | 2 | Pretzel Logic | 1974-02-20 +-- 3 | 3 | Under Construction | 2002-11-12 +-- 4 | 4 | Return to Wherever | 2019-07-11 +-- 5 | 5 | The Nightfly | 1982-10-01 +-- 6 | 6 | It's Alive | 2013-10-15 +-- 7 | 7 | Pure Ella | 1994-02-15 +-- (7 rows) +-- +-- SELECT * FROM albums where album_id > 5; +-- album_id | artist_id | title | released +-- ----------+-----------+------------+------------ +-- 6 | 6 | It's Alive | 2013-10-15 +-- 7 | 7 | Pure Ella | 1994-02-15 +-- (2 rows) +-- +DROP TABLE albums; + +-- multi_insert2 +-- more data to take multiple pages +CREATE TABLE Towns ( + id SERIAL UNIQUE NOT NULL, + code VARCHAR(10) NOT NULL, + article TEXT, + name TEXT NOT NULL, + department VARCHAR(4) NOT NULL, + UNIQUE (code, department) +) USING pg_tde; + +COPY towns (id, code, article, name, department) FROM stdin; +1 001 some_text Abergement-Clémenciat 01 +2 002 some_text Abergement-de-Varey 01 +3 004 some_text Ambérieu-en-Bugey 01 +4 005 some_text Ambérieux-en-Dombes 01 +5 006 some_text Ambléon 01 +6 007 some_text Ambronay 01 +7 008 some_text Ambutrix 01 +8 009 some_text Andert-et-Condon 01 +9 010 some_text Anglefort 01 +10 011 some_text Apremont 01 +11 012 some_text Aranc 01 +12 013 some_text Arandas 01 +13 014 some_text Arbent 01 +14 015 some_text Arbignieu 01 +15 016 some_text Arbigny 01 +16 017 some_text Argis 01 +17 019 some_text Armix 01 +18 021 some_text Ars-sur-Formans 01 +19 022 some_text Artemare 01 +20 023 some_text Asnières-sur-Saône 01 +21 024 some_text Attignat 01 +22 025 some_text Bâgé-la-Ville 01 +23 026 some_text Bâgé-le-Châtel 01 +24 027 some_text Balan 01 +25 028 some_text Baneins 01 +26 029 some_text Beaupont 01 +27 030 some_text Beauregard 01 +28 031 some_text Bellignat 01 +29 032 some_text Béligneux 01 +30 033 some_text Bellegarde-sur-Valserine 01 +31 034 some_text Belley 01 +32 035 some_text Belleydoux 01 +33 036 some_text Belmont-Luthézieu 01 +34 037 some_text Bénonces 01 +35 038 some_text Bény 01 +36 039 some_text Béon 01 +37 040 some_text Béréziat 01 +38 041 some_text Bettant 01 +39 042 some_text Bey 01 +40 043 some_text Beynost 01 +41 044 some_text Billiat 01 +42 045 some_text Birieux 01 +43 046 some_text Biziat 01 +44 047 some_text Blyes 01 +45 049 some_text Boisse 01 +46 050 some_text Boissey 01 +47 051 some_text Bolozon 01 +48 052 some_text Bouligneux 01 +49 053 some_text Bourg-en-Bresse 01 +50 054 some_text Bourg-Saint-Christophe 01 +51 056 some_text Boyeux-Saint-Jérôme 01 +52 057 some_text Boz 01 +53 058 some_text Brégnier-Cordon 01 +54 059 some_text Brénaz 01 +55 060 some_text Brénod 01 +56 061 some_text Brens 01 +57 062 some_text Bressolles 01 +58 063 some_text Brion 01 +59 064 some_text Briord 01 +60 065 some_text Buellas 01 +61 066 some_text Burbanche 01 +62 067 some_text Ceignes 01 +63 068 some_text Cerdon 01 +64 069 some_text Certines 01 +65 071 some_text Cessy 01 +66 072 some_text Ceyzériat 01 +67 073 some_text Ceyzérieu 01 +68 074 some_text Chalamont 01 +69 075 some_text Chaleins 01 +70 076 some_text Chaley 01 +71 077 some_text Challes 01 +72 078 some_text Challex 01 +73 079 some_text Champagne-en-Valromey 01 +74 080 some_text Champdor 01 +75 081 some_text Champfromier 01 +76 082 some_text Chanay 01 +77 083 some_text Chaneins 01 +78 084 some_text Chanoz-Châtenay 01 +79 085 some_text Chapelle-du-Châtelard 01 +80 087 some_text Charix 01 +81 088 some_text Charnoz-sur-Ain 01 +82 089 some_text Château-Gaillard 01 +83 090 some_text Châtenay 01 +84 091 some_text Châtillon-en-Michaille 01 +85 092 some_text Châtillon-la-Palud 01 +86 093 some_text Châtillon-sur-Chalaronne 01 +87 094 some_text Chavannes-sur-Reyssouze 01 +88 095 some_text Chavannes-sur-Suran 01 +89 096 some_text Chaveyriat 01 +90 097 some_text Chavornay 01 +91 098 some_text Chazey-Bons 01 +92 099 some_text Chazey-sur-Ain 01 +93 100 some_text Cheignieu-la-Balme 01 +94 101 some_text Chevillard 01 +95 102 some_text Chevroux 01 +96 103 some_text Chevry 01 +97 104 some_text Chézery-Forens 01 +98 105 some_text Civrieux 01 +99 106 some_text Cize 01 +100 107 some_text Cleyzieu 01 +101 108 some_text Coligny 01 +102 109 some_text Collonges 01 +103 110 some_text Colomieu 01 +104 111 some_text Conand 01 +105 112 some_text Condamine 01 +106 113 some_text Condeissiat 01 +107 114 some_text Confort 01 +108 115 some_text Confrançon 01 +109 116 some_text Contrevoz 01 +110 117 some_text Conzieu 01 +111 118 some_text Corbonod 01 +112 119 some_text Corcelles 01 +113 121 some_text Corlier 01 +114 122 some_text Cormaranche-en-Bugey 01 +115 123 some_text Cormoranche-sur-Saône 01 +116 124 some_text Cormoz 01 +117 125 some_text Corveissiat 01 +118 127 some_text Courmangoux 01 +119 128 some_text Courtes 01 +120 129 some_text Crans 01 +121 130 some_text Cras-sur-Reyssouze 01 +122 133 some_text Cressin-Rochefort 01 +123 134 some_text Crottet 01 +124 135 some_text Crozet 01 +125 136 some_text Cruzilles-lès-Mépillat 01 +126 138 some_text Culoz 01 +127 139 some_text Curciat-Dongalon 01 +128 140 some_text Curtafond 01 +129 141 some_text Cuzieu 01 +130 142 some_text Dagneux 01 +131 143 some_text Divonne-les-Bains 01 +132 144 some_text Dommartin 01 +133 145 some_text Dompierre-sur-Veyle 01 +134 146 some_text Dompierre-sur-Chalaronne 01 +135 147 some_text Domsure 01 +136 148 some_text Dortan 01 +137 149 some_text Douvres 01 +138 150 some_text Drom 01 +139 151 some_text Druillat 01 +140 152 some_text Échallon 01 +141 153 some_text Échenevex 01 +142 154 some_text Étrez 01 +143 155 some_text Évosges 01 +144 156 some_text Faramans 01 +145 157 some_text Fareins 01 +146 158 some_text Farges 01 +147 159 some_text Feillens 01 +148 160 some_text Ferney-Voltaire 01 +149 162 some_text Flaxieu 01 +150 163 some_text Foissiat 01 +151 165 some_text Francheleins 01 +152 166 some_text Frans 01 +153 167 some_text Garnerans 01 +154 169 some_text Genouilleux 01 +155 170 some_text Géovreissiat 01 +156 171 some_text Géovreisset 01 +157 172 some_text Germagnat 01 +158 173 some_text Gex 01 +159 174 some_text Giron 01 +160 175 some_text Gorrevod 01 +161 176 some_text Grand-Abergement 01 +162 177 some_text Grand-Corent 01 +163 179 some_text Grièges 01 +164 180 some_text Grilly 01 +165 181 some_text Groissiat 01 +166 182 some_text Groslée 01 +167 183 some_text Guéreins 01 +168 184 some_text Hautecourt-Romanèche 01 +169 185 some_text Hauteville-Lompnes 01 +170 186 some_text Hostias 01 +171 187 some_text Hotonnes 01 +172 188 some_text Illiat 01 +173 189 some_text Injoux-Génissiat 01 +174 190 some_text Innimond 01 +175 191 some_text Izenave 01 +176 192 some_text Izernore 01 +177 193 some_text Izieu 01 +178 194 some_text Jassans-Riottier 01 +179 195 some_text Jasseron 01 +180 196 some_text Jayat 01 +181 197 some_text Journans 01 +182 198 some_text Joyeux 01 +183 199 some_text Jujurieux 01 +184 200 some_text Labalme 01 +185 202 some_text Lagnieu 01 +186 203 some_text Laiz 01 +187 204 some_text Lalleyriat 01 +188 205 some_text Lancrans 01 +189 206 some_text Lantenay 01 +190 207 some_text Lapeyrouse 01 +191 208 some_text Lavours 01 +192 209 some_text Léaz 01 +193 210 some_text Lélex 01 +194 211 some_text Lent 01 +195 212 some_text Lescheroux 01 +196 213 some_text Leyment 01 +197 214 some_text Leyssard 01 +198 215 some_text Lhôpital 01 +199 216 some_text Lhuis 01 +200 218 some_text Lochieu 01 +201 219 some_text Lompnas 01 +202 221 some_text Lompnieu 01 +203 224 some_text Loyettes 01 +204 225 some_text Lurcy 01 +205 227 some_text Magnieu 01 +206 228 some_text Maillat 01 +207 229 some_text Malafretaz 01 +208 230 some_text Mantenay-Montlin 01 +209 231 some_text Manziat 01 +210 232 some_text Marboz 01 +211 233 some_text Marchamp 01 +212 234 some_text Marignieu 01 +213 235 some_text Marlieux 01 +214 236 some_text Marsonnas 01 +215 237 some_text Martignat 01 +216 238 some_text Massieux 01 +217 239 some_text Massignieu-de-Rives 01 +218 240 some_text Matafelon-Granges 01 +219 241 some_text Meillonnas 01 +220 242 some_text Mérignat 01 +221 243 some_text Messimy-sur-Saône 01 +222 244 some_text Meximieux 01 +223 245 some_text Bohas-Meyriat-Rignat 01 +224 246 some_text Mézériat 01 +225 247 some_text Mijoux 01 +226 248 some_text Mionnay 01 +227 249 some_text Miribel 01 +228 250 some_text Misérieux 01 +229 252 some_text Mogneneins 01 +230 254 some_text Montagnat 01 +231 255 some_text Montagnieu 01 +232 257 some_text Montanges 01 +233 258 some_text Montceaux 01 +234 259 some_text Montcet 01 +235 260 some_text Montellier 01 +236 261 some_text Monthieux 01 +237 262 some_text Montluel 01 +238 263 some_text Montmerle-sur-Saône 01 +239 264 some_text Montracol 01 +240 265 some_text Montréal-la-Cluse 01 +241 266 some_text Montrevel-en-Bresse 01 +242 267 some_text Nurieux-Volognat 01 +243 268 some_text Murs-et-Gélignieux 01 +244 269 some_text Nantua 01 +245 271 some_text Nattages 01 +246 272 some_text Neuville-les-Dames 01 +247 273 some_text Neuville-sur-Ain 01 +248 274 some_text Neyrolles 01 +249 275 some_text Neyron 01 +250 276 some_text Niévroz 01 +251 277 some_text Nivollet-Montgriffon 01 +252 279 some_text Oncieu 01 +253 280 some_text Ordonnaz 01 +254 281 some_text Ornex 01 +255 282 some_text Outriaz 01 +256 283 some_text Oyonnax 01 +257 284 some_text Ozan 01 +258 285 some_text Parcieux 01 +259 286 some_text Parves 01 +260 288 some_text Péron 01 +261 289 some_text Péronnas 01 +262 290 some_text Pérouges 01 +263 291 some_text Perrex 01 +264 292 some_text Petit-Abergement 01 +265 293 some_text Peyriat 01 +266 294 some_text Peyrieu 01 +267 295 some_text Peyzieux-sur-Saône 01 +268 296 some_text Pirajoux 01 +269 297 some_text Pizay 01 +270 298 some_text Plagne 01 +271 299 some_text Plantay 01 +272 300 some_text Poizat 01 +273 301 some_text Polliat 01 +274 302 some_text Pollieu 01 +275 303 some_text Poncin 01 +276 304 some_text Pont-d'Ain 01 +277 305 some_text Pont-de-Vaux 01 +278 306 some_text Pont-de-Veyle 01 +279 307 some_text Port 01 +280 308 some_text Pougny 01 +281 309 some_text Pouillat 01 +282 310 some_text Prémeyzel 01 +283 311 some_text Prémillieu 01 +284 312 some_text Pressiat 01 +285 313 some_text Prévessin-Moëns 01 +286 314 some_text Priay 01 +287 316 some_text Pugieu 01 +288 317 some_text Ramasse 01 +289 318 some_text Rancé 01 +290 319 some_text Relevant 01 +291 320 some_text Replonges 01 +292 321 some_text Revonnas 01 +293 322 some_text Reyrieux 01 +294 323 some_text Reyssouze 01 +295 325 some_text Rignieux-le-Franc 01 +296 328 some_text Romans 01 +297 329 some_text Rossillon 01 +298 330 some_text Ruffieu 01 +299 331 some_text Saint-Alban 01 +300 332 some_text Saint-André-de-Bâgé 01 +301 333 some_text Saint-André-de-Corcy 01 +302 334 some_text Saint-André-d'Huiriat 01 +303 335 some_text Saint-André-le-Bouchoux 01 +304 336 some_text Saint-André-sur-Vieux-Jonc 01 +305 337 some_text Saint-Bénigne 01 +306 338 some_text Saint-Benoît 01 +307 339 some_text Saint-Bernard 01 +308 340 some_text Saint-Bois 01 +309 341 some_text Saint-Champ 01 +310 342 some_text Sainte-Croix 01 +311 343 some_text Saint-Cyr-sur-Menthon 01 +312 344 some_text Saint-Denis-lès-Bourg 01 +313 345 some_text Saint-Denis-en-Bugey 01 +314 346 some_text Saint-Didier-d'Aussiat 01 +315 347 some_text Saint-Didier-de-Formans 01 +316 348 some_text Saint-Didier-sur-Chalaronne 01 +317 349 some_text Saint-Éloi 01 +318 350 some_text Saint-Étienne-du-Bois 01 +319 351 some_text Saint-Étienne-sur-Chalaronne 01 +320 352 some_text Saint-Étienne-sur-Reyssouze 01 +321 353 some_text Sainte-Euphémie 01 +322 354 some_text Saint-Genis-Pouilly 01 +323 355 some_text Saint-Genis-sur-Menthon 01 +324 356 some_text Saint-Georges-sur-Renon 01 +325 357 some_text Saint-Germain-de-Joux 01 +326 358 some_text Saint-Germain-les-Paroisses 01 +327 359 some_text Saint-Germain-sur-Renon 01 +328 360 some_text Saint-Jean-de-Gonville 01 +329 361 some_text Saint-Jean-de-Niost 01 +330 362 some_text Saint-Jean-de-Thurigneux 01 +331 363 some_text Saint-Jean-le-Vieux 01 +332 364 some_text Saint-Jean-sur-Reyssouze 01 +333 365 some_text Saint-Jean-sur-Veyle 01 +334 366 some_text Sainte-Julie 01 +335 367 some_text Saint-Julien-sur-Reyssouze 01 +336 368 some_text Saint-Julien-sur-Veyle 01 +337 369 some_text Saint-Just 01 +338 370 some_text Saint-Laurent-sur-Saône 01 +339 371 some_text Saint-Marcel 01 +340 372 some_text Saint-Martin-de-Bavel 01 +341 373 some_text Saint-Martin-du-Frêne 01 +342 374 some_text Saint-Martin-du-Mont 01 +343 375 some_text Saint-Martin-le-Châtel 01 +344 376 some_text Saint-Maurice-de-Beynost 01 +345 378 some_text Saint-Maurice-de-Gourdans 01 +346 379 some_text Saint-Maurice-de-Rémens 01 +347 380 some_text Saint-Nizier-le-Bouchoux 01 +348 381 some_text Saint-Nizier-le-Désert 01 +349 382 some_text Sainte-Olive 01 +350 383 some_text Saint-Paul-de-Varax 01 +351 384 some_text Saint-Rambert-en-Bugey 01 +352 385 some_text Saint-Rémy 01 +353 386 some_text Saint-Sorlin-en-Bugey 01 +354 387 some_text Saint-Sulpice 01 +355 388 some_text Saint-Trivier-de-Courtes 01 +356 389 some_text Saint-Trivier-sur-Moignans 01 +357 390 some_text Saint-Vulbas 01 +358 391 some_text Salavre 01 +359 392 some_text Samognat 01 +360 393 some_text Sandrans 01 +361 396 some_text Sault-Brénaz 01 +362 397 some_text Sauverny 01 +363 398 some_text Savigneux 01 +364 399 some_text Ségny 01 +365 400 some_text Seillonnaz 01 +366 401 some_text Sergy 01 +367 402 some_text Sermoyer 01 +368 403 some_text Serrières-de-Briord 01 +369 404 some_text Serrières-sur-Ain 01 +370 405 some_text Servas 01 +371 406 some_text Servignat 01 +372 407 some_text Seyssel 01 +373 408 some_text Simandre-sur-Suran 01 +374 409 some_text Songieu 01 +375 410 some_text Sonthonnax-la-Montagne 01 +376 411 some_text Souclin 01 +377 412 some_text Sulignat 01 +378 413 some_text Surjoux 01 +379 414 some_text Sutrieu 01 +380 415 some_text Talissieu 01 +381 416 some_text Tenay 01 +382 417 some_text Thézillieu 01 +383 418 some_text Thil 01 +384 419 some_text Thoiry 01 +385 420 some_text Thoissey 01 +386 421 some_text Torcieu 01 +387 422 some_text Tossiat 01 +388 423 some_text Toussieux 01 +389 424 some_text Tramoyes 01 +390 425 some_text Tranclière 01 +391 426 some_text Treffort-Cuisiat 01 +392 427 some_text Trévoux 01 +393 428 some_text Valeins 01 +394 429 some_text Vandeins 01 +395 430 some_text Varambon 01 +396 431 some_text Vaux-en-Bugey 01 +397 432 some_text Verjon 01 +398 433 some_text Vernoux 01 +399 434 some_text Versailleux 01 +400 435 some_text Versonnex 01 +401 436 some_text Vesancy 01 +402 437 some_text Vescours 01 +403 439 some_text Vésines 01 +404 441 some_text Vieu-d'Izenave 01 +405 442 some_text Vieu 01 +406 443 some_text Villars-les-Dombes 01 +407 444 some_text Villebois 01 +408 445 some_text Villemotier 01 +409 446 some_text Villeneuve 01 +410 447 some_text Villereversure 01 +411 448 some_text Villes 01 +412 449 some_text Villette-sur-Ain 01 +413 450 some_text Villieu-Loyes-Mollon 01 +414 451 some_text Viriat 01 +415 452 some_text Virieu-le-Grand 01 +416 453 some_text Virieu-le-Petit 01 +417 454 some_text Virignin 01 +418 456 some_text Vongnes 01 +419 457 some_text Vonnas 01 +420 001 some_text Abbécourt 02 +421 002 some_text Achery 02 +422 003 some_text Acy 02 +423 004 some_text Agnicourt-et-Séchelles 02 +424 005 some_text Aguilcourt 02 +425 006 some_text Aisonville-et-Bernoville 02 +426 007 some_text Aizelles 02 +427 008 some_text Aizy-Jouy 02 +428 009 some_text Alaincourt 02 +429 010 some_text Allemant 02 +430 011 some_text Ambleny 02 +431 012 some_text Ambrief 02 +432 013 some_text Amifontaine 02 +433 014 some_text Amigny-Rouy 02 +434 015 some_text Ancienville 02 +435 016 some_text Andelain 02 +436 017 some_text Anguilcourt-le-Sart 02 +437 018 some_text Anizy-le-Château 02 +438 019 some_text Annois 02 +439 020 some_text Any-Martin-Rieux 02 +440 021 some_text Archon 02 +441 022 some_text Arcy-Sainte-Restitue 02 +442 023 some_text Armentières-sur-Ourcq 02 +443 024 some_text Arrancy 02 +444 025 some_text Artemps 02 +445 026 some_text Artonges 02 +446 027 some_text Assis-sur-Serre 02 +447 028 some_text Athies-sous-Laon 02 +448 029 some_text Attilly 02 +449 030 some_text Aubencheul-aux-Bois 02 +450 031 some_text Aubenton 02 +451 032 some_text Aubigny-aux-Kaisnes 02 +452 033 some_text Aubigny-en-Laonnois 02 +453 034 some_text Audignicourt 02 +454 035 some_text Audigny 02 +455 036 some_text Augy 02 +456 037 some_text Aulnois-sous-Laon 02 +457 038 some_text Autels 02 +458 039 some_text Autremencourt 02 +459 040 some_text Autreppes 02 +460 041 some_text Autreville 02 +461 042 some_text Azy-sur-Marne 02 +462 043 some_text Bagneux 02 +463 044 some_text Bancigny 02 +464 046 some_text Barenton-Bugny 02 +465 047 some_text Barenton-Cel 02 +466 048 some_text Barenton-sur-Serre 02 +467 049 some_text Barisis 02 +468 050 some_text Barzy-en-Thiérache 02 +469 051 some_text Barzy-sur-Marne 02 +470 052 some_text Bassoles-Aulers 02 +471 053 some_text Baulne-en-Brie 02 +472 054 some_text Bazoches-sur-Vesles 02 +473 055 some_text Beaumé 02 +474 056 some_text Beaumont-en-Beine 02 +475 057 some_text Beaurevoir 02 +476 058 some_text Beaurieux 02 +477 059 some_text Beautor 02 +478 060 some_text Beauvois-en-Vermandois 02 +479 061 some_text Becquigny 02 +480 062 some_text Belleau 02 +481 063 some_text Bellenglise 02 +482 064 some_text Belleu 02 +483 065 some_text Bellicourt 02 +484 066 some_text Benay 02 +485 067 some_text Bergues-sur-Sambre 02 +486 068 some_text Berlancourt 02 +487 069 some_text Berlise 02 +488 070 some_text Bernot 02 +489 071 some_text Berny-Rivière 02 +490 072 some_text Berrieux 02 +491 073 some_text Berry-au-Bac 02 +492 074 some_text Bertaucourt-Epourdon 02 +493 075 some_text Berthenicourt 02 +494 076 some_text Bertricourt 02 +495 077 some_text Berzy-le-Sec 02 +496 078 some_text Besmé 02 +497 079 some_text Besmont 02 +498 080 some_text Besny-et-Loizy 02 +499 081 some_text Béthancourt-en-Vaux 02 +500 082 some_text Beugneux 02 +501 083 some_text Beuvardes 02 +502 084 some_text Bézu-le-Guéry 02 +503 085 some_text Bézu-Saint-Germain 02 +504 086 some_text Bichancourt 02 +505 087 some_text Bieuxy 02 +506 088 some_text Bièvres 02 +507 089 some_text Billy-sur-Aisne 02 +508 090 some_text Billy-sur-Ourcq 02 +509 091 some_text Blanzy-lès-Fismes 02 +510 093 some_text Blérancourt 02 +511 094 some_text Blesmes 02 +512 095 some_text Bohain-en-Vermandois 02 +513 096 some_text Bois-lès-Pargny 02 +514 097 some_text Boncourt 02 +515 098 some_text Bonneil 02 +516 099 some_text Bonnesvalyn 02 +517 100 some_text Bony 02 +518 101 some_text Bosmont-sur-Serre 02 +519 102 some_text Bouconville-Vauclair 02 +520 103 some_text Boué 02 +521 104 some_text Bouffignereux 02 +522 105 some_text Bouresches 02 +523 106 some_text Bourg-et-Comin 02 +524 107 some_text Bourguignon-sous-Coucy 02 +525 108 some_text Bourguignon-sous-Montbavin 02 +526 109 some_text Bouteille 02 +527 110 some_text Braine 02 +528 111 some_text Brancourt-en-Laonnois 02 +529 112 some_text Brancourt-le-Grand 02 +530 114 some_text Brasles 02 +531 115 some_text Braye-en-Laonnois 02 +532 116 some_text Braye-en-Thiérache 02 +533 117 some_text Bray-Saint-Christophe 02 +534 118 some_text Braye 02 +535 119 some_text Brécy 02 +536 120 some_text Brenelle 02 +537 121 some_text Breny 02 +538 122 some_text Brie 02 +539 123 some_text Brissay-Choigny 02 +540 124 some_text Brissy-Hamégicourt 02 +541 125 some_text Brumetz 02 +542 126 some_text Brunehamel 02 +543 127 some_text Bruyères-sur-Fère 02 +544 128 some_text Bruyères-et-Montbérault 02 +545 129 some_text Bruys 02 +546 130 some_text Bucilly 02 +547 131 some_text Bucy-le-Long 02 +548 132 some_text Bucy-lès-Cerny 02 +549 133 some_text Bucy-lès-Pierrepont 02 +550 134 some_text Buire 02 +551 135 some_text Buironfosse 02 +552 136 some_text Burelles 02 +553 137 some_text Bussiares 02 +554 138 some_text Buzancy 02 +555 139 some_text Caillouël-Crépigny 02 +556 140 some_text Camelin 02 +557 141 some_text Capelle 02 +558 142 some_text Castres 02 +559 143 some_text Catelet 02 +560 144 some_text Caulaincourt 02 +561 145 some_text Caumont 02 +562 146 some_text Celles-lès-Condé 02 +563 147 some_text Celle-sous-Montmirail 02 +564 148 some_text Celles-sur-Aisne 02 +565 149 some_text Cerizy 02 +566 150 some_text Cerny-en-Laonnois 02 +567 151 some_text Cerny-lès-Bucy 02 +568 152 some_text Cerseuil 02 +569 153 some_text Cessières 02 +570 154 some_text Chacrise 02 +571 155 some_text Chaillevois 02 +572 156 some_text Chalandry 02 +573 157 some_text Chambry 02 +574 158 some_text Chamouille 02 +575 159 some_text Champs 02 +576 160 some_text Chaourse 02 +577 161 some_text Chapelle-Monthodon 02 +578 162 some_text Chapelle-sur-Chézy 02 +579 163 some_text Charly 02 +580 164 some_text Charmel 02 +581 165 some_text Charmes 02 +582 166 some_text Chartèves 02 +583 167 some_text Chassemy 02 +584 168 some_text Château-Thierry 02 +585 169 some_text Châtillon-lès-Sons 02 +586 170 some_text Châtillon-sur-Oise 02 +587 171 some_text Chaudardes 02 +588 172 some_text Chaudun 02 +589 173 some_text Chauny 02 +590 174 some_text Chavignon 02 +591 175 some_text Chavigny 02 +592 176 some_text Chavonne 02 +593 177 some_text Chérêt 02 +594 178 some_text Chermizy-Ailles 02 +595 179 some_text Chéry-Chartreuve 02 +596 180 some_text Chéry-lès-Pouilly 02 +597 181 some_text Chéry-lès-Rozoy 02 +598 182 some_text Chevennes 02 +599 183 some_text Chevregny 02 +600 184 some_text Chevresis-Monceau 02 +601 185 some_text Chézy-en-Orxois 02 +602 186 some_text Chézy-sur-Marne 02 +603 187 some_text Chierry 02 +604 188 some_text Chigny 02 +605 189 some_text Chivres-en-Laonnois 02 +606 190 some_text Chivres-Val 02 +607 191 some_text Chivy-lès-Étouvelles 02 +608 192 some_text Chouy 02 +609 193 some_text Cierges 02 +610 194 some_text Cilly 02 +611 195 some_text Ciry-Salsogne 02 +612 196 some_text Clacy-et-Thierret 02 +613 197 some_text Clairfontaine 02 +614 198 some_text Clamecy 02 +615 199 some_text Clastres 02 +616 200 some_text Clermont-les-Fermes 02 +617 201 some_text Cœuvres-et-Valsery 02 +618 203 some_text Coincy 02 +619 204 some_text Coingt 02 +620 205 some_text Colligis-Crandelain 02 +621 206 some_text Colonfay 02 +622 207 some_text Commenchon 02 +623 208 some_text Concevreux 02 +624 209 some_text Condé-en-Brie 02 +625 210 some_text Condé-sur-Aisne 02 +626 211 some_text Condé-sur-Suippe 02 +627 212 some_text Condren 02 +628 213 some_text Connigis 02 +629 214 some_text Contescourt 02 +630 215 some_text Corbeny 02 +631 216 some_text Corcy 02 +632 217 some_text Coucy-le-Château-Auffrique 02 +633 218 some_text Coucy-lès-Eppes 02 +634 219 some_text Coucy-la-Ville 02 +635 220 some_text Coulonges-Cohan 02 +636 221 some_text Coupru 02 +637 222 some_text Courbes 02 +638 223 some_text Courboin 02 +639 224 some_text Courcelles-sur-Vesles 02 +640 225 some_text Courchamps 02 +641 226 some_text Courmelles 02 +642 227 some_text Courmont 02 +643 228 some_text Courtemont-Varennes 02 +644 229 some_text Courtrizy-et-Fussigny 02 +645 230 some_text Couvrelles 02 +646 231 some_text Couvron-et-Aumencourt 02 +647 232 some_text Coyolles 02 +648 233 some_text Cramaille 02 +649 234 some_text Craonne 02 +650 235 some_text Craonnelle 02 +651 236 some_text Crécy-au-Mont 02 +652 237 some_text Crécy-sur-Serre 02 +653 238 some_text Crépy 02 +654 239 some_text Crézancy 02 +655 240 some_text Croix-Fonsommes 02 +656 241 some_text Croix-sur-Ourcq 02 +657 242 some_text Crouttes-sur-Marne 02 +658 243 some_text Crouy 02 +659 244 some_text Crupilly 02 +660 245 some_text Cuffies 02 +661 246 some_text Cugny 02 +662 248 some_text Cuirieux 02 +663 249 some_text Cuiry-Housse 02 +664 250 some_text Cuiry-lès-Chaudardes 02 +665 251 some_text Cuiry-lès-Iviers 02 +666 252 some_text Cuissy-et-Geny 02 +667 253 some_text Cuisy-en-Almont 02 +668 254 some_text Cutry 02 +669 255 some_text Cys-la-Commune 02 +670 256 some_text Dagny-Lambercy 02 +671 257 some_text Dallon 02 +672 258 some_text Dammard 02 +673 259 some_text Dampleux 02 +674 260 some_text Danizy 02 +675 261 some_text Dercy 02 +676 262 some_text Deuillet 02 +677 263 some_text Dhuizel 02 +678 264 some_text Dizy-le-Gros 02 +679 265 some_text Dohis 02 +680 266 some_text Dolignon 02 +681 267 some_text Dommiers 02 +682 268 some_text Domptin 02 +683 269 some_text Dorengt 02 +684 270 some_text Douchy 02 +685 271 some_text Dravegny 02 +686 272 some_text Droizy 02 +687 273 some_text Dury 02 +688 274 some_text Ébouleau 02 +689 275 some_text Effry 02 +690 276 some_text Englancourt 02 +691 277 some_text Épagny 02 +692 278 some_text Éparcy 02 +693 279 some_text Épaux-Bézu 02 +694 280 some_text Épieds 02 +695 281 some_text Épine-aux-Bois 02 +696 282 some_text Eppes 02 +697 283 some_text Erlon 02 +698 284 some_text Erloy 02 +699 286 some_text Esquéhéries 02 +700 287 some_text Essigny-le-Grand 02 +701 288 some_text Essigny-le-Petit 02 +702 289 some_text Essises 02 +703 290 some_text Essômes-sur-Marne 02 +704 291 some_text Estrées 02 +705 292 some_text Étampes-sur-Marne 02 +706 293 some_text Étaves-et-Bocquiaux 02 +707 294 some_text Étouvelles 02 +708 295 some_text Étréaupont 02 +709 296 some_text Étreillers 02 +710 297 some_text Étrépilly 02 +711 298 some_text Étreux 02 +712 299 some_text Évergnicourt 02 +713 301 some_text Faucoucourt 02 +714 302 some_text Faverolles 02 +715 303 some_text Fayet 02 +716 304 some_text Fère 02 +717 305 some_text Fère-en-Tardenois 02 +718 306 some_text Ferté-Chevresis 02 +719 307 some_text Ferté-Milon 02 +720 308 some_text Fesmy-le-Sart 02 +721 309 some_text Festieux 02 +722 310 some_text Fieulaine 02 +723 311 some_text Filain 02 +724 312 some_text Flamengrie 02 +725 313 some_text Flavigny-le-Grand-et-Beaurain 02 +726 315 some_text Flavy-le-Martel 02 +727 316 some_text Fleury 02 +728 317 some_text Fluquières 02 +729 318 some_text Folembray 02 +730 319 some_text Fonsommes 02 +731 320 some_text Fontaine-lès-Clercs 02 +732 321 some_text Fontaine-lès-Vervins 02 +733 322 some_text Fontaine-Notre-Dame 02 +734 323 some_text Fontaine-Uterte 02 +735 324 some_text Fontenelle 02 +736 325 some_text Fontenelle-en-Brie 02 +737 326 some_text Fontenoy 02 +738 327 some_text Foreste 02 +739 328 some_text Fossoy 02 +740 329 some_text Fourdrain 02 +741 330 some_text Francilly-Selency 02 +742 331 some_text Franqueville 02 +743 332 some_text Fresnes-en-Tardenois 02 +744 333 some_text Fresnes 02 +745 334 some_text Fresnoy-le-Grand 02 +746 335 some_text Fressancourt 02 +747 336 some_text Frières-Faillouël 02 +748 337 some_text Froidestrées 02 +749 338 some_text Froidmont-Cohartille 02 +750 339 some_text Gandelu 02 +751 340 some_text Gauchy 02 +752 341 some_text Gercy 02 +753 342 some_text Gergny 02 +754 343 some_text Germaine 02 +755 344 some_text Gernicourt 02 +756 345 some_text Gibercourt 02 +757 346 some_text Gizy 02 +758 347 some_text Gland 02 +759 348 some_text Glennes 02 +760 349 some_text Goudelancourt-lès-Berrieux 02 +761 350 some_text Goudelancourt-lès-Pierrepont 02 +762 351 some_text Goussancourt 02 +763 352 some_text Gouy 02 +764 353 some_text Grandlup-et-Fay 02 +765 354 some_text Grandrieux 02 +766 355 some_text Gricourt 02 +767 356 some_text Grisolles 02 +768 357 some_text Gronard 02 +769 358 some_text Grougis 02 +770 359 some_text Grugies 02 +771 360 some_text Guignicourt 02 +772 361 some_text Guise 02 +773 362 some_text Guivry 02 +774 363 some_text Guny 02 +775 364 some_text Guyencourt 02 +776 366 some_text Hannapes 02 +777 367 some_text Happencourt 02 +778 368 some_text Haramont 02 +779 369 some_text Harcigny 02 +780 370 some_text Hargicourt 02 +781 371 some_text Harly 02 +782 372 some_text Hartennes-et-Taux 02 +783 373 some_text Hary 02 +784 374 some_text Lehaucourt 02 +785 375 some_text Hautevesnes 02 +786 376 some_text Hauteville 02 +787 377 some_text Haution 02 +788 378 some_text Hérie 02 +789 379 some_text Hérie-la-Viéville 02 +790 380 some_text Hinacourt 02 +791 381 some_text Hirson 02 +792 382 some_text Holnon 02 +793 383 some_text Homblières 02 +794 384 some_text Houry 02 +795 385 some_text Housset 02 +796 386 some_text Iron 02 +797 387 some_text Itancourt 02 +798 388 some_text Iviers 02 +799 389 some_text Jaulgonne 02 +800 390 some_text Jeancourt 02 +801 391 some_text Jeantes 02 +802 392 some_text Joncourt 02 +803 393 some_text Jouaignes 02 +804 395 some_text Jumencourt 02 +805 396 some_text Jumigny 02 +806 397 some_text Jussy 02 +807 398 some_text Juvigny 02 +808 399 some_text Juvincourt-et-Damary 02 +809 400 some_text Laffaux 02 +810 401 some_text Laigny 02 +811 402 some_text Lanchy 02 +812 403 some_text Landifay-et-Bertaignemont 02 +813 404 some_text Landouzy-la-Cour 02 +814 405 some_text Landouzy-la-Ville 02 +815 406 some_text Landricourt 02 +816 407 some_text Laniscourt 02 +817 408 some_text Laon 02 +818 409 some_text Lappion 02 +819 410 some_text Largny-sur-Automne 02 +820 411 some_text Latilly 02 +821 412 some_text Launoy 02 +822 413 some_text Laval-en-Laonnois 02 +823 414 some_text Lavaqueresse 02 +824 415 some_text Laversine 02 +825 416 some_text Lemé 02 +826 417 some_text Lempire 02 +827 418 some_text Lerzy 02 +828 419 some_text Leschelles 02 +829 420 some_text Lesdins 02 +830 421 some_text Lesges 02 +831 422 some_text Lesquielles-Saint-Germain 02 +832 423 some_text Leuilly-sous-Coucy 02 +833 424 some_text Leury 02 +834 425 some_text Leuze 02 +835 426 some_text Levergies 02 +836 427 some_text Lhuys 02 +837 428 some_text Licy-Clignon 02 +838 429 some_text Lierval 02 +839 430 some_text Liesse-Notre-Dame 02 +840 431 some_text Liez 02 +841 432 some_text Limé 02 +842 433 some_text Lislet 02 +843 434 some_text Lizy 02 +844 435 some_text Logny-lès-Aubenton 02 +845 438 some_text Longpont 02 +846 439 some_text Longueval-Barbonval 02 +847 440 some_text Lor 02 +848 441 some_text Louâtre 02 +849 442 some_text Loupeigne 02 +850 443 some_text Lucy-le-Bocage 02 +851 444 some_text Lugny 02 +852 445 some_text Luzoir 02 +853 446 some_text Ly-Fontaine 02 +854 447 some_text Maast-et-Violaine 02 +855 448 some_text Mâchecourt 02 +856 449 some_text Macogny 02 +857 450 some_text Macquigny 02 +858 451 some_text Magny-la-Fosse 02 +859 452 some_text Maissemy 02 +860 453 some_text Maizy 02 +861 454 some_text Malmaison 02 +862 455 some_text Malzy 02 +863 456 some_text Manicamp 02 +864 457 some_text Marchais 02 +865 458 some_text Marchais-en-Brie 02 +866 459 some_text Marcy 02 +867 460 some_text Marcy-sous-Marle 02 +868 461 some_text Marest-Dampcourt 02 +869 462 some_text Mareuil-en-Dôle 02 +870 463 some_text Marfontaine 02 +871 464 some_text Margival 02 +872 465 some_text Marigny-en-Orxois 02 +873 466 some_text Marizy-Sainte-Geneviève 02 +874 467 some_text Marizy-Saint-Mard 02 +875 468 some_text Marle 02 +876 469 some_text Marly-Gomont 02 +877 470 some_text Martigny 02 +878 471 some_text Martigny-Courpierre 02 +879 472 some_text Mauregny-en-Haye 02 +880 473 some_text Mayot 02 +881 474 some_text Mennessis 02 +882 475 some_text Menneville 02 +883 476 some_text Mennevret 02 +884 477 some_text Mercin-et-Vaux 02 +885 478 some_text Merlieux-et-Fouquerolles 02 +886 479 some_text Merval 02 +887 480 some_text Mesbrecourt-Richecourt 02 +888 481 some_text Mesnil-Saint-Laurent 02 +889 482 some_text Meurival 02 +890 483 some_text Mézières-sur-Oise 02 +891 484 some_text Mézy-Moulins 02 +892 485 some_text Missy-aux-Bois 02 +893 486 some_text Missy-lès-Pierrepont 02 +894 487 some_text Missy-sur-Aisne 02 +895 488 some_text Molain 02 +896 489 some_text Molinchart 02 +897 490 some_text Monampteuil 02 +898 491 some_text Monceau-le-Neuf-et-Faucouzy 02 +899 492 some_text Monceau-lès-Leups 02 +900 493 some_text Monceau-le-Waast 02 +901 494 some_text Monceau-sur-Oise 02 +902 495 some_text Mondrepuis 02 +903 496 some_text Monnes 02 +904 497 some_text Mons-en-Laonnois 02 +905 498 some_text Montaigu 02 +906 499 some_text Montbavin 02 +907 500 some_text Montbrehain 02 +908 501 some_text Montchâlons 02 +909 502 some_text Montcornet 02 +910 503 some_text Mont-d'Origny 02 +911 504 some_text Montescourt-Lizerolles 02 +912 505 some_text Montfaucon 02 +913 506 some_text Montgobert 02 +914 507 some_text Montgru-Saint-Hilaire 02 +915 508 some_text Monthenault 02 +916 509 some_text Monthiers 02 +917 510 some_text Monthurel 02 +918 511 some_text Montigny-en-Arrouaise 02 +919 512 some_text Montigny-l'Allier 02 +920 513 some_text Montigny-le-Franc 02 +921 514 some_text Montigny-Lengrain 02 +922 515 some_text Montigny-lès-Condé 02 +923 516 some_text Montigny-sous-Marle 02 +924 517 some_text Montigny-sur-Crécy 02 +925 518 some_text Montlevon 02 +926 519 some_text Montloué 02 +927 520 some_text Mont-Notre-Dame 02 +928 521 some_text Montreuil-aux-Lions 02 +929 522 some_text Mont-Saint-Jean 02 +930 523 some_text Mont-Saint-Martin 02 +931 524 some_text Mont-Saint-Père 02 +932 525 some_text Morcourt 02 +933 526 some_text Morgny-en-Thiérache 02 +934 527 some_text Morsain 02 +935 528 some_text Mortefontaine 02 +936 529 some_text Mortiers 02 +937 530 some_text Moulins 02 +938 531 some_text Moussy-Verneuil 02 +939 532 some_text Moÿ-de-l'Aisne 02 +940 533 some_text Muret-et-Crouttes 02 +941 534 some_text Muscourt 02 +942 535 some_text Nampcelles-la-Cour 02 +943 536 some_text Nampteuil-sous-Muret 02 +944 537 some_text Nanteuil-la-Fosse 02 +945 538 some_text Nanteuil-Notre-Dame 02 +946 539 some_text Nauroy 02 +947 540 some_text Nesles-la-Montagne 02 +948 541 some_text Neufchâtel-sur-Aisne 02 +949 542 some_text Neuflieux 02 +950 543 some_text Neuilly-Saint-Front 02 +951 544 some_text Neuve-Maison 02 +952 545 some_text Neuville-Bosmont 02 +953 546 some_text Neuville-en-Beine 02 +954 547 some_text Neuville-Housset 02 +955 548 some_text Neuville-lès-Dorengt 02 +956 549 some_text Neuville-Saint-Amand 02 +957 550 some_text Neuville-sur-Ailette 02 +958 551 some_text Neuville-sur-Margival 02 +959 552 some_text Neuvillette 02 +960 553 some_text Nizy-le-Comte 02 +961 554 some_text Nogentel 02 +962 555 some_text Nogent-l'Artaud 02 +963 556 some_text Noircourt 02 +964 557 some_text Noroy-sur-Ourcq 02 +965 558 some_text Nouvion-en-Thiérache 02 +966 559 some_text Nouvion-et-Catillon 02 +967 560 some_text Nouvion-le-Comte 02 +968 561 some_text Nouvion-le-Vineux 02 +969 562 some_text Nouvron-Vingré 02 +970 563 some_text Noyales 02 +971 564 some_text Noyant-et-Aconin 02 +972 565 some_text Œuilly 02 +973 566 some_text Ognes 02 +974 567 some_text Ohis 02 +975 568 some_text Oigny-en-Valois 02 +976 569 some_text Oisy 02 +977 570 some_text Ollezy 02 +978 571 some_text Omissy 02 +979 572 some_text Orainville 02 +980 573 some_text Orgeval 02 +981 574 some_text Origny-en-Thiérache 02 +982 575 some_text Origny-Sainte-Benoite 02 +983 576 some_text Osly-Courtil 02 +984 577 some_text Ostel 02 +985 578 some_text Oulches-la-Vallée-Foulon 02 +986 579 some_text Oulchy-la-Ville 02 +987 580 some_text Oulchy-le-Château 02 +988 581 some_text Paars 02 +989 582 some_text Paissy 02 +990 583 some_text Pancy-Courtecon 02 +991 584 some_text Papleux 02 +992 585 some_text Parcy-et-Tigny 02 +993 586 some_text Parfondeval 02 +994 587 some_text Parfondru 02 +995 588 some_text Pargnan 02 +996 589 some_text Pargny-Filain 02 +997 590 some_text Pargny-la-Dhuys 02 +998 591 some_text Pargny-les-Bois 02 +999 592 some_text Parpeville 02 +1000 593 some_text Pasly 02 +1001 594 some_text Passy-en-Valois 02 +1002 595 some_text Passy-sur-Marne 02 +1003 596 some_text Pavant 02 +1004 597 some_text Perles 02 +1005 598 some_text Pernant 02 +1006 599 some_text Pierremande 02 +1007 600 some_text Pierrepont 02 +1008 601 some_text Pignicourt 02 +1009 602 some_text Pinon 02 +1010 604 some_text Pithon 02 +1011 605 some_text Pleine-Selve 02 +1012 606 some_text Plessier-Huleu 02 +1013 607 some_text Ploisy 02 +1014 608 some_text Plomion 02 +1015 609 some_text Ployart-et-Vaurseine 02 +1016 610 some_text Pommiers 02 +1017 612 some_text Pont-Arcy 02 +1018 613 some_text Pontavert 02 +1019 614 some_text Pontru 02 +1020 615 some_text Pontruet 02 +1021 616 some_text Pont-Saint-Mard 02 +1022 617 some_text Pouilly-sur-Serre 02 +1023 618 some_text Prémont 02 +1024 619 some_text Prémontré 02 +1025 620 some_text Presles-et-Boves 02 +1026 621 some_text Presles-et-Thierny 02 +1027 622 some_text Priez 02 +1028 623 some_text Prisces 02 +1029 624 some_text Proisy 02 +1030 625 some_text Proix 02 +1031 626 some_text Prouvais 02 +1032 627 some_text Proviseux-et-Plesnoy 02 +1033 628 some_text Puiseux-en-Retz 02 +1034 629 some_text Puisieux-et-Clanlieu 02 +1035 631 some_text Quierzy 02 +1036 632 some_text Quincy-Basse 02 +1037 633 some_text Quincy-sous-le-Mont 02 +1038 634 some_text Raillimont 02 +1039 635 some_text Ramicourt 02 +1040 636 some_text Regny 02 +1041 637 some_text Remaucourt 02 +1042 638 some_text Remies 02 +1043 639 some_text Remigny 02 +1044 640 some_text Renansart 02 +1045 641 some_text Renneval 02 +1046 642 some_text Résigny 02 +1047 643 some_text Ressons-le-Long 02 +1048 644 some_text Retheuil 02 +1049 645 some_text Reuilly-Sauvigny 02 +1050 646 some_text Révillon 02 +1051 647 some_text Ribeauville 02 +1052 648 some_text Ribemont 02 +1053 649 some_text Rocourt-Saint-Martin 02 +1054 650 some_text Rocquigny 02 +1055 651 some_text Rogécourt 02 +1056 652 some_text Rogny 02 +1057 653 some_text Romeny-sur-Marne 02 +1058 654 some_text Romery 02 +1059 655 some_text Ronchères 02 +1060 656 some_text Roucy 02 +1061 657 some_text Rougeries 02 +1062 658 some_text Roupy 02 +1063 659 some_text Rouvroy 02 +1064 660 some_text Rouvroy-sur-Serre 02 +1065 661 some_text Royaucourt-et-Chailvet 02 +1066 662 some_text Rozet-Saint-Albin 02 +1067 663 some_text Rozières-sur-Crise 02 +1068 664 some_text Rozoy-Bellevalle 02 +1069 665 some_text Grand-Rozoy 02 +1070 666 some_text Rozoy-sur-Serre 02 +1071 667 some_text Saconin-et-Breuil 02 +1072 668 some_text Sains-Richaumont 02 +1073 669 some_text Saint-Agnan 02 +1074 670 some_text Saint-Algis 02 +1075 671 some_text Saint-Aubin 02 +1076 672 some_text Saint-Bandry 02 +1077 673 some_text Saint-Christophe-à-Berry 02 +1078 674 some_text Saint-Clément 02 +1079 675 some_text Sainte-Croix 02 +1080 676 some_text Saint-Erme-Outre-et-Ramecourt 02 +1081 677 some_text Saint-Eugène 02 +1082 678 some_text Sainte-Geneviève 02 +1083 679 some_text Saint-Gengoulph 02 +1084 680 some_text Saint-Gobain 02 +1085 681 some_text Saint-Gobert 02 +1086 682 some_text Saint-Mard 02 +1087 683 some_text Saint-Martin-Rivière 02 +1088 684 some_text Saint-Michel 02 +1089 685 some_text Saint-Nicolas-aux-Bois 02 +1090 686 some_text Saint-Paul-aux-Bois 02 +1091 687 some_text Saint-Pierre-Aigle 02 +1092 688 some_text Saint-Pierre-lès-Franqueville 02 +1093 689 some_text Saint-Pierremont 02 +1094 690 some_text Sainte-Preuve 02 +1095 691 some_text Saint-Quentin 02 +1096 693 some_text Saint-Rémy-Blanzy 02 +1097 694 some_text Saint-Simon 02 +1098 695 some_text Saint-Thibaut 02 +1099 696 some_text Saint-Thomas 02 +1100 697 some_text Samoussy 02 +1101 698 some_text Sancy-les-Cheminots 02 +1102 699 some_text Saponay 02 +1103 701 some_text Saulchery 02 +1104 702 some_text Savy 02 +1105 703 some_text Seboncourt 02 +1106 704 some_text Selens 02 +1107 705 some_text Selve 02 +1108 706 some_text Septmonts 02 +1109 707 some_text Septvaux 02 +1110 708 some_text Sequehart 02 +1111 709 some_text Serain 02 +1112 710 some_text Seraucourt-le-Grand 02 +1113 711 some_text Serches 02 +1114 712 some_text Sergy 02 +1115 713 some_text Seringes-et-Nesles 02 +1116 714 some_text Sermoise 02 +1117 715 some_text Serval 02 +1118 716 some_text Servais 02 +1119 717 some_text Séry-lès-Mézières 02 +1120 718 some_text Silly-la-Poterie 02 +1121 719 some_text Sinceny 02 +1122 720 some_text Sissonne 02 +1123 721 some_text Sissy 02 +1124 722 some_text Soissons 02 +1125 723 some_text Soize 02 +1126 724 some_text Sommelans 02 +1127 725 some_text Sommeron 02 +1128 726 some_text Sommette-Eaucourt 02 +1129 727 some_text Sons-et-Ronchères 02 +1130 728 some_text Sorbais 02 +1131 729 some_text Soucy 02 +1132 730 some_text Soupir 02 +1133 731 some_text Sourd 02 +1134 732 some_text Surfontaine 02 +1135 733 some_text Suzy 02 +1136 734 some_text Taillefontaine 02 +1137 735 some_text Tannières 02 +1138 736 some_text Tartiers 02 +1139 737 some_text Tavaux-et-Pontséricourt 02 +1140 738 some_text Tergnier 02 +1141 739 some_text Terny-Sorny 02 +1142 740 some_text Thenailles 02 +1143 741 some_text Thenelles 02 +1144 742 some_text Thiernu 02 +1145 743 some_text Thuel 02 +1146 744 some_text Torcy-en-Valois 02 +1147 745 some_text Toulis-et-Attencourt 02 +1148 746 some_text Travecy 02 +1149 747 some_text Trefcon 02 +1150 748 some_text Trélou-sur-Marne 02 +1151 749 some_text Troësnes 02 +1152 750 some_text Trosly-Loire 02 +1153 751 some_text Trucy 02 +1154 752 some_text Tugny-et-Pont 02 +1155 753 some_text Tupigny 02 +1156 754 some_text Ugny-le-Gay 02 +1157 755 some_text Urcel 02 +1158 756 some_text Urvillers 02 +1159 757 some_text Vadencourt 02 +1160 758 some_text Vailly-sur-Aisne 02 +1161 759 some_text Vallée-au-Blé 02 +1162 760 some_text Vallée-Mulâtre 02 +1163 761 some_text Variscourt 02 +1164 762 some_text Vassens 02 +1165 763 some_text Vasseny 02 +1166 764 some_text Vassogne 02 +1167 765 some_text Vaucelles-et-Beffecourt 02 +1168 766 some_text Vaudesson 02 +1169 767 some_text Vauxrezis 02 +1170 768 some_text Vauxaillon 02 +1171 769 some_text Vaux-Andigny 02 +1172 770 some_text Vauxbuin 02 +1173 771 some_text Vauxcéré 02 +1174 772 some_text Vaux-en-Vermandois 02 +1175 773 some_text Vauxtin 02 +1176 774 some_text Vendelles 02 +1177 775 some_text Vendeuil 02 +1178 776 some_text Vendhuile 02 +1179 777 some_text Vendières 02 +1180 778 some_text Vendresse-Beaulne 02 +1181 779 some_text Vénérolles 02 +1182 780 some_text Venizel 02 +1183 781 some_text Verdilly 02 +1184 782 some_text Verguier 02 +1185 783 some_text Grand-Verly 02 +1186 784 some_text Petit-Verly 02 +1187 785 some_text Vermand 02 +1188 786 some_text Verneuil-sous-Coucy 02 +1189 787 some_text Verneuil-sur-Serre 02 +1190 788 some_text Versigny 02 +1191 789 some_text Vervins 02 +1192 790 some_text Vesles-et-Caumont 02 +1193 791 some_text Veslud 02 +1194 792 some_text Veuilly-la-Poterie 02 +1195 793 some_text Vézaponin 02 +1196 794 some_text Vézilly 02 +1197 795 some_text Vic-sur-Aisne 02 +1198 796 some_text Vichel-Nanteuil 02 +1199 797 some_text Viel-Arcy 02 +1200 798 some_text Viels-Maisons 02 +1201 799 some_text Vierzy 02 +1202 800 some_text Viffort 02 +1203 801 some_text Vigneux-Hocquet 02 +1204 802 some_text Ville-aux-Bois-lès-Dizy 02 +1205 803 some_text Ville-aux-Bois-lès-Pontavert 02 +1206 804 some_text Villemontoire 02 +1207 805 some_text Villeneuve-Saint-Germain 02 +1208 806 some_text Villeneuve-sur-Fère 02 +1209 807 some_text Villequier-Aumont 02 +1210 808 some_text Villeret 02 +1211 809 some_text Villers-Agron-Aiguizy 02 +1212 810 some_text Villers-Cotterêts 02 +1213 811 some_text Villers-en-Prayères 02 +1214 812 some_text Villers-Hélon 02 +1215 813 some_text Villers-le-Sec 02 +1216 814 some_text Villers-lès-Guise 02 +1217 815 some_text Villers-Saint-Christophe 02 +1218 816 some_text Villers-sur-Fère 02 +1219 817 some_text Ville-Savoye 02 +1220 818 some_text Villiers-Saint-Denis 02 +1221 819 some_text Vincy-Reuil-et-Magny 02 +1222 820 some_text Viry-Noureuil 02 +1223 821 some_text Vivaise 02 +1224 822 some_text Vivières 02 +1225 823 some_text Voharies 02 +1226 824 some_text Vorges 02 +1227 826 some_text Voulpaix 02 +1228 827 some_text Voyenne 02 +1229 828 some_text Vregny 02 +1230 829 some_text Vuillery 02 +1231 830 some_text Wassigny 02 +1232 831 some_text Watigny 02 +1233 832 some_text Wiège-Faty 02 +1234 833 some_text Wimy 02 +1235 834 some_text Wissignicourt 02 +1236 001 some_text Abrest 03 +1237 002 some_text Agonges 03 +1238 003 some_text Ainay-le-Château 03 +1239 004 some_text Andelaroche 03 +1240 005 some_text Archignat 03 +1241 006 some_text Arfeuilles 03 +1242 007 some_text Arpheuilles-Saint-Priest 03 +1243 008 some_text Arronnes 03 +1244 009 some_text Aubigny 03 +1245 010 some_text Audes 03 +1246 011 some_text Aurouër 03 +1247 012 some_text Autry-Issards 03 +1248 013 some_text Avermes 03 +1249 014 some_text Avrilly 03 +1250 015 some_text Bagneux 03 +1251 016 some_text Barberier 03 +1252 017 some_text Barrais-Bussolles 03 +1253 018 some_text Bayet 03 +1254 019 some_text Beaulon 03 +1255 020 some_text Beaune-d'Allier 03 +1256 021 some_text Bègues 03 +1257 022 some_text Bellenaves 03 +1258 023 some_text Bellerive-sur-Allier 03 +1259 024 some_text Bert 03 +1260 025 some_text Bessay-sur-Allier 03 +1261 026 some_text Besson 03 +1262 027 some_text Bézenet 03 +1263 028 some_text Billezois 03 +1264 029 some_text Billy 03 +1265 030 some_text Biozat 03 +1266 031 some_text Bizeneuille 03 +1267 032 some_text Blomard 03 +1268 033 some_text Bost 03 +1269 034 some_text Boucé 03 +1270 035 some_text Bouchaud 03 +1271 036 some_text Bourbon-l'Archambault 03 +1272 037 some_text Braize 03 +1273 038 some_text Bransat 03 +1274 039 some_text Bresnay 03 +1275 040 some_text Bressolles 03 +1276 041 some_text Brethon 03 +1277 042 some_text Breuil 03 +1278 043 some_text Broût-Vernet 03 +1279 044 some_text Brugheas 03 +1280 045 some_text Busset 03 +1281 046 some_text Buxières-les-Mines 03 +1282 047 some_text Celle 03 +1283 048 some_text Cérilly 03 +1284 049 some_text Cesset 03 +1285 050 some_text Chabanne 03 +1286 051 some_text Chambérat 03 +1287 052 some_text Chamblet 03 +1288 053 some_text Chantelle 03 +1289 054 some_text Chapeau 03 +1290 055 some_text Chapelaude 03 +1291 056 some_text Chapelle 03 +1292 057 some_text Chapelle-aux-Chasses 03 +1293 058 some_text Chappes 03 +1294 059 some_text Chareil-Cintrat 03 +1295 060 some_text Charmeil 03 +1296 061 some_text Charmes 03 +1297 062 some_text Charroux 03 +1298 063 some_text Chassenard 03 +1299 064 some_text Château-sur-Allier 03 +1300 065 some_text Châtel-de-Neuvre 03 +1301 066 some_text Châtel-Montagne 03 +1302 067 some_text Châtelperron 03 +1303 068 some_text Châtelus 03 +1304 069 some_text Châtillon 03 +1305 070 some_text Chavenon 03 +1306 071 some_text Chavroches 03 +1307 072 some_text Chazemais 03 +1308 073 some_text Chemilly 03 +1309 074 some_text Chevagnes 03 +1310 075 some_text Chezelle 03 +1311 076 some_text Chézy 03 +1312 077 some_text Chirat-l'Église 03 +1313 078 some_text Chouvigny 03 +\. + +SELECT count(*) FROM towns; +SELECT * FROM towns where id in (13, 666); +-- ON REPLICA +-- +-- select count(*) from towns; +-- count +-- ------- +-- 1313 +-- (1 row) +-- +-- select * from towns where id in (13, 666); +-- id | code | article | name | department +-- -----+------+-----------+----------------+------------ +-- 13 | 014 | some_text | Arbent | 01 +-- 666 | 252 | some_text | Cuissy-et-Geny | 02 +-- (2 rows) +-- + +DROP TABLE towns; +DROP EXTENSION pg_tde; diff --git a/src/access/pg_tde_tdemap.c b/src/access/pg_tde_tdemap.c index 06fccc58..250cc952 100644 --- a/src/access/pg_tde_tdemap.c +++ b/src/access/pg_tde_tdemap.c @@ -16,6 +16,9 @@ #include "storage/fd.h" #include "utils/wait_event.h" #include "utils/memutils.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xloginsert.h" #include "access/pg_tde_tdemap.h" #include "encryption/enc_aes.h" @@ -26,8 +29,13 @@ #include "pg_tde_defines.h" +/* TODO: should be a user defined */ +static const char *MasterKeyName = "master-key"; + static inline char* pg_tde_get_key_file_path(const RelFileLocator *newrlocator); static void put_keys_into_map(Oid rel_id, RelKeysData *keys); +static void pg_tde_write_key_fork(RelFileLocator *rlocator, InternalKey *key, const char *MasterKeyName); +static void pg_tde_xlog_create_fork(XLogReaderState *record); void pg_tde_delete_key_fork(Relation rel) @@ -42,6 +50,7 @@ pg_tde_delete_key_fork(Relation rel) RegisterFileForDeletion(key_file_path, true); pfree(key_file_path); } + /* * Creates a relation fork file relfilenode.tde that contains the * encryption key for the relation. @@ -49,12 +58,36 @@ pg_tde_delete_key_fork(Relation rel) void pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) { - /* TODO: should be a user defined */ - static const char *MasterKeyName = "master-key"; + InternalKey int_key = {0}; + + if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) + { + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("could not generate internal key for relation \"%s\": %s", + RelationGetRelationName(rel), ERR_error_string(ERR_get_error(), NULL)))); + } + + + /* XLOG internal keys */ + XLogBeginInsert(); + XLogRegisterData((char *) newrlocator, sizeof(RelFileLocator)); + XLogRegisterData((char *) &int_key, sizeof(InternalKey)); + XLogInsert(RM_TDERMGR_ID, XLOG_TDE_CREATE_FORK); + + /* TODO: should DB crash after sending XLog, secondaries would create a fork + * file but the relation won't be created either on primary or secondaries. + * Hence, the *.tde file will remain as garbage on secondaries. + */ + pg_tde_write_key_fork(newrlocator, &int_key, MasterKeyName); +} + +void +pg_tde_write_key_fork(RelFileLocator *rlocator, InternalKey *key, const char *MasterKeyName) +{ char *key_file_path; File file = -1; - InternalKey int_key = {0}; RelKeysData *data; unsigned char dataEnc[1024]; RelKeysData *encData = dataEnc; @@ -75,51 +108,28 @@ pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) (errmsg("failed to retrieve master key"))); } - key_file_path = pg_tde_get_key_file_path(newrlocator); + key_file_path = pg_tde_get_key_file_path(rlocator); if (!key_file_path) - { ereport(ERROR, (errmsg("failed to get key file path"))); - } file = PathNameOpenFile(key_file_path, O_RDWR | O_CREAT | PG_BINARY); if (file < 0) - { ereport(FATAL, (errcode_for_file_access(), errmsg("could not open tde key file \"%s\": %m", key_file_path))); - } - - - if (!RAND_bytes(int_key.key, INTERNAL_KEY_LEN)) - { - ereport(FATAL, - (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("could not generate internal key for relation \"%s\": %s", - RelationGetRelationName(rel), ERR_error_string(ERR_get_error(), NULL)))); - } /* Allocate in TopMemoryContext and don't pfree sice we add it to * the cache as well */ data = (RelKeysData *) MemoryContextAlloc(TopMemoryContext, SizeOfRelKeysData(1)); strcpy(data->master_key_name, MasterKeyName); - data->internal_key[0] = int_key; + data->internal_key[0] = *key; data->internal_keys_len = 1; -#if TDE_FORK_DEBUG - ereport(DEBUG2, - (errmsg("internal_key: %s", tde_sprint_key(&data->internal_key[0])))); -#endif - sz = SizeOfRelKeysData(data->internal_keys_len); - -#if TDE_FORK_DEBUG - ereport(DEBUG2, - (errmsg("fork file master key: %s: %s", master_key_info->name.name, tde_sprint_masterkey(&master_key_info->data)))); -#endif - + memcpy(dataEnc, data, sz); AesEncrypt(master_key_info->data.data, iv, (unsigned char*)data + SizeOfRelKeysDataHeader, INTERNAL_KEY_LEN, dataEnc + SizeOfRelKeysDataHeader, &encsz); @@ -129,18 +139,16 @@ pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation rel) #endif if (FileWrite(file, dataEnc, sz, 0, WAIT_EVENT_DATA_FILE_WRITE) != sz) - { ereport(FATAL, (errcode_for_file_access(), errmsg("could not write key data to file \"%s\": %m", key_file_path))); - } /* Register the file for delete in case transaction Aborts */ RegisterFileForDeletion(key_file_path, false); /* Add to the cache */ - put_keys_into_map(newrlocator->relNumber, data); + put_keys_into_map(rlocator->relNumber, data); pfree(key_file_path); FileClose(file); @@ -340,3 +348,70 @@ pg_tde_get_key_file_path(const RelFileLocator *newrlocator) } return key_file_path; } + +/* + * TDE fork XLog + */ +void +pg_tde_rmgr_redo(XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + switch (info) + { + case XLOG_TDE_CREATE_FORK: + pg_tde_xlog_create_fork(record); + break; + default: + elog(PANIC, "pg_tde_redo: unknown op code %u", info); + } +} + +void +pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record) +{ + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + char *rec = XLogRecGetData(record); + RelFileLocator rlocator; + + if (info == XLOG_TDE_CREATE_FORK) + { + memcpy(&rlocator, rec, sizeof(RelFileLocator)); + appendStringInfo(buf, "create tde fork for relation %u/%u", rlocator.dbOid, rlocator.relNumber); + } +} + +const char * +pg_tde_rmgr_identify(uint8 info) +{ + if ((info & ~XLR_INFO_MASK) == XLOG_TDE_CREATE_FORK) + return "TDE_CREATE_FORK"; + + return NULL; +} + +static void +pg_tde_xlog_create_fork(XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + RelFileLocator rlocator; + InternalKey int_key = {0}; + + if (XLogRecGetDataLen(record) < sizeof(InternalKey)+sizeof(RelFileLocator)) + { + ereport(FATAL, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("corrupted XLOG_TDE_CREATE_FORK data"))); + } + + /* Format [RelFileLocator][InternalKey] */ + memcpy(&rlocator, rec, sizeof(RelFileLocator)); + memcpy(&int_key, rec+sizeof(RelFileLocator), sizeof(InternalKey)); + +#if TDE_FORK_DEBUG + ereport(DEBUG2, + (errmsg("xlog internal_key: %s", tde_sprint_key(&int_key)))); +#endif + + pg_tde_write_key_fork(&rlocator, &int_key, MasterKeyName); +} \ No newline at end of file diff --git a/src/access/pg_tdeam.c b/src/access/pg_tdeam.c index 3be71ce0..99e9643e 100644 --- a/src/access/pg_tdeam.c +++ b/src/access/pg_tdeam.c @@ -1975,9 +1975,11 @@ pg_tde_insert(Relation relation, HeapTuple tup, CommandId cid, */ XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags); XLogRegisterBufData(0, (char *) &xlhdr, SizeOfHeapHeader); + /* register encrypted tuple data from the buffer */ + PageHeader phdr = (PageHeader) BufferGetPage(buffer); /* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */ XLogRegisterBufData(0, - (char *) heaptup->t_data + SizeofHeapTupleHeader, + ((char *) phdr) + phdr->pd_upper + SizeofHeapTupleHeader, heaptup->t_len - SizeofHeapTupleHeader); /* filtering by origin on a row level is much more efficient */ @@ -2341,10 +2343,12 @@ pg_tde_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, tuphdr->t_infomask = heaptup->t_data->t_infomask; tuphdr->t_hoff = heaptup->t_data->t_hoff; + /* Point to an encrypted tuple data in the Buffer */ + char *tup_data_on_page = (char *) page + ItemIdGetOffset(PageGetItemId(page, heaptup->t_self.ip_posid)); /* write bitmap [+ padding] [+ oid] + data */ datalen = heaptup->t_len - SizeofHeapTupleHeader; memcpy(scratchptr, - (char *) heaptup->t_data + SizeofHeapTupleHeader, + tup_data_on_page + SizeofHeapTupleHeader, datalen); tuphdr->datalen = datalen; scratchptr += datalen; @@ -8400,6 +8404,7 @@ log_pg_tde_update(Relation reln, Buffer oldbuf, suffixlen = 0; XLogRecPtr recptr; Page page = BufferGetPage(newbuf); + PageHeader phdr = (PageHeader) page; bool need_tuple_data = RelationIsLogicallyLogged(reln); bool init; int bufflags; @@ -8551,11 +8556,12 @@ log_pg_tde_update(Relation reln, Buffer oldbuf, * * The 'data' doesn't include the common prefix or suffix. */ + /* We write an encrypted newtuple data from the buffer */ XLogRegisterBufData(0, (char *) &xlhdr, SizeOfHeapHeader); if (prefixlen == 0) { XLogRegisterBufData(0, - ((char *) newtup->t_data) + SizeofHeapTupleHeader, + ((char *) phdr) + phdr->pd_upper + SizeofHeapTupleHeader, newtup->t_len - SizeofHeapTupleHeader - suffixlen); } else @@ -8568,13 +8574,13 @@ log_pg_tde_update(Relation reln, Buffer oldbuf, if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0) { XLogRegisterBufData(0, - ((char *) newtup->t_data) + SizeofHeapTupleHeader, + ((char *) phdr) + phdr->pd_upper + SizeofHeapTupleHeader, newtup->t_data->t_hoff - SizeofHeapTupleHeader); } /* data after common prefix */ XLogRegisterBufData(0, - ((char *) newtup->t_data) + newtup->t_data->t_hoff + prefixlen, + ((char *) phdr) + phdr->pd_upper + newtup->t_data->t_hoff + prefixlen, newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen); } diff --git a/src/include/access/pg_tde_tdemap.h b/src/include/access/pg_tde_tdemap.h index d3a8a702..9071c93e 100644 --- a/src/include/access/pg_tde_tdemap.h +++ b/src/include/access/pg_tde_tdemap.h @@ -10,6 +10,7 @@ #include "utils/rel.h" #include "storage/relfilelocator.h" +#include "access/xlog_internal.h" #define TDE_FORK_EXT "tde" @@ -56,4 +57,22 @@ extern void pg_tde_create_key_fork(const RelFileLocator *newrlocator, Relation r extern RelKeysData *pg_tde_get_keys_from_fork(const RelFileLocator *rlocator); extern RelKeysData *GetRelationKeys(RelFileLocator rel); const char * tde_sprint_key(InternalKey *k); + +/* TDE XLOG resource manager */ +#define XLOG_TDE_CREATE_FORK 0x00 +/* TODO: ID has to be registedred and changed: https://wiki.postgresql.org/wiki/CustomWALResourceManagers */ +#define RM_TDERMGR_ID RM_EXPERIMENTAL_ID +#define RM_TDERMGR_NAME "test_pg_tde_custom_rmgr" + +extern void pg_tde_rmgr_redo(XLogReaderState *record); +extern void pg_tde_rmgr_desc(StringInfo buf, XLogReaderState *record); +extern const char * pg_tde_rmgr_identify(uint8 info); + +static const RmgrData pg_tde_rmgr = { + .rm_name = RM_TDERMGR_NAME, + .rm_redo = pg_tde_rmgr_redo, + .rm_desc = pg_tde_rmgr_desc, + .rm_identify = pg_tde_rmgr_identify +}; + #endif /* PG_TDE_MAP_H */ diff --git a/src/pg_tde.c b/src/pg_tde.c index ad9e7e48..c16432e4 100644 --- a/src/pg_tde.c +++ b/src/pg_tde.c @@ -18,6 +18,7 @@ #include "storage/shmem.h" #include "access/pg_tde_ddl.h" #include "encryption/enc_aes.h" +#include "access/pg_tde_tdemap.h" #include "keyring/keyring_config.h" #include "keyring/keyring_api.h" @@ -67,4 +68,6 @@ void RegisterXactCallback(pg_tde_xact_callback, NULL); RegisterSubXactCallback(pg_tde_subxact_callback, NULL); SetupTdeDDLHooks(); + + RegisterCustomRmgr(RM_TDERMGR_ID, &pg_tde_rmgr); }