Merge pull request percona#55 from dutow/ciperf

Basic performance tests on github.

.github/workflows/postgresql-16-src-meson-perf.yml

@@ -0,0 +1,121 @@
+name: Perf test
+on: [pull_request]
+permissions:
+  contents: write
+  pull-requests: write
+  repository-projects: write
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+
+
+      - name: Remove old postgres
+        run: |
+          sudo apt purge postgresql-client-common postgresql-common \
+            postgresql postgresql*
+          sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
+           /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
+           /etc/postgresql
+          sudo rm -f /usr/bin/pg_config
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
+            zlib1g-dev libssl-dev libpam0g-dev bison flex \
+            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
+            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
+            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
+            llvm-11 llvm-11-dev libselinux1-dev python3-dev \
+            uuid-dev liblz4-dev meson ninja-build libjson-c-dev \
+            sysbench
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN'
+          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+
+      - name: Clone postgres repository
+        uses: actions/checkout@v2
+        with:
+          repository: 'postgres/postgres'
+          ref: 'a81e5516fa4bc53e332cb35eefe231147c0e1749'
+          path: 'src'
+
+      - name: Clone postgres-tde-ext repository
+        uses: actions/checkout@v2
+        with:
+          path: 'src/contrib/postgres-tde-ext'
+
+      - name: Include postgres-tde-ext in meson build
+        run: |
+          echo "subdir('postgres-tde-ext')" >> src/contrib/meson.build
+
+      - name: Build postgres
+        run: |
+          meson setup build --prefix `pwd`/../inst --buildtype=release
+          cd build && ninja && ninja install
+        working-directory: src
+
+      - name: Test postgres-tde-ext
+        run: |
+          cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json
+          meson test --suite setup -v
+          meson test --suite postgres-tde-ext -v --num-processes 1
+        working-directory: src/build
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/build/testrun/postgres-tde-ext/regress/
+          retention-days: 3
+
+      - name: Setup test environment
+        run: |  
+          bin/initdb -D data
+          echo "shared_preload_libraries = 'pg_tde'" >> data/postgresql.conf
+          echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> data/postgresql.conf
+          bin/pg_ctl -D data start
+          bin/createdb sbtest
+          bin/createdb sbtest2
+          bin/createuser sbtest -s
+          bin/psql sbtest2 <<< "CREATE EXTENSION pg_tde;"
+          cp -r ../src/contrib/postgres-tde-ext/sysbench .
+        working-directory: inst
+
+      - name: Run baseline performance tests
+        run: |  
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_insert.lua --tables=1 --table-size=10000 --pgsql-db=sbtest prepare
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest run
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest run | tee perf_norm
+        working-directory: inst
+
+      - name: Run TDE performance tests
+        run: |  
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_common_tde.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 prepare
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 run
+          sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 run | tee perf_tde
+        working-directory: inst
+
+      - name: Print results
+        run: |  
+          NORM_Q=$(cat perf_norm | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')
+          TDE_Q=$(cat perf_tde | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')
+          echo "Norm queries: $NORM_Q"
+          echo "TDE queries: $TDE_Q"
+          echo "Performance test results:" >> pr_perf_results
+          echo "Normal queries: $(cat perf_norm | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')" >>  pr_perf_results
+          echo "TDE queries: $(cat perf_tde | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')" >>  pr_perf_results
+          echo "Percentage: $(($TDE_Q*100/$NORM_Q))%" >> pr_perf_results
+          PERF_RESULT=$(cat pr_perf_results)
+          echo "PERF_RESULT<<EOF" >> $GITHUB_ENV
+          echo "$PERF_RESULT" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+        working-directory: inst
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: pr_perf_results
+          path: inst/pr_perf_results

.github/workflows/postgresql-perf-results.yml

@@ -0,0 +1,51 @@
+name: Perf test results
+
+on:
+  workflow_run:
+    workflows: [Perf test]
+    types:
+      - completed
+
+jobs:
+  download:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Download artifact'
+        uses: actions/github-script@v5
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "pr_perf_results"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_perf_results.zip`, Buffer.from(download.data));
+
+      - name: 'Unzip artifact'
+        run: |
+          unzip pr_perf_results.zip
+
+      - name: Clone postgres-tde-ext repository
+        uses: actions/checkout@v2
+        with:
+          path: 'src'
+          ref: ${{ github.event.workflow_run.head_branch }}
+
+      - name: 'Create comment'
+        run: |
+          gh pr comment ${PR_NUMBER} -F ../pr_perf_results --edit-last || \
+            gh pr comment ${PR_NUMBER} -F ../pr_perf_results
+        env:
+          PR_NUMBER: ${{ github.event.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        working-directory: src

Makefile

@@ -6,7 +6,7 @@ EXTENSION = pg_tde
 DATA = pg_tde--1.0.sql
 
 REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/postgres-tde-ext/postgres-tde-ext.conf
-REGRESS = non_sorted_off_compact update_compare_indexes pgtde_is_encrypted
+REGRESS = non_sorted_off_compact update_compare_indexes pgtde_is_encrypted trigger_on_view
 TAP_TESTS = 1
 
 OBJS = src/encryption/enc_tuple.o \

expected/trigger_on_view.out

@@ -0,0 +1,204 @@
+CREATE extension pg_tde;
+--
+-- 2 -- Test triggers on a join view
+--
+SET default_table_access_method TO 'pg_tde';
+DROP VIEW IF EXISTS city_view CASCADE;
+NOTICE:  view "city_view" does not exist, skipping
+DROP TABLE IF exists country_table CASCADE;
+NOTICE:  table "country_table" does not exist, skipping
+DROP TABLE IF exists city_table cascade;
+NOTICE:  table "city_table" does not exist, skipping
+ CREATE TABLE country_table (
+     country_id        serial primary key,
+     country_name    text unique not null,
+     continent        text not null
+ ) using pg_tde;
+
+ INSERT INTO country_table (country_name, continent)
+     VALUES ('Japan', 'Asia'),
+            ('UK', 'Europe'),
+            ('USA', 'North America')
+     RETURNING *;
+ country_id | country_name |   continent   
+------------+--------------+---------------
+          1 | Japan        | Asia
+          2 | UK           | Europe
+          3 | USA          | North America
+(3 rows)
+
+
+ CREATE TABLE city_table (
+     city_id        serial primary key,
+     city_name    text not null,
+     population    bigint, 
+     country_id    int references country_table
+ ) using pg_tde;
+
+ CREATE VIEW city_view AS
+     SELECT city_id, city_name, population, country_name, continent
+     FROM city_table ci
+     LEFT JOIN country_table co ON co.country_id = ci.country_id;
+
+CREATE OR REPLACE FUNCTION city_insert() RETURNS trigger LANGUAGE plpgsql AS $$
+ declare
+     ctry_id int;
+ begin
+     if NEW.country_name IS NOT NULL then
+         SELECT country_id, continent INTO ctry_id, NEW.continent
+             FROM country_table WHERE country_name = NEW.country_name;
+         if NOT FOUND then 
+             raise exception 'No such country: "%"', NEW.country_name;
+         end if;
+     else
+         NEW.continent := NULL;
+     end if;
+
+     if NEW.city_id IS NOT NULL then
+         INSERT INTO city_table
+             VALUES(NEW.city_id, NEW.city_name, NEW.population, ctry_id);
+     else
+         INSERT INTO city_table(city_name, population, country_id)
+             VALUES(NEW.city_name, NEW.population, ctry_id)
+             RETURNING city_id INTO NEW.city_id;
+     end if;
+
+     RETURN NEW;
+ end;
+ $$;
+  CREATE TRIGGER city_insert_trig INSTEAD OF INSERT ON city_view
+ FOR EACH ROW EXECUTE PROCEDURE city_insert();
+
+ CREATE OR REPLACE FUNCTION city_delete() RETURNS trigger LANGUAGE plpgsql AS $$
+ begin
+     DELETE FROM city_table WHERE city_id = OLD.city_id;
+     if NOT FOUND then RETURN NULL; end if;
+     RETURN OLD;
+ end;
+ $$;
+
+ CREATE TRIGGER city_delete_trig INSTEAD OF DELETE ON city_view
+ FOR EACH ROW EXECUTE PROCEDURE city_delete();
+
+ CREATE OR REPLACE FUNCTION city_update() RETURNS trigger LANGUAGE plpgsql AS $$
+ declare
+     ctry_id int;
+ begin
+     if NEW.country_name IS DISTINCT FROM OLD.country_name then
+         SELECT country_id, continent INTO ctry_id, NEW.continent
+             FROM country_table WHERE country_name = NEW.country_name;
+         if NOT FOUND then
+             raise exception 'No such country: "%"', NEW.country_name;
+         end if;
+
+         UPDATE city_table SET city_name = NEW.city_name,
+                               population = NEW.population,
+                               country_id = ctry_id
+             WHERE city_id = OLD.city_id;
+     else
+         UPDATE city_table SET city_name = NEW.city_name,
+                               population = NEW.population
+             WHERE city_id = OLD.city_id;
+         NEW.continent := OLD.continent;
+     end if;
+
+     if NOT FOUND then RETURN NULL; end if;
+     RETURN NEW;
+ end;
+ $$;
+ CREATE TRIGGER city_update_trig INSTEAD OF UPDATE ON city_view
+ FOR EACH ROW EXECUTE PROCEDURE city_update();
+
+-- INSERT .. RETURNING
+ INSERT INTO city_view(city_name) VALUES('Tokyo') RETURNING *;
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+       1 | Tokyo     |            |              | 
+(1 row)
+
+ INSERT INTO city_view(city_name, population) VALUES('London', 7556900) RETURNING *;
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+       2 | London    |    7556900 |              | 
+(1 row)
+
+ INSERT INTO city_view(city_name, country_name) VALUES('Washington DC', 'USA') RETURNING *;
+ city_id |   city_name   | population | country_name |   continent   
+---------+---------------+------------+--------------+---------------
+       3 | Washington DC |            | USA          | North America
+(1 row)
+
+ INSERT INTO city_view(city_id, city_name) VALUES(123456, 'New York') RETURNING *;
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+  123456 | New York  |            |              | 
+(1 row)
+
+ INSERT INTO city_view VALUES(234567, 'Birmingham', 1016800, 'UK', 'EU') RETURNING *;
+ city_id | city_name  | population | country_name | continent 
+---------+------------+------------+--------------+-----------
+  234567 | Birmingham |    1016800 | UK           | Europe
+(1 row)
+
+
+ -- UPDATE .. RETURNING
+ UPDATE city_view SET country_name = 'Japon' WHERE city_name = 'Tokyo'; -- error
+ERROR:  No such country: "Japon"
+CONTEXT:  PL/pgSQL function city_update() line 9 at RAISE
+ UPDATE city_view SET country_name = 'Japan' WHERE city_name = 'Takyo'; -- no match
+ UPDATE city_view SET country_name = 'Japan' WHERE city_name = 'Tokyo' RETURNING *; -- OK
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+       1 | Tokyo     |            | Japan        | Asia
+(1 row)
+
+
+ UPDATE city_view SET population = 13010279 WHERE city_name = 'Tokyo' RETURNING *;
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+       1 | Tokyo     |   13010279 | Japan        | Asia
+       1 | Tokyo     |   13010279 |              | 
+(2 rows)
+
+ UPDATE city_view SET country_name = 'UK' WHERE city_name = 'New York' RETURNING *;
+ city_id | city_name | population | country_name | continent 
+---------+-----------+------------+--------------+-----------
+  123456 | New York  |            | UK           | Europe
+(1 row)
+
+ UPDATE city_view SET country_name = 'USA', population = 8391881 WHERE city_name = 'New York' RETURNING *;
+ city_id | city_name | population | country_name |   continent   
+---------+-----------+------------+--------------+---------------
+  123456 | New York  |    8391881 | USA          | North America
+  123456 | New York  |    8391881 | USA          | North America
+(2 rows)
+
+ UPDATE city_view SET continent = 'EU' WHERE continent = 'Europe' RETURNING *;
+ city_id | city_name  | population | country_name | continent 
+---------+------------+------------+--------------+-----------
+  234567 | Birmingham |    1016800 | UK           | Europe
+  123456 | New York   |            | UK           | Europe
+(2 rows)
+
+ UPDATE city_view v1 SET country_name = v2.country_name FROM city_view v2
+     WHERE v2.city_name = 'Birmingham' AND v1.city_name = 'London' RETURNING *;
+ city_id | city_name | population | country_name | continent | city_id | city_name  | population | country_name | continent 
+---------+-----------+------------+--------------+-----------+---------+------------+------------+--------------+-----------
+       2 | London    |    7556900 | UK           | Europe    |  234567 | Birmingham |    1016800 | UK           | Europe
+       2 | London    |    7556900 | UK           | Europe    |  234567 | Birmingham |    1016800 | UK           | Europe
+(2 rows)
+
+
+ -- DELETE .. RETURNING
+ DELETE FROM city_view WHERE city_name = 'Birmingham' RETURNING *;
+ city_id | city_name  | population | country_name | continent 
+---------+------------+------------+--------------+-----------
+  234567 | Birmingham |    1016800 | UK           | Europe
+(1 row)
+
+
+DROP extension pg_tde CASCADE;
+NOTICE:  drop cascades to 3 other objects
+DETAIL:  drop cascades to table country_table
+drop cascades to table city_table
+drop cascades to view city_view

meson.build

@@ -71,6 +71,7 @@ tests += {
       'update_compare_indexes',
       'pgtde_is_encrypted',
       'multi_insert',
+      'trigger_on_view',
     ],
     'regress_args': ['--temp-config', files('postgres-tde-ext.conf')],
     'runningcheck': false,