forked from percona/pg_tde
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request percona#55 from dutow/ciperf
Basic performance tests on github.
- Loading branch information
Showing
23 changed files
with
2,105 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
name: Perf test | ||
on: [pull_request] | ||
permissions: | ||
contents: write | ||
pull-requests: write | ||
repository-projects: write | ||
|
||
jobs: | ||
build: | ||
runs-on: ubuntu-22.04 | ||
steps: | ||
|
||
|
||
- name: Remove old postgres | ||
run: | | ||
sudo apt purge postgresql-client-common postgresql-common \ | ||
postgresql postgresql* | ||
sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \ | ||
/usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \ | ||
/etc/postgresql | ||
sudo rm -f /usr/bin/pg_config | ||
- name: Install dependencies | ||
run: | | ||
sudo apt-get update | ||
sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \ | ||
zlib1g-dev libssl-dev libpam0g-dev bison flex \ | ||
libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \ | ||
libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \ | ||
libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \ | ||
llvm-11 llvm-11-dev libselinux1-dev python3-dev \ | ||
uuid-dev liblz4-dev meson ninja-build libjson-c-dev \ | ||
sysbench | ||
sudo /usr/bin/perl -MCPAN -e 'install IPC::RUN' | ||
sudo /usr/bin/perl -MCPAN -e 'install Text::Trim' | ||
- name: Clone postgres repository | ||
uses: actions/checkout@v2 | ||
with: | ||
repository: 'postgres/postgres' | ||
ref: 'a81e5516fa4bc53e332cb35eefe231147c0e1749' | ||
path: 'src' | ||
|
||
- name: Clone postgres-tde-ext repository | ||
uses: actions/checkout@v2 | ||
with: | ||
path: 'src/contrib/postgres-tde-ext' | ||
|
||
- name: Include postgres-tde-ext in meson build | ||
run: | | ||
echo "subdir('postgres-tde-ext')" >> src/contrib/meson.build | ||
- name: Build postgres | ||
run: | | ||
meson setup build --prefix `pwd`/../inst --buildtype=release | ||
cd build && ninja && ninja install | ||
working-directory: src | ||
|
||
- name: Test postgres-tde-ext | ||
run: | | ||
cp ../contrib/postgres-tde-ext/keyring.json /tmp/keyring.json | ||
meson test --suite setup -v | ||
meson test --suite postgres-tde-ext -v --num-processes 1 | ||
working-directory: src/build | ||
|
||
- name: Report on test fail | ||
uses: actions/upload-artifact@v2 | ||
if: ${{ failure() }} | ||
with: | ||
name: Regressions diff and postgresql log | ||
path: | | ||
src/build/testrun/postgres-tde-ext/regress/ | ||
retention-days: 3 | ||
|
||
- name: Setup test environment | ||
run: | | ||
bin/initdb -D data | ||
echo "shared_preload_libraries = 'pg_tde'" >> data/postgresql.conf | ||
echo "pg_tde.keyringConfigFile = '/tmp/keyring.json'" >> data/postgresql.conf | ||
bin/pg_ctl -D data start | ||
bin/createdb sbtest | ||
bin/createdb sbtest2 | ||
bin/createuser sbtest -s | ||
bin/psql sbtest2 <<< "CREATE EXTENSION pg_tde;" | ||
cp -r ../src/contrib/postgres-tde-ext/sysbench . | ||
working-directory: inst | ||
|
||
- name: Run baseline performance tests | ||
run: | | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_insert.lua --tables=1 --table-size=10000 --pgsql-db=sbtest prepare | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest run | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest run | tee perf_norm | ||
working-directory: inst | ||
|
||
- name: Run TDE performance tests | ||
run: | | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_common_tde.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 prepare | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 run | ||
sysbench --db-driver=pgsql --threads=1 sysbench/oltp_read_only.lua --tables=1 --table-size=10000 --pgsql-db=sbtest2 run | tee perf_tde | ||
working-directory: inst | ||
|
||
- name: Print results | ||
run: | | ||
NORM_Q=$(cat perf_norm | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g') | ||
TDE_Q=$(cat perf_tde | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g') | ||
echo "Norm queries: $NORM_Q" | ||
echo "TDE queries: $TDE_Q" | ||
echo "Performance test results:" >> pr_perf_results | ||
echo "Normal queries: $(cat perf_norm | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')" >> pr_perf_results | ||
echo "TDE queries: $(cat perf_tde | grep 'total number of events' | cut -d ':' -f 2 | sed 's/ //g')" >> pr_perf_results | ||
echo "Percentage: $(($TDE_Q*100/$NORM_Q))%" >> pr_perf_results | ||
PERF_RESULT=$(cat pr_perf_results) | ||
echo "PERF_RESULT<<EOF" >> $GITHUB_ENV | ||
echo "$PERF_RESULT" >> $GITHUB_ENV | ||
echo "EOF" >> $GITHUB_ENV | ||
working-directory: inst | ||
|
||
- uses: actions/upload-artifact@v3 | ||
with: | ||
name: pr_perf_results | ||
path: inst/pr_perf_results |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
name: Perf test results | ||
|
||
on: | ||
workflow_run: | ||
workflows: [Perf test] | ||
types: | ||
- completed | ||
|
||
jobs: | ||
download: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: 'Download artifact' | ||
uses: actions/github-script@v5 | ||
with: | ||
script: | | ||
let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ | ||
owner: context.repo.owner, | ||
repo: context.repo.repo, | ||
run_id: context.payload.workflow_run.id, | ||
}); | ||
let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { | ||
return artifact.name == "pr_perf_results" | ||
})[0]; | ||
let download = await github.rest.actions.downloadArtifact({ | ||
owner: context.repo.owner, | ||
repo: context.repo.repo, | ||
artifact_id: matchArtifact.id, | ||
archive_format: 'zip', | ||
}); | ||
let fs = require('fs'); | ||
fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/pr_perf_results.zip`, Buffer.from(download.data)); | ||
- name: 'Unzip artifact' | ||
run: | | ||
unzip pr_perf_results.zip | ||
- name: Clone postgres-tde-ext repository | ||
uses: actions/checkout@v2 | ||
with: | ||
path: 'src' | ||
ref: ${{ github.event.workflow_run.head_branch }} | ||
|
||
- name: 'Create comment' | ||
run: | | ||
gh pr comment ${PR_NUMBER} -F ../pr_perf_results --edit-last || \ | ||
gh pr comment ${PR_NUMBER} -F ../pr_perf_results | ||
env: | ||
PR_NUMBER: ${{ github.event.number }} | ||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
working-directory: src |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,204 @@ | ||
CREATE extension pg_tde; | ||
-- | ||
-- 2 -- Test triggers on a join view | ||
-- | ||
SET default_table_access_method TO 'pg_tde'; | ||
DROP VIEW IF EXISTS city_view CASCADE; | ||
NOTICE: view "city_view" does not exist, skipping | ||
DROP TABLE IF exists country_table CASCADE; | ||
NOTICE: table "country_table" does not exist, skipping | ||
DROP TABLE IF exists city_table cascade; | ||
NOTICE: table "city_table" does not exist, skipping | ||
CREATE TABLE country_table ( | ||
country_id serial primary key, | ||
country_name text unique not null, | ||
continent text not null | ||
) using pg_tde; | ||
|
||
INSERT INTO country_table (country_name, continent) | ||
VALUES ('Japan', 'Asia'), | ||
('UK', 'Europe'), | ||
('USA', 'North America') | ||
RETURNING *; | ||
country_id | country_name | continent | ||
------------+--------------+--------------- | ||
1 | Japan | Asia | ||
2 | UK | Europe | ||
3 | USA | North America | ||
(3 rows) | ||
|
||
|
||
CREATE TABLE city_table ( | ||
city_id serial primary key, | ||
city_name text not null, | ||
population bigint, | ||
country_id int references country_table | ||
) using pg_tde; | ||
|
||
CREATE VIEW city_view AS | ||
SELECT city_id, city_name, population, country_name, continent | ||
FROM city_table ci | ||
LEFT JOIN country_table co ON co.country_id = ci.country_id; | ||
|
||
CREATE OR REPLACE FUNCTION city_insert() RETURNS trigger LANGUAGE plpgsql AS $$ | ||
declare | ||
ctry_id int; | ||
begin | ||
if NEW.country_name IS NOT NULL then | ||
SELECT country_id, continent INTO ctry_id, NEW.continent | ||
FROM country_table WHERE country_name = NEW.country_name; | ||
if NOT FOUND then | ||
raise exception 'No such country: "%"', NEW.country_name; | ||
end if; | ||
else | ||
NEW.continent := NULL; | ||
end if; | ||
|
||
if NEW.city_id IS NOT NULL then | ||
INSERT INTO city_table | ||
VALUES(NEW.city_id, NEW.city_name, NEW.population, ctry_id); | ||
else | ||
INSERT INTO city_table(city_name, population, country_id) | ||
VALUES(NEW.city_name, NEW.population, ctry_id) | ||
RETURNING city_id INTO NEW.city_id; | ||
end if; | ||
|
||
RETURN NEW; | ||
end; | ||
$$; | ||
CREATE TRIGGER city_insert_trig INSTEAD OF INSERT ON city_view | ||
FOR EACH ROW EXECUTE PROCEDURE city_insert(); | ||
|
||
CREATE OR REPLACE FUNCTION city_delete() RETURNS trigger LANGUAGE plpgsql AS $$ | ||
begin | ||
DELETE FROM city_table WHERE city_id = OLD.city_id; | ||
if NOT FOUND then RETURN NULL; end if; | ||
RETURN OLD; | ||
end; | ||
$$; | ||
|
||
CREATE TRIGGER city_delete_trig INSTEAD OF DELETE ON city_view | ||
FOR EACH ROW EXECUTE PROCEDURE city_delete(); | ||
|
||
CREATE OR REPLACE FUNCTION city_update() RETURNS trigger LANGUAGE plpgsql AS $$ | ||
declare | ||
ctry_id int; | ||
begin | ||
if NEW.country_name IS DISTINCT FROM OLD.country_name then | ||
SELECT country_id, continent INTO ctry_id, NEW.continent | ||
FROM country_table WHERE country_name = NEW.country_name; | ||
if NOT FOUND then | ||
raise exception 'No such country: "%"', NEW.country_name; | ||
end if; | ||
|
||
UPDATE city_table SET city_name = NEW.city_name, | ||
population = NEW.population, | ||
country_id = ctry_id | ||
WHERE city_id = OLD.city_id; | ||
else | ||
UPDATE city_table SET city_name = NEW.city_name, | ||
population = NEW.population | ||
WHERE city_id = OLD.city_id; | ||
NEW.continent := OLD.continent; | ||
end if; | ||
|
||
if NOT FOUND then RETURN NULL; end if; | ||
RETURN NEW; | ||
end; | ||
$$; | ||
CREATE TRIGGER city_update_trig INSTEAD OF UPDATE ON city_view | ||
FOR EACH ROW EXECUTE PROCEDURE city_update(); | ||
|
||
-- INSERT .. RETURNING | ||
INSERT INTO city_view(city_name) VALUES('Tokyo') RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
1 | Tokyo | | | | ||
(1 row) | ||
|
||
INSERT INTO city_view(city_name, population) VALUES('London', 7556900) RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
2 | London | 7556900 | | | ||
(1 row) | ||
|
||
INSERT INTO city_view(city_name, country_name) VALUES('Washington DC', 'USA') RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+---------------+------------+--------------+--------------- | ||
3 | Washington DC | | USA | North America | ||
(1 row) | ||
|
||
INSERT INTO city_view(city_id, city_name) VALUES(123456, 'New York') RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
123456 | New York | | | | ||
(1 row) | ||
|
||
INSERT INTO city_view VALUES(234567, 'Birmingham', 1016800, 'UK', 'EU') RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+------------+------------+--------------+----------- | ||
234567 | Birmingham | 1016800 | UK | Europe | ||
(1 row) | ||
|
||
|
||
-- UPDATE .. RETURNING | ||
UPDATE city_view SET country_name = 'Japon' WHERE city_name = 'Tokyo'; -- error | ||
ERROR: No such country: "Japon" | ||
CONTEXT: PL/pgSQL function city_update() line 9 at RAISE | ||
UPDATE city_view SET country_name = 'Japan' WHERE city_name = 'Takyo'; -- no match | ||
UPDATE city_view SET country_name = 'Japan' WHERE city_name = 'Tokyo' RETURNING *; -- OK | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
1 | Tokyo | | Japan | Asia | ||
(1 row) | ||
|
||
|
||
UPDATE city_view SET population = 13010279 WHERE city_name = 'Tokyo' RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
1 | Tokyo | 13010279 | Japan | Asia | ||
1 | Tokyo | 13010279 | | | ||
(2 rows) | ||
|
||
UPDATE city_view SET country_name = 'UK' WHERE city_name = 'New York' RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+----------- | ||
123456 | New York | | UK | Europe | ||
(1 row) | ||
|
||
UPDATE city_view SET country_name = 'USA', population = 8391881 WHERE city_name = 'New York' RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+--------------- | ||
123456 | New York | 8391881 | USA | North America | ||
123456 | New York | 8391881 | USA | North America | ||
(2 rows) | ||
|
||
UPDATE city_view SET continent = 'EU' WHERE continent = 'Europe' RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+------------+------------+--------------+----------- | ||
234567 | Birmingham | 1016800 | UK | Europe | ||
123456 | New York | | UK | Europe | ||
(2 rows) | ||
|
||
UPDATE city_view v1 SET country_name = v2.country_name FROM city_view v2 | ||
WHERE v2.city_name = 'Birmingham' AND v1.city_name = 'London' RETURNING *; | ||
city_id | city_name | population | country_name | continent | city_id | city_name | population | country_name | continent | ||
---------+-----------+------------+--------------+-----------+---------+------------+------------+--------------+----------- | ||
2 | London | 7556900 | UK | Europe | 234567 | Birmingham | 1016800 | UK | Europe | ||
2 | London | 7556900 | UK | Europe | 234567 | Birmingham | 1016800 | UK | Europe | ||
(2 rows) | ||
|
||
|
||
-- DELETE .. RETURNING | ||
DELETE FROM city_view WHERE city_name = 'Birmingham' RETURNING *; | ||
city_id | city_name | population | country_name | continent | ||
---------+------------+------------+--------------+----------- | ||
234567 | Birmingham | 1016800 | UK | Europe | ||
(1 row) | ||
|
||
|
||
DROP extension pg_tde CASCADE; | ||
NOTICE: drop cascades to 3 other objects | ||
DETAIL: drop cascades to table country_table | ||
drop cascades to table city_table | ||
drop cascades to view city_view |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.