From 6b111cda177704fae6a66933192a90a083fc4ee8 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 4 Oct 2023 13:59:38 +0200 Subject: [PATCH 01/36] remove function from tests --- test/test_config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index cd6458c..9db73ba 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -8,7 +8,6 @@ import pytest from config import ConfigError, config, get_ignored_tables, validate_config -from smtp_functions import can_send_email from .conftest import _reset_config @@ -257,7 +256,6 @@ def test_config_notification_setup(): # no NOTIFICATIONS set should pass but cannot send email validate_config(config) - assert can_send_email(config) is False # incomplete setting config.update( @@ -347,6 +345,3 @@ def test_config_notification_setup(): with pytest.raises(ConfigError, match="Config SMTP Error"): validate_config(config) - - # notifications are set, emails can be send - but this config was not validated, as it would be in real run - assert can_send_email(config) From 6fdb96fc3775234c23d4678f1bded5ccb767234a Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 4 Oct 2023 14:05:06 +0200 Subject: [PATCH 02/36] fix test --- test/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_config.py b/test/test_config.py index 9db73ba..c47db69 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -324,7 +324,7 @@ def test_config_notification_setup(): } ) - with pytest.raises(ConfigError, match="Config error: `smtp_port` must be set an integer"): + with pytest.raises(ConfigError, match="Config error: `smtp_port` must be set to an integer"): validate_config(config) # complete setting but does not work config.update( From 35ad0de54462934ba1e1736b3db4c3e3feccd682 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 4 Oct 2023 14:11:47 +0200 Subject: [PATCH 03/36] cleanup update with delete_project_now --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d13863a..e96a3f0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -63,14 +63,14 @@ def _reset_config( def cleanup( - mc, + mc: MerginClient, project, dirs, ): """cleanup leftovers from previous test if needed such as remote project and local directories""" try: print("Deleting project on Mergin Maps server: " + project) - mc.delete_project(project) + mc.delete_project_now(project) except ClientError as e: print("Deleting project error: " + str(e)) pass From 11c082350b3ca2a5e23e2fd4f3ff803642ceadc6 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 4 Oct 2023 14:11:58 +0200 Subject: [PATCH 04/36] black style --- test/conftest.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index e96a3f0..9e7b8a0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,25 +1,15 @@ -import pytest import os -import tempfile import shutil +import tempfile import psycopg2 import psycopg2.extensions -from psycopg2 import ( - sql, -) - -from mergin import ( - MerginClient, - ClientError, -) +import pytest +from mergin import ClientError, MerginClient +from psycopg2 import sql -from dbsync import ( - dbsync_init, -) -from config import ( - config, -) +from config import config +from dbsync import dbsync_init GEODIFF_EXE = os.environ.get("TEST_GEODIFF_EXE") DB_CONNINFO = os.environ.get("TEST_DB_CONNINFO") From 3dcaa241a6b62da7d873785c3031923dfbc4a32f Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 4 Oct 2023 14:39:09 +0200 Subject: [PATCH 05/36] update project delete --- test/test_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_basic.py b/test/test_basic.py index 0af19a8..b853cf9 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -657,7 +657,7 @@ def test_recreated_project_ids( source_gpkg_path, ) # delete remote project - mc.delete_project(full_project_name) + mc.delete_project_now(full_project_name) # recreate project with the same name mc.create_project( project_name, From b5076e053c63e3c9ae0d40773461ca5ac7f153a5 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Oct 2023 14:35:17 +0200 Subject: [PATCH 06/36] add init_sync_from_db and functions --- test/conftest.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 9e7b8a0..d893f51 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,7 @@ import os import shutil import tempfile +from typing import List import psycopg2 import psycopg2.extensions @@ -195,3 +196,127 @@ def mc(): @pytest.fixture(scope="function") def db_connection() -> psycopg2.extensions.connection: return psycopg2.connect(DB_CONNINFO) + + +def name_db_schema_main(project_name: str) -> str: + return project_name + "_main" + + +def name_db_schema_base(project_name: str) -> str: + return project_name + "_base" + + +def name_project_dir(project_name: str) -> str: + return os.path.join( + TMP_DIR, + project_name + "_work", + ) + + +def name_project_sync_dir(project_name: str) -> str: + return os.path.join( + TMP_DIR, + project_name + "_dbsync", + ) + + +def complete_project_name(project_name: str) -> str: + return WORKSPACE + "/" + project_name + + +def path_test_data(filename: str) -> str: + return os.path.join( + TEST_DATA_DIR, + filename, + ) + + +def path_sync_gpkg() -> str: + return "test_sync.gpkg" + + +def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[str] = None): + """ + Initialize sync from given database file: + - (re)create Mergin Maps project with the file + - (re)create local project working directory and sync directory + - configure DB sync and let it do the init (make copies to the database) + """ + if ignored_tables is None: + ignored_tables = [] + + full_project_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) # working directory + sync_project_dir = name_project_sync_dir(project_name) # used by dbsync + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + + conn = psycopg2.connect(DB_CONNINFO) + + cleanup( + mc, + full_project_name, + [ + project_dir, + sync_project_dir, + ], + ) + cleanup_db( + conn, + db_schema_base, + db_schema_main, + ) + + with open( + path_test_data("create_base.sql"), + encoding="utf-8", + ) as file: + base_table_dump = file.read() + + base_table_dump = base_table_dump.replace("default-schema-name", db_schema_main) + base_table_dump = base_table_dump.replace("default-table-name", "simple") + + cur = conn.cursor() + cur.execute(base_table_dump) + + # prepare a new Mergin Maps project + mc.create_project( + project_name, + namespace=WORKSPACE, + ) + + # prepare dbsync config + # patch config to fit testing purposes + if ignored_tables: + connection = { + "driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": path_sync_gpkg(), + "skip_tables": ignored_tables, + } + else: + connection = { + "driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": path_sync_gpkg(), + } + + config.update( + { + "GEODIFF_EXE": GEODIFF_EXE, + "WORKING_DIR": sync_project_dir, + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "CONNECTIONS": [connection], + "init_from": "db", + } + ) + + dbsync_init(mc) From 78de23ed9ba7c4bb0c8edd16767aadeceb23d7dd Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Oct 2023 14:36:23 +0200 Subject: [PATCH 07/36] tests for init from db --- test/test_init_db.py | 142 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 test/test_init_db.py diff --git a/test/test_init_db.py b/test/test_init_db.py new file mode 100644 index 0000000..1e90475 --- /dev/null +++ b/test/test_init_db.py @@ -0,0 +1,142 @@ +import os +import shutil +import sqlite3 + + +import psycopg2 +from psycopg2 import ( + sql, +) + +from mergin import ( + MerginClient, +) + +from dbsync import ( + dbsync_pull, + dbsync_push, +) + +from .conftest import ( + DB_CONNINFO, + init_sync_from_db, + name_project_dir, + name_db_schema_main, + name_db_schema_base, + complete_project_name, + path_sync_gpkg, + path_test_data, +) + + +def test_init_from_db( + mc: MerginClient, +): + project_name = "test_db_init" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + + path_synced_gpkg = project_dir + "/" + path_sync_gpkg() + + init_sync_from_db(mc, project_name) + + # test that database schemas are created + tables are populated + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) + assert cur.fetchone()[0] == 3 + + cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_base)).as_string(conn)) + assert cur.fetchone()[0] == 3 + + # download project and validate that the path synced file exist + mc.download_project(project_full_name, project_dir) + assert os.path.exists(path_synced_gpkg) + + # connect to sync file + gpkg_conn = sqlite3.connect(path_synced_gpkg) + gpkg_cur = gpkg_conn.cursor() + + # validate that simple table exists + gpkg_cur.execute( + "SELECT name FROM sqlite_schema WHERE type ='table' AND " + " name NOT LIKE 'sqlite_%' AND name NOT LIKE 'gpkg_%' AND name NOT LIKE 'rtree_%';" + ) + assert gpkg_cur.fetchone()[0] == "simple" + + # validate number of elements in simple + gpkg_cur.execute("SELECT count(*) FROM simple") + assert gpkg_cur.fetchone()[0] == 3 + + +def test_local_changes_to_db( + mc: MerginClient, +): + project_name = "test_mergin_changes_to_db" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + + path_synced_gpkg = project_dir + "/" + path_sync_gpkg() + + init_sync_from_db(mc, project_name) + + # connecto to db + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + # check that there are 3 features prior to changes + cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"') + assert cur.fetchone()[0] == 3 + + mc.download_project(project_full_name, project_dir) + + # make changes in GPKG + shutil.copy(path_test_data("inserted_point_from_db.gpkg"), path_synced_gpkg) + + # push project + mc.push_project(project_dir) + + # run sync + dbsync_pull(mc) + dbsync_push(mc) + + # check that new feature was added + cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"') + assert cur.fetchone()[0] == 4 + + +def test_db_changes_to_mergin( + mc: MerginClient, +): + project_name = "test_db_changes_mergin" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + + init_sync_from_db(mc, project_name) + + # connecto to db + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + + cur.execute( + f'INSERT INTO "{db_schema_main}"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES (\'0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F\', 4, \'new feature\', 4);' + ) + cur.execute("COMMIT") + + dbsync_pull(mc) + dbsync_push(mc) + + mc.download_project(project_full_name, project_dir) + + # look at changes in in GPKG + gpkg_conn = sqlite3.connect(project_dir + "/" + path_sync_gpkg()) + gpkg_cur = gpkg_conn.cursor() + gpkg_cur.execute("SELECT COUNT(*) FROM simple") + assert gpkg_cur.fetchone()[0] == 4 From bde3d47d53aa4de0c47e7cb0c477a6b0cf83fe3c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Oct 2023 14:36:57 +0200 Subject: [PATCH 08/36] tests data --- test/test_data/create_base.sql | 17 +++++++++++++++++ test/test_data/inserted_point_from_db.gpkg | Bin 0 -> 98304 bytes 2 files changed, 17 insertions(+) create mode 100644 test/test_data/create_base.sql create mode 100644 test/test_data/inserted_point_from_db.gpkg diff --git a/test/test_data/create_base.sql b/test/test_data/create_base.sql new file mode 100644 index 0000000..141e769 --- /dev/null +++ b/test/test_data/create_base.sql @@ -0,0 +1,17 @@ +SET standard_conforming_strings = OFF; + +CREATE SCHEMA IF NOT EXISTS default-schema-name; + +DROP TABLE IF EXISTS "default-schema-name"."default-table-name" CASCADE; + +CREATE TABLE "default-schema-name"."default-table-name" ( "ogc_fid" SERIAL, CONSTRAINT "base_pk" PRIMARY KEY ("ogc_fid") ); + +SELECT AddGeometryColumn('default-schema-name','default-table-name','wkb_geometry',4326,'POINT',2); +ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "fid" NUMERIC(20,0); +ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "name" VARCHAR; +ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "rating" NUMERIC(10,0); + +INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000001E78CBA1366CF1BF70E6AAC83981DD3F', 1, 'feature1', 1); +INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E6100000F0431AAFE449D7BFF874B615E6FDE13F', 2, 'feature2', 2); +INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F', 3, 'feature3', 3); + diff --git a/test/test_data/inserted_point_from_db.gpkg b/test/test_data/inserted_point_from_db.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..0e7fe2d7a0f43163012cd7b76e04f46167ea7bdb GIT binary patch literal 98304 zcmeI)PfQ!x9S86kAT~Auvww(}taDq?QLEMV&Hq3ClT?aT zg1#dNnfHG4Jo9;P=KV2v_4=ZuicF;;*94VGQKJ-1Q|~bhMNtFf`z`WqZ8Z6hmS$~a zfAWvoPIl#5)=KpK9H7qJ_}(#+NJsvoWsMm>d{Nki00bZa0SG_<0uX=z1Rwwb2tc5R z0&j+gN$GUtR}}fe1_1~_00Izz00bZa0SG_<0uX?}VHQZ!L*pmMhu<9}Tc0A;npLqP z3hK5jD#l(io=nXo;xqBNVj^*2ZvMh-;!Hdq|6ueWOs`NY3}F6$q$(B{0|5v?00Izz00bZa0SG_<0uXQsIP-t)_y5Ts z{bPdw1Rwwb2tWV=5P$##AOHafK;Q@qXz%}H{(pok7?%bC2tWV=5P$##AOHafKmY;| zAOhO^|NSEmsmO05kH`;f5P$##AOHafKmY;|fB*y_0D&F~?7jV2WpJDt>w7Rx-N>#o z=V!T<)$Hg&mRrs)tlngeG@+(cPsQ0=x7uZ$OC`@~Su2Iy5=UMzD~n=% zM_S)xr{mMDB{ApI@q`v%%IAtV*|NAn;z{V4cw&BbE|p9t=Mw4kY%0~}*mNS9>~I#d z##u~J!8vMd@ExjDFN+naE|!_B*w~PT=BBisU(R#eR{GS}))q`rW3>B-1wmFtMH1@T z0bkok;P?M~*sM`B1Rwwb2tWV=5P$##AOHafK;V!F;P?LzNk!l^AOHafKmY;|fB*y_ z009U<00KP}!2AC_#6{5%fB*y_009U<00Izz00bZafkPsI_x}$`Mc_0b009U<00Izz z00bZa0SG_<0zDMK`~N+}MbQv|00bZa0SG_<0uX=z1Rwx`Ln46r{~@UeoCX9S009U< z00Izz00bZa0SG{#hXR=Y_YfCFLjVF0fB*y_009U<00Izz00a(+fHwc9BY&aD7d8k$ z00Izz00bZa0SG_<0uX=z1YSu2h7OKTMEeE@6{*&&iaT4k`3(3Gb$YXgc|-~1XN&ipjB3-nO!O!jjk!OL6-6N0-0SG_<0ub;B?CpetQI4Y@ z+);(wRdJ)awZWGKRp8ef)$LlH{HQCcEJ$@#>6RB>C~%n~#}qT~FLDgqEtF-Z!VJSY z1@m>GCbCSC`+kwhFO%=m;^MR~M7`H+8**2uZkrkUAvLMak^+{vLTg&~Q zMY>C3-(Ox{X-T>Sb|Pxl@%?kr%erhQ7~t6XX-!a)Cl^Q*-|hJ3ygQ`zF3INrDnQq`(R8jvf8DR5V~0+(OlRvGWR;kijk4k9oR3I3go1^z7{greZr75VAP~~N@!YlWT`tzj) z+f<=1#H>X{R$Le8z2USxsmwB@BDpMAuru8TI#xs85bDx>a$44%V8^!PlSfEHE|c?a zTeKct7AsO+>Qq=u9fo6j;elZE^l5tknxiD{l18SG6Ec9D-D7QRoZJqLTR*Rswx_4s z$98n=em%;1qwT}Yevou-W2DLXUY)eMcaH_4*H3qCb@oA4n{yHZ`*k`Pot&gU{)S^| z-Ho1Gv#*V#71%-1ZWS#pNbh5uThI>Fg{3liTu2p3l%2;c+tLh}%iNVrX|c#mPU=Th z1x4l8H--8Jc>-J}y-_i@#MuSg(Nm;9tEl7&Ff}=`HZxJ1nJ5=0u3nf}x-e1sU~+n) zR3Oi#BCj31TFfl1#Ej<<=~;NvSzKN&kseNeJuT&)H-+>bz8I3?jq%@O-e;#GOJ(z^ z>AQV-E2-DBmZcr{y(q7X2f#}j?*Y;vnE1&eVfPr-1iyvxqQdUX*WYjUJ zjXZ{f|E7kH%}^s>j{NBO&%=+xYvGZ>YlD5qejogTBx8dB1RwwbP2i(%j|HRST>B@yC-2XG?r6f>X*%PjmoeeE z-)PkMs(4qdwhn}ieqX;Nen%>+o83b;MQLMG?GU4LbSp#;=vqWm+7+w(F0V-UwIPo_ zG_eOf`nehr_c}*(o87o1$gttkn}O)+OxIz9tA@?FjVmQe+Os|a$i5zk-kf{kf)r70 zEsQ6rZ|uMKnm$*0c&crq(_;KBOG|sbd*jnWp4$N3Y`tuZ{?VefKH{5l8(}YP;7BIO zuEDOiMR`WOzAX3fw>8Ti2isZUSZ*{By*1ypEm)r)ZLUZu=9tu<{8}#ZzZYeUQT;`Y zJ_w8iqx19h$ETf!FxiiQQjw$ z!CZb#TV=9t1LJwzKHYtbc3rp83bCh3PG7>3XWqUz9E@JPNIxViqSopt`7m=qpS_ah z7*Q7M>&BvKYw}2&#?TsNHV3;fzSHu6d+Etud?bsG&I**1DmQk>Y71|!j+)CkI+ix^ zC4F?QN&1VIK1D8*fd}dH+YLT^4=gi|=WP(F+%mPb@}{J+IR`KEO@WZBo|o|6FLuC- zmmQ9+91m#ANo#s7n`^O(cxPLyUEl7or&fKLQ`+EZnj9J1J29lMs_bvIwjBM@VO`vD zPkF}5xs_+Ojkccd=A9`O$%pHcf|kh2g>g@m+qNTn5oA(p!K9-zR!(XpS{3SLK`!&# zvSci}_--=qlBu2Pl&F$5rL()Ws#&CYm-xyG$ME<6nA$)90uX=z1Rwwb2tWV=5P$## zj=li?{{QILF|H5-5P$##AOHafKmY;|fB*y_puPV;5Luuizl%JIY?J?Bg8&2|009U< z00Izz00bZa0SG{#oj`mbKwY6v(b*M>J|G{ZPtzpo$6tI$&VTsjNCTWeWz&> g9y%HFoXXXOKgP;96{n4K=0>Z>RumAu6 literal 0 HcmV?d00001 From 2105ea2404e0463d75126a3180a13b62c44314db Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 6 Oct 2023 10:10:47 +0200 Subject: [PATCH 09/36] update tests --- test/test_init_db.py | 86 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/test/test_init_db.py b/test/test_init_db.py index 1e90475..83b974e 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -1,6 +1,7 @@ import os import shutil import sqlite3 +import pytest import psycopg2 @@ -12,13 +13,15 @@ MerginClient, ) -from dbsync import ( - dbsync_pull, - dbsync_push, -) +from dbsync import dbsync_pull, dbsync_push, dbsync_init, config, DbSyncError from .conftest import ( + GEODIFF_EXE, + API_USER, + USER_PWD, + SERVER_URL, DB_CONNINFO, + WORKSPACE, init_sync_from_db, name_project_dir, name_db_schema_main, @@ -26,6 +29,9 @@ complete_project_name, path_sync_gpkg, path_test_data, + cleanup, + cleanup_db, + name_project_sync_dir, ) @@ -72,7 +78,7 @@ def test_init_from_db( assert gpkg_cur.fetchone()[0] == 3 -def test_local_changes_to_db( +def test_with_local_changes( mc: MerginClient, ): project_name = "test_mergin_changes_to_db" @@ -110,7 +116,7 @@ def test_local_changes_to_db( assert cur.fetchone()[0] == 4 -def test_db_changes_to_mergin( +def test_with_db_changes( mc: MerginClient, ): project_name = "test_db_changes_mergin" @@ -140,3 +146,71 @@ def test_db_changes_to_mergin( gpkg_cur = gpkg_conn.cursor() gpkg_cur.execute("SELECT COUNT(*) FROM simple") assert gpkg_cur.fetchone()[0] == 4 + + +def test_missing_table(mc: MerginClient): + project_name = "test_db_missing_table" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + sync_project_dir = name_project_sync_dir(project_name) # used by dbsync + + conn = psycopg2.connect(DB_CONNINFO) + + cleanup( + mc, + project_full_name, + [ + project_dir, + sync_project_dir, + ], + ) + cleanup_db( + conn, + db_schema_base, + db_schema_main, + ) + + with open( + path_test_data("create_base.sql"), + encoding="utf-8", + ) as file: + base_table_dump = file.read() + + base_table_dump = base_table_dump.replace("default-schema-name", db_schema_main) + base_table_dump = base_table_dump.replace("default-table-name", "simple") + + cur = conn.cursor() + cur.execute(base_table_dump) + + # prepare a new Mergin Maps project + mc.create_project( + project_name, + namespace=WORKSPACE, + ) + + connection = { + "driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": "non_existing_schema", + "base": db_schema_base, + "mergin_project": project_full_name, + "sync_file": path_sync_gpkg(), + } + + config.update( + { + "GEODIFF_EXE": GEODIFF_EXE, + "WORKING_DIR": sync_project_dir, + "MERGIN__USERNAME": API_USER, + "MERGIN__PASSWORD": USER_PWD, + "MERGIN__URL": SERVER_URL, + "CONNECTIONS": [connection], + "init_from": "db", + } + ) + + with pytest.raises(DbSyncError) as err: + dbsync_init(mc) + assert "The 'modified' schema does not exist" in str(err.value) From d8efbec81c75de4b90fd3011a36b98f81f64d358 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 6 Oct 2023 10:11:00 +0200 Subject: [PATCH 10/36] update description --- test/conftest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index d893f51..3311a92 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -238,9 +238,9 @@ def path_sync_gpkg() -> str: def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[str] = None): """ Initialize sync from given database file: - - (re)create Mergin Maps project with the file - - (re)create local project working directory and sync directory - - configure DB sync and let it do the init (make copies to the database) + - prepare schema with simple table + - create MM project + - configure DB sync and let it do the init """ if ignored_tables is None: ignored_tables = [] From 89b2d90fb8906979f09ff7bd9bdf9595b78a5996 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 6 Oct 2023 10:47:03 +0200 Subject: [PATCH 11/36] use functions to make test more general --- test/conftest.py | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 3311a92..be2fef1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,20 +25,18 @@ ) -def _reset_config( - project_name: str = "mergin", -): +def _reset_config(project_name: str = "mergin", init_from: str = "gpkg"): """helper to reset config settings to ensure valid config""" - db_schema_main = project_name + "_main" - db_schema_base = project_name + "_base" - full_project_name = WORKSPACE + "/" + project_name + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) + full_project_name = complete_project_name(project_name) config.update( { "MERGIN__USERNAME": API_USER, "MERGIN__PASSWORD": USER_PWD, "MERGIN__URL": SERVER_URL, - "init_from": "gpkg", + "init_from": init_from, "CONNECTIONS": [ { "driver": "postgres", @@ -46,7 +44,7 @@ def _reset_config( "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, - "sync_file": "test_sync.gpkg", + "sync_file": filename_sync_gpkg(), } ], } @@ -89,17 +87,11 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables - (re)create local project working directory and sync directory - configure DB sync and let it do the init (make copies to the database) """ - full_project_name = WORKSPACE + "/" + project_name - project_dir = os.path.join( - TMP_DIR, - project_name + "_work", - ) # working directory - sync_project_dir = os.path.join( - TMP_DIR, - project_name + "_dbsync", - ) # used by dbsync - db_schema_main = project_name + "_main" - db_schema_base = project_name + "_base" + full_project_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) # working directory + sync_project_dir = name_project_sync_dir(project_name) # used by dbsync + db_schema_main = name_db_schema_main(project_name) + db_schema_base = name_db_schema_base(project_name) conn = psycopg2.connect(DB_CONNINFO) @@ -130,7 +122,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables source_gpkg_path, os.path.join( project_dir, - "test_sync.gpkg", + filename_sync_gpkg(), ), ) for extra_filepath in extra_init_files: @@ -154,7 +146,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, - "sync_file": "test_sync.gpkg", + "sync_file": filename_sync_gpkg(), "skip_tables": ignored_tables, } else: @@ -164,7 +156,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, - "sync_file": "test_sync.gpkg", + "sync_file": filename_sync_gpkg(), } config.update( @@ -231,7 +223,7 @@ def path_test_data(filename: str) -> str: ) -def path_sync_gpkg() -> str: +def filename_sync_gpkg() -> str: return "test_sync.gpkg" @@ -294,7 +286,7 @@ def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[ "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, - "sync_file": path_sync_gpkg(), + "sync_file": filename_sync_gpkg(), "skip_tables": ignored_tables, } else: @@ -304,7 +296,7 @@ def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[ "modified": db_schema_main, "base": db_schema_base, "mergin_project": full_project_name, - "sync_file": path_sync_gpkg(), + "sync_file": filename_sync_gpkg(), } config.update( From 5dbfba484b0704bc8508b39aef72041c6a2c93bf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 6 Oct 2023 10:47:36 +0200 Subject: [PATCH 12/36] skip test due to changes in MM --- test/test_basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_basic.py b/test/test_basic.py index b853cf9..1b88bfc 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -638,6 +638,7 @@ def test_with_local_changes( dbsync_status(mc) +@pytest.mark.skip(reason="Currently being reworked in MM") def test_recreated_project_ids( mc: MerginClient, ): From d21d46d3f6b3d43cf5fe4dc1ee58c3af356a47c9 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 6 Oct 2023 10:47:59 +0200 Subject: [PATCH 13/36] use functions --- test/test_init_db.py | 49 ++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/test/test_init_db.py b/test/test_init_db.py index 83b974e..160bb8d 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -27,7 +27,7 @@ name_db_schema_main, name_db_schema_base, complete_project_name, - path_sync_gpkg, + filename_sync_gpkg, path_test_data, cleanup, cleanup_db, @@ -35,27 +35,28 @@ ) -def test_init_from_db( - mc: MerginClient, -): +def test_init_from_db(mc: MerginClient, db_connection): project_name = "test_db_init" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) db_schema_main = name_db_schema_main(project_name) db_schema_base = name_db_schema_base(project_name) - path_synced_gpkg = project_dir + "/" + path_sync_gpkg() + path_synced_gpkg = project_dir + "/" + filename_sync_gpkg() init_sync_from_db(mc, project_name) # test that database schemas are created + tables are populated - conn = psycopg2.connect(DB_CONNINFO) - cur = conn.cursor() + cur = db_connection.cursor() - cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(conn)) + cur.execute( + sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_main)).as_string(db_connection) + ) assert cur.fetchone()[0] == 3 - cur.execute(sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_base)).as_string(conn)) + cur.execute( + sql.SQL("SELECT count(*) from {}.simple").format(sql.Identifier(db_schema_base)).as_string(db_connection) + ) assert cur.fetchone()[0] == 3 # download project and validate that the path synced file exist @@ -78,22 +79,18 @@ def test_init_from_db( assert gpkg_cur.fetchone()[0] == 3 -def test_with_local_changes( - mc: MerginClient, -): +def test_with_local_changes(mc: MerginClient, db_connection): project_name = "test_mergin_changes_to_db" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) db_schema_main = name_db_schema_main(project_name) db_schema_base = name_db_schema_base(project_name) - path_synced_gpkg = project_dir + "/" + path_sync_gpkg() + path_synced_gpkg = project_dir + "/" + filename_sync_gpkg() init_sync_from_db(mc, project_name) - # connecto to db - conn = psycopg2.connect(DB_CONNINFO) - cur = conn.cursor() + cur = db_connection.cursor() # check that there are 3 features prior to changes cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"') @@ -116,9 +113,7 @@ def test_with_local_changes( assert cur.fetchone()[0] == 4 -def test_with_db_changes( - mc: MerginClient, -): +def test_with_db_changes(mc: MerginClient, db_connection): project_name = "test_db_changes_mergin" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) @@ -127,9 +122,7 @@ def test_with_db_changes( init_sync_from_db(mc, project_name) - # connecto to db - conn = psycopg2.connect(DB_CONNINFO) - cur = conn.cursor() + cur = db_connection.cursor() cur.execute( f'INSERT INTO "{db_schema_main}"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES (\'0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F\', 4, \'new feature\', 4);' @@ -142,13 +135,13 @@ def test_with_db_changes( mc.download_project(project_full_name, project_dir) # look at changes in in GPKG - gpkg_conn = sqlite3.connect(project_dir + "/" + path_sync_gpkg()) + gpkg_conn = sqlite3.connect(project_dir + "/" + filename_sync_gpkg()) gpkg_cur = gpkg_conn.cursor() gpkg_cur.execute("SELECT COUNT(*) FROM simple") assert gpkg_cur.fetchone()[0] == 4 -def test_missing_table(mc: MerginClient): +def test_missing_table(mc: MerginClient, db_connection): project_name = "test_db_missing_table" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) @@ -156,8 +149,6 @@ def test_missing_table(mc: MerginClient): db_schema_base = name_db_schema_base(project_name) sync_project_dir = name_project_sync_dir(project_name) # used by dbsync - conn = psycopg2.connect(DB_CONNINFO) - cleanup( mc, project_full_name, @@ -167,7 +158,7 @@ def test_missing_table(mc: MerginClient): ], ) cleanup_db( - conn, + db_connection, db_schema_base, db_schema_main, ) @@ -181,7 +172,7 @@ def test_missing_table(mc: MerginClient): base_table_dump = base_table_dump.replace("default-schema-name", db_schema_main) base_table_dump = base_table_dump.replace("default-table-name", "simple") - cur = conn.cursor() + cur = db_connection.cursor() cur.execute(base_table_dump) # prepare a new Mergin Maps project @@ -196,7 +187,7 @@ def test_missing_table(mc: MerginClient): "modified": "non_existing_schema", "base": db_schema_base, "mergin_project": project_full_name, - "sync_file": path_sync_gpkg(), + "sync_file": filename_sync_gpkg(), } config.update( From b915dced20e55dd7493ef10b22d593a42a6403de Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 11:56:38 +0200 Subject: [PATCH 14/36] simplify --- test/conftest.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index be2fef1..f574fa1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -139,25 +139,17 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables # prepare dbsync config # patch config to fit testing purposes + connection = { + "driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": full_project_name, + "sync_file": "test_sync.gpkg", + } + if ignored_tables: - connection = { - "driver": "postgres", - "conn_info": DB_CONNINFO, - "modified": db_schema_main, - "base": db_schema_base, - "mergin_project": full_project_name, - "sync_file": filename_sync_gpkg(), - "skip_tables": ignored_tables, - } - else: - connection = { - "driver": "postgres", - "conn_info": DB_CONNINFO, - "modified": db_schema_main, - "base": db_schema_base, - "mergin_project": full_project_name, - "sync_file": filename_sync_gpkg(), - } + connection["skip_tables"] = (ignored_tables,) config.update( { From 4fe240bf1f8a3a5db3d93b374c26fa2181702cd2 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 12:01:28 +0200 Subject: [PATCH 15/36] add docstrings --- test/test_init_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_init_db.py b/test/test_init_db.py index 160bb8d..e9e4dee 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -36,6 +36,7 @@ def test_init_from_db(mc: MerginClient, db_connection): + """Test that init from db happens correctly, with the tables in sync GPKG created a populated correctly""" project_name = "test_db_init" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) @@ -80,6 +81,7 @@ def test_init_from_db(mc: MerginClient, db_connection): def test_with_local_changes(mc: MerginClient, db_connection): + """Test that after init and local changes the changes are correctly pushed to database""" project_name = "test_mergin_changes_to_db" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) @@ -114,6 +116,7 @@ def test_with_local_changes(mc: MerginClient, db_connection): def test_with_db_changes(mc: MerginClient, db_connection): + """Test that after init and DB changes the changes are correctly pulled to the MM GPKG""" project_name = "test_db_changes_mergin" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) From e08b734405c90db738692c828aa310fba4bbb663 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 12:02:45 +0200 Subject: [PATCH 16/36] docstring --- test/test_init_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_init_db.py b/test/test_init_db.py index e9e4dee..3a3f121 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -145,6 +145,7 @@ def test_with_db_changes(mc: MerginClient, db_connection): def test_missing_table(mc: MerginClient, db_connection): + """Test that if the schema is missing in DB the sync init raises correct DbSyncError""" project_name = "test_db_missing_table" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) From 05eaeb3cb834cdbab7e53b29b84e9ae5c124815c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 13:02:51 +0200 Subject: [PATCH 17/36] sql file as parameter --- test/conftest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index f574fa1..9c0d9e3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -219,7 +219,7 @@ def filename_sync_gpkg() -> str: return "test_sync.gpkg" -def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[str] = None): +def init_sync_from_db(mc: MerginClient, project_name: str, path_sql_file: str, ignored_tables: List[str] = None): """ Initialize sync from given database file: - prepare schema with simple table @@ -252,14 +252,11 @@ def init_sync_from_db(mc: MerginClient, project_name: str, ignored_tables: List[ ) with open( - path_test_data("create_base.sql"), + path_sql_file, encoding="utf-8", ) as file: base_table_dump = file.read() - base_table_dump = base_table_dump.replace("default-schema-name", db_schema_main) - base_table_dump = base_table_dump.replace("default-table-name", "simple") - cur = conn.cursor() cur.execute(base_table_dump) From 7e3e746812f35bcf60e86ff3637bf51181791732 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 13:03:03 +0200 Subject: [PATCH 18/36] constant schema names --- test/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9c0d9e3..f94eda3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -232,8 +232,8 @@ def init_sync_from_db(mc: MerginClient, project_name: str, path_sql_file: str, i full_project_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) # working directory sync_project_dir = name_project_sync_dir(project_name) # used by dbsync - db_schema_main = name_db_schema_main(project_name) - db_schema_base = name_db_schema_base(project_name) + db_schema_main = "test_init_from_db_main" + db_schema_base = "test_init_from_db_base" conn = psycopg2.connect(DB_CONNINFO) From 70e9f9ca4d0d5269d6b4befc5e66cb107921ce48 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 13:03:19 +0200 Subject: [PATCH 19/36] update sql files for tests --- test/test_data/create_another_schema.sql | 17 +++++++++++++++++ test/test_data/create_base.sql | 20 ++++++++++---------- 2 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 test/test_data/create_another_schema.sql diff --git a/test/test_data/create_another_schema.sql b/test/test_data/create_another_schema.sql new file mode 100644 index 0000000..4479a09 --- /dev/null +++ b/test/test_data/create_another_schema.sql @@ -0,0 +1,17 @@ +SET standard_conforming_strings = OFF; + +CREATE SCHEMA IF NOT EXISTS test_init_from_db_another; + +DROP TABLE IF EXISTS "test_init_from_db_another"."simple" CASCADE; + +CREATE TABLE "test_init_from_db_another"."simple" ( "ogc_fid" SERIAL, CONSTRAINT "base_pk" PRIMARY KEY ("ogc_fid") ); + +SELECT AddGeometryColumn('test_init_from_db_another','simple','wkb_geometry',4326,'POINT',2); +ALTER TABLE "test_init_from_db_another"."simple" ADD COLUMN "fid" NUMERIC(20,0); +ALTER TABLE "test_init_from_db_another"."simple" ADD COLUMN "name" VARCHAR; +ALTER TABLE "test_init_from_db_another"."simple" ADD COLUMN "rating" NUMERIC(10,0); + +INSERT INTO "test_init_from_db_another"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000001E78CBA1366CF1BF70E6AAC83981DD3F', 1, 'feature1', 1); +INSERT INTO "test_init_from_db_another"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E6100000F0431AAFE449D7BFF874B615E6FDE13F', 2, 'feature2', 2); +INSERT INTO "test_init_from_db_another"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F', 3, 'feature3', 3); + diff --git a/test/test_data/create_base.sql b/test/test_data/create_base.sql index 141e769..5f1958e 100644 --- a/test/test_data/create_base.sql +++ b/test/test_data/create_base.sql @@ -1,17 +1,17 @@ SET standard_conforming_strings = OFF; -CREATE SCHEMA IF NOT EXISTS default-schema-name; +CREATE SCHEMA IF NOT EXISTS test_init_from_db_main; -DROP TABLE IF EXISTS "default-schema-name"."default-table-name" CASCADE; +DROP TABLE IF EXISTS "test_init_from_db_main"."simple" CASCADE; -CREATE TABLE "default-schema-name"."default-table-name" ( "ogc_fid" SERIAL, CONSTRAINT "base_pk" PRIMARY KEY ("ogc_fid") ); +CREATE TABLE "test_init_from_db_main"."simple" ( "ogc_fid" SERIAL, CONSTRAINT "base_pk" PRIMARY KEY ("ogc_fid") ); -SELECT AddGeometryColumn('default-schema-name','default-table-name','wkb_geometry',4326,'POINT',2); -ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "fid" NUMERIC(20,0); -ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "name" VARCHAR; -ALTER TABLE "default-schema-name"."default-table-name" ADD COLUMN "rating" NUMERIC(10,0); +SELECT AddGeometryColumn('test_init_from_db_main','simple','wkb_geometry',4326,'POINT',2); +ALTER TABLE "test_init_from_db_main"."simple" ADD COLUMN "fid" NUMERIC(20,0); +ALTER TABLE "test_init_from_db_main"."simple" ADD COLUMN "name" VARCHAR; +ALTER TABLE "test_init_from_db_main"."simple" ADD COLUMN "rating" NUMERIC(10,0); -INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000001E78CBA1366CF1BF70E6AAC83981DD3F', 1, 'feature1', 1); -INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E6100000F0431AAFE449D7BFF874B615E6FDE13F', 2, 'feature2', 2); -INSERT INTO "default-schema-name"."default-table-name" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F', 3, 'feature3', 3); +INSERT INTO "test_init_from_db_main"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000001E78CBA1366CF1BF70E6AAC83981DD3F', 1, 'feature1', 1); +INSERT INTO "test_init_from_db_main"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E6100000F0431AAFE449D7BFF874B615E6FDE13F', 2, 'feature2', 2); +INSERT INTO "test_init_from_db_main"."simple" ("wkb_geometry" , "fid", "name", "rating") VALUES ('0101000020E61000009CB92A724E60E7BFE0FDF1F774B6A53F', 3, 'feature3', 3); From 412de1c3494e56ed8149cd4c2729c3ce002ba0b7 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 13:03:57 +0200 Subject: [PATCH 20/36] simplify tests according to review --- test/test_init_db.py | 87 +++++++------------------------------------- 1 file changed, 13 insertions(+), 74 deletions(-) diff --git a/test/test_init_db.py b/test/test_init_db.py index 3a3f121..bf784f2 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -13,7 +13,7 @@ MerginClient, ) -from dbsync import dbsync_pull, dbsync_push, dbsync_init, config, DbSyncError +from dbsync import dbsync_pull, dbsync_push, config, DbSyncError from .conftest import ( GEODIFF_EXE, @@ -24,13 +24,9 @@ WORKSPACE, init_sync_from_db, name_project_dir, - name_db_schema_main, - name_db_schema_base, complete_project_name, filename_sync_gpkg, path_test_data, - cleanup, - cleanup_db, name_project_sync_dir, ) @@ -40,12 +36,12 @@ def test_init_from_db(mc: MerginClient, db_connection): project_name = "test_db_init" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) - db_schema_main = name_db_schema_main(project_name) - db_schema_base = name_db_schema_base(project_name) + db_schema_main = "test_init_from_db_main" + db_schema_base = "test_init_from_db_base" path_synced_gpkg = project_dir + "/" + filename_sync_gpkg() - init_sync_from_db(mc, project_name) + init_sync_from_db(mc, project_name, path_test_data("create_base.sql")) # test that database schemas are created + tables are populated cur = db_connection.cursor() @@ -85,12 +81,12 @@ def test_with_local_changes(mc: MerginClient, db_connection): project_name = "test_mergin_changes_to_db" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) - db_schema_main = name_db_schema_main(project_name) - db_schema_base = name_db_schema_base(project_name) + db_schema_main = "test_init_from_db_main" + db_schema_base = "test_init_from_db_base" path_synced_gpkg = project_dir + "/" + filename_sync_gpkg() - init_sync_from_db(mc, project_name) + init_sync_from_db(mc, project_name, path_test_data("create_base.sql")) cur = db_connection.cursor() @@ -120,10 +116,10 @@ def test_with_db_changes(mc: MerginClient, db_connection): project_name = "test_db_changes_mergin" project_full_name = complete_project_name(project_name) project_dir = name_project_dir(project_name) - db_schema_main = name_db_schema_main(project_name) - db_schema_base = name_db_schema_base(project_name) + db_schema_main = "test_init_from_db_main" + db_schema_base = "test_init_from_db_base" - init_sync_from_db(mc, project_name) + init_sync_from_db(mc, project_name, path_test_data("create_base.sql")) cur = db_connection.cursor() @@ -144,68 +140,11 @@ def test_with_db_changes(mc: MerginClient, db_connection): assert gpkg_cur.fetchone()[0] == 4 -def test_missing_table(mc: MerginClient, db_connection): +def test_missing_table(mc: MerginClient): """Test that if the schema is missing in DB the sync init raises correct DbSyncError""" project_name = "test_db_missing_table" - project_full_name = complete_project_name(project_name) - project_dir = name_project_dir(project_name) - db_schema_main = name_db_schema_main(project_name) - db_schema_base = name_db_schema_base(project_name) - sync_project_dir = name_project_sync_dir(project_name) # used by dbsync - - cleanup( - mc, - project_full_name, - [ - project_dir, - sync_project_dir, - ], - ) - cleanup_db( - db_connection, - db_schema_base, - db_schema_main, - ) - - with open( - path_test_data("create_base.sql"), - encoding="utf-8", - ) as file: - base_table_dump = file.read() - - base_table_dump = base_table_dump.replace("default-schema-name", db_schema_main) - base_table_dump = base_table_dump.replace("default-table-name", "simple") - - cur = db_connection.cursor() - cur.execute(base_table_dump) - - # prepare a new Mergin Maps project - mc.create_project( - project_name, - namespace=WORKSPACE, - ) - - connection = { - "driver": "postgres", - "conn_info": DB_CONNINFO, - "modified": "non_existing_schema", - "base": db_schema_base, - "mergin_project": project_full_name, - "sync_file": filename_sync_gpkg(), - } - - config.update( - { - "GEODIFF_EXE": GEODIFF_EXE, - "WORKING_DIR": sync_project_dir, - "MERGIN__USERNAME": API_USER, - "MERGIN__PASSWORD": USER_PWD, - "MERGIN__URL": SERVER_URL, - "CONNECTIONS": [connection], - "init_from": "db", - } - ) with pytest.raises(DbSyncError) as err: - dbsync_init(mc) + init_sync_from_db(mc, project_name, path_test_data("create_another_schema.sql")) + assert "The 'modified' schema does not exist" in str(err.value) From 5169f7fe323d0944a2909e986d9e816173ba3a9b Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 11 Oct 2023 13:04:10 +0200 Subject: [PATCH 21/36] do not skip test --- test/test_basic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_basic.py b/test/test_basic.py index 1b88bfc..b853cf9 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -638,7 +638,6 @@ def test_with_local_changes( dbsync_status(mc) -@pytest.mark.skip(reason="Currently being reworked in MM") def test_recreated_project_ids( mc: MerginClient, ): From 4cc4bb8ff690fa8e192a8ec8b2a8f817055654eb Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 14:52:36 +0100 Subject: [PATCH 22/36] install from requirements files --- .github/workflows/tests_mergin_db_sync.yaml | 8 +++++--- requirements-dev.txt | 3 +++ requirements.txt | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 requirements-dev.txt diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 6d6d5ee..a64638f 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -46,14 +46,16 @@ jobs: - name: Check Geodiff version run: geodiff version + + - name: Checkout + uses: actions/checkout@v4 - name: Install Python dependencies run: | python3 -m pip install --upgrade pip - python3 -m pip install mergin-client pytest pytest-cov dynaconf psycopg2 + python3 -m pip install -r requirements.txt + python3 -m pip install -r requirements-dev.txt - - name: Checkout - uses: actions/checkout@v2 - name: Run tests run: | diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..6144456 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +pytest +pytest-cov +psycopg2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f6ceeb5..f527d24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -mergin-client>=0.8.3 +mergin-client>=0.9.0 dynaconf>=3.1 From 5cb924fd3c1ac0133cdab8abec6b56ad2013c5f1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 16:06:30 +0100 Subject: [PATCH 23/36] remove --- requirements-dev.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6144456..98837e0 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,2 @@ -pytest -pytest-cov -psycopg2 \ No newline at end of file +pytest>=6.2 +pytest-cov>=3.0 \ No newline at end of file From d659e93a5e7632ca325c5502d0ad7515b892a6a6 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 16:06:40 +0100 Subject: [PATCH 24/36] require versions --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f527d24..7fad60f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -mergin-client>=0.9.0 +mergin-client==0.9.0 dynaconf>=3.1 +psycopg2>=2.9 \ No newline at end of file From 5d956e27264186450fbf3a2ccab1baa94a3c0b28 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 16:08:04 +0100 Subject: [PATCH 25/36] files that trigger testing --- .github/workflows/tests_mergin_db_sync.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index a64638f..50a0e53 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -5,6 +5,9 @@ on: paths: - "test/**" - "**.py" + - "requirements.txt" + - "requirements-dev.txt" + - "pyproject.toml" - ".github/workflows/tests_mergin_db_sync.yaml" env: From 57bcc9384952dc04af134560c597af97baf031d0 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 16:19:04 +0100 Subject: [PATCH 26/36] run only basic tests --- .github/workflows/tests_mergin_db_sync.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 50a0e53..08180ff 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -59,10 +59,13 @@ jobs: python3 -m pip install -r requirements.txt python3 -m pip install -r requirements-dev.txt - - name: Run tests run: | - pytest test --cov=. --cov-report=term-missing:skip-covered -vv + pytest test/test_basic.py --cov=. --cov-report=term-missing:skip-covered -vv + + # - name: Run tests + # run: | + # pytest test --cov=. --cov-report=term-missing:skip-covered -vv - name: Check files using the black formatter uses: rickstaa/action-black@v1 From 82e07a9d637f718a27b5868ceecb18fde32a4f88 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 17:13:47 +0100 Subject: [PATCH 27/36] fix project info obtaining using updated mergin-client api --- dbsync.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbsync.py b/dbsync.py index 8472536..8fb7fd6 100644 --- a/dbsync.py +++ b/dbsync.py @@ -754,12 +754,11 @@ def pull(conn_cfg, mc): # Make sure that local project ID (if available) is the same as on the server _validate_local_project_id(mp, mc) - project_path = mp.metadata["name"] - local_version = mp.metadata["version"] + local_version = mp.version() try: - projects = mc.get_projects_by_names([project_path]) - server_version = projects[project_path]["version"] + projects = mc.get_projects_by_names([mp.project_full_name()]) + server_version = projects[mp.project_full_name()]["version"] except ClientError as e: # this could be e.g. DNS error raise DbSyncError("Mergin Maps client error: " + str(e)) From 3e7f934100f8031b411e2cc473e7ccf56b0cbd53 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 22:56:48 +0100 Subject: [PATCH 28/36] correctly access project info --- dbsync.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dbsync.py b/dbsync.py index 8fb7fd6..89468f5 100644 --- a/dbsync.py +++ b/dbsync.py @@ -899,8 +899,8 @@ def status(conn_cfg, mc): mp.set_tables_to_skip(ignored_tables) if mp.geodiff is None: raise DbSyncError("Mergin Maps client installation problem: geodiff not available") - project_path = mp.metadata["name"] - local_version = mp.metadata["version"] + project_path = mp.project_full_name() + local_version = mp.version() logging.debug("Checking status...") try: server_info = mc.project_info( @@ -1008,12 +1008,11 @@ def push(conn_cfg, mc): # Make sure that local project ID (if available) is the same as on the server _validate_local_project_id(mp, mc) - project_path = mp.metadata["name"] - local_version = mp.metadata["version"] + local_version = mp.version() try: - projects = mc.get_projects_by_names([project_path]) - server_version = projects[project_path]["version"] + projects = mc.get_projects_by_names([mp.project_full_name()]) + server_version = projects[mp.project_full_name()]["version"] except ClientError as e: # this could be e.g. DNS error raise DbSyncError("Mergin Maps client error: " + str(e)) From 72a99f6fe2e066769eac233bc73ab3a09eff617e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 22:57:29 +0100 Subject: [PATCH 29/36] need to actively read project metadata, otherwise old info is stored --- dbsync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbsync.py b/dbsync.py index 89468f5..2e8b31f 100644 --- a/dbsync.py +++ b/dbsync.py @@ -538,6 +538,7 @@ def _get_mergin_project( """ if work_path not in cached_mergin_project_objects: cached_mergin_project_objects[work_path] = MerginProject(work_path) + cached_mergin_project_objects[work_path]._read_metadata() return cached_mergin_project_objects[work_path] From c16e04a6a0b8b4ff8f805ed9785e585f5ea048c1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 22:58:01 +0100 Subject: [PATCH 30/36] correctly convert to list --- config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.py b/config.py index e2380cc..47b539b 100644 --- a/config.py +++ b/config.py @@ -13,6 +13,7 @@ import tempfile from dynaconf import Dynaconf +import dynaconf from smtp_functions import create_connection_and_log_user @@ -158,6 +159,8 @@ def get_ignored_tables( connection.skip_tables, list, ): + if isinstance(connection.skip_tables, dynaconf.vendor.box.box_list.BoxList): + return connection.skip_tables.to_list()[0] return connection.skip_tables else: return [] From fa64ffb1e408331559787f77287c629a62d0c75b Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Fri, 8 Dec 2023 23:01:51 +0100 Subject: [PATCH 31/36] Revert "run only basic tests" This reverts commit 57bcc9384952dc04af134560c597af97baf031d0. --- .github/workflows/tests_mergin_db_sync.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests_mergin_db_sync.yaml b/.github/workflows/tests_mergin_db_sync.yaml index 08180ff..50a0e53 100644 --- a/.github/workflows/tests_mergin_db_sync.yaml +++ b/.github/workflows/tests_mergin_db_sync.yaml @@ -59,13 +59,10 @@ jobs: python3 -m pip install -r requirements.txt python3 -m pip install -r requirements-dev.txt + - name: Run tests run: | - pytest test/test_basic.py --cov=. --cov-report=term-missing:skip-covered -vv - - # - name: Run tests - # run: | - # pytest test --cov=. --cov-report=term-missing:skip-covered -vv + pytest test --cov=. --cov-report=term-missing:skip-covered -vv - name: Check files using the black formatter uses: rickstaa/action-black@v1 From fe907a3f191be4548c463fbd7665d89f0f88d131 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Sat, 9 Dec 2023 13:36:50 +0100 Subject: [PATCH 32/36] removed deprecated metadata calls --- dbsync.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/dbsync.py b/dbsync.py index 2e8b31f..d36fd6a 100644 --- a/dbsync.py +++ b/dbsync.py @@ -525,9 +525,7 @@ def _print_mergin_changes( cached_mergin_project_objects = {} -def _get_mergin_project( - work_path, -): +def _get_mergin_project(work_path) -> MerginProject: """ Returns a cached MerginProject object or creates one if it does not exist yet. This is to avoid creating many of these objects (e.g. every pull/push) because it does @@ -542,20 +540,16 @@ def _get_mergin_project( return cached_mergin_project_objects[work_path] -def _get_project_version( - work_path, -): +def _get_project_version(work_path) -> str: """Returns the current version of the project""" mp = _get_mergin_project(work_path) - return mp.metadata["version"] + return mp.version() -def _get_project_id( - mp, -): +def _get_project_id(mp: MerginProject): """Returns the project ID""" try: - project_id = uuid.UUID(mp.metadata["project_id"]) + project_id = uuid.UUID(mp.project_id()) except ( KeyError, ValueError, @@ -637,10 +631,9 @@ def _validate_local_project_id( local_project_id = _get_project_id(mp) if local_project_id is None: return - project_path = mp.metadata["name"] if server_info is None: try: - server_info = mc.project_info(project_path) + server_info = mc.project_info(mp.project_full_name()) except ClientError as e: raise DbSyncError("Mergin Maps client error: " + str(e)) @@ -719,7 +712,7 @@ def revert_local_changes( mp.dir, update_delete_file, update_delete_filepath, - mp.metadata["version"], + mp.version(), ) except ClientError as e: raise DbSyncError("Mergin Maps client error: " + str(e)) From 93bda813cdaf1a35602a2bbf99b64088e7aa81ba Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Sat, 9 Dec 2023 13:38:35 +0100 Subject: [PATCH 33/36] use full project name --- test/conftest.py | 11 +++-------- test/test_basic.py | 6 ++---- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index f94eda3..45bd5d1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -110,10 +110,8 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables ) # prepare a new Mergin Maps project - mc.create_project( - project_name, - namespace=WORKSPACE, - ) + mc.create_project(full_project_name) + mc.download_project( full_project_name, project_dir, @@ -261,10 +259,7 @@ def init_sync_from_db(mc: MerginClient, project_name: str, path_sql_file: str, i cur.execute(base_table_dump) # prepare a new Mergin Maps project - mc.create_project( - project_name, - namespace=WORKSPACE, - ) + mc.create_project(full_project_name) # prepare dbsync config # patch config to fit testing purposes diff --git a/test/test_basic.py b/test/test_basic.py index b853cf9..64220a3 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -659,10 +659,8 @@ def test_recreated_project_ids( # delete remote project mc.delete_project_now(full_project_name) # recreate project with the same name - mc.create_project( - project_name, - namespace=WORKSPACE, - ) + mc.create_project(full_project_name) + # comparing project IDs after recreating it with the same name mp = _get_mergin_project(project_dir) local_project_id = _get_project_id(mp) From 04d0a5191f5f1de7450d71e023cdc5a2321e4baa Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Sat, 9 Dec 2023 13:54:19 +0100 Subject: [PATCH 34/36] fix skip tables --- config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config.py b/config.py index 47b539b..f1d84c0 100644 --- a/config.py +++ b/config.py @@ -159,8 +159,10 @@ def get_ignored_tables( connection.skip_tables, list, ): - if isinstance(connection.skip_tables, dynaconf.vendor.box.box_list.BoxList): - return connection.skip_tables.to_list()[0] + if len(connection.skip_tables) < 1: + return [] + elif isinstance(connection.skip_tables, dynaconf.vendor.box.box_list.BoxList): + return connection.skip_tables.to_list() return connection.skip_tables else: return [] From cfabb8b916f10dc57d1a32a9498cc4e04012628e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Sat, 9 Dec 2023 14:07:57 +0100 Subject: [PATCH 35/36] fix ignored tables setup --- test/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 45bd5d1..49f9af5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -147,7 +147,10 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables } if ignored_tables: - connection["skip_tables"] = (ignored_tables,) + if isinstance(ignored_tables, str): + connection["skip_tables"] = [ignored_tables] + elif isinstance(ignored_tables, list): + connection["skip_tables"] = ignored_tables config.update( { From 59cf6c5f924d648f82d140f2ee5b0dcd90812bb2 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Sat, 9 Dec 2023 14:57:24 +0100 Subject: [PATCH 36/36] install using requirements.txt --- .github/workflows/build_windows.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_windows.yaml b/.github/workflows/build_windows.yaml index 6b7311b..e6d8b37 100644 --- a/.github/workflows/build_windows.yaml +++ b/.github/workflows/build_windows.yaml @@ -32,7 +32,8 @@ jobs: - name: Install dependencies run: | - python -m pip install dynaconf pyinstaller mergin-client psycopg2 + python -m pip install -r requirements.txt + python -m pip install pyinstaller - name: Build Binary run: |