feat: APP-353 retirements multiple batches (#60)

.github/workflows/test.yml

@@ -12,14 +12,14 @@ jobs:
       - name: checkout
         uses: actions/checkout@v3
       - name: docker-test
-        run: docker-compose up --abort-on-container-exit --exit-code-from tester
+        run: docker compose up --abort-on-container-exit --exit-code-from tester
       - name: docker-logs
-        run: docker-compose ps
+        run: docker compose ps
       - name: docker-logs-db
-        run: docker-compose logs db
+        run: docker compose logs db
       - name: docker-logs-ledger
-        run: docker-compose logs ledger
+        run: docker compose logs ledger
       - name: docker-logs-indexer
-        run: docker-compose logs indexer
+        run: docker compose logs indexer
       - name: docker-logs-tester
-        run: docker-compose logs tester
+        run: docker compose logs tester

index_retires.py

@@ -29,20 +29,34 @@ def _index_retires(pg_conn, _client, _chain_num):
             normalize["chain_num"] = chain_num
             normalize["timestamp"] = timestamp
             normalize["tx_hash"] = tx_hash
+            normalize["batch_denoms"] = []
+            normalize["amount"] = 0
+
             for entry in event:
                 (_, _, _, _, key, value, _, _, _) = entry
                 value = value.strip('"')
                 if "v1alpha1.EventRetire" in entry[0]:
                     if key == "amount":
-                        normalize["amount"] = value
+                        normalize["amount"] = normalize["amount"] + float(value)
                     elif key == "batch_denom":
-                        normalize["batch_denom"] = value
+                        normalize["batch_denoms"].append(value)
+                        normalize["batch_denom"] = value # TODO remove once app fully migrated
                     elif key == "location":
                         normalize["jurisdiction"] = value
                     elif key == "retirer":
                         normalize["owner"] = value
                 elif "v1.EventRetire" in entry[0]:
-                    normalize[key] = value
+                    if key == "amount":
+                        normalize["amount"] = normalize["amount"] + float(value)
+                    elif key == "batch_denom":
+                        normalize["batch_denoms"].append(value)
+                        normalize["batch_denom"] = value # TODO remove once app fully migrated
+                    elif key == "jurisdiction":
+                        normalize["jurisdiction"] = value
+                    elif key == "owner":
+                        normalize["owner"] = value
+                    elif key == "reason":
+                        normalize["reason"] = value
             with pg_conn.cursor() as _cur:
                 _cur.execute(
                     """SELECT TRIM(BOTH '"' FROM (tx.data -> 'tx' -> 'body' -> 'memo')::text) AS memo FROM tx WHERE block_height=%s AND chain_num=%s AND tx_idx=%s""",
@@ -54,7 +68,8 @@ def _index_retires(pg_conn, _client, _chain_num):
                 retirement = (
                     normalize["type"],
                     normalize["amount"],
-                    normalize["batch_denom"],
+                    normalize["batch_denom"], # TODO remove once app fully migrated
+                    normalize["batch_denoms"],
                     normalize["jurisdiction"],
                     normalize["owner"],
                     normalize.get("reason", ""),
@@ -66,7 +81,7 @@ def _index_retires(pg_conn, _client, _chain_num):
                     normalize["tx_hash"],
                 )
                 _cur.execute(
-                    "INSERT INTO retirements (type, amount, batch_denom, jurisdiction, owner, reason, block_height, chain_num, tx_idx, msg_idx, timestamp, tx_hash) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
+                    "INSERT INTO retirements (type, amount, batch_denom, batch_denoms, jurisdiction, owner, reason, block_height, chain_num, tx_idx, msg_idx, timestamp, tx_hash) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
                     retirement,
                 )
                 pg_conn.commit()

migrations/README.md

@@ -16,11 +16,11 @@ $ docker-compose up --build postgres
 
 Then, you must initialize the database:
 
-```
-$ export DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer"
-$ export SHADOW_DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer_shadow"
-$ export ROOT_DATABASE_URL="postgres://postgres:postgres@localhost:5432/postgres"
-$ yarn run graphile-migrate reset --erase
+```sh
+export DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer"
+export SHADOW_DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer_shadow"
+export ROOT_DATABASE_URL="postgres://postgres:postgres@localhost:5432/postgres"
+yarn run graphile-migrate reset --erase
 ```
 
 Now, we set up a watch process that will monitor `migrations/current.sql` for your changes as well as apply them to your local database:

migrations/committed/000003.sql

@@ -0,0 +1,12 @@
+--! Previous: sha1:80425329666e5d20b5fbb7b6179f3d19acceeb76
+--! Hash: sha1:23379758e3a742625d5262c472693435196be95f
+
+ALTER TABLE public.retirements
+ADD COLUMN IF NOT EXISTS batch_denoms text[] DEFAULT ARRAY[]::text[] NOT NULL;
+
+UPDATE public.retirements
+SET batch_denoms = ARRAY[batch_denom];
+
+-- TODO later once app fully migrated
+-- ALTER TABLE public.retirements
+-- DROP COLUMN IF EXISTS batch_denom;

migrations/committed/000004.sql

@@ -0,0 +1,8 @@
+--! Previous: sha1:23379758e3a742625d5262c472693435196be95f
+--! Hash: sha1:8aa6f840171d1c0076e70fbee8dd9b56f794a03d
+
+ALTER TABLE retirements
+DROP CONSTRAINT IF EXISTS retirements_tx_hash_key;
+
+ALTER TABLE retirements
+ADD CONSTRAINT retirements_tx_hash_key UNIQUE (tx_hash);

migrations/schema_snapshot.sql

@@ -3,11 +3,12 @@
 --
 
 -- Dumped from database version 14.9 (Debian 14.9-1.pgdg110+1)
--- Dumped by pg_dump version 15.4
+-- Dumped by pg_dump version 17.0
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
+SET transaction_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
@@ -405,7 +406,8 @@ CREATE TABLE public.retirements (
     chain_num smallint NOT NULL,
     tx_idx smallint NOT NULL,
     msg_idx smallint NOT NULL,
-    tx_hash text NOT NULL
+    tx_hash text NOT NULL,
+    batch_denoms text[] DEFAULT ARRAY[]::text[] NOT NULL
 );
 
 
@@ -532,6 +534,14 @@ ALTER TABLE ONLY public.retirements
     ADD CONSTRAINT retirements_pkey PRIMARY KEY (chain_num, block_height, tx_idx, msg_idx);
 
 
+--
+-- Name: retirements retirements_tx_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
+--
+
+ALTER TABLE ONLY public.retirements
+    ADD CONSTRAINT retirements_tx_hash_key UNIQUE (tx_hash);
+
+
 --
 -- Name: tx tx_hash_key; Type: CONSTRAINT; Schema: public; Owner: -
 --