test: add foundation for e2e testing using docker (#41)

.github/workflows/test.yml

@@ -0,0 +1,28 @@
+name: test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: docker-test
+        run: docker-compose up --abort-on-container-exit --exit-code-from tester
+      - name: docker-logs
+        run: docker-compose ps
+      - name: docker-logs-db
+        run: docker-compose logs db
+      - name: docker-logs-ledger
+        run: docker-compose logs ledger
+      - name: docker-logs-indexer
+        run: docker-compose logs indexer
+      - name: docker-logs-tester
+        run: docker-compose logs tester

README.md

@@ -0,0 +1,31 @@
+# indexer
+
+Blockchain indexer and database
+
+## E2E Testing
+
+This project uses docker for e2e testing.
+
+Build docker containers (this only needs to be run once or after making changes):
+
+```sh
+docker-compose build
+```
+
+Run docker containers (and continue running non-tester containers after test scripts):
+
+```sh
+docker-compose up
+```
+
+Run docker containers (and stop all containers after test scripts):
+
+```sh
+docker-compose up --abort-on-container-exit --exit-code-from tester
+```
+
+Stop and remove containers (clean up previous run before running again):
+
+```sh
+docker-compose down
+```

docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  db:
+    image: postgres:14
+    container_name: db_container
+    restart: always
+    user: postgres
+    environment:
+      POSTGRES_PASSWORD: password
+    network_mode: host
+    expose:
+      - 5432
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 1s
+      timeout: 10s
+      retries: 100
+
+  ledger:
+    build:
+      context: .
+      dockerfile: docker/ledger.Dockerfile
+    container_name: ledger_container
+    entrypoint: ["/bin/sh", "-c", "./scripts/ledger_start.sh"]
+    network_mode: host
+    expose:
+      - 1317
+      - 26657
+    healthcheck:
+      test: ["CMD-SHELL", "curl -f http://127.0.0.1:26657 || exit 1"]
+      interval: 1s
+      timeout: 10s
+      retries: 100
+
+  indexer:
+    build:
+      context: .
+      dockerfile: docker/indexer.Dockerfile
+    container_name: indexer_container
+    environment:
+      DATABASE_URL: postgres://postgres:password@localhost:5432
+      REGEN_API: http://localhost:1317
+      REGEN_RPC: http://localhost:26657
+    entrypoint: ["/bin/sh", "-c", "./docker/scripts/indexer_start.sh"]
+    network_mode: host
+    depends_on:
+      db:
+        condition: service_healthy
+      ledger:
+        condition: service_healthy
+
+  tester:
+    build:
+      context: .
+      dockerfile: docker/tester.Dockerfile
+    container_name: tester_container
+    entrypoint: ["/bin/sh", "-c", "./scripts/tester_start.sh"]
+    network_mode: host
+    depends_on:
+      db:
+        condition: service_healthy
+      ledger:
+        condition: service_healthy
+      indexer:
+        condition: service_started

docker/data/ledger_group.json

@@ -0,0 +1,55 @@
+{
+  "group_members": [
+    {
+      "group_id": "1",
+      "member": {
+        "added_at": "2023-01-01T00:00:00Z",
+        "address": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk",
+        "metadata": "",
+        "weight": "1"
+      }
+    },
+    {
+      "group_id": "1",
+      "member": {
+        "added_at": "2023-01-01T00:00:00Z",
+        "address": "regen14v5z5yyl5unnyu6q3ele8ze9jev6y0m7tx6gct",
+        "metadata": "",
+        "weight": "1"
+      }
+    }
+  ],
+  "group_policies": [
+    {
+      "address": "regen1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzs475lmr",
+      "admin": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk",
+      "created_at": "2023-01-01T00:00:00Z",
+      "decision_policy": {
+        "@type": "/cosmos.group.v1.ThresholdDecisionPolicy",
+        "threshold": "1",
+        "windows": {
+          "min_execution_period": "0s",
+          "voting_period": "20s"
+        }
+      },
+      "group_id": "1",
+      "metadata": "",
+      "version": "1"
+    }
+  ],
+  "group_policy_seq": "1",
+  "group_seq": "1",
+  "groups": [
+    {
+      "admin": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk",
+      "created_at": "2023-01-01T00:00:00Z",
+      "id": "1",
+      "metadata": "",
+      "total_weight": "2",
+      "version": "1"
+    }
+  ],
+  "proposal_seq": "0",
+  "proposals": [],
+  "votes": []
+}

docker/indexer.Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9
+
+# Install dependencies
+RUN apt-get update
+RUN apt-get install libpq-dev postgresql-client -y
+
+# Set working directory
+WORKDIR /home/indexer
+
+# Copy source code
+COPY . .
+
+# Install python dependencies
+RUN pip install poetry
+RUN pip install load_dotenv
+RUN pip install psycopg2
+RUN pip install sentry_sdk
+RUN pip install tenacity
+
+# Install indexer
+RUN poetry install

docker/ledger.Dockerfile

@@ -0,0 +1,67 @@
+FROM golang:1.19
+
+# Install dependencies
+RUN apt-get update
+RUN apt-get install jq -y
+
+# Set ledger version
+ENV GIT_CHECKOUT='v5.1.2'
+
+# Clone regen ledger
+RUN git clone https://github.com/regen-network/regen-ledger/ /home/ledger
+
+# Set working directory
+WORKDIR /home/ledger
+
+# Use provided version
+RUN git checkout $GIT_CHECKOUT
+
+# Build regen binary
+RUN make install
+
+# Setup moniker, chain, homedir
+RUN regen --chain-id regen-local init validator
+
+# Set configuration
+RUN regen config chain-id regen-local
+RUN regen config keyring-backend test
+
+# Update stake to uregen
+RUN sed -i "s/stake/uregen/g" /root/.regen/config/genesis.json
+
+# Add accounts
+RUN printf "trouble alarm laptop turn call stem lend brown play planet grocery survey smooth seed describe hood praise whale smile repeat dry sauce front future\n\n" | regen keys --keyring-backend test add validator -i
+RUN printf "cool trust waste core unusual report duck amazing fault juice wish century across ghost cigar diary correct draw glimpse face crush rapid quit equip\n\n" | regen keys --keyring-backend test add user1 -i
+RUN printf "music debris chicken erode flag law demise over fall always put bounce ring school dumb ivory spin saddle ostrich better seminar heart beach kingdom\n\n" | regen keys --keyring-backend test add user2 -i
+
+# Set up validator
+RUN regen add-genesis-account validator 1000000000uregen --keyring-backend test
+RUN regen gentx validator 1000000uregen
+
+# Set up user acounts
+RUN regen add-genesis-account user1 1000000000uregen --keyring-backend test
+RUN regen add-genesis-account user2 1000000000uregen --keyring-backend test
+
+# Prepare genesis file
+RUN regen collect-gentxs
+
+# Set minimum gas price
+RUN sed -i "s/minimum-gas-prices = \"\"/minimum-gas-prices = \"0uregen\"/" /root/.regen/config/app.toml
+
+# Set cors allow all origins
+RUN sed -i "s/cors_allowed_origins = \[\]/cors_allowed_origins = [\"*\"]/" /root/.regen/config/config.toml
+
+# Copy genesis state files
+COPY docker/data /home/ledger/data
+
+# Add group state to genesis
+RUN jq '.app_state.group |= . + input' /root/.regen/config/genesis.json /home/ledger/data/ledger_group.json > genesis-tmp.json
+
+# Overwrite genesis file with updated genesis file
+RUN mv -f genesis-tmp.json /root/.regen/config/genesis.json
+
+# Copy regen start script
+COPY docker/scripts/ledger_start.sh /home/ledger/scripts/
+
+# Make start script executable
+RUN ["chmod", "+x", "/home/ledger/scripts/ledger_start.sh"]

docker/scripts/indexer_init.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# run migrations
+(cd sql && ./run_all_migrations.sh)
+
+# workaround for indexer starting with new chain
+psql "$DATABASE_URL" -c "INSERT INTO chain (
+  num,
+  chain_id
+) VALUES (
+  1,
+  'regen-local'
+)"
+psql "$DATABASE_URL" -c "INSERT INTO block (
+  chain_num,
+  height,
+  data,
+  time
+) VALUES (
+  1,
+  0,
+  '{}',
+  now()
+)"

docker/scripts/indexer_start.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+INDEXER_INITIALIZED="INDEXER_INITIALIZED"
+
+# initialize indexer if not yet initialized
+if [ ! -e $INDEXER_INITIALIZED ]; then
+
+  # set indexer initialized
+  touch $INDEXER_INITIALIZED
+
+  echo "First start, running init script..."
+
+  # run indexer init script
+  /home/indexer/docker/scripts/indexer_init.sh
+fi
+
+# start indexer
+python main.py

docker/scripts/ledger_start.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# start regen with api and unsafe CORS enabled
+regen start --api.enable=true --api.enabled-unsafe-cors=true

docker/scripts/tester_start.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# wait for indexer to start
+sleep 5
+
+# run tester test scripts
+/home/tester/scripts/test_index_proposals.sh
+/home/tester/scripts/test_index_votes.sh
+
+# exit without error
+exit 0

docker/tester.Dockerfile

@@ -0,0 +1,52 @@
+FROM golang:1.19
+
+# Install dependencies
+RUN apt-get update
+RUN apt-get install jq libpq-dev postgresql-client -y
+
+# Set ledger version
+ENV GIT_CHECKOUT='v5.1.2'
+
+# Set database url
+ENV DATABASE_URL='postgres://postgres:password@localhost:5432/postgres'
+
+# Set test addresses
+ENV TEST_USER_ADDRESS_1=regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk
+ENV TEST_USER_ADDRESS_2=regen14v5z5yyl5unnyu6q3ele8ze9jev6y0m7tx6gct
+ENV TEST_POLICY_ADDRESS_1=regen1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzs475lmr
+
+# Set transaction flags
+ENV REGEN_TX_FLAGS="--keyring-backend test --chain-id regen-local --yes"
+
+# Clone regen ledger
+RUN git clone https://github.com/regen-network/regen-ledger/ /home/tester
+
+# Set working directory
+WORKDIR /home/tester
+
+# Use provided version
+RUN git checkout $GIT_CHECKOUT
+
+# Build regen binary
+RUN make install
+
+# Set configuration
+RUN regen config chain-id regen-local
+RUN regen config keyring-backend test
+
+# Add accounts
+RUN printf "cool trust waste core unusual report duck amazing fault juice wish century across ghost cigar diary correct draw glimpse face crush rapid quit equip\n\n" | regen keys --keyring-backend test add user1 -i
+RUN printf "music debris chicken erode flag law demise over fall always put bounce ring school dumb ivory spin saddle ostrich better seminar heart beach kingdom\n\n" | regen keys --keyring-backend test add user2 -i
+
+# Copy tester start script
+COPY docker/scripts/tester_start.sh /home/tester/scripts/
+
+# Copy tester test scripts
+COPY docker/tester/ /home/tester/scripts/
+
+# Make start script executable
+RUN ["chmod", "+x", "/home/tester/scripts/tester_start.sh"]
+
+# Make test scripts executable
+RUN ["chmod", "+x", "/home/tester/scripts/test_index_proposals.sh"]
+RUN ["chmod", "+x", "/home/tester/scripts/test_index_votes.sh"]