diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..3ea7c9f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,28 @@ +name: test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: docker-test + run: docker-compose up --abort-on-container-exit --exit-code-from tester + - name: docker-logs + run: docker-compose ps + - name: docker-logs-db + run: docker-compose logs db + - name: docker-logs-ledger + run: docker-compose logs ledger + - name: docker-logs-indexer + run: docker-compose logs indexer + - name: docker-logs-tester + run: docker-compose logs tester diff --git a/README.md b/README.md index e69de29..e26d95e 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,31 @@ +# indexer + +Blockchain indexer and database + +## E2E Testing + +This project uses docker for e2e testing. + +Build docker containers (this only needs to be run once or after making changes): + +```sh +docker-compose build +``` + +Run docker containers (and continue running non-tester containers after test scripts): + +```sh +docker-compose up +``` + +Run docker containers (and stop all containers after test scripts): + +```sh +docker-compose up --abort-on-container-exit --exit-code-from tester +``` + +Stop and remove containers (clean up previous run before running again): + +```sh +docker-compose down +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6e8bb8b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +services: + db: + image: postgres:14 + container_name: db_container + restart: always + user: postgres + environment: + POSTGRES_PASSWORD: password + network_mode: host + expose: + - 5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 1s + timeout: 10s + retries: 100 + + ledger: + build: + context: . + dockerfile: docker/ledger.Dockerfile + container_name: ledger_container + entrypoint: ["/bin/sh", "-c", "./scripts/ledger_start.sh"] + network_mode: host + expose: + - 1317 + - 26657 + healthcheck: + test: ["CMD-SHELL", "curl -f http://127.0.0.1:26657 || exit 1"] + interval: 1s + timeout: 10s + retries: 100 + + indexer: + build: + context: . + dockerfile: docker/indexer.Dockerfile + container_name: indexer_container + environment: + DATABASE_URL: postgres://postgres:password@localhost:5432 + REGEN_API: http://localhost:1317 + REGEN_RPC: http://localhost:26657 + entrypoint: ["/bin/sh", "-c", "./docker/scripts/indexer_start.sh"] + network_mode: host + depends_on: + db: + condition: service_healthy + ledger: + condition: service_healthy + + tester: + build: + context: . + dockerfile: docker/tester.Dockerfile + container_name: tester_container + entrypoint: ["/bin/sh", "-c", "./scripts/tester_start.sh"] + network_mode: host + depends_on: + db: + condition: service_healthy + ledger: + condition: service_healthy + indexer: + condition: service_started diff --git a/docker/data/ledger_group.json b/docker/data/ledger_group.json new file mode 100644 index 0000000..283bd87 --- /dev/null +++ b/docker/data/ledger_group.json @@ -0,0 +1,55 @@ +{ + "group_members": [ + { + "group_id": "1", + "member": { + "added_at": "2023-01-01T00:00:00Z", + "address": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk", + "metadata": "", + "weight": "1" + } + }, + { + "group_id": "1", + "member": { + "added_at": "2023-01-01T00:00:00Z", + "address": "regen14v5z5yyl5unnyu6q3ele8ze9jev6y0m7tx6gct", + "metadata": "", + "weight": "1" + } + } + ], + "group_policies": [ + { + "address": "regen1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzs475lmr", + "admin": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk", + "created_at": "2023-01-01T00:00:00Z", + "decision_policy": { + "@type": "/cosmos.group.v1.ThresholdDecisionPolicy", + "threshold": "1", + "windows": { + "min_execution_period": "0s", + "voting_period": "20s" + } + }, + "group_id": "1", + "metadata": "", + "version": "1" + } + ], + "group_policy_seq": "1", + "group_seq": "1", + "groups": [ + { + "admin": "regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk", + "created_at": "2023-01-01T00:00:00Z", + "id": "1", + "metadata": "", + "total_weight": "2", + "version": "1" + } + ], + "proposal_seq": "0", + "proposals": [], + "votes": [] +} diff --git a/docker/indexer.Dockerfile b/docker/indexer.Dockerfile new file mode 100644 index 0000000..be73ab9 --- /dev/null +++ b/docker/indexer.Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.9 + +# Install dependencies +RUN apt-get update +RUN apt-get install libpq-dev postgresql-client -y + +# Set working directory +WORKDIR /home/indexer + +# Copy source code +COPY . . + +# Install python dependencies +RUN pip install poetry +RUN pip install load_dotenv +RUN pip install psycopg2 +RUN pip install sentry_sdk +RUN pip install tenacity + +# Install indexer +RUN poetry install diff --git a/docker/ledger.Dockerfile b/docker/ledger.Dockerfile new file mode 100644 index 0000000..26a4473 --- /dev/null +++ b/docker/ledger.Dockerfile @@ -0,0 +1,67 @@ +FROM golang:1.19 + +# Install dependencies +RUN apt-get update +RUN apt-get install jq -y + +# Set ledger version +ENV GIT_CHECKOUT='v5.1.2' + +# Clone regen ledger +RUN git clone https://github.com/regen-network/regen-ledger/ /home/ledger + +# Set working directory +WORKDIR /home/ledger + +# Use provided version +RUN git checkout $GIT_CHECKOUT + +# Build regen binary +RUN make install + +# Setup moniker, chain, homedir +RUN regen --chain-id regen-local init validator + +# Set configuration +RUN regen config chain-id regen-local +RUN regen config keyring-backend test + +# Update stake to uregen +RUN sed -i "s/stake/uregen/g" /root/.regen/config/genesis.json + +# Add accounts +RUN printf "trouble alarm laptop turn call stem lend brown play planet grocery survey smooth seed describe hood praise whale smile repeat dry sauce front future\n\n" | regen keys --keyring-backend test add validator -i +RUN printf "cool trust waste core unusual report duck amazing fault juice wish century across ghost cigar diary correct draw glimpse face crush rapid quit equip\n\n" | regen keys --keyring-backend test add user1 -i +RUN printf "music debris chicken erode flag law demise over fall always put bounce ring school dumb ivory spin saddle ostrich better seminar heart beach kingdom\n\n" | regen keys --keyring-backend test add user2 -i + +# Set up validator +RUN regen add-genesis-account validator 1000000000uregen --keyring-backend test +RUN regen gentx validator 1000000uregen + +# Set up user acounts +RUN regen add-genesis-account user1 1000000000uregen --keyring-backend test +RUN regen add-genesis-account user2 1000000000uregen --keyring-backend test + +# Prepare genesis file +RUN regen collect-gentxs + +# Set minimum gas price +RUN sed -i "s/minimum-gas-prices = \"\"/minimum-gas-prices = \"0uregen\"/" /root/.regen/config/app.toml + +# Set cors allow all origins +RUN sed -i "s/cors_allowed_origins = \[\]/cors_allowed_origins = [\"*\"]/" /root/.regen/config/config.toml + +# Copy genesis state files +COPY docker/data /home/ledger/data + +# Add group state to genesis +RUN jq '.app_state.group |= . + input' /root/.regen/config/genesis.json /home/ledger/data/ledger_group.json > genesis-tmp.json + +# Overwrite genesis file with updated genesis file +RUN mv -f genesis-tmp.json /root/.regen/config/genesis.json + +# Copy regen start script +COPY docker/scripts/ledger_start.sh /home/ledger/scripts/ + +# Make start script executable +RUN ["chmod", "+x", "/home/ledger/scripts/ledger_start.sh"] diff --git a/docker/scripts/indexer_init.sh b/docker/scripts/indexer_init.sh new file mode 100755 index 0000000..2739906 --- /dev/null +++ b/docker/scripts/indexer_init.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# run migrations +(cd sql && ./run_all_migrations.sh) + +# workaround for indexer starting with new chain +psql "$DATABASE_URL" -c "INSERT INTO chain ( + num, + chain_id +) VALUES ( + 1, + 'regen-local' +)" +psql "$DATABASE_URL" -c "INSERT INTO block ( + chain_num, + height, + data, + time +) VALUES ( + 1, + 0, + '{}', + now() +)" diff --git a/docker/scripts/indexer_start.sh b/docker/scripts/indexer_start.sh new file mode 100755 index 0000000..8cf1da7 --- /dev/null +++ b/docker/scripts/indexer_start.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +INDEXER_INITIALIZED="INDEXER_INITIALIZED" + +# initialize indexer if not yet initialized +if [ ! -e $INDEXER_INITIALIZED ]; then + + # set indexer initialized + touch $INDEXER_INITIALIZED + + echo "First start, running init script..." + + # run indexer init script + /home/indexer/docker/scripts/indexer_init.sh +fi + +# start indexer +python main.py diff --git a/docker/scripts/ledger_start.sh b/docker/scripts/ledger_start.sh new file mode 100755 index 0000000..438acb9 --- /dev/null +++ b/docker/scripts/ledger_start.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# start regen with api and unsafe CORS enabled +regen start --api.enable=true --api.enabled-unsafe-cors=true diff --git a/docker/scripts/tester_start.sh b/docker/scripts/tester_start.sh new file mode 100755 index 0000000..ff68bc3 --- /dev/null +++ b/docker/scripts/tester_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -eo pipefail + +# wait for indexer to start +sleep 5 + +# run tester test scripts +/home/tester/scripts/test_index_proposals.sh +/home/tester/scripts/test_index_votes.sh + +# exit without error +exit 0 diff --git a/docker/tester.Dockerfile b/docker/tester.Dockerfile new file mode 100644 index 0000000..b455c53 --- /dev/null +++ b/docker/tester.Dockerfile @@ -0,0 +1,52 @@ +FROM golang:1.19 + +# Install dependencies +RUN apt-get update +RUN apt-get install jq libpq-dev postgresql-client -y + +# Set ledger version +ENV GIT_CHECKOUT='v5.1.2' + +# Set database url +ENV DATABASE_URL='postgres://postgres:password@localhost:5432/postgres' + +# Set test addresses +ENV TEST_USER_ADDRESS_1=regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk +ENV TEST_USER_ADDRESS_2=regen14v5z5yyl5unnyu6q3ele8ze9jev6y0m7tx6gct +ENV TEST_POLICY_ADDRESS_1=regen1afk9zr2hn2jsac63h4hm60vl9z3e5u69gndzf7c99cqge3vzwjzs475lmr + +# Set transaction flags +ENV REGEN_TX_FLAGS="--keyring-backend test --chain-id regen-local --yes" + +# Clone regen ledger +RUN git clone https://github.com/regen-network/regen-ledger/ /home/tester + +# Set working directory +WORKDIR /home/tester + +# Use provided version +RUN git checkout $GIT_CHECKOUT + +# Build regen binary +RUN make install + +# Set configuration +RUN regen config chain-id regen-local +RUN regen config keyring-backend test + +# Add accounts +RUN printf "cool trust waste core unusual report duck amazing fault juice wish century across ghost cigar diary correct draw glimpse face crush rapid quit equip\n\n" | regen keys --keyring-backend test add user1 -i +RUN printf "music debris chicken erode flag law demise over fall always put bounce ring school dumb ivory spin saddle ostrich better seminar heart beach kingdom\n\n" | regen keys --keyring-backend test add user2 -i + +# Copy tester start script +COPY docker/scripts/tester_start.sh /home/tester/scripts/ + +# Copy tester test scripts +COPY docker/tester/ /home/tester/scripts/ + +# Make start script executable +RUN ["chmod", "+x", "/home/tester/scripts/tester_start.sh"] + +# Make test scripts executable +RUN ["chmod", "+x", "/home/tester/scripts/test_index_proposals.sh"] +RUN ["chmod", "+x", "/home/tester/scripts/test_index_votes.sh"] diff --git a/docker/tester/test_index_proposals.sh b/docker/tester/test_index_proposals.sh new file mode 100644 index 0000000..be88d86 --- /dev/null +++ b/docker/tester/test_index_proposals.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# set group proposal json +cat > proposal.json < proposal.json <