feat: add graphile-migrate for managing migrations

.gitignore

@@ -1,3 +1,5 @@
+migrations/current.sql
+
 *.sw[pno]
 .envrc
 
@@ -232,3 +234,26 @@ cython_debug/
 # vendor/
 
 /.idea/
+
+# Logs
+yarn-debug.log*
+yarn-error.log*
+
+# Dependency directories
+node_modules/
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variable files
+.env
+.env.development.local
+.env.test.local
+.env.production.local
+.env.local
+
+# yarn v2
+.yarn/cache
+.yarn/unplugged
+.yarn/build-state.yml
+.yarn/install-state.gz

.gmrc

@@ -0,0 +1,140 @@
+/*
+ * Graphile Migrate configuration.
+ *
+ * If you decide to commit this file (recommended) please ensure that it does
+ * not contain any secrets (passwords, etc) - we recommend you manage these
+ * with environmental variables instead.
+ *
+ * This file is in JSON5 format, in VSCode you can use "JSON with comments" as
+ * the file format.
+ */
+{
+  /*
+   * connectionString: this tells Graphile Migrate where to find the database
+   * to run the migrations against.
+   *
+   * RECOMMENDATION: use `DATABASE_URL` envvar instead.
+   */
+  // "connectionString": "postgres://appuser:apppassword@host:5432/appdb",
+
+  /*
+   * shadowConnectionString: like connectionString, but this is used for the
+   * shadow database (which will be reset frequently).
+   *
+   * RECOMMENDATION: use `SHADOW_DATABASE_URL` envvar instead.
+   */
+  // "shadowConnectionString": "postgres://appuser:apppassword@host:5432/appdb_shadow",
+
+  /*
+   * rootConnectionString: like connectionString, but this is used for
+   * dropping/creating the database in `graphile-migrate reset`. This isn't
+   * necessary, shouldn't be used in production, but helps during development.
+   *
+   * RECOMMENDATION: use `ROOT_DATABASE_URL` envvar instead.
+   */
+  // "rootConnectionString": "postgres://adminuser:adminpassword@host:5432/postgres",
+
+  /*
+   * pgSettings: key-value settings to be automatically loaded into PostgreSQL
+   * before running migrations, using an equivalent of `SET LOCAL <key> TO
+   * <value>`
+   */
+  "pgSettings": {
+    // "search_path": "app_public,app_private,app_hidden,public",
+  },
+
+  /*
+   * placeholders: substituted in SQL files when compiled/executed. Placeholder
+   * keys should be prefixed with a colon and in all caps, like
+   * `:COLON_PREFIXED_ALL_CAPS`. Placeholder values should be strings. They
+   * will be replaced verbatim with NO ESCAPING AT ALL (this differs from how
+   * psql handles placeholders) so should only be used with "safe" values. This
+   * is useful for committing migrations where certain parameters can change
+   * between environments (development, staging, production) but you wish to
+   * use the same signed migration files for all.
+   *
+   * The special value "!ENV" can be used to indicate an environmental variable
+   * of the same name should be used.
+   *
+   * Graphile Migrate automatically sets the `:DATABASE_NAME` and
+   * `:DATABASE_OWNER` placeholders, and you should not attempt to override
+   * these.
+   */
+  "placeholders": {
+    // ":DATABASE_VISITOR": "!ENV", // Uses process.env.DATABASE_VISITOR
+  },
+
+  /*
+   * Actions allow you to run scripts or commands at certain points in the
+   * migration lifecycle. SQL files are ran against the database directly.
+   * "command" actions are ran with the following environmental variables set:
+   *
+   * - GM_DBURL: the PostgreSQL URL of the database being migrated
+   * - GM_DBNAME: the name of the database from GM_DBURL
+   * - GM_DBUSER: the user from GM_DBURL
+   * - GM_SHADOW: set to 1 if the shadow database is being migrated, left unset
+   *   otherwise
+   *
+   * If "shadow" is unspecified, the actions will run on events to both shadow
+   * and normal databases. If "shadow" is true the action will only run on
+   * actions to the shadow DB, and if false only on actions to the main DB.
+   */
+
+  /*
+   * afterReset: actions executed after a `graphile-migrate reset` command.
+   */
+  "afterReset": [
+    // "afterReset.sql",
+    // { "_": "command", "command": "graphile-worker --schema-only" },
+    "initial_schema.sql"
+  ],
+
+  /*
+   * afterAllMigrations: actions executed once all migrations are complete.
+   */
+  "afterAllMigrations": [
+    {
+      "_": "command",
+      "shadow": true,
+      "command": "pg_dump --no-sync --schema-only --no-owner --file=./migrations/schema_snapshot.sql $SHADOW_DATABASE_URL",
+    },
+  ],
+
+  /*
+   * afterCurrent: actions executed once the current migration has been
+   * evaluated (i.e. in watch mode).
+   */
+  "afterCurrent": [
+    // {
+    //   "_": "command",
+    //   "shadow": true,
+    //   "command": "if [ \"$IN_TESTS\" = \"1\" ]; then ./scripts/test-seed; fi",
+    // },
+  ],
+
+  /*
+   * blankMigrationContent: content to be written to the current migration
+   * after commit. NOTE: this should only contain comments.
+   */
+  // "blankMigrationContent": "-- Write your migration here\n",
+
+  /****************************************************************************\
+  ***                                                                        ***
+  ***         You probably don't want to edit anything below here.           ***
+  ***                                                                        ***
+  \****************************************************************************/
+
+  /*
+   * manageGraphileMigrateSchema: if you set this false, you must be sure to
+   * keep the graphile_migrate schema up to date yourself. We recommend you
+   * leave it at its default.
+   */
+  // "manageGraphileMigrateSchema": true,
+
+  /*
+   * migrationsFolder: path to the folder in which to store your migrations.
+   */
+  // migrationsFolder: "./migrations",
+
+  "//generatedWith": "1.4.1"
+}

docker-compose.yml

@@ -37,7 +37,9 @@ services:
       dockerfile: docker/indexer.Dockerfile
     container_name: indexer_container
     environment:
-      DATABASE_URL: postgres://postgres:password@localhost:5432
+      DATABASE_URL: postgres://postgres:password@localhost:5432/indexer
+      SHADOW_DATABASE_URL: postgres://postgres:password@localhost:5432/indexer_shadow
+      ROOT_DATABASE_URL: postgres://postgres:password@localhost:5432/postgres
       REGEN_API: http://localhost:1317
       REGEN_RPC: http://localhost:26657
     entrypoint: ["/bin/sh", "-c", "./docker/scripts/indexer_start.sh"]

docker/indexer.Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.9
 
 # Install dependencies
 RUN apt-get update
-RUN apt-get install libpq-dev postgresql-client python3-poetry -y
+RUN apt-get install libpq-dev postgresql-client nodejs python3-poetry yarnpkg -y
 
 # Set working directory
 WORKDIR /home/indexer
@@ -12,3 +12,4 @@ COPY . .
 
 # Install indexer
 RUN poetry install
+RUN yarnpkg install 

docker/scripts/indexer_init.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 
+psql "$DATABASE_URL" -c "CREATE DATABASE indexer"
+psql "$DATABASE_URL" -c "CREATE DATABASE indexer_shadow"
+
 # run migrations
-(cd sql && ./run_all_migrations.sh)
+yarnpkg run db-init
 
 # workaround for indexer starting with new chain
 psql "$DATABASE_URL" -c "INSERT INTO chain (

docker/tester.Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get install jq libpq-dev postgresql-client -y
 ENV GIT_CHECKOUT='v5.1.2'
 
 # Set database url
-ENV DATABASE_URL='postgres://postgres:password@localhost:5432/postgres'
+ENV DATABASE_URL='postgres://postgres:password@localhost:5432/indexer'
 
 # Set test addresses
 ENV TEST_USER_ADDRESS_1=regen1l2pwmzk96ftmmt5egpjulyqtneygmmzndf7csk

migrations/README.md

@@ -0,0 +1,72 @@
+# Migrations
+
+This readme provides info about how to work with migrations in this repo.
+
+## Local development
+
+In order to develop this project locally we must use the following commands.
+If this is your first time setting up the project locally, we need to initialize your database.
+First, you must run the local database:
+
+```
+$ pwd
+/Users/kyle/regen/indexer
+$ docker-compose up --build postgres
+```
+
+Then, you must initialize the database:
+
+```
+$ export DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer"
+$ export SHADOW_DATABASE_URL="postgres://postgres:postgres@localhost:5432/indexer_shadow"
+$ export ROOT_DATABASE_URL="postgres://postgres:postgres@localhost:5432/postgres"
+$ yarn run graphile-migrate reset --erase
+```
+
+Now, we set up a watch process that will monitor `migrations/current.sql` for your changes as well as apply them to your local database:
+
+```
+$ yarn run graphile-migrate watch
+```
+
+When you are satisfied with the changes in `migration/current.sql`, you commit them:
+
+```
+$ yarn run graphile-migrate commit
+```
+
+By committing your changes you should see a new SQL file in `migration/committed/`.
+
+## Schema Snapshot
+
+The schema snapshot is stored and tracked in version control as `migrations/schema_snapshot.sql`.
+Each time you apply a migration in local development this snapshot will be automatically updated.
+See `.gmrc` and `afterAllMigrations` from [the `graphile-migrate` configuration docs](https://github.com/graphile/migrate#configuration) for how this is done.
+This allows us to keep track of the changes being introduced to the schema.
+You must commit your changes to this file.
+
+This is a helpful file to keep in mind when you have questions about entities in the database.
+For example, it allows you to also view the functions in the database being used in various RLS policies.
+Similarly, you can view the various policies in the database or which tables have RLS enabled.
+
+## Deploying to staging or production
+
+The migrations are always automatically run in Heroku for staging and production.
+See the `migrate` command in `package.json` and `Procfile` for Heroku.
+
+## Debugging
+
+This section contains some notes that may be useful for debugging common scenarios.
+
+### Viewing migrations applied in a particular database
+
+Our migration tool tracks which migrations have been applied in the following table:
+
+```
+regen_registry=# select * from graphile_migrate.migrations;
+                     hash                      | previous_hash |  filename  |             date
+-----------------------------------------------+---------------+------------+-------------------------------
+ sha1:28ab5499d9a4520daa9428681a9bf1152f9887af |               | 000001.sql | 2023-05-08 20:20:31.213547+00
+```
+
+This is one way that you can track the migrations that will be deployed to staging or production.

migrations/current.sql

@@ -0,0 +1 @@
+-- Enter migration here