[RFR] use pgmigrate for schema migrations

.gitignore

@@ -25,6 +25,9 @@ nytprof*
 # Devel::Cover output
 cover_db/
 
+# Schema file dumped for reference in development
+mediawords.sql
+
 # MediaWords::TM::Dump output
 /dump_layout_*
 

apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml

@@ -51,8 +51,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/common/docker-compose.tests.yml

@@ -94,8 +94,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/crawler-ap/docker-compose.tests.yml

@@ -90,8 +90,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/crawler-fetcher/docker-compose.tests.yml

@@ -53,8 +53,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/crawler-provider/docker-compose.tests.yml

@@ -46,8 +46,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/create-missing-partitions/docker-compose.tests.yml

@@ -40,8 +40,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-generate-media-health/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-generate-user-summary/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-print-long-running-job-states/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-refresh-stats/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-rescrape-due-media/docker-compose.tests.yml

@@ -44,8 +44,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-rescraping-changes/docker-compose.tests.yml

@@ -43,8 +43,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-set-media-primary-language/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/cron-set-media-subject-country/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/dump-table/docker-compose.tests.yml

@@ -40,8 +40,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/export-tables-to-backup-crawler/docker-compose.tests.yml

@@ -46,8 +46,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/extract-and-vector/docker-compose.tests.yml

@@ -66,8 +66,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/facebook-fetch-story-stats/docker-compose.tests.yml

@@ -50,8 +50,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/import-solr-data-for-testing/docker-compose.tests.yml

@@ -47,8 +47,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/import-solr-data/docker-compose.tests.yml

@@ -72,8 +72,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/import-stories-feedly/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/import-stories-scrapehtml/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/munin-cron/docker-compose.tests.yml

@@ -60,8 +60,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/munin-httpd/docker-compose.tests.yml

@@ -87,8 +87,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/munin-node/docker-compose.tests.yml

@@ -40,8 +40,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml

@@ -49,8 +49,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/podcast-transcribe-episode/docker-compose.tests.yml

@@ -65,8 +65,8 @@ services:
               source: ./../postgresql-server/bin/
               target: /opt/mediacloud/bin/
             - type: bind
-              source: ./../postgresql-server/schema/
-              target: /opt/mediacloud/schema/
+              source: ./../postgresql-server/pgmigrate/
+              target: /opt/postgresql-server/pgmigrate/
             - type: bind
               source: ./../postgresql-base/conf/
               target: /etc/postgresql/13/main/

apps/postgresql-server/Dockerfile

@@ -5,30 +5,57 @@
 FROM gcr.io/mcback/postgresql-base:latest
 
 USER root
+RUN \
+    apt-get -y --no-install-recommends install python3 python3-pip python3-setuptools && \
+    #
+    # Upgrade Pip
+    pip3 install -U pip && \
+    #
+    # https://github.com/pypa/pip/issues/5221#issuecomment-382069604
+    hash -r pip3 && \
+    apt-get -y --no-install-recommends install build-essential python3-dev libffi-dev libpq-dev && \
+    #
+    # Install package to manage schema migrations
+    pip3 install yandex-pgmigrate==1.0.6 && \
+    #
+    # Remove temporary dependencies
+    apt-get -y remove build-essential python3-dev libffi-dev libpq-dev && \
+    #
+    # Cleanup
+    apt-get -y autoremove && \
+    apt-get -y clean && \
+    rm -rf /root/.cache/ && \
+    #
+    true
+
+# Copy helper scripts, migrations, pgmigrate callbacks/config, create schema dir
 RUN \
     mkdir -p \
         /opt/postgresql-server/bin/ \
-        /opt/postgresql-server/schema/ \
+        /opt/postgresql-server/pgmigrate \
     && \
+    chmod ugo+rw /opt/postgresql-server/pgmigrate && \
     true
 
-# Copy helper scripts, schema, migrations
 COPY bin/* /opt/postgresql-server/bin/
-COPY schema/ /opt/postgresql-server/schema/
+COPY pgmigrate/ /opt/postgresql-server/pgmigrate
 
-# Initialize data volume, create users, a database, and initialize it with
-# schema
+# Initialize data volume, create users + database
 # If a new empty volume gets mounted to /var/lib/postgresql/ upon
 # container start, Docker will copy the files from the container to the volume
+
 USER postgres
-RUN /opt/postgresql-server/bin/initialize_schema.sh
+RUN /opt/postgresql-server/bin/initialize_db.sh
 
 # Remove the init script so that someone doesn't accidentally run it in
 # production
 USER root
-RUN rm /opt/postgresql-server/bin/initialize_schema.sh
-USER postgres
+RUN rm /opt/postgresql-server/bin/initialize_db.sh
+
+# dump schema file for reference in development (run ./dev/get_schema.sh to get local copy)
+RUN mv /opt/postgresql-server/pgmigrate/mediawords.sql /opt/mediawords.sql
 
+USER postgres
 ENV \
     PATH="/opt/postgresql-server/bin:${PATH}" \
     #
@@ -43,4 +70,4 @@ ENV \
 VOLUME /var/lib/postgresql/
 
 # Use our own wrapper script which runs schema upgrades first
-CMD ["/opt/postgresql-server/bin/postgresql.sh"]
+CMD ["/opt/postgresql-server/bin/postgresql.sh"]