diff --git a/.gitignore b/.gitignore index 1b978effae..6bd5677b9e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,9 @@ nytprof* # Devel::Cover output cover_db/ +# Schema file dumped for reference in development +mediawords.sql + # MediaWords::TM::Dump output /dump_layout_* diff --git a/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml b/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml index a97835a29f..ce2028d844 100644 --- a/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml +++ b/apps/cliff-fetch-annotation-and-tag/docker-compose.tests.yml @@ -51,8 +51,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/common/docker-compose.tests.yml b/apps/common/docker-compose.tests.yml index 17126383ee..512d478d8b 100644 --- a/apps/common/docker-compose.tests.yml +++ b/apps/common/docker-compose.tests.yml @@ -94,8 +94,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/crawler-ap/docker-compose.tests.yml b/apps/crawler-ap/docker-compose.tests.yml index 2ea0f17570..d43c351f3c 100644 --- a/apps/crawler-ap/docker-compose.tests.yml +++ b/apps/crawler-ap/docker-compose.tests.yml @@ -90,8 +90,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/crawler-fetcher/docker-compose.tests.yml b/apps/crawler-fetcher/docker-compose.tests.yml index fb3cb718a7..ae3ad80d71 100644 --- a/apps/crawler-fetcher/docker-compose.tests.yml +++ b/apps/crawler-fetcher/docker-compose.tests.yml @@ -53,8 +53,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/crawler-provider/docker-compose.tests.yml b/apps/crawler-provider/docker-compose.tests.yml index 0cfd0cbdfd..6145990302 100644 --- a/apps/crawler-provider/docker-compose.tests.yml +++ b/apps/crawler-provider/docker-compose.tests.yml @@ -46,8 +46,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/create-missing-partitions/docker-compose.tests.yml b/apps/create-missing-partitions/docker-compose.tests.yml index a454e2a84b..d9db7c2e53 100644 --- a/apps/create-missing-partitions/docker-compose.tests.yml +++ b/apps/create-missing-partitions/docker-compose.tests.yml @@ -40,8 +40,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml b/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml index 56fdc8b91e..4f554d3ab4 100644 --- a/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml +++ b/apps/cron-generate-daily-rss-dumps/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-generate-media-health/docker-compose.tests.yml b/apps/cron-generate-media-health/docker-compose.tests.yml index 74b389d3e9..56d2a317d4 100644 --- a/apps/cron-generate-media-health/docker-compose.tests.yml +++ b/apps/cron-generate-media-health/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-generate-user-summary/docker-compose.tests.yml b/apps/cron-generate-user-summary/docker-compose.tests.yml index 1ac0b72ada..bec9e68355 100644 --- a/apps/cron-generate-user-summary/docker-compose.tests.yml +++ b/apps/cron-generate-user-summary/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-print-long-running-job-states/docker-compose.tests.yml b/apps/cron-print-long-running-job-states/docker-compose.tests.yml index f2eb667e72..24431fde60 100644 --- a/apps/cron-print-long-running-job-states/docker-compose.tests.yml +++ b/apps/cron-print-long-running-job-states/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-refresh-stats/docker-compose.tests.yml b/apps/cron-refresh-stats/docker-compose.tests.yml index 9af8e30be6..f381c76f5c 100644 --- a/apps/cron-refresh-stats/docker-compose.tests.yml +++ b/apps/cron-refresh-stats/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-rescrape-due-media/docker-compose.tests.yml b/apps/cron-rescrape-due-media/docker-compose.tests.yml index 1b321172a7..5163cd2fc8 100644 --- a/apps/cron-rescrape-due-media/docker-compose.tests.yml +++ b/apps/cron-rescrape-due-media/docker-compose.tests.yml @@ -44,8 +44,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-rescraping-changes/docker-compose.tests.yml b/apps/cron-rescraping-changes/docker-compose.tests.yml index a563decac7..729095d782 100644 --- a/apps/cron-rescraping-changes/docker-compose.tests.yml +++ b/apps/cron-rescraping-changes/docker-compose.tests.yml @@ -43,8 +43,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-set-media-primary-language/docker-compose.tests.yml b/apps/cron-set-media-primary-language/docker-compose.tests.yml index f4e0ca5bf7..e152c23ad5 100644 --- a/apps/cron-set-media-primary-language/docker-compose.tests.yml +++ b/apps/cron-set-media-primary-language/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/cron-set-media-subject-country/docker-compose.tests.yml b/apps/cron-set-media-subject-country/docker-compose.tests.yml index 01864f5a44..2a2872d961 100644 --- a/apps/cron-set-media-subject-country/docker-compose.tests.yml +++ b/apps/cron-set-media-subject-country/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/dump-table/docker-compose.tests.yml b/apps/dump-table/docker-compose.tests.yml index 7940757394..5f53609d5d 100644 --- a/apps/dump-table/docker-compose.tests.yml +++ b/apps/dump-table/docker-compose.tests.yml @@ -40,8 +40,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/export-tables-to-backup-crawler/docker-compose.tests.yml b/apps/export-tables-to-backup-crawler/docker-compose.tests.yml index bec36c4c46..31f916dbdd 100644 --- a/apps/export-tables-to-backup-crawler/docker-compose.tests.yml +++ b/apps/export-tables-to-backup-crawler/docker-compose.tests.yml @@ -46,8 +46,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/extract-and-vector/docker-compose.tests.yml b/apps/extract-and-vector/docker-compose.tests.yml index dc15a36094..e3c8b5d01f 100644 --- a/apps/extract-and-vector/docker-compose.tests.yml +++ b/apps/extract-and-vector/docker-compose.tests.yml @@ -66,8 +66,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/facebook-fetch-story-stats/docker-compose.tests.yml b/apps/facebook-fetch-story-stats/docker-compose.tests.yml index 917c869c08..b5a8d3f727 100644 --- a/apps/facebook-fetch-story-stats/docker-compose.tests.yml +++ b/apps/facebook-fetch-story-stats/docker-compose.tests.yml @@ -50,8 +50,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/import-solr-data-for-testing/docker-compose.tests.yml b/apps/import-solr-data-for-testing/docker-compose.tests.yml index 27fc041b8c..0b7e048a91 100644 --- a/apps/import-solr-data-for-testing/docker-compose.tests.yml +++ b/apps/import-solr-data-for-testing/docker-compose.tests.yml @@ -47,8 +47,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/import-solr-data/docker-compose.tests.yml b/apps/import-solr-data/docker-compose.tests.yml index ebf9bfbd6d..2527cc48ba 100644 --- a/apps/import-solr-data/docker-compose.tests.yml +++ b/apps/import-solr-data/docker-compose.tests.yml @@ -72,8 +72,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/import-stories-feedly/docker-compose.tests.yml b/apps/import-stories-feedly/docker-compose.tests.yml index dfc441ac98..5398a84035 100644 --- a/apps/import-stories-feedly/docker-compose.tests.yml +++ b/apps/import-stories-feedly/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/import-stories-scrapehtml/docker-compose.tests.yml b/apps/import-stories-scrapehtml/docker-compose.tests.yml index f48a47cda7..d42815d441 100644 --- a/apps/import-stories-scrapehtml/docker-compose.tests.yml +++ b/apps/import-stories-scrapehtml/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/munin-cron/docker-compose.tests.yml b/apps/munin-cron/docker-compose.tests.yml index 055e88673f..e3a52eec00 100644 --- a/apps/munin-cron/docker-compose.tests.yml +++ b/apps/munin-cron/docker-compose.tests.yml @@ -60,8 +60,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/munin-httpd/docker-compose.tests.yml b/apps/munin-httpd/docker-compose.tests.yml index 0c8403be69..d50207945e 100644 --- a/apps/munin-httpd/docker-compose.tests.yml +++ b/apps/munin-httpd/docker-compose.tests.yml @@ -87,8 +87,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/munin-node/docker-compose.tests.yml b/apps/munin-node/docker-compose.tests.yml index d9bdc527e2..d6d94c6287 100644 --- a/apps/munin-node/docker-compose.tests.yml +++ b/apps/munin-node/docker-compose.tests.yml @@ -40,8 +40,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml b/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml index 785e7aacb4..b9ed705eea 100644 --- a/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml +++ b/apps/nytlabels-fetch-annotation-and-tag/docker-compose.tests.yml @@ -49,8 +49,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/podcast-transcribe-episode/docker-compose.tests.yml b/apps/podcast-transcribe-episode/docker-compose.tests.yml index 3ec64d2a21..55de596db7 100644 --- a/apps/podcast-transcribe-episode/docker-compose.tests.yml +++ b/apps/podcast-transcribe-episode/docker-compose.tests.yml @@ -65,8 +65,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/postgresql-server/Dockerfile b/apps/postgresql-server/Dockerfile index e5609decb9..617fccdf62 100644 --- a/apps/postgresql-server/Dockerfile +++ b/apps/postgresql-server/Dockerfile @@ -5,30 +5,57 @@ FROM gcr.io/mcback/postgresql-base:latest USER root +RUN \ + apt-get -y --no-install-recommends install python3 python3-pip python3-setuptools && \ + # + # Upgrade Pip + pip3 install -U pip && \ + # + # https://github.com/pypa/pip/issues/5221#issuecomment-382069604 + hash -r pip3 && \ + apt-get -y --no-install-recommends install build-essential python3-dev libffi-dev libpq-dev && \ + # + # Install package to manage schema migrations + pip3 install yandex-pgmigrate==1.0.6 && \ + # + # Remove temporary dependencies + apt-get -y remove build-essential python3-dev libffi-dev libpq-dev && \ + # + # Cleanup + apt-get -y autoremove && \ + apt-get -y clean && \ + rm -rf /root/.cache/ && \ + # + true + +# Copy helper scripts, migrations, pgmigrate callbacks/config, create schema dir RUN \ mkdir -p \ /opt/postgresql-server/bin/ \ - /opt/postgresql-server/schema/ \ + /opt/postgresql-server/pgmigrate \ && \ + chmod ugo+rw /opt/postgresql-server/pgmigrate && \ true -# Copy helper scripts, schema, migrations COPY bin/* /opt/postgresql-server/bin/ -COPY schema/ /opt/postgresql-server/schema/ +COPY pgmigrate/ /opt/postgresql-server/pgmigrate -# Initialize data volume, create users, a database, and initialize it with -# schema +# Initialize data volume, create users + database # If a new empty volume gets mounted to /var/lib/postgresql/ upon # container start, Docker will copy the files from the container to the volume + USER postgres -RUN /opt/postgresql-server/bin/initialize_schema.sh +RUN /opt/postgresql-server/bin/initialize_db.sh # Remove the init script so that someone doesn't accidentally run it in # production USER root -RUN rm /opt/postgresql-server/bin/initialize_schema.sh -USER postgres +RUN rm /opt/postgresql-server/bin/initialize_db.sh + +# dump schema file for reference in development (run ./dev/get_schema.sh to get local copy) +RUN mv /opt/postgresql-server/pgmigrate/mediawords.sql /opt/mediawords.sql +USER postgres ENV \ PATH="/opt/postgresql-server/bin:${PATH}" \ # @@ -43,4 +70,4 @@ ENV \ VOLUME /var/lib/postgresql/ # Use our own wrapper script which runs schema upgrades first -CMD ["/opt/postgresql-server/bin/postgresql.sh"] +CMD ["/opt/postgresql-server/bin/postgresql.sh"] \ No newline at end of file diff --git a/apps/postgresql-server/bin/apply_migrations.sh b/apps/postgresql-server/bin/apply_migrations.sh index 9435299dd6..8d037a0b2b 100755 --- a/apps/postgresql-server/bin/apply_migrations.sh +++ b/apps/postgresql-server/bin/apply_migrations.sh @@ -7,9 +7,7 @@ MC_POSTGRESQL_BIN_DIR="/usr/lib/postgresql/13/bin/" MC_POSTGRESQL_DATA_DIR="/var/lib/postgresql/13/main/" MC_POSTGRESQL_CONF_PATH="/etc/postgresql/13/main/postgresql.conf" -SCHEMA_DIR="/opt/postgresql-server/schema/" -SCHEMA_PATH="${SCHEMA_DIR}/mediawords.sql" -MIGRATIONS_DIR="${SCHEMA_DIR}/migrations/" +MIGRATIONS_DIR="/opt/postgresql-server/pgmigratemigrations" # Apply migrations when running on a different port so that clients don't end # up connecting in the middle of migrating @@ -18,35 +16,11 @@ TEMP_PORT=12345 # In case the database is in recovery, wait for up to 1 hour for it to complete PGCTL_START_TIMEOUT=3600 -if [ ! -f "${SCHEMA_PATH}" ]; then - echo "Schema ${SCHEMA_PATH} does not exist." - exit 1 -fi - if [ ! -d "${MIGRATIONS_DIR}" ]; then echo "Migrations directory ${MIGRATIONS_DIR} does not exist." exit 1 fi - -validate_schema_version() { - schema_version="$1" - if [[ ! "${schema_version}" =~ ^[0-9]+$ ]]; then - echo "Invalid schema version; got: ${schema_version}" - exit 1 - fi -} - -# Read new (mediawords.sql) schema version -NEW_SCHEMA_VERSION=$(cat ${SCHEMA_PATH} | \ - grep "MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT" | \ - awk '{ print $5 }' | \ - sed 's/;//') - -validate_schema_version "${NEW_SCHEMA_VERSION}" - -echo "New schema version: ${NEW_SCHEMA_VERSION}" - # Start PostgreSQL on a temporary port "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ -o "-c config_file=${MC_POSTGRESQL_CONF_PATH} -p ${TEMP_PORT}" \ @@ -55,67 +29,9 @@ echo "New schema version: ${NEW_SCHEMA_VERSION}" -w \ start -# Read old (database) schema version -OLD_SCHEMA_VERSION=$(psql -v ON_ERROR_STOP=1 -p "${TEMP_PORT}" -t -c \ - "SELECT value FROM database_variables WHERE name = 'database-schema-version';" | \ - xargs) - -validate_schema_version "${OLD_SCHEMA_VERSION}" - -echo "Old schema version: ${OLD_SCHEMA_VERSION}" - -# Run migration if needed -if (( "${OLD_SCHEMA_VERSION}" == "${NEW_SCHEMA_VERSION}" )); then - echo "Schema is up-to-date, nothing to do." -elif (( "${OLD_SCHEMA_VERSION}" > "${NEW_SCHEMA_VERSION}" )); then - echo "Schema is newer in the database, go write a migration and rebuild this image." - echo "old version: $OLD_SCHEMA_VERSION, new_version: $NEW_SCHEMA_VERSION" - exit 1 -else - echo "Upgrading from ${OLD_SCHEMA_VERSION} to ${NEW_SCHEMA_VERSION}..." - CONCAT_MIGRATION=$(mktemp /var/tmp/migration-${OLD_SCHEMA_VERSION}-${NEW_SCHEMA_VERSION}.XXXXXX) - - # Run concatenated migration files in a single transaction - echo "BEGIN;" >> "${CONCAT_MIGRATION}" - - for SCHEMA_VERSION in $(seq "${OLD_SCHEMA_VERSION}" "$(( ${NEW_SCHEMA_VERSION} - 1 ))"); do - MIGRATION_FILENAME="mediawords-${SCHEMA_VERSION}-$(( ${SCHEMA_VERSION} + 1 )).sql" - MIGRATION_PATH="${MIGRATIONS_DIR}/${MIGRATION_FILENAME}" - - if [ ! -f "${MIGRATION_PATH}" ]; then - echo "Migration at ${MIGRATION_PATH} does not exist." - exit 1 - fi - - cat "${MIGRATION_PATH}" >> "${CONCAT_MIGRATION}" - - done - - # Add shim which verifies that schema version has been set to correct value - cat >> "${CONCAT_MIGRATION}" << EOF - - DO \$\$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM database_variables - WHERE name = 'database-schema-version' - AND value = '${NEW_SCHEMA_VERSION}' - ) THEN - RAISE EXCEPTION 'Schema version has not been updated after applying the migration'; - END IF; - END\$\$; - -EOF - - # Commit transaction - echo "COMMIT;" >> "${CONCAT_MIGRATION}" - - # Apply migration - psql -v ON_ERROR_STOP=1 -p "${TEMP_PORT}" -f "${CONCAT_MIGRATION}" - - rm "${CONCAT_MIGRATION}" -fi +# apply migrations +cd /opt/postgresql-server/pgmigrate +pgmigrate migrate --target latest --conn "dbname=mediacloud user=mediacloud password=mediacloud host=localhost port=${TEMP_PORT}" # Stop PostgreSQL "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ diff --git a/apps/postgresql-server/bin/generate_empty_sql_migration.sh b/apps/postgresql-server/bin/generate_empty_sql_migration.sh deleted file mode 100755 index 9225e7e64d..0000000000 --- a/apps/postgresql-server/bin/generate_empty_sql_migration.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -set -u -set -e - -SCHEMA_DIR="apps/postgresql-server/schema" -MEDIAWORDS_SQL_PATH="$SCHEMA_DIR/mediawords.sql" - -if [ ! -e "$MEDIAWORDS_SQL_PATH" ]; then - echo "Can't find mediawords.sql." - exit 1 -fi - -NEW_SCHEMA_VERSION=`cat $MEDIAWORDS_SQL_PATH \ - | perl -lne 'print if /(MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT)/' \ - | perl -lpe 's/.*MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := (\d+?);.*/$1/'` - -if [ "$NEW_SCHEMA_VERSION" == '' ]; then - echo "Unable to find MEDIACLOUD_DATABASE_SCHEMA_VERSION in mediawords.sql" - exit -fi - -OLD_SCHEMA_VERSION=`expr "$NEW_SCHEMA_VERSION" - 1` - -if [ "$OLD_SCHEMA_VERSION" == '' ]; then - echo "Unable to generate old schema version from new schema version" - exit -fi - -MIGRATION_FILE="$SCHEMA_DIR/migrations/mediawords-$OLD_SCHEMA_VERSION-$NEW_SCHEMA_VERSION.sql" - -if [ -e "$MIGRATION_FILE" ]; then - echo "'$MIGRATION_FILE' already exists. Cowardly refusing to overwrite it." - exit 1 -fi - -SQL="-- --- This is a Media Cloud PostgreSQL schema difference file (a \"diff\") between schema --- versions ${OLD_SCHEMA_VERSION} and ${NEW_SCHEMA_VERSION}. --- --- If you are running Media Cloud with a database that was set up with a schema version --- ${OLD_SCHEMA_VERSION}, and you would like to upgrade both the Media Cloud and the --- database to be at version ${NEW_SCHEMA_VERSION}, import this SQL file: --- --- psql mediacloud < mediawords-${OLD_SCHEMA_VERSION}-${NEW_SCHEMA_VERSION}.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS \$\$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := ${NEW_SCHEMA_VERSION}; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -\$\$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - -" - -echo "$SQL" > "$MIGRATION_FILE" - -git add "$MIGRATION_FILE" - -echo "Generated $MIGRATION_FILE" diff --git a/apps/postgresql-server/bin/initialize_schema.sh b/apps/postgresql-server/bin/initialize_db.sh similarity index 90% rename from apps/postgresql-server/bin/initialize_schema.sh rename to apps/postgresql-server/bin/initialize_db.sh index dc41a31672..921a2410c7 100755 --- a/apps/postgresql-server/bin/initialize_schema.sh +++ b/apps/postgresql-server/bin/initialize_db.sh @@ -35,8 +35,12 @@ CREATE DATABASE mediacloud WITH EOF psql -v ON_ERROR_STOP=1 -c "${CREATE_DB_SQL}" -# Initialize with schema -psql -v ON_ERROR_STOP=1 -d mediacloud -f /opt/postgresql-server/schema/mediawords.sql +# run migrations with pgmigrate package +cd /opt/postgresql-server/pgmigrate +pgmigrate migrate --target latest + +# dump schema for reference in development +pg_dump mediacloud > ./mediawords.sql # Stop PostgreSQL "${MC_POSTGRESQL_BIN_DIR}/pg_ctl" \ diff --git a/apps/postgresql-server/pgmigrate/callbacks/beforeEach/00_set_search_path.sql b/apps/postgresql-server/pgmigrate/callbacks/beforeEach/00_set_search_path.sql new file mode 100644 index 0000000000..09e5a7994e --- /dev/null +++ b/apps/postgresql-server/pgmigrate/callbacks/beforeEach/00_set_search_path.sql @@ -0,0 +1,3 @@ +/* pgmigrate-encoding: utf-8 */ + +SET search_path = public, pg_catalog; diff --git a/apps/postgresql-server/pgmigrate/migrations.yml b/apps/postgresql-server/pgmigrate/migrations.yml new file mode 100644 index 0000000000..9c3202b2ae --- /dev/null +++ b/apps/postgresql-server/pgmigrate/migrations.yml @@ -0,0 +1,4 @@ +callbacks: + beforeEach: + - callbacks/beforeEach +conn: dbname=mediacloud user=mediacloud password=mediacloud host=localhost \ No newline at end of file diff --git a/apps/postgresql-server/schema/mediawords.sql b/apps/postgresql-server/pgmigrate/migrations/V0001__initial_schema.sql similarity index 86% rename from apps/postgresql-server/schema/mediawords.sql rename to apps/postgresql-server/pgmigrate/migrations/V0001__initial_schema.sql index 72d7c56608..b3e8930978 100644 --- a/apps/postgresql-server/schema/mediawords.sql +++ b/apps/postgresql-server/pgmigrate/migrations/V0001__initial_schema.sql @@ -4,7 +4,7 @@ -- main schema CREATE SCHEMA IF NOT EXISTS public; - +COMMENT ON SCHEMA public IS 'main schema for MediaWords database'; CREATE OR REPLACE LANGUAGE plpgsql; @@ -59,6 +59,8 @@ $$ LANGUAGE 'plpgsql' IMMUTABLE COST 10; +COMMENT ON FUNCTION week_start_date (day date) IS 'Need b/c date_trunc("week", date) is not immutable; +see http://www.mentby.com/Group/pgsql-general/datetrunc-on-date-is-immutable.html'; -- Returns first 64 bits (16 characters) of MD5 hash -- @@ -69,6 +71,9 @@ CREATE OR REPLACE FUNCTION half_md5(string TEXT) RETURNS bytea AS $$ SELECT SUBSTRING(public.digest(string, 'md5'::text), 0, 9); $$ LANGUAGE SQL; +COMMENT ON FUNCTION half_md5 (string TEXT) IS 'Returns first 64 bits (16 characters) of MD5 hash; +useful for reducing index sizes (e.g. in story_sentences.sentence) where 64 bits of entropy is not enough +pgcrypto functions are being referred with public schema prefix to make pg_upgrade work'; -- Returns true if table exists (and user has access to it) -- Table name might be with ("public.stories") or without ("stories") schema. @@ -100,6 +105,8 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION table_exists (target_table_name VARCHAR) IS 'Returns true if table exists +(and user has access to it). Table name might be with ("public.stories") or without ("stories") schema.'; -- -- Common partitioning tools @@ -164,6 +171,15 @@ create table media ( CONSTRAINT media_self_dup CHECK ( dup_media_id IS NULL OR dup_media_id <> media_id ) ); +COMMENT ON COLUMN media.foreign_rss_links IS 'ndicates that the media source includes a substantial +number of links in its feeds that are not its own. These media sources cause problems for the +topic mapper spider, which finds those foreign rss links an thinks that the urls belong to the +parent media source.'; +COMMENT ON COLUMN media.content_delay IS 'Delay content downloads for this media source for (int) hours'; +COMMENT ON COLUMN media.editor_notes IS 'notes for internal MC consumption (e.g. "added this for yochai")'; +COMMENT ON COLUMN media.public_notes IS 'notes for public consumption (e.g. "leading dissident paper in antarctica")'; +COMMENT ON COLUMN media.is_monitored IS 'if true, indicates that MC closely monitors health of this source'; + create unique index media_name on media(name); create unique index media_url on media(url); create index media_normalized_url on media(normalized_url); @@ -194,6 +210,9 @@ $$ $$ LANGUAGE 'plpgsql'; +COMMENT ON FUNCTION media_rescraping_add_initial_state_trigger () IS 'Insert new rows to "media_rescraping" +for each new row in "media"'; + CREATE TRIGGER media_rescraping_add_initial_state_trigger AFTER INSERT ON media FOR EACH ROW EXECUTE PROCEDURE media_rescraping_add_initial_state_trigger(); @@ -249,6 +268,7 @@ END; $$ LANGUAGE 'plpgsql'; +COMMENT ON FUNCTION media_has_active_syndicated_feeds (param_media_id INT) IS 'true if media has active rss feeds'; create type feed_type AS ENUM ( @@ -316,6 +336,9 @@ CREATE TABLE feeds_after_rescraping ( url VARCHAR(1024) NOT NULL, type feed_type NOT NULL DEFAULT 'syndicated' ); + +COMMENT ON TABLE feeds_after_rescraping IS 'feeds for media item discovered after (re)scraping'; + CREATE INDEX feeds_after_rescraping_media_id ON feeds_after_rescraping(media_id); CREATE INDEX feeds_after_rescraping_name ON feeds_after_rescraping(name); CREATE UNIQUE INDEX feeds_after_rescraping_url ON feeds_after_rescraping(url, media_id); @@ -355,6 +378,8 @@ END; $$ LANGUAGE 'plpgsql'; +COMMENT ON FUNCTION feed_is_stale (param_feeds_id INT) IS '-- Feed is "stale" (has not provided a new story in some time) +-- Not to be confused with "stale feeds" in extractor!'; create table tag_sets ( tag_sets_id serial primary key, @@ -377,6 +402,11 @@ create table tag_sets ( CONSTRAINT tag_sets_name_not_empty CHECK (((name)::text <> ''::text)) ); +COMMENT ON COLUMN tag_sets.show_on_media IS 'should public interfaces show this as an option for +searching media sources?'; +COMMENT ON COLUMN tag_sets.show_on_stories IS 'should public interfaces show this as an option +for search stories?'; + create unique index tag_sets_name on tag_sets (name); create table tags ( @@ -407,6 +437,13 @@ create table tags ( CONSTRAINT tag_not_empty CHECK (((tag)::text <> ''::text)) ); +COMMENT ON COLUMN tags.show_on_media IS 'should public interfaces show this as an option for +searching media sources?'; +COMMENT ON COLUMN tags.show_on_stories IS 'should public interfaces show this as an option +for search stories?'; +COMMENT ON COLUMN tags.is_static IS 'if true, users can expect this tag and its associations +not to change in major ways'; + create index tags_tag_sets_id ON tags (tag_sets_id); create unique index tags_tag on tags (tag, tag_sets_id); create index tags_label on tags (label); @@ -486,6 +523,10 @@ create table stories ( language varchar(3) null -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset ); +COMMENT ON TABLE stories IS 'stories (news articles)'; +COMMENT ON COLUMN stories.language IS '2- or 3-character ISO 690 +language code; empty if unknown, NULL if unset'; + create index stories_media_id on stories (media_id); create unique index stories_guid on stories(guid, media_id); create index stories_url on stories (url); @@ -544,6 +585,11 @@ begin end $function$ language plpgsql; +COMMENT ON FUNCTION get_normalized_title (title text, title_media_id int) IS 'get normalized story title by breaking the title into parts by +the separator characters :-| and using the longest single part. longest part must be at least 32 characters, +cannot be the same as the media source name. also remove all html, punctuation and repeated spaces, +lowecase, and limit to 1024 characters.'; + create function add_normalized_title_hash() returns trigger as $function$ BEGIN @@ -616,6 +662,8 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +COMMENT ON FUNCTION partition_by_stories_id_chunk_size () IS 'Return partition size for every table +that is partitioned by "stories_id"'; -- Return partition table name for a given base table name and "stories_id" CREATE OR REPLACE FUNCTION partition_by_stories_id_partition_name( @@ -634,6 +682,9 @@ END; $$ LANGUAGE plpgsql IMMUTABLE; +COMMENT ON FUNCTION partition_by_stories_id_partition_name (base_table_name TEXT, stories_id BIGINT) IS 'Return +partition table name for a given base table name and "stories_id"'; + -- Create missing partitions for tables partitioned by "stories_id", returning -- a list of created partition tables CREATE OR REPLACE FUNCTION partition_by_stories_id_create_partitions(base_table_name TEXT) @@ -724,6 +775,9 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION partition_by_stories_id_create_partitions (base_table_name TEXT) IS 'Create missing partitions for +tables partitioned by "stories_id", returning a list of created partition tables'; + -- list of all url or guid identifiers for each story create table story_urls ( story_urls_id bigserial primary key, @@ -734,6 +788,8 @@ create table story_urls ( create unique index story_urls_url on story_urls ( url, stories_id ); create index stories_story on story_urls ( stories_id ); +COMMENT ON TABLE story_urls IS 'list of all url or guid identifiers for each story'; + -- -- Downloads -- @@ -832,6 +888,11 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION test_referenced_download_trigger () IS 'Imitate a foreign key by testing if a download +with an INSERTed / UPDATEd "downloads_id" exists in "downloads." Partitioned tables do not support foreign +keys being pointed to them, so this trigger achieves the same referential integrity for tables that point +to "downloads". Column name from NEW (NEW.) that contains the INSERTed / UPDATEd "downloads_id" +should be passed as an trigger argument.'; CREATE INDEX downloads_parent ON downloads (parent); @@ -914,6 +975,9 @@ CREATE TABLE downloads_success_content CREATE UNIQUE INDEX downloads_success_content_downloads_id ON downloads_success_content (downloads_id); +COMMENT ON INDEX downloads_success_content_downloads_id IS 'We need a separate unique index for the +"download_texts" foreign key to be able to point to "downloads_success_content" partitions'; + CREATE INDEX downloads_success_content_extracted ON downloads_success_content (extracted); @@ -962,6 +1026,8 @@ BEGIN END; $$ LANGUAGE plpgsql IMMUTABLE; +COMMENT ON FUNCTION partition_by_downloads_id_chunk_size () IS +'Return partition size for every table that is partitioned by "downloads_id"'; -- Return partition table name for a given base table name and "downloads_id" CREATE OR REPLACE FUNCTION partition_by_downloads_id_partition_name( @@ -980,6 +1046,9 @@ END; $$ LANGUAGE plpgsql IMMUTABLE; +COMMENT ON FUNCTION partition_by_downloads_id_partition_name (base_table_name TEXT, downloads_id BIGINT) IS 'Return +partition table name for a given base table name and "downloads_id"'; + -- Create missing partitions for tables partitioned by "downloads_id", returning -- a list of created partition tables CREATE OR REPLACE FUNCTION partition_by_downloads_id_create_partitions(base_table_name TEXT) @@ -1063,6 +1132,9 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION partition_by_downloads_id_create_partitions (base_table_name TEXT) IS 'Create +missing partitions for tables partitioned by "downloads_id", returning a list of +created partition tables'; -- Create subpartitions of "downloads_success_feed" or "downloads_success_content" CREATE OR REPLACE FUNCTION downloads_create_subpartitions(base_table_name TEXT) @@ -1092,6 +1164,8 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION downloads_create_subpartitions (base_table_name TEXT) IS 'Create +subpartitions of "downloads_success_feed" or "downloads_success_content"'; -- Create missing "downloads_success_content" partitions CREATE OR REPLACE FUNCTION downloads_success_content_create_partitions() @@ -1103,6 +1177,9 @@ $$ $$ LANGUAGE SQL; +COMMENT ON FUNCTION downloads_success_content_create_partitions () IS 'Create +missing "downloads_success_content" partitions'; + -- Create initial "downloads_success_content" partitions for empty database SELECT downloads_success_content_create_partitions(); @@ -1117,12 +1194,16 @@ $$ $$ LANGUAGE SQL; +COMMENT ON FUNCTION downloads_success_feed_create_partitions () IS 'Create missing +"downloads_success_feed" partitions'; + -- Create initial "downloads_success_feed" partitions for empty database SELECT downloads_success_feed_create_partitions(); -- table for object types used for mediawords.util.public_store create schema public_store; +COMMENT ON SCHEMA public_store IS 'table for object types used for mediawords.util.public_store'; create table public_store.timespan_files ( timespan_files_id bigserial primary key, @@ -1160,6 +1241,11 @@ CREATE TABLE raw_downloads ( raw_data BYTEA NOT NULL ); + +COMMENT ON TABLE raw_downloads IS 'Raw downloads stored in the database (if +the "postgresql" download storage method is enabled)'; +COMMENT ON COLUMN raw_downloads.object_id IS '"downloads_id" from "downloads"'; + CREATE UNIQUE INDEX raw_downloads_object_id ON raw_downloads (object_id); @@ -1173,6 +1259,8 @@ CREATE TRIGGER raw_downloads_test_referenced_download_trigger FOR EACH ROW EXECUTE PROCEDURE test_referenced_download_trigger('object_id'); +COMMENT ON COLUMN raw_downloads.raw_data IS 'Do not attempt to compress BLOBs in +"raw_data" because they are going to becompressed already'; -- -- Feed -> story map @@ -1188,6 +1276,11 @@ CREATE TABLE feeds_stories_map_p ( stories_id INT NOT NULL ); +COMMENT ON TABLE feeds_stories_map_p IS '"Master" table (no indexes, no foreign keys as +they will be ineffective)'; +COMMENT ON COLUMN feeds_stories_map_p.feeds_stories_map_p_id IS 'PRIMARY KEY on master table +needed for database handler primary_key_column() method to work'; + -- Note: "INSERT ... RETURNING *" doesn't work with the trigger, please use -- "feeds_stories_map" view instead CREATE OR REPLACE FUNCTION feeds_stories_map_p_insert_trigger() @@ -1208,6 +1301,10 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION feeds_stories_map_p_insert_trigger () IS 'Note: "INSERT ... RETURNING *" does not +work with the trigger, please use "feeds_stories_map" view instead. target_table_name += partition table name (e.g. "feeds_stories_map_01")'; + CREATE TRIGGER feeds_stories_map_p_insert_trigger BEFORE INSERT ON feeds_stories_map_p FOR EACH ROW EXECUTE PROCEDURE feeds_stories_map_p_insert_trigger(); @@ -1246,6 +1343,9 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION feeds_stories_map_create_partitions () IS 'Create missing +"feeds_stories_map_p" partitions'; + -- Create initial "feeds_stories_map_p" partitions for empty database SELECT feeds_stories_map_create_partitions(); @@ -1259,6 +1359,9 @@ CREATE OR REPLACE VIEW feeds_stories_map AS stories_id FROM feeds_stories_map_p; +COMMENT ON VIEW feeds_stories_map IS 'Proxy view to "feeds_stories_map_p" +to make RETURNING work with partitioned tables + (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning)'; -- Make RETURNING work with partitioned tables -- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) @@ -1310,6 +1413,10 @@ BEGIN END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION feeds_stories_map_view_insert_update_delete () IS 'Trigger that +implements INSERT / UPDATE / DELETE behavior on "feeds_stories_map" view. By INSERTing +into the master table (feeds_stories_map_p), we are letting triggers choose the correct partition.'; + CREATE TRIGGER feeds_stories_map_view_insert_update_delete_trigger INSTEAD OF INSERT OR UPDATE OR DELETE ON feeds_stories_map FOR EACH ROW EXECUTE PROCEDURE feeds_stories_map_view_insert_update_delete(); @@ -1330,6 +1437,10 @@ CREATE TABLE stories_tags_map_p ( tags_id INT NOT NULL ); +COMMENT ON TABLE stories_tags_map_p IS '"Master" table (no indexes, +no foreign keys as they will be ineffective)'; +COMMENT ON COLUMN stories_tags_map_p.stories_tags_map_p_id IS 'PRIMARY KEY on +master table needed for database handler primary_key_column() method to work'; -- Create missing "stories_tags_map" partitions CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() @@ -1365,6 +1476,9 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION stories_tags_map_create_partitions () IS 'Create missing "stories_tags_map" +partitions, add extra foreign keys / constraints to the newly created partitions'; + -- Create initial "stories_tags_map" partitions for empty database SELECT stories_tags_map_create_partitions(); @@ -1389,6 +1503,8 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION stories_tags_map_p_upsert_trigger () IS 'Upsert row into correct partition'; + CREATE TRIGGER stories_tags_map_p_upsert_trigger BEFORE INSERT ON stories_tags_map_p FOR EACH ROW EXECUTE PROCEDURE stories_tags_map_p_upsert_trigger(); @@ -1408,6 +1524,8 @@ CREATE OR REPLACE VIEW stories_tags_map AS tags_id FROM stories_tags_map_p; +COMMENT ON VIEW stories_tags_map IS 'Proxy view to "stories_tags_map_p" to make RETURNING work +with partitioned tables'; -- Make RETURNING work with partitioned tables -- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) @@ -1458,6 +1576,10 @@ BEGIN END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION stories_tags_map_view_insert_update_delete () IS 'Trigger +that implements INSERT / UPDATE / DELETE behavior on "stories_tags_map" view. By +INSERTing into the master table, we are letting triggers choose the correct partition.'; + CREATE TRIGGER stories_tags_map_view_insert_update_delete INSTEAD OF INSERT OR UPDATE OR DELETE ON stories_tags_map FOR EACH ROW EXECUTE PROCEDURE stories_tags_map_view_insert_update_delete(); @@ -1492,6 +1614,9 @@ end; $$ language plpgsql; +COMMENT ON FUNCTION pop_queued_download () IS 'do this as a plpgsql function +because it wraps it in the necessary transaction without having to know whether +the calling context is in a transaction'; -- -- Extracted plain text from every download @@ -1553,6 +1678,8 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION download_texts_create_partitions () IS 'Create missing "download_texts" partitions'; + -- Create initial "download_texts" partitions for empty database SELECT download_texts_create_partitions(); @@ -1588,6 +1715,18 @@ CREATE TABLE story_sentences_p ( is_dup BOOLEAN NULL ); +COMMENT ON TABLE story_sentences_p IS 'Master table for individual sentences of stories +(no indexes, no foreign keys as they will be ineffective)'; +COMMENT ON COLUMN story_sentences_p.language IS '2- or 3-character ISO 690 +language code; empty if unknown, NULL if unset'; +COMMENT ON COLUMN story_sentences_p.is_dup IS 'Set to true for every sentence for +which a duplicate sentence was found in a future story (even though that duplicate sentence +was not added to the table). We only use is_dup in the topic spidering, but I think it is critical +there. It is there because the first time I tried to run a spider on a broadly popular topic, +it was unusable because of the amount of irrelevant content. When I dug in, I found that stories +were getting included because of matches on boilerplate content that was getting duped out of +most stories but not the first time it appeared. So I added the check to remove stories that match +on a dup sentence, even if it is the dup sentence, and things cleaned up.'; -- Note: "INSERT ... RETURNING *" doesn't work with the trigger, please use -- "story_sentences" view instead @@ -1609,6 +1748,9 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION story_sentences_p_insert_trigger () IS 'Note: "INSERT ... RETURNING *" +does not work with the trigger, please use "story_sentences" view instead'; + CREATE TRIGGER story_sentences_p_insert_trigger BEFORE INSERT ON story_sentences_p FOR EACH ROW EXECUTE PROCEDURE story_sentences_p_insert_trigger(); @@ -1647,6 +1789,8 @@ END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION story_sentences_create_partitions () IS 'Create missing "story_sentences_p" partitions'; + -- Create initial "story_sentences_p" partitions for empty database SELECT story_sentences_create_partitions(); @@ -1665,6 +1809,8 @@ CREATE OR REPLACE VIEW story_sentences AS is_dup FROM story_sentences_p; +COMMENT ON VIEW story_sentences IS 'Proxy view to "story_sentences_p" to make RETURNING work +with partitioned tables'; -- Make RETURNING work with partitioned tables -- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) @@ -1720,6 +1866,10 @@ BEGIN END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION story_sentences_view_insert_update_delete () IS 'Trigger that +implements INSERT / UPDATE / DELETE behavior on "story_sentences" view. By INSERTing +into the master table, we are letting triggers choose the correct partition.'; + CREATE TRIGGER story_sentences_view_insert_update_delete_trigger INSTEAD OF INSERT OR UPDATE OR DELETE ON story_sentences FOR EACH ROW EXECUTE PROCEDURE story_sentences_view_insert_update_delete(); @@ -1737,6 +1887,9 @@ create index solr_imports_date on solr_imports ( import_date ); create table solr_import_stories ( stories_id int not null references stories on delete cascade ); + +COMMENT ON TABLE solr_import_stories IS 'Extra stories to import into'; + create index solr_import_stories_story on solr_import_stories ( stories_id ); -- log of all stories import into solr, with the import date @@ -1745,6 +1898,8 @@ create table solr_imported_stories ( import_date timestamp not null ); +COMMENT ON TABLE solr_imported_stories IS 'log of all stories import into solr, with the import date'; + create index solr_imported_stories_story on solr_imported_stories ( stories_id ); create index solr_imported_stories_day on solr_imported_stories ( date_trunc( 'day', import_date ) ); @@ -1757,6 +1912,9 @@ create table topic_modes ( description text not null ); +COMMENT ON TABLE topic_modes IS 'the mode is how we analyze the data from the platform +(as web pages, social media posts, url sharing posts, etc)'; + create unique index topic_modes_name on topic_modes(name); insert into topic_modes ( name, description ) values @@ -1771,6 +1929,8 @@ create table topic_platforms ( description text not null ); +COMMENT ON TABLE topic_platforms IS 'the platform is where the analyzed data lives (web, twitter, reddit, etc)'; + create unique index topic_platforms_name on topic_platforms(name); insert into topic_platforms (name, description) values @@ -1786,6 +1946,9 @@ create table topic_sources ( description text not null ); +COMMENT ON TABLE topic_sources IS 'the source is where we get the +platforn data from (a particular database, api, csv, etc)'; + create unique index topic_sources_name on topic_sources(name); insert into topic_sources ( name, description ) values @@ -1803,6 +1966,9 @@ create table topic_platforms_sources_map ( topic_sources_id int not null references topic_sources on delete cascade ); +COMMENT ON TABLE topic_platforms_sources_map IS 'the pairs of platforms/sources +for which the platform can fetch data'; + create unique index topic_platforms_sources_map_ps on topic_platforms_sources_map ( topic_platforms_id, topic_sources_id ); @@ -1820,6 +1986,8 @@ create function insert_platform_source_pair( text, text ) returns void as $$ ts.name = $2 $$ language sql; +COMMENT ON FUNCTION insert_platform_source_pair ( text, text ) IS 'easily create platform source pairs'; + select insert_platform_source_pair( 'web', 'mediacloud' ); select insert_platform_source_pair( 'twitter', 'crimson_hexagon' ); select insert_platform_source_pair( 'generic_post', 'csv' ); @@ -1870,6 +2038,17 @@ create table topics ( only_snapshot_engaged_stories boolean not null default false ); +COMMENT ON COLUMN topics.respider_stories IS 'if true, the topic_stories +associated with this topic wilbe set to link_mined = "f" on the next mining job'; +COMMENT ON COLUMN topics.snapshot_periods IS 'space-separated list of periods to snapshot'; +COMMENT ON COLUMN topics.platform IS 'platform that topic is analyzing'; +COMMENT ON COLUMN topics.mode IS 'mode of analysis'; +COMMENT ON COLUMN topics.job_queue IS 'job queue to use for spider and snapshot jobs for this topic'; +COMMENT ON COLUMN topics.is_story_index_ready IS 'if false, we should refuse to spider +this topic because the use has not confirmed the new story query syntax'; +COMMENT ON COLUMN topics.only_snapshot_engaged_stories IS 'if true, snapshots +are pruned to only stories with a minimum level of engagements (links, shares, etc)'; + create unique index topics_name on topics( name ); create unique index topics_media_type_tag_set on topics( media_type_tag_sets_id ); @@ -1930,8 +2109,9 @@ create table topic_domains ( self_links int not null default 0 ); -create unique index topic_domains_domain on topic_domains (topics_id, md5(domain)); +COMMENT ON TABLE topic_domains IS 'track self liks and all links for a given domain within a given topic'; +create unique index topic_domains_domain on topic_domains (topics_id, md5(domain)); create table topic_stories ( topic_stories_id serial primary key, @@ -1956,6 +2136,8 @@ create table topic_dead_links ( url text not null ); +COMMENT ON TABLE topic_dead_links IS 'topic links for which the http request failed'; + -- no foreign key constraints on topics_id and stories_id because -- we have the combined foreign key constraint pointing to topic_stories -- below @@ -1969,6 +2151,9 @@ create table topic_links ( link_spidered boolean default 'f' ); +COMMENT ON TABLE topic_links IS 'no foreign key constraints on topics_id and stories_id +because we have the combined foreign key constraint pointing to topic_stories below'; + alter table topic_links add constraint topic_links_topic_story_stories_id foreign key ( stories_id, topics_id ) references topic_stories ( stories_id, topics_id ) on delete cascade; @@ -2117,6 +2302,10 @@ create table timespans ( ( snapshots_id is not null and archive_snapshots_id is null ) ) ); +COMMENT ON COLUMN timespans.snapshots_id IS 'individual timespans within a snapshot'; +COMMENT ON COLUMN timespans.archive_snapshots_id IS 'timespan is an archived part of +this snapshot (and thus mostly not visible)'; +COMMENT ON COLUMN timespans.tags_id IS 'keep on cascade to avoid accidental deletion'; create index timespans_snapshot on timespans ( snapshots_id ); create unique index timespans_unique on timespans ( snapshots_id, foci_id, start_date, end_date, period ); @@ -2153,6 +2342,7 @@ create unique index snapshot_files_snapshot_name on snapshot_files ( snapshots_i -- schema to hold the various snapshot snapshot tables CREATE SCHEMA snap; +COMMENT ON SCHEMA snap IS 'schema to hold the various snapshot snapshot tables'; CREATE OR REPLACE LANGUAGE plpgsql; @@ -2175,6 +2365,12 @@ create table snap.stories ( full_text_rss boolean not null default 'f', language varchar(3) null -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset ); + +COMMENT ON TABLE snap.stories IS 'create a table for each of these tables to hold a snapshot of stories +relevant to a topic for each snapshot for that topic'; +COMMENT ON COLUMN snap.stories.language IS '2- or 3-character ISO 690 +language code; empty if unknown, NULL if unset'; + create index stories_id on snap.stories ( snapshots_id, stories_id ); -- stats for various externally dervied statistics about a story. @@ -2189,6 +2385,8 @@ create table story_statistics ( facebook_api_error text null ); +COMMENT ON TABLE story_statistics IS 'stats for various externally dervied statistics about a story.'; + create unique index story_statistics_story on story_statistics ( stories_id ); @@ -2202,8 +2400,9 @@ create table story_statistics_twitter ( twitter_api_error text null ); -create unique index story_statistics_twitter_story on story_statistics_twitter ( stories_id ); +COMMENT ON TABLE story_statistics_twitter IS 'stats for deprecated Twitter share counts'; +create unique index story_statistics_twitter_story on story_statistics_twitter ( stories_id ); create table snap.topic_stories ( snapshots_id int not null references snapshots on delete cascade, @@ -2277,8 +2476,14 @@ create table snap.story_links ( ref_stories_id int not null ); +COMMENT ON TABLE snap.story_links IS 'story -> story links within a timespan'; + -- TODO: add complex foreign key to check that *_stories_id exist for the snapshot stories snapshot create index story_links_source on snap.story_links( timespans_id, source_stories_id ); + +COMMENT ON INDEX snap.story_links_source IS 'TODO: add complex foreign key to check that +*_stories_id exist for the snapshot stories snapshot'; + create index story_links_ref on snap.story_links( timespans_id, ref_stories_id ); -- link counts for stories within a timespan @@ -2297,8 +2502,14 @@ create table snap.story_link_counts ( channel_count int null ); +COMMENT ON TABLE snap.story_link_counts IS 'link counts for stories within a timespan'; + -- TODO: add complex foreign key to check that stories_id exists for the snapshot stories snapshot create index story_link_counts_ts on snap.story_link_counts ( timespans_id, stories_id ); + +COMMENT ON INDEX snap.story_link_counts_ts IS 'TODO: add complex foreign key to check that stories_id +exists for the snapshot stories snapshot'; + create index story_link_counts_story on snap.story_link_counts ( stories_id ); create index story_link_counts_fb on snap.story_link_counts ( timespans_id, facebook_share_count desc nulls last ); create index story_link_counts_post on snap.story_link_counts ( timespans_id, post_count desc nulls last); @@ -2323,8 +2534,14 @@ create table snap.medium_link_counts ( sum_channel_count int null ); +COMMENT ON TABLE snap.medium_link_counts IS 'links counts for media within a timespan'; + -- TODO: add complex foreign key to check that media_id exists for the snapshot media snapshot create index medium_link_counts_medium on snap.medium_link_counts ( timespans_id, media_id ); + +COMMENT ON INDEX snap.medium_link_counts_medium IS 'TODO: add complex foreign key +to check that media_id exists for the snapshot media snapshot'; + create index medium_link_counts_fb on snap.medium_link_counts ( timespans_id, facebook_share_count desc nulls last); create index medium_link_counts_sum_post on snap.medium_link_counts ( timespans_id, sum_post_count desc nulls last); create index medium_link_counts_sum_author on snap.medium_link_counts ( timespans_id, sum_author_count desc nulls last); @@ -2340,6 +2557,10 @@ create table snap.medium_links ( -- TODO: add complex foreign key to check that *_media_id exist for the snapshot media snapshot create index medium_links_source on snap.medium_links( timespans_id, source_media_id ); + +COMMENT ON INDEX snap.medium_links_source IS 'TODO: add complex foreign key to check that +*_media_id exist for the snapshot media snapshot'; + create index medium_links_ref on snap.medium_links( timespans_id, ref_media_id ); -- create a mirror of the stories table with the stories for each topic. this is to make @@ -2362,6 +2583,13 @@ create table snap.live_stories ( language varchar(3) null -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset ); +COMMENT ON TABLE snap.live_stories IS 'create a mirror of the stories table with the stories +for each topic. this is to make it much faster to query the stories associated with a given topic, +rather than querying the contested and bloated stories table. only inserts and updates on stories +are triggered, because deleted cascading stories_id and topics_id fields take care of deletes.'; +COMMENT ON COLUMN snap.live_stories.language IS '2- or 3-character ISO 690 language code; +empty if unknown, NULL if unset'; + create index live_story_topic on snap.live_stories ( topics_id ); create unique index live_stories_story on snap.live_stories ( topics_id, stories_id ); create index live_stories_story_solo on snap.live_stories ( stories_id ); @@ -2428,6 +2656,8 @@ CREATE TABLE snap.word2vec_models ( -- We'll need to find the latest word2vec model CREATE INDEX snap_word2vec_models_object_id_creation_date ON snap.word2vec_models (object_id, creation_date); +COMMENT ON INDEX snap.snap_word2vec_models_object_id_creation_date IS 'We need to find the latest word2vec model'; + CREATE TABLE snap.word2vec_models_data ( word2vec_models_data_id SERIAL PRIMARY KEY, object_id INTEGER NOT NULL @@ -2442,6 +2672,8 @@ CREATE UNIQUE INDEX snap_word2vec_models_data_object_id ON snap.word2vec_models_ ALTER TABLE snap.word2vec_models_data ALTER COLUMN raw_data SET STORAGE EXTERNAL; +COMMENT ON TABLE snap.word2vec_models_data IS 'Do not (attempt to) compress BLOBs in "raw_data" because +they are going to be compressed already'; create table processed_stories ( processed_stories_id bigserial primary key, @@ -2460,6 +2692,8 @@ create table scraped_stories ( import_module text not null ); +COMMENT ON TABLE scraped_stories IS 'list of stories that have been scraped and the source'; + create index scraped_stories_story on scraped_stories ( stories_id ); -- dates on which feeds have been scraped with MediaWords::ImportStories and the module used for scraping @@ -2470,6 +2704,9 @@ create table scraped_feeds ( import_module text not null ); +COMMENT ON TABLE scraped_feeds IS 'dates on which feeds have been scraped with MediaWords::ImportStories +and the module used for scraping'; + create index scraped_feeds_feed on scraped_feeds ( feeds_id ); create view feedly_unscraped_feeds as @@ -2557,10 +2794,16 @@ CREATE TABLE auth_users ( has_consented BOOLEAN NOT NULL DEFAULT false ); +COMMENT ON COLUMN auth_users.email IS 'Emails are case-insensitive'; +COMMENT ON COLUMN auth_users.password_hash IS 'salted hash of a password'; +COMMENT ON COLUMN auth_users.password_reset_token_hash IS 'Salted hash of a +password reset token (with Crypt::SaltedHash, algorithm => "SHA-256", salt_len=>64) or NULL'; +COMMENT ON COLUMN auth_users.has_consented IS 'Whether user has consented to the privacy policy'; -- Used by daily stats script CREATE INDEX auth_users_created_day ON auth_users (date_trunc('day', created_date)); +COMMENT ON INDEX auth_users_created_day IS 'used by daily stats script'; -- Generate random API key CREATE FUNCTION generate_api_key() RETURNS VARCHAR(64) LANGUAGE plpgsql AS $$ @@ -2589,6 +2832,9 @@ CREATE TABLE auth_user_api_keys ( ip_address INET NULL ); +COMMENT ON COLUMN auth_user_api_keys.api_key IS 'must be 64 bytes in order to prevent someone +from resetting it to empty string somehow'; + CREATE UNIQUE INDEX auth_user_api_keys_api_key_ip_address ON auth_user_api_keys (api_key, ip_address); @@ -2669,7 +2915,7 @@ CREATE TABLE auth_user_request_daily_counts ( -- Single index to enforce upsert uniqueness CREATE UNIQUE INDEX auth_user_request_daily_counts_email_day ON auth_user_request_daily_counts (email, day); - +COMMENT ON INDEX auth_user_request_daily_counts_email_day IS 'Single index to enforce upsert uniqueness'; -- User limits for logged + throttled controller actions CREATE TABLE auth_user_limits ( @@ -2691,6 +2937,10 @@ CREATE TABLE auth_user_limits ( ); +COMMENT ON TABLE auth_user_limits IS 'User limits for logged + throttled controller actions'; +COMMENT ON COLUMN auth_user_limits.weekly_requests_limit IS 'Request limit (0 or belonging to +"admin"/"admin-readonly" group = no limit)'; + CREATE UNIQUE INDEX auth_user_limits_auth_users_id ON auth_user_limits (auth_users_id); -- Set the default limits for newly created users @@ -2777,6 +3027,14 @@ CREATE TABLE activities ( ); +COMMENT ON COLUMN activities.name IS 'activity name, e.g. "tm_snapshot_topic"'; +COMMENT ON COLUMN activities.object_id IS 'Indexed ID of the object that was modified +in some way by the activity'; +COMMENT ON COLUMN activities.reason IS 'user-provided reason why the activity was made'; +COMMENT ON COLUMN activities.description_json IS 'Other free-form data describing the +action in the JSON format (e.g.: { "field": "name", "old_value": "Foo.", "new_value": "Bar." }). +FIXME: has potential to use JSON type instead of TEXT in PostgreSQL 9.2+'; + CREATE INDEX activities_name ON activities (name); CREATE INDEX activities_creation_date ON activities (creation_date); CREATE INDEX activities_user_identifier ON activities (user_identifier); @@ -2816,6 +3074,9 @@ CREATE TABLE feeds_from_yesterday ( active BOOLEAN NOT NULL ); +COMMENT ON TABLE feeds_from_yesterday IS 'Copy of "feeds" table from yesterday; +used for generating reports for rescraping efforts'; + CREATE INDEX feeds_from_yesterday_feeds_id ON feeds_from_yesterday(feeds_id); CREATE INDEX feeds_from_yesterday_media_id ON feeds_from_yesterday(media_id); CREATE INDEX feeds_from_yesterday_name ON feeds_from_yesterday(name); @@ -3042,6 +3303,8 @@ $$ LANGUAGE 'plpgsql'; +COMMENT ON FUNCTION rescraping_changes () IS 'Print out a diff between "feeds" and "feeds_from_yesterday"'; + -- implements link_id as documented in the topics api spec create table api_links ( api_links_id bigserial primary key, @@ -3051,6 +3314,8 @@ create table api_links ( previous_link_id bigint null references api_links on delete set null deferrable ); +COMMENT ON TABLE api_links IS 'implements link_id as documented in the topics api spec'; + create unique index api_links_params on api_links ( path, md5( params_json ) ); -- Create missing partitions for partitioned tables @@ -3155,6 +3420,12 @@ create or replace view topics_with_user_permission as join auth_users u on ( true ) left join topic_permissions tp using ( topics_id, auth_users_id ); +COMMENT ON VIEW topics_with_user_permission IS 'topics table with auth_users_id and user_permission fields +that indicate the permission level for the user for the topic. permissions in decreasing order are admin, +write, read, none. users with the admin role have admin permission for every topic. users with admin-readonly +role have at least read access to every topic. all users have read access to every is_public topic. otherwise, +the topic_permissions tableis used, with "none" for no topic_permission.'; + -- list of tweet counts and fetching statuses for each day of each topic create table topic_post_days ( topic_post_days_id serial primary key, @@ -3165,6 +3436,8 @@ create table topic_post_days ( posts_fetched boolean not null default false ); +COMMENT ON TABLE topic_post_days IS 'list of tweet counts and fetching statuses for each day of each topic'; + create index topic_post_days_td on topic_post_days ( topic_seed_queries_id, day ); -- list of posts associated with a given topic @@ -3180,6 +3453,8 @@ create table topic_posts ( url text null ); +COMMENT ON TABLE topic_posts IS 'list of posts associated with a given topic'; + create unique index topic_posts_id on topic_posts( topic_post_days_id, post_id ); create index topic_post_topic_author on topic_posts( topic_post_days_id, author ); create index topic_post_topic_channel on topic_posts( topic_post_days_id, channel ); @@ -3191,6 +3466,8 @@ create table topic_post_urls ( url varchar (1024) not null ); +COMMENT ON TABLE topic_post_urls IS 'urls parsed from topic tweets and imported into topic_seed_urls'; + create index topic_post_urls_url on topic_post_urls ( url ); create unique index topic_post_urls_tt on topic_post_urls ( topic_posts_id, url ); @@ -3234,7 +3511,9 @@ create view topic_post_stories as join topic_stories ts on ( ts.topics_id = tsq.topics_id and ts.stories_id = tsu.stories_id ); - +COMMENT ON VIEW topic_post_stories IS 'view that joins together the chain of tables from topic_seed_queries +all the way through to topic_stories, so that you get back a topics_id, topic_posts_id stories_id, and +topic_seed_queries_id in each row to track which stories came from which posts in which seed queries'; create table snap.timespan_posts ( topic_posts_id int not null references topic_posts on delete cascade, @@ -3339,6 +3618,8 @@ create table mediacloud_stats ( total_sentences bigint not null ); +COMMENT ON TABLE mediacloud_stats IS 'keep track of basic high level stats for mediacloud for access through api'; + -- job states as implemented in mediawords.job.StatefulJobBroker create table job_states ( job_states_id serial primary key, @@ -3364,6 +3645,12 @@ create table job_states ( process_id int not null ); +COMMENT ON TABLE job_states IS 'job states as implemented in mediawords.job.StatefulJobBroker'; +COMMENT ON COLUMN job_states.class IS 'MediaWords::Job::* class implementing the job'; +COMMENT ON COLUMN job_states.state IS 'short class-specific state'; +COMMENT ON COLUMN job_states.message IS 'optional longer message describing the state, such +as a stack trace for an error'; + create index job_states_class_date on job_states( class, last_updated ); create view pending_job_states as select * from job_states where state in ( 'running', 'queued' ); @@ -3383,6 +3670,8 @@ create table retweeter_scores ( match_type retweeter_scores_match_type not null default 'retweet' ); +COMMENT ON TABLE retweeter_scores IS 'definition of bipolar comparisons for retweeter polarization scores'; + -- group retweeters together so that we an compare, for example, sanders/warren retweeters to cruz/kasich retweeters create table retweeter_groups ( retweeter_groups_id serial primary key, @@ -3390,6 +3679,9 @@ create table retweeter_groups ( name text not null ); +COMMENT ON TABLE retweeter_groups IS 'group retweeters together so that we +can compare, for example, sanders/warren retweeters to cruz/kasich retweeters'; + alter table retweeter_scores add constraint retweeter_scores_group_a foreign key ( group_a_id ) references retweeter_groups on delete cascade; alter table retweeter_scores add constraint retweeter_scores_group_b @@ -3403,6 +3695,8 @@ create table retweeters ( retweeted_user varchar(1024) not null ); +COMMENT ON TABLE retweeters IS 'list of twitter users within a given topic that have retweeted the given user'; + create unique index retweeters_user on retweeters( retweeter_scores_id, twitter_user, retweeted_user ); create table retweeter_groups_users_map ( @@ -3420,6 +3714,8 @@ create table retweeter_stories ( share_count int not null ); +COMMENT ON TABLE retweeter_stories IS 'count of shares by retweeters for each retweeted_user in retweeters'; + create unique index retweeter_stories_psu on retweeter_stories ( retweeter_scores_id, stories_id, retweeted_user ); @@ -3435,6 +3731,9 @@ create table retweeter_media ( partition int not null ); +COMMENT ON TABLE retweeter_media IS 'polarization scores for media within a topic for the given +retweeter_scores definition'; + create unique index retweeter_media_score on retweeter_media ( retweeter_scores_id, media_id ); create table retweeter_partition_matrix ( @@ -3456,6 +3755,8 @@ create index retweeter_partition_matrix_score on retweeter_partition_matrix ( re CREATE SCHEMA cache; +COMMENT ON SCHEMA cache IS 'schema to hold object caches'; + CREATE OR REPLACE LANGUAGE plpgsql; @@ -3468,6 +3769,8 @@ BEGIN END; $$ LANGUAGE 'plpgsql'; +COMMENT ON FUNCTION cache.update_cache_db_row_last_updated () IS 'Trigger +to update "db_row_last_updated" for cache tables'; -- Helper to purge object caches CREATE OR REPLACE FUNCTION cache.purge_object_caches() @@ -3507,6 +3810,12 @@ CREATE UNLOGGED TABLE cache.s3_raw_downloads_cache ( raw_data BYTEA NOT NULL ); + + +COMMENT ON COLUMN cache.s3_raw_downloads_cache.object_id IS '"downloads_id" from "downloads"'; +COMMENT ON COLUMN cache.s3_raw_downloads_cache.db_row_last_updated IS 'Will be used to purge old cache objects; +do not forget to update cache.purge_object_caches()'; + CREATE UNIQUE INDEX s3_raw_downloads_cache_object_id ON cache.s3_raw_downloads_cache (object_id); CREATE INDEX s3_raw_downloads_cache_db_row_last_updated @@ -3538,6 +3847,12 @@ CREATE UNLOGGED TABLE cache.extractor_results_cache ( -- don't forget to update cache.purge_object_caches() db_row_last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() ); + +COMMENT ON TABLE cache.extractor_results_cache IS 'Cached extractor results for +extraction jobs with use_cache set to true'; +COMMENT ON COLUMN cache.extractor_results_cache.db_row_last_updated IS 'Will be used to purge old cache objects; +do not forget to update cache.purge_object_caches()'; + CREATE UNIQUE INDEX extractor_results_cache_downloads_id ON cache.extractor_results_cache (downloads_id); CREATE INDEX extractor_results_cache_db_row_last_updated @@ -3597,6 +3912,11 @@ create unlogged table domain_web_requests ( request_time timestamp not null default now() ); +COMMENT ON TABLE domain_web_requests IS 'keep track of per domain web requests so that we can throttle them +using mediawords.util.web.user_agent.throttled. this is unlogged because we do not care about anything more +than about 10 seconds old. we do not have a primary key because we want it just to be a fast table for +temporary storage.'; + create index domain_web_requests_domain on domain_web_requests ( domain ); -- return false if there is a request for the given domain within the last domain_timeout_arg milliseconds. otherwise @@ -3632,6 +3952,12 @@ return true; end $$ language plpgsql; +COMMENT ON FUNCTION get_domain_web_requests_lock ( domain_arg text, domain_timeout_arg float ) IS 'return +false if there is a request for the given domain within the last domain_timeout_arg milliseconds. otherwise +return true and insert a row into domain_web_request for the domain. this function does not lock the table and +so may allow some parallel requests through. we do not want this table to grow forever or to have to manage +it externally, so just truncate about every 1 million requests. only do this if there are more than 1000 rows +in the table so that unit tests will not randomly fail.'; CREATE TYPE media_sitemap_pages_change_frequency AS ENUM ( 'always', @@ -3672,6 +3998,8 @@ CREATE TABLE media_sitemap_pages ( ); +COMMENT ON TABLE media_sitemap_pages IS 'Pages derived from XML sitemaps (stories or not)'; + CREATE INDEX media_sitemap_pages_media_id ON media_sitemap_pages (media_id); @@ -3697,6 +4025,13 @@ CREATE TABLE similarweb_domains ( ); +COMMENT ON TABLE similarweb_domains IS 'Domains for which we have tried to fetch SimilarWeb stats. +Every media source domain for which we have tried to fetch estimated visits from SimilarWeb gets +stored here. The domain might have been invalid or unpopular enough so "similarweb_estimated_visits" +might not necessarily store stats for every domain in this table.'; +COMMENT ON COLUMN similarweb_domains.domain IS 'Top-level (e.g. cnn.com) or second-level +(e.g. edition.cnn.com) domain'; + CREATE UNIQUE INDEX similarweb_domains_domain ON similarweb_domains (domain); @@ -3715,10 +4050,16 @@ CREATE TABLE media_similarweb_domains_map ( similarweb_domains_id INT NOT NULL REFERENCES similarweb_domains (similarweb_domains_id) ON DELETE CASCADE ); +COMMENT ON TABLE media_similarweb_domains_map IS 'Media - SimilarWeb domain map. A few media sources +might be pointing to one or more domains due to code differences in how domain was extracted from media +source URL between various implementations.'; + -- Different media sources can point to the same domain CREATE UNIQUE INDEX media_similarweb_domains_map_media_id_sdi ON media_similarweb_domains_map (media_id, similarweb_domains_id); +COMMENT ON INDEX media_similarweb_domains_map_media_id_sdi IS 'Different media sources can point +to the same domain'; -- -- SimilarWeb estimated visits for domain @@ -3741,6 +4082,8 @@ CREATE TABLE similarweb_estimated_visits ( ); +COMMENT ON TABLE similarweb_estimated_visits IS 'https://www.similarweb.com/corp/developer/estimated_visits_api'; + CREATE UNIQUE INDEX similarweb_estimated_visits_domain_month_mdo ON similarweb_estimated_visits (similarweb_domains_id, month, main_domain_only); @@ -3766,10 +4109,11 @@ CREATE TABLE story_enclosures ( length BIGINT NULL ); +COMMENT ON TABLE story_enclosures IS 'Enclosures added to feed item of the story'; + CREATE UNIQUE INDEX story_enclosures_stories_id_url ON story_enclosures (stories_id, url); - -- -- Celery job results -- (configured as self.__app.conf.database_table_names; schema is dictated by Celery + SQLAlchemy) @@ -3782,6 +4126,9 @@ CREATE TABLE celery_groups ( date_done TIMESTAMP WITHOUT TIME ZONE NULL ); +COMMENT ON TABLE celery_groups IS 'Celery job results (configured as self.__app.conf.database_table_names; +schema is dictated by Celery + SQLAlchemy)'; + CREATE TABLE celery_tasks ( id BIGINT NOT NULL PRIMARY KEY, task_id CHARACTER VARYING(155) NULL UNIQUE, @@ -3792,4 +4139,3 @@ CREATE TABLE celery_tasks ( ); CREATE SEQUENCE task_id_sequence AS BIGINT; - diff --git a/apps/postgresql-server/pgmigrate/migrations/V0002__drop_db_version_var.sql b/apps/postgresql-server/pgmigrate/migrations/V0002__drop_db_version_var.sql new file mode 100644 index 0000000000..6f6e738a89 --- /dev/null +++ b/apps/postgresql-server/pgmigrate/migrations/V0002__drop_db_version_var.sql @@ -0,0 +1 @@ +DELETE FROM database_variables WHERE name = 'database-schema-version'; \ No newline at end of file diff --git a/apps/postgresql-server/pgmigrate/migrations/V0003__drop_db_version_func.sql b/apps/postgresql-server/pgmigrate/migrations/V0003__drop_db_version_func.sql new file mode 100644 index 0000000000..d4f63aa687 --- /dev/null +++ b/apps/postgresql-server/pgmigrate/migrations/V0003__drop_db_version_func.sql @@ -0,0 +1 @@ +DROP FUNCTION set_database_schema_version(); \ No newline at end of file diff --git a/apps/postgresql-server/schema/migrations/mediawords-4379-4388.sql b/apps/postgresql-server/schema/migrations/mediawords-4379-4388.sql deleted file mode 100644 index 6ce7d3b734..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4379-4388.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4379 and 4388. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4379, and you would like to upgrade both the Media Cloud and the --- database to be at version 4388, import this SQL file: --- --- psql mediacloud < mediawords-4379-4388.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- -SET search_path = public, pg_catalog; - -CREATE TABLE example ( - one varchar(512) NOT NULL, - two varchar(512) NOT NULL, - three varchar(512) NOT NULL -); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes; for example, if you're currently at - -- SVN revision 4379, set it to 4380 (which would be the future SVN revision when committed) - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4388; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4388-4389.sql b/apps/postgresql-server/schema/migrations/mediawords-4388-4389.sql deleted file mode 100644 index 1a6e6a7f42..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4388-4389.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4388 and 4389. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4388, and you would like to upgrade both the Media Cloud and the --- database to be at version 4389, import this SQL file: --- --- psql mediacloud < mediawords-4388-4389.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- -SET search_path = public, pg_catalog; - -DROP TABLE example; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes; for example, if you're currently at - -- SVN revision 4379, set it to 4380 (which would be the future SVN revision when committed) - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4389; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4389-4390.sql b/apps/postgresql-server/schema/migrations/mediawords-4389-4390.sql deleted file mode 100644 index abbc3f64f0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4389-4390.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4389 and 4390. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4389, and you would like to upgrade both the Media Cloud and the --- database to be at version 4390, import this SQL file: --- --- psql mediacloud < mediawords-4389-4390.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes; for example, if you're currently at - -- SVN revision 4379, set it to 4380 (which would be the future SVN revision when committed) - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4390; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4390-4391.sql b/apps/postgresql-server/schema/migrations/mediawords-4390-4391.sql deleted file mode 100644 index d74bfd0d68..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4390-4391.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4390 and 4391. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4390, and you would like to upgrade both the Media Cloud and the --- database to be at version 4391, import this SQL file: --- --- psql mediacloud < mediawords-4390-4391.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes; for example, if you're currently at - -- SVN revision 4379, set it to 4380 (which would be the future SVN revision when committed) - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4391; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4391-4392.sql b/apps/postgresql-server/schema/migrations/mediawords-4391-4392.sql deleted file mode 100644 index f0a4fdf2df..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4391-4392.sql +++ /dev/null @@ -1,69 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4391 and 4392. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4391, and you would like to upgrade both the Media Cloud and the --- database to be at version 4392, import this SQL file: --- --- psql mediacloud < mediawords-4391-4392.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4392; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION media_set_retains_sw_data_for_date(v_media_sets_id int, test_date date, default_start_day date, default_end_day date) RETURNS BOOLEAN AS -$$ -DECLARE - media_rec record; - current_time timestamp; - start_date date; - end_date date; -BEGIN - current_time := timeofday()::timestamp; - - -- RAISE NOTICE 'time - %', current_time; - - media_rec = media_set_sw_data_retention_dates( v_media_sets_id, default_start_day, default_end_day ); -- INTO (media_rec); - - start_date = media_rec.start_date; - end_date = media_rec.end_date; - - -- RAISE NOTICE 'start date - %', start_date; - -- RAISE NOTICE 'end date - %', end_date; - - return ( ( start_date is null ) OR ( start_date <= test_date ) ) AND ( (end_date is null ) OR ( end_date >= test_date ) ); -END; -$$ -LANGUAGE 'plpgsql' STABLE - ; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4392-4393.sql b/apps/postgresql-server/schema/migrations/mediawords-4392-4393.sql deleted file mode 100644 index 56392c82b9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4392-4393.sql +++ /dev/null @@ -1,95 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4392 and 4393. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4392, and you would like to upgrade both the Media Cloud and the --- database to be at version 4393, import this SQL file: --- --- psql mediacloud < mediawords-4392-4393.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP INDEX IF EXISTS relative_file_paths_to_verify; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4393; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION cat(text, text) RETURNS text - LANGUAGE plpgsql - AS $_$ - DECLARE - t text; - BEGIN -return coalesce($1) || ' | ' || coalesce($2); - END; -$_$; - -CREATE INDEX tags_tag_sets_id ON tags USING btree (tag_sets_id); - -CREATE INDEX queries_description ON queries USING btree (description); - -CREATE INDEX queries_hash ON queries USING btree (md5(description)); - -CREATE UNIQUE INDEX queries_hash_version ON queries USING btree (md5(description), query_version); - -CREATE INDEX queries_md5_signature ON queries USING btree (md5_signature); - -CREATE INDEX stories_guid_non_unique ON stories USING btree (guid, media_id); - -CREATE UNIQUE INDEX stories_guid_unique_temp ON stories USING btree (guid, media_id) WHERE (stories_id > 72728270); - -CREATE INDEX relative_file_paths_to_verify ON downloads USING btree (relative_file_path) WHERE (((((file_status = 'tbd'::download_file_status) AND (relative_file_path <> 'tbd'::text)) AND (relative_file_path <> 'error'::text)) AND (relative_file_path <> 'na'::text)) AND (relative_file_path <> 'inline'::text)); - -CREATE INDEX downloads_in_old_format ON downloads USING btree (downloads_id) WHERE ((state = 'success'::download_state) AND (path ~~ 'content/%'::text)); - -CREATE INDEX file_status_downloads_time_new_format ON downloads USING btree (file_status, download_time) WHERE (relative_file_path ~~ 'mediacloud-%'::text); - -CREATE INDEX relative_file_paths_new_format_to_verify ON downloads USING btree (relative_file_path) WHERE ((((((file_status = 'tbd'::download_file_status) AND (relative_file_path <> 'tbd'::text)) AND (relative_file_path <> 'error'::text)) AND (relative_file_path <> 'na'::text)) AND (relative_file_path <> 'inline'::text)) AND (relative_file_path ~~ 'mediacloud-%'::text)); - -CREATE INDEX relative_file_paths_old_format_to_verify ON downloads USING btree (relative_file_path) WHERE ((((((file_status = 'tbd'::download_file_status) AND (relative_file_path <> 'tbd'::text)) AND (relative_file_path <> 'error'::text)) AND (relative_file_path <> 'na'::text)) AND (relative_file_path <> 'inline'::text)) AND (NOT (relative_file_path ~~ 'mediacloud-%'::text))); - -CREATE INDEX story_sentence_words_dm ON story_sentence_words USING btree (publish_day, media_id); - -CREATE INDEX weekly_words_topic ON weekly_words USING btree (publish_week, dashboard_topics_id); - -CREATE INDEX top_500_weekly_words_dmds ON top_500_weekly_words USING btree (publish_week, media_sets_id, dashboard_topics_id, stem); - -CREATE INDEX total_daily_words_date ON total_daily_words USING btree (publish_day); - -CREATE INDEX total_daily_words_date_dt ON total_daily_words USING btree (publish_day, dashboard_topics_id); - -CREATE INDEX ssw_queue_stories_id ON ssw_queue USING btree (stories_id); - -CREATE INDEX top_500_weekly_author_words_publish_week ON top_500_weekly_author_words USING btree (publish_week); - -CREATE INDEX query_story_searches_stories_map_qss ON query_story_searches_stories_map USING btree (query_story_searches_id); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4393-4394.sql b/apps/postgresql-server/schema/migrations/mediawords-4393-4394.sql deleted file mode 100644 index fb05dec2fb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4393-4394.sql +++ /dev/null @@ -1,58 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4393 and 4394. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4393, and you would like to upgrade both the Media Cloud and the --- database to be at version 4394, import this SQL file: --- --- psql mediacloud < mediawords-4393-4394.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE TABLE feedless_stories ( - stories_id integer, - media_id integer -); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4394; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION cancel_pg_process(cancel_pid integer) RETURNS boolean - LANGUAGE plpgsql SECURITY DEFINER - AS $$ -BEGIN -return pg_cancel_backend(cancel_pid); -END; -$$; - -CREATE INDEX feedless_stories_story ON feedless_stories USING btree (stories_id); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4394-4395.sql b/apps/postgresql-server/schema/migrations/mediawords-4394-4395.sql deleted file mode 100644 index ecfb173f78..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4394-4395.sql +++ /dev/null @@ -1,145 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4394 and 4395. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4394, and you would like to upgrade both the Media Cloud and the --- database to be at version 4395, import this SQL file: --- --- psql mediacloud < mediawords-4394-4395.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP VIEW IF EXISTS controversy_links_cross_media CASCADE ; -DROP VIEW IF EXISTS media_dups_transitive CASCADE ; -DROP VIEW IF EXISTS tags_with_sets CASCADE ; -DROP VIEW IF EXISTS media_sets_tt2_locale_format CASCADE ; -DROP VIEW IF EXISTS media_sets_explict_sw_data_dates CASCADE ; -DROP VIEW IF EXISTS media_with_collections CASCADE ; -DROP VIEW IF EXISTS dashboard_topics_tt2_locale_format CASCADE ; -DROP VIEW IF EXISTS downloads_media CASCADE ; -DROP VIEW IF EXISTS downloads_non_media CASCADE ; -DROP VIEW IF EXISTS downloads_sites CASCADE ; -DROP VIEW IF EXISTS media_extractor_training_downloads_count CASCADE ; -DROP VIEW IF EXISTS yahoo_top_political_2008_media CASCADE ; -DROP VIEW IF EXISTS technorati_top_political_2008_media CASCADE ; -DROP VIEW IF EXISTS media_extractor_training_downloads_count_adjustments CASCADE ; -DROP VIEW IF EXISTS media_adjusted_extractor_training_downloads_count CASCADE ; -DROP VIEW IF EXISTS top_500_weekly_words_with_totals CASCADE ; -DROP VIEW IF EXISTS top_500_weekly_words_normalized CASCADE ; -DROP VIEW IF EXISTS daily_words_with_totals CASCADE ; -DROP VIEW IF EXISTS story_extracted_texts CASCADE ; -DROP VIEW IF EXISTS media_feed_counts CASCADE ; -DROP VIEW IF EXISTS story_similarities_transitive CASCADE ; -DROP VIEW IF EXISTS controversy_links_cross_media CASCADE ; -DROP VIEW IF EXISTS stories_collected_in_past_day CASCADE ; -DROP VIEW IF EXISTS downloads_to_be_extracted CASCADE ; -DROP VIEW IF EXISTS downloads_in_past_day CASCADE ; -DROP VIEW IF EXISTS downloads_with_error_in_past_day CASCADE ; -DROP VIEW IF EXISTS daily_stats CASCADE ; - -ALTER TABLE daily_words - ALTER COLUMN daily_words_id TYPE bigint /* TYPE change - table: daily_words original: serial primary key new: bigserial primary key */; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4395; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -create view media_dups_transitive as select distinct media_id, main_media_id from ( ( select media_id, main_media_id from media where main_media_id is not null ) union ( select main_media_id as media_id, media_id as main_media_id from media where main_media_id is not null ) ) q; - --- -create view tags_with_sets as select t.*, ts.name as tag_set_name from tags t, tag_sets ts where t.tag_sets_id = ts.tag_sets_id; - --- -CREATE VIEW media_sets_tt2_locale_format as select '[% c.loc("' || COALESCE( name, '') || '") %]' || E'\n' || '[% c.loc("' || COALESCE (description, '') || '") %] ' as tt2_value from media_sets where set_type = 'collection' order by media_sets_id; - --- -CREATE VIEW media_sets_explict_sw_data_dates as select media_sets_id, min(media.sw_data_start_date) as sw_data_start_date, max( media.sw_data_end_date) as sw_data_end_date from media_sets_media_map join media on (media_sets_media_map.media_id = media.media_id ) group by media_sets_id; - -CREATE VIEW media_with_collections AS - SELECT t.tag, m.media_id, m.url, m.name, m.moderated, m.feeds_added, m.moderation_notes, m.full_text_rss FROM media m, tags t, tag_sets ts, media_tags_map mtm WHERE (((((ts.name)::text = 'collection'::text) AND (ts.tag_sets_id = t.tag_sets_id)) AND (mtm.tags_id = t.tags_id)) AND (mtm.media_id = m.media_id)) ORDER BY m.media_id; --- -CREATE VIEW dashboard_topics_tt2_locale_format as select distinct on (tt2_value) '[% c.loc("' || name || '") %]' || ' - ' || '[% c.loc("' || lower(name) || '") %]' as tt2_value from (select * from dashboard_topics order by name, dashboard_topics_id) AS dashboard_topic_names order by tt2_value; - --- -create view downloads_media as select d.*, f.media_id as _media_id from downloads d, feeds f where d.feeds_id = f.feeds_id; - -create view downloads_non_media as select d.* from downloads d where d.feeds_id is null; - --- -CREATE VIEW downloads_sites as select regexp_replace(host, $q$^(.)*?([^.]+)\.([^.]+)$$q$ ,E'\\2.\\3') as site, * from downloads_media; - --- -CREATE VIEW media_extractor_training_downloads_count AS - SELECT media.media_id, COALESCE(foo.extractor_training_downloads_for_media_id, (0)::bigint) AS extractor_training_download_count FROM (media LEFT JOIN (SELECT stories.media_id, count(stories.media_id) AS extractor_training_downloads_for_media_id FROM extractor_training_lines, downloads, stories WHERE ((extractor_training_lines.downloads_id = downloads.downloads_id) AND (downloads.stories_id = stories.stories_id)) GROUP BY stories.media_id ORDER BY stories.media_id) foo ON ((media.media_id = foo.media_id))); --- -CREATE VIEW yahoo_top_political_2008_media AS - SELECT DISTINCT media_tags_map.media_id FROM media_tags_map, (SELECT tags.tags_id FROM tags, (SELECT DISTINCT media_tags_map.tags_id FROM media_tags_map ORDER BY media_tags_map.tags_id) media_tags WHERE ((tags.tags_id = media_tags.tags_id) AND ((tags.tag)::text ~~ 'yahoo_top_political_2008'::text))) interesting_media_tags WHERE (media_tags_map.tags_id = interesting_media_tags.tags_id) ORDER BY media_tags_map.media_id; --- -CREATE VIEW technorati_top_political_2008_media AS - SELECT DISTINCT media_tags_map.media_id FROM media_tags_map, (SELECT tags.tags_id FROM tags, (SELECT DISTINCT media_tags_map.tags_id FROM media_tags_map ORDER BY media_tags_map.tags_id) media_tags WHERE ((tags.tags_id = media_tags.tags_id) AND ((tags.tag)::text ~~ 'technorati_top_political_2008'::text))) interesting_media_tags WHERE (media_tags_map.tags_id = interesting_media_tags.tags_id) ORDER BY media_tags_map.media_id; --- -CREATE VIEW media_extractor_training_downloads_count_adjustments AS - SELECT yahoo.media_id, yahoo.yahoo_count_adjustment, tech.technorati_count_adjustment FROM (SELECT media_extractor_training_downloads_count.media_id, COALESCE(foo.yahoo_count_adjustment, 0) AS yahoo_count_adjustment FROM (media_extractor_training_downloads_count LEFT JOIN (SELECT yahoo_top_political_2008_media.media_id, 1 AS yahoo_count_adjustment FROM yahoo_top_political_2008_media) foo ON ((foo.media_id = media_extractor_training_downloads_count.media_id)))) yahoo, (SELECT media_extractor_training_downloads_count.media_id, COALESCE(foo.count_adjustment, 0) AS technorati_count_adjustment FROM (media_extractor_training_downloads_count LEFT JOIN (SELECT technorati_top_political_2008_media.media_id, 1 AS count_adjustment FROM technorati_top_political_2008_media) foo ON ((foo.media_id = media_extractor_training_downloads_count.media_id)))) tech WHERE (tech.media_id = yahoo.media_id); --- -CREATE VIEW media_adjusted_extractor_training_downloads_count AS - SELECT media_extractor_training_downloads_count.media_id, ((media_extractor_training_downloads_count.extractor_training_download_count - (2 * media_extractor_training_downloads_count_adjustments.yahoo_count_adjustment)) - (2 * media_extractor_training_downloads_count_adjustments.technorati_count_adjustment)) AS count FROM (media_extractor_training_downloads_count JOIN media_extractor_training_downloads_count_adjustments ON ((media_extractor_training_downloads_count.media_id = media_extractor_training_downloads_count_adjustments.media_id))) ORDER BY ((media_extractor_training_downloads_count.extractor_training_download_count - (2 * media_extractor_training_downloads_count_adjustments.yahoo_count_adjustment)) - (2 * media_extractor_training_downloads_count_adjustments.technorati_count_adjustment)); --- -create view top_500_weekly_words_with_totals as select t5.*, tt5.total_count from top_500_weekly_words t5, total_top_500_weekly_words tt5 where t5.media_sets_id = tt5.media_sets_id and t5.publish_week = tt5.publish_week and ( ( t5.dashboard_topics_id = tt5.dashboard_topics_id ) or ( t5.dashboard_topics_id is null and tt5.dashboard_topics_id is null ) ); - -create view top_500_weekly_words_normalized - as select t5.stem, min(t5.term) as term, ( least( 0.01, sum(t5.stem_count)::numeric / sum(t5.total_count)::numeric ) * count(*) ) as stem_count, t5.media_sets_id, t5.publish_week, t5.dashboard_topics_id from top_500_weekly_words_with_totals t5 group by t5.stem, t5.publish_week, t5.media_sets_id, t5.dashboard_topics_id; --- -create view daily_words_with_totals as select d.*, t.total_count from daily_words d, total_daily_words t where d.media_sets_id = t.media_sets_id and d.publish_day = t.publish_day and ( ( d.dashboard_topics_id = t.dashboard_topics_id ) or ( d.dashboard_topics_id is null and t.dashboard_topics_id is null ) ); - --- -create view story_extracted_texts as select stories_id, array_to_string(array_agg(download_text), ' ') as extracted_text - from (select * from downloads natural join download_texts order by downloads_id) as downloads group by stories_id; --- -CREATE VIEW media_feed_counts as (SELECT media_id, count(*) as feed_count FROM feeds GROUP by media_id); - --- -create view story_similarities_transitive as - ( select story_similarities_id, stories_id_a, publish_day_a, stories_id_b, publish_day_b, similarity from story_similarities ) union ( select story_similarities_id, stories_id_b as stories_id_a, publish_day_b as publish_day_a, stories_id_a as stories_id_b, publish_day_a as publish_day_b, similarity from story_similarities ); --- -create view controversy_links_cross_media as - select s.stories_id, substr(sm.name::text, 0, 24) as media_name, r.stories_id as ref_stories_id, substr(rm.name::text, 0, 24) as ref_media_name, substr(cl.url, 0, 144) as url, cs.controversies_id from media sm, media rm, controversy_links cl, stories s, stories r, controversy_stories cs where cl.ref_stories_id <> cl.stories_id and s.stories_id = cl.stories_id and cl.ref_stories_id = r.stories_id and s.media_id <> r.media_id and sm.media_id = s.media_id and rm.media_id = r.media_id and cs.stories_id = cl.ref_stories_id and cs.controversies_id = cl.controversies_id; --- -CREATE VIEW stories_collected_in_past_day as select * from stories where collect_date > now() - interval '1 day'; - -CREATE VIEW downloads_to_be_extracted as select * from downloads where extracted = 'f' and state = 'success' and type = 'content'; - -CREATE VIEW downloads_in_past_day as select * from downloads where download_time > now() - interval '1 day'; -CREATE VIEW downloads_with_error_in_past_day as select * from downloads_in_past_day where state = 'error'; - -CREATE VIEW daily_stats as select * from (SELECT count(*) as daily_downloads from downloads_in_past_day) as dd, (select count(*) as daily_stories from stories_collected_in_past_day) ds , (select count(*) as downloads_to_be_extracted from downloads_to_be_extracted) dex, (select count(*) as download_errors from downloads_with_error_in_past_day ) er; - - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4395-4396.sql b/apps/postgresql-server/schema/migrations/mediawords-4395-4396.sql deleted file mode 100644 index 9604e4514f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4395-4396.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4395 and 4396. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4395, and you would like to upgrade both the Media Cloud and the --- database to be at version 4396, import this SQL file: --- --- psql mediacloud < mediawords-4395-4396.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE weekly_words - ALTER COLUMN weekly_words_id TYPE bigserial primary key /* TYPE change - table: weekly_words original: serial primary key new: bigserial primary key */; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4396; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4396-4397.sql b/apps/postgresql-server/schema/migrations/mediawords-4396-4397.sql deleted file mode 100644 index b9279b2a59..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4396-4397.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4396 and 4397. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4396, and you would like to upgrade both the Media Cloud and the --- database to be at version 4397, import this SQL file: --- --- psql mediacloud < mediawords-4396-4397.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP TABLE feedless_stories; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4397; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4397-4398.sql b/apps/postgresql-server/schema/migrations/mediawords-4397-4398.sql deleted file mode 100644 index b35bde1e8c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4397-4398.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4397 and 4398. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4397, and you would like to upgrade both the Media Cloud and the --- database to be at version 4398, import this SQL file: --- --- psql mediacloud < mediawords-4397-4398.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE UNIQUE INDEX database_variables_name_key2 on database_variables( name ); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4398; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - -INSERT INTO database_variables( name, value ) values ( 'LAST_STORY_SENTENCES_ID_PROCESSED', '0' ); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4398-4399.sql b/apps/postgresql-server/schema/migrations/mediawords-4398-4399.sql deleted file mode 100644 index 6831b9a8e6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4398-4399.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4398 and 4399. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4398, and you would like to upgrade both the Media Cloud and the --- database to be at version 4399, import this SQL file: --- --- psql mediacloud < mediawords-4398-4399.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE TABLE processed_stories ( - processed_stories_id bigserial primary key, - stories_id bigint not null references stories on delete cascade -); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4399; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4399-4400.sql b/apps/postgresql-server/schema/migrations/mediawords-4399-4400.sql deleted file mode 100644 index 1455e6613d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4399-4400.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4399 and 4400. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4399, and you would like to upgrade both the Media Cloud and the --- database to be at version 4400, import this SQL file: --- --- psql mediacloud < mediawords-4399-4400.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE TABLE story_subsets ( - story_subsets_id bigserial primary key, - start_date timestamp with time zone, - end_date timestamp with time zone, - media_id int references media_sets, - media_sets_id int references media_sets, - ready boolean DEFAULT 'false', - last_processed_stories_id bigint references processed_stories(processed_stories_id) -); - -CREATE TABLE story_subsets_processed_stories_map ( - story_subsets_processed_stories_map_id bigserial primary key, - story_subsets_id bigint NOT NULL references story_subsets on delete cascade, - processed_stories_id bigint NOT NULL references processed_stories on delete cascade -); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4400; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4400-4401.sql b/apps/postgresql-server/schema/migrations/mediawords-4400-4401.sql deleted file mode 100644 index 296a1bd707..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4400-4401.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4400 and 4401. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4400, and you would like to upgrade both the Media Cloud and the --- database to be at version 4401, import this SQL file: --- --- psql mediacloud < mediawords-4400-4401.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4401; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE INDEX story_sentences_media_id ON story_sentences ( media_id ); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4401-4402.sql b/apps/postgresql-server/schema/migrations/mediawords-4401-4402.sql deleted file mode 100644 index 577f84b82a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4401-4402.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4401 and 4402. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4401, and you would like to upgrade both the Media Cloud and the --- database to be at version 4402, import this SQL file: --- --- psql mediacloud < mediawords-4401-4402.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE database_variables - RENAME COLUMN variables_id to database_variables_id; - -ALTER SEQUENCE database_variables_variables_id_seq RENAME TO database_variables_datebase_variables_id_seq; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4402; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4402-4403.sql b/apps/postgresql-server/schema/migrations/mediawords-4402-4403.sql deleted file mode 100644 index 632b46e8c5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4402-4403.sql +++ /dev/null @@ -1,68 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4402 and 4403. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4402, and you would like to upgrade both the Media Cloud and the --- database to be at version 4403, import this SQL file: --- --- psql mediacloud < mediawords-4402-4403.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP VIEW downloads_sites; - -DROP INDEX downloads_sites_index; - -DROP INDEX downloads_sites_pending; - -DROP INDEX downloads_sites_downloads_id_pending; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4403; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION site_from_host("host" varchar) RETURNS varchar AS -$$ -BEGIN - RETURN regexp_replace(host, E'^(.)*?([^.]+)\\.([^.]+)$' ,E'\\2.\\3'); -END; -$$ -LANGUAGE 'plpgsql' IMMUTABLE; - -CREATE INDEX downloads_sites_index ON downloads ( site_from_host(host) ); - -CREATE INDEX downloads_sites_pending ON downloads ( site_from_host( host ) ) where state='pending'; - -CREATE UNIQUE INDEX downloads_sites_downloads_id_pending ON downloads ( site_from_host(host), downloads_id ) WHERE (state = 'pending'); - -CREATE VIEW downloads_sites AS - select site_from_host( host ) as site, * from downloads_media; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4403-4404.sql b/apps/postgresql-server/schema/migrations/mediawords-4403-4404.sql deleted file mode 100644 index f03a2bf088..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4403-4404.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4403 and 4404. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4403, and you would like to upgrade both the Media Cloud and the --- database to be at version 4404, import this SQL file: --- --- psql mediacloud < mediawords-4403-4404.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4404; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE UNIQUE INDEX downloads_for_extractor_trainer ON downloads ( downloads_id, feeds_id) where file_status <> 'missing' and type = 'content' and state = 'success'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4404-4405.sql b/apps/postgresql-server/schema/migrations/mediawords-4404-4405.sql deleted file mode 100644 index 8b0d518347..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4404-4405.sql +++ /dev/null @@ -1,670 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4404 and 4405. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4404, and you would like to upgrade both the Media Cloud and the --- database to be at version 4405, import this SQL file: --- --- psql mediacloud < mediawords-4404-4405.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -DROP SCHEMA stories_tags_map_media_sub_tables CASCADE; - -DROP VIEW media_dups_transitive; - -DROP VIEW stories_collected_in_past_day; - -ALTER TABLE media - DROP CONSTRAINT media_dup; - -ALTER TABLE tags - DROP CONSTRAINT no_lead_or_trailing_whitspace; - -ALTER TABLE media_cluster_runs - DROP CONSTRAINT media_cluster_runs_state; - -ALTER TABLE media_cluster_maps - DROP CONSTRAINT media_cluster_maps_type; - -DROP INDEX queries_signature_version; - -DROP INDEX queries_signature; - -DROP INDEX stories_guid; - -DROP INDEX downloads_sites_index; - -DROP INDEX story_sentence_words_day; - -DROP INDEX story_sentence_words_media_day; - -DROP INDEX daily_words_unique; - -DROP INDEX weekly_words_publish_week; - -DROP INDEX story_similarities_a_b; - -DROP INDEX story_similarities_a_s; - -DROP INDEX story_similarities_b_s; - -DROP INDEX controversy_links_story; - -DROP TABLE sopa_links; - -DROP TABLE sopa_stories; - -DROP TABLE controversy_merged_media; - -CREATE SEQUENCE database_variables_datebase_variables_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE media_cluster_map_pole_simila_media_cluster_map_pole_simila_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE total_top_500_weekly_words_total_top_500_words_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE total_top_500_weekly_author_words_total_top_500_words_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE story_similarities_story_similarities_id_seq1 - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE story_similarities_story_similarities_id_seq2 - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE story_similarities_100_short_story_similarities_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE controversy_dates_controversy_dates_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE controversy_seed_urls_controversy_seed_urls_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE SEQUENCE media_alexa_stats_media_alexa_stats_id_seq - START WITH 1 - INCREMENT BY 1 - NO MAXVALUE - NO MINVALUE - CACHE 1; - -CREATE TABLE media_cluster_map_pole_similarities ( - media_cluster_map_pole_similarities_id integer primary key DEFAULT nextval('media_cluster_map_pole_simila_media_cluster_map_pole_simila_seq'::regclass) NOT NULL, - media_id integer NOT NULL REFERENCES media(media_id), - queries_id integer NOT NULL REFERENCES queries(queries_id), - similarity integer NOT NULL, - media_cluster_maps_id integer NOT NULL REFERENCES media_cluster_maps(media_cluster_maps_id) -); - -CREATE TABLE story_similarities_1000_tiny_idf ( - story_similarities_id integer primary key DEFAULT nextval('story_similarities_story_similarities_id_seq2'::regclass) NOT NULL, - stories_id_a integer, - publish_day_a date, - stories_id_b integer, - publish_day_b date, - similarity integer, - "method" character varying(1024) -); - -CREATE TABLE story_similarities_100_short ( - story_similarities_id integer PRIMARY KEY DEFAULT nextval('story_similarities_100_short_story_similarities_id_seq'::regclass) NOT NULL, - stories_id_a integer, - publish_day_a date, - stories_id_b integer, - publish_day_b date, - similarity integer -); - -CREATE TABLE tar_downloads_queue ( - downloads_id integer -); - -CREATE TABLE controversy_dates ( - controversy_dates_id integer primary key DEFAULT nextval('controversy_dates_controversy_dates_id_seq'::regclass) NOT NULL, - controversies_id integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE, - start_date date NOT NULL, - end_date date NOT NULL -); - -CREATE TABLE controversy_seed_urls ( - controversy_seed_urls_id integer primary key DEFAULT nextval('controversy_seed_urls_controversy_seed_urls_id_seq'::regclass) NOT NULL, - controversies_id integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE, - url text, - source text, - stories_id integer REFERENCES stories(stories_id) ON DELETE CASCADE, - processed boolean DEFAULT false NOT NULL -); - -CREATE TABLE controversy_merged_stories_map ( - source_stories_id integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE, - target_stories_id integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE -); - -CREATE TABLE controversy_query_story_searches_imported_stories_map ( - controversies_id integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE, - stories_id integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE -); - -CREATE TABLE controversy_unmerged_media ( - media_id integer NOT NULL REFERENCES media(media_id) ON DELETE CASCADE -); - -CREATE TABLE adhoc_momentum ( - publish_day date, - stem character varying(1024), - num_sentences integer, - media_set_name character varying(1024), - media_sets_id integer -); - -CREATE TABLE controversy_links_copy ( - controversy_links_id integer, - controversies_id integer, - stories_id integer, - url text, - redirect_url text, - ref_stories_id integer, - link_spidered boolean -); - -CREATE TABLE controversy_links_copy_20120920 ( - controversy_links_id integer, - controversies_id integer, - stories_id integer, - url text, - redirect_url text, - ref_stories_id integer, - link_spidered boolean -); - -CREATE TABLE controversy_media_codes_20121020 ( - controversies_id integer, - media_id integer, - code_type text, - code text -); - -CREATE TABLE controversy_stories_20121018 ( - controversy_stories_id integer, - controversies_id integer, - stories_id integer, - link_mined boolean, - iteration integer, - link_weight real, - redirect_url text -); - -CREATE TABLE controversy_links_20121018 ( - controversy_links_id integer, - controversies_id integer, - stories_id integer, - url text, - redirect_url text, - ref_stories_id integer, - link_spidered boolean -); - -CREATE TABLE controversy_stories_copy ( - controversy_stories_id integer, - controversies_id integer, - stories_id integer, - link_mined boolean, - iteration integer, - link_weight real, - redirect_url text -); - -CREATE TABLE controversy_stories_copy_20120920 ( - controversy_stories_id integer, - controversies_id integer, - stories_id integer, - link_mined boolean, - iteration integer, - link_weight real, - redirect_url text -); - -CREATE TABLE controversy_links_distinct ( - controversy_links_id integer, - controversies_id integer, - stories_id integer, - url text, - redirect_url text, - ref_stories_id integer, - link_spidered boolean -); - -CREATE TABLE extractor_training_lines_corrupted_download_content ( - extractor_training_lines_id integer, - line_number integer, - required boolean, - downloads_id integer, - "time" timestamp without time zone, - submitter character varying(256) -); - -CREATE TABLE hr_pilot_study_stories ( - media_id integer, - stories_id integer, - title text, - url character varying(1024) -); - -CREATE TABLE india_million ( - stories_id integer, - publish_date timestamp without time zone, - title text, - url character varying(1024), - media_name character varying(128) -); - -CREATE TABLE ma_ms_queue ( - media_sets_id integer -); - -CREATE TABLE pilot_story_sims ( - similarity text, - title_1 text, - title_2 text, - url_1 text, - url_2 text, - stories_id_1 text, - stories_id_2 text, - include boolean -); - -CREATE TABLE pilot_story_sims_code ( - similarity text, - title_1 text, - title_2 text, - url_1 text, - url_2 text, - stories_id_1 text, - stories_id_2 text, - include boolean -); - -CREATE TABLE pilot_study_stories ( - media_id integer, - stories_id integer, - title text, - url text -); - -CREATE TABLE questionable_downloads_rows ( - downloads_id integer, - feeds_id integer, - stories_id integer, - parent integer, - url character varying(1024), - "host" character varying(1024), - download_time timestamp without time zone, - type download_type, - "state" download_state, - "path" text, - error_message text, - priority integer, - "sequence" integer, - extracted boolean, - old_download_time timestamp without time zone, - old_state download_state -); - -CREATE TABLE ssw_dump ( - stories_id integer, - term character varying(256), - stem character varying(256), - stem_count smallint, - sentence_number smallint, - media_id integer, - publish_day date -); - -CREATE TABLE stories_description_not_salvaged ( - stories_id integer, - media_id integer, - url character varying(1024), - guid character varying(1024), - title text, - description text, - publish_date timestamp without time zone, - collect_date timestamp without time zone, - story_texts_id integer, - full_text_rss boolean -); - -CREATE TABLE total_daily_media_words ( - media_id integer, - publish_day date, - stem_count bigint, - dashboard_topics_id integer -); - -CREATE TABLE valid_trayvon_stories ( - stories_id integer -); - -CREATE TABLE media_rss_full_text_detection_data ( - media_id integer, - max_similarity real, - avg_similarity double precision, - min_similarity real, - avg_extracted_length numeric, - avg_rss_length numeric, - avg_rss_discription numeric, - "count" bigint -); - -CREATE TABLE media_alexa_stats ( - media_alexa_stats_id integer primary key DEFAULT nextval('media_alexa_stats_media_alexa_stats_id_seq'::regclass) NOT NULL, - media_id integer REFERENCES media(media_id) ON DELETE CASCADE, - "day" date, - reach_per_million double precision, - page_views_per_million double precision, - page_views_per_user double precision, - "rank" integer -); - -ALTER TABLE database_variables - ALTER COLUMN database_variables_id TYPE integer /* TYPE change - table: database_variables original: serial primary key new: integer */, - ALTER COLUMN database_variables_id SET DEFAULT nextval('database_variables_datebase_variables_id_seq'::regclass), - ALTER COLUMN database_variables_id SET NOT NULL; - -ALTER TABLE media - DROP COLUMN main_media_id, - DROP COLUMN is_dup, - ADD COLUMN dup_media_id integer REFERENCES media(media_id) ON DELETE SET, - ADD COLUMN is_not_dup boolean; - -ALTER TABLE media_clusters - ALTER COLUMN media_clusters_id TYPE serial primary key /* TYPE change - table: media_clusters original: serial primary key new: serial primary key */, - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_clusters original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE media_cluster_words - ALTER COLUMN media_cluster_words_id TYPE serial primary key /* TYPE change - table: media_cluster_words original: serial primary key new: serial primary key */, - ALTER COLUMN media_clusters_id TYPE int not null references media_clusters on delete cascade /* TYPE change - table: media_cluster_words original: int not null references media_clusters on delete cascade new: int not null references media_clusters on delete cascade */; - -ALTER TABLE media_cluster_links - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_cluster_links original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE media_cluster_zscores - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_cluster_zscores original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE stories - ADD COLUMN story_texts_id integer; - -ALTER TABLE ONLY downloads ALTER COLUMN "host" SET STATISTICS 10000; - -ALTER TABLE stories_tags_map - ALTER COLUMN stories_id TYPE int /* TYPE change - table: stories_tags_map original: int not null references stories on delete cascade new: int */, - ALTER COLUMN stories_id SET NOT NULL, - ALTER COLUMN tags_id TYPE int /* TYPE change - table: stories_tags_map original: int not null references tags on delete cascade new: int */, - ALTER COLUMN tags_id SET NOT NULL; - -ALTER TABLE top_ten_tags_for_media - ALTER COLUMN media_id TYPE integer NOT NULL REFERENCES media(media_id) ON DELETE CASCADE /* TYPE change - table: top_ten_tags_for_media original: integer new: integer NOT NULL REFERENCES media(media_id) ON DELETE CASCADE */, - ALTER COLUMN media_id DROP NOT NULL, - ALTER COLUMN tags_id TYPE integer NOT NULL REFERENCES tags(tags_id) /* TYPE change - table: top_ten_tags_for_media original: integer new: integer NOT NULL REFERENCES tags(tags_id) */, - ALTER COLUMN tags_id DROP NOT NULL, - ALTER COLUMN tag_sets_id TYPE integer NOT NULL REFERENCES tag_sets(tag_sets_id) /* TYPE change - table: top_ten_tags_for_media original: integer new: integer NOT NULL REFERENCES tag_sets(tag_sets_id) */, - ALTER COLUMN tag_sets_id DROP NOT NULL; - -ALTER TABLE word_cloud_topics - ALTER COLUMN source_tags_id TYPE int /* TYPE change - table: word_cloud_topics original: int not null references tags new: int */, - ALTER COLUMN source_tags_id SET NOT NULL; - -ALTER TABLE total_top_500_weekly_words - ALTER COLUMN total_top_500_weekly_words_id TYPE int primary key /* TYPE change - table: total_top_500_weekly_words original: serial primary key new: int primary key */, - ALTER COLUMN total_top_500_weekly_words_id SET DEFAULT nextval('total_top_500_weekly_words_total_top_500_words_id_seq'::regclass), - ALTER COLUMN media_sets_id TYPE int not null references media_sets(media_sets_id) on delete cascade /* TYPE change - table: total_top_500_weekly_words original: int not null references media_sets on delete cascade new: int not null references media_sets(media_sets_id) on delete cascade */, - ALTER COLUMN dashboard_topics_id TYPE int null references dashboard_topics(dashboard_topics_id) on delete cascade /* TYPE change - table: total_top_500_weekly_words original: int null references dashboard_topics new: int null references dashboard_topics(dashboard_topics_id) on delete cascade */; - -ALTER TABLE total_top_500_weekly_author_words - ALTER COLUMN total_top_500_weekly_author_words_id TYPE integer primary key /* TYPE change - table: total_top_500_weekly_author_words original: serial primary key new: integer primary key */, - ALTER COLUMN total_top_500_weekly_author_words_id SET DEFAULT nextval('total_top_500_weekly_author_words_total_top_500_words_id_seq'::regclass); - -ALTER TABLE query_story_searches_stories_map - ALTER COLUMN query_story_searches_id TYPE int REFERENCES query_story_searches(query_story_searches_id) ON DELETE CASCADE /* TYPE change - table: query_story_searches_stories_map original: int new: int REFERENCES query_story_searches(query_story_searches_id) ON DELETE CASCADE */, - ALTER COLUMN stories_id TYPE int REFERENCES stories(stories_id) ON DELETE CASCADE /* TYPE change - table: query_story_searches_stories_map original: int new: int REFERENCES stories(stories_id) ON DELETE CASCADE */; - -ALTER TABLE story_similarities - ALTER COLUMN story_similarities_id TYPE integer primary key /* TYPE change - table: story_similarities original: serial primary key new: integer primary key */, - ALTER COLUMN story_similarities_id SET DEFAULT nextval('story_similarities_story_similarities_id_seq1'::regclass); - -ALTER TABLE controversy_stories - ALTER COLUMN controversies_id TYPE int not null references controversies(controversies_id) on delete cascade /* TYPE change - table: controversy_stories original: int not null references controversies on delete cascade new: int not null references controversies(controversies_id) on delete cascade */, - ALTER COLUMN stories_id TYPE int not null references stories(stories_id) on delete cascade /* TYPE change - table: controversy_stories original: int not null references stories on delete cascade new: int not null references stories(stories_id) on delete cascade */; - -ALTER TABLE controversy_links - ALTER COLUMN controversies_id TYPE int /* TYPE change - table: controversy_links original: int not null references controversies on delete cascade new: int */, - ALTER COLUMN controversies_id SET NOT NULL, - ALTER COLUMN stories_id TYPE int /* TYPE change - table: controversy_links original: int not null references stories on delete cascade new: int */, - ALTER COLUMN stories_id SET NOT NULL; - -ALTER SEQUENCE database_variables_datebase_variables_id_seq - OWNED BY database_variables.database_variables_id; - -ALTER SEQUENCE media_cluster_map_pole_simila_media_cluster_map_pole_simila_seq - OWNED BY media_cluster_map_pole_similarities.media_cluster_map_pole_similarities_id; - -ALTER SEQUENCE total_top_500_weekly_words_total_top_500_words_id_seq - OWNED BY total_top_500_weekly_words.total_top_500_weekly_words_id; - -ALTER SEQUENCE total_top_500_weekly_author_words_total_top_500_words_id_seq - OWNED BY total_top_500_weekly_author_words.total_top_500_weekly_author_words_id; - -ALTER SEQUENCE story_similarities_story_similarities_id_seq1 - OWNED BY story_similarities.story_similarities_id; - -ALTER SEQUENCE story_similarities_story_similarities_id_seq2 - OWNED BY story_similarities_1000_tiny_idf.story_similarities_id; - -ALTER SEQUENCE story_similarities_100_short_story_similarities_id_seq - OWNED BY story_similarities_100_short.story_similarities_id; - -ALTER SEQUENCE controversy_dates_controversy_dates_id_seq - OWNED BY controversy_dates.controversy_dates_id; - -ALTER SEQUENCE controversy_seed_urls_controversy_seed_urls_id_seq - OWNED BY controversy_seed_urls.controversy_seed_urls_id; - -ALTER SEQUENCE media_alexa_stats_media_alexa_stats_id_seq - OWNED BY media_alexa_stats.media_alexa_stats_id; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4405; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION download_relative_file_path_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - IF TG_OP = 'UPDATE' then - -- RAISE NOTICE 'UPDATE '; - - -- The second part is needed because of the way comparisons with null are handled. - path_change := ( OLD.path <> NEW.path ) AND ( ( OLD.path is not null) <> (NEW.path is not null) ) ; - -- RAISE NOTICE 'test result % ', path_change; - - IF path_change is null THEN - -- RAISE NOTICE 'Path change % != %', OLD.path, NEW.path; - NEW.relative_file_path = get_relative_file_path(NEW.path); - - IF NEW.relative_file_path = 'inline' THEN - NEW.file_status = 'inline'; - END IF; - ELSE - -- RAISE NOTICE 'NO path change % = %', OLD.path, NEW.path; - END IF; - ELSIF TG_OP = 'INSERT' then - NEW.relative_file_path = get_relative_file_path(NEW.path); - - IF NEW.relative_file_path = 'inline' THEN - NEW.file_status = 'inline'; - END IF; - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE database_variables - ADD CONSTRAINT database_variables_pkey PRIMARY KEY (database_variables_id); - -ALTER TABLE media - ADD CONSTRAINT media_self_dup CHECK (((dup_media_id IS NULL) OR (dup_media_id <> media_id))); - -ALTER TABLE tags - ADD CONSTRAINT no_lead_or_trailing_whitspace CHECK - (tag_sets_id = 13 OR tag_sets_id = 9 OR tag_sets_id = 8 OR tag_sets_id = 6 OR tag::text = btrim(tag::text, ' - '::text)); - -ALTER TABLE media_cluster_runs - ADD CONSTRAINT media_cluster_runs_state CHECK (((state)::text = ANY (ARRAY[('pending'::character varying)::text, ('executing'::character varying)::text, ('completed'::character varying)::text]))); - -ALTER TABLE media_cluster_maps - ADD CONSTRAINT media_cluster_maps_type CHECK (((map_type)::text = ANY (ARRAY[('cluster'::character varying)::text, ('polar'::character varying)::text]))); - -ALTER TABLE stories_tags_map - ADD CONSTRAINT stories_tags_map_tag FOREIGN KEY (tags_id) REFERENCES tags(tags_id) ON DELETE CASCADE; - -ALTER TABLE stories_tags_map - ADD CONSTRAINT stories_tags_map_story FOREIGN KEY (stories_id) REFERENCES stories(stories_id) ON DELETE CASCADE; - -ALTER TABLE stories_tags_map - ADD CONSTRAINT stories_tags_map_stories_id_fkey FOREIGN KEY (stories_id) REFERENCES stories(stories_id) ON DELETE CASCADE; - -ALTER TABLE stories_tags_map - ADD CONSTRAINT stories_tags_map_tags_id_fkey FOREIGN KEY (tags_id) REFERENCES tags(tags_id) ON DELETE CASCADE; - -ALTER TABLE word_cloud_topics - ADD CONSTRAINT word_cloud_topics_source_tag_fk FOREIGN KEY (source_tags_id) REFERENCES tags(tags_id); - -ALTER TABLE word_cloud_topics - ADD CONSTRAINT word_cloud_topics_source_tags_id_fkey FOREIGN KEY (source_tags_id) REFERENCES tags(tags_id); - -ALTER TABLE controversy_links - ADD CONSTRAINT controversy_links_controversy_story_stories_id FOREIGN KEY (stories_id, controversies_id) REFERENCES controversy_stories(stories_id, controversies_id) ON DELETE CASCADE; - -CREATE UNIQUE INDEX database_variables_name_key_index ON database_variables USING btree (name); - -CREATE INDEX media_cluster_map_pole_similarities_map ON media_cluster_map_pole_similarities USING btree (media_cluster_maps_id); - -CREATE INDEX story_similarities_1000_tiny_a_b ON story_similarities USING btree (stories_id_a, stories_id_b); - -CREATE INDEX story_similarities_1000_tiny_a_s ON story_similarities USING btree (stories_id_a, similarity, publish_day_b); - -CREATE INDEX story_similarities_1000_tiny_b_s ON story_similarities USING btree (stories_id_b, similarity, publish_day_a); - -CREATE INDEX story_similarities_a_b ON story_similarities_1000_tiny_idf USING btree (stories_id_a, stories_id_b); - -CREATE INDEX story_similarities_a_s ON story_similarities_1000_tiny_idf USING btree (stories_id_a, similarity, publish_day_b); - -CREATE INDEX story_similarities_b_s ON story_similarities_1000_tiny_idf USING btree (stories_id_b, similarity, publish_day_a); - -CREATE INDEX story_similarities_100_short_a_s ON story_similarities_100_short USING btree (stories_id_a, similarity, publish_day_b); - -CREATE INDEX story_similarities_100_short_b_s ON story_similarities_100_short USING btree (stories_id_b, similarity, publish_day_a); - -CREATE INDEX tar_downloads_queue_download ON tar_downloads_queue USING btree (downloads_id); - -CREATE INDEX controversy_seed_urls_controversy ON controversy_seed_urls USING btree (controversies_id); - -CREATE INDEX controversy_merged_stories_map_source ON controversy_merged_stories_map USING btree (source_stories_id); - -CREATE INDEX cqssism_c ON controversy_query_story_searches_imported_stories_map USING btree (controversies_id); - -CREATE INDEX cqssism_s ON controversy_query_story_searches_imported_stories_map USING btree (stories_id); - -CREATE INDEX controversy_unmerged_media_media ON controversy_unmerged_media USING btree (media_id); - -CREATE UNIQUE INDEX controversy_stories_sc ON controversy_stories USING btree (stories_id, controversies_id); - -CREATE UNIQUE INDEX controversy_links_scr ON controversy_links USING btree (stories_id, controversies_id, ref_stories_id); - -CREATE INDEX processed_stories_story ON processed_stories USING btree (stories_id); - -CREATE INDEX media_rss_full_text_detection_data_media_1305311081 ON media_rss_full_text_detection_data USING btree (media_id); - -CREATE INDEX media_alexa_stats_medium ON media_alexa_stats USING btree (media_id); - -ALTER TABLE story_sentence_counts CLUSTER ON story_sentence_counts_pkey; - -ALTER TABLE total_daily_words CLUSTER ON total_daily_words_pkey; - -CREATE VIEW media_no_dups AS - SELECT * - FROM media - WHERE dup_media_id IS NULL; - -CREATE VIEW stories_collected_in_past_day AS - SELECT stories.stories_id, stories.media_id, stories.url, stories.guid, stories.title, stories.description, stories.publish_date, stories.collect_date, stories.story_texts_id, stories.full_text_rss FROM stories WHERE (stories.collect_date > (now() - '1 day'::interval)); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4405-4406.sql b/apps/postgresql-server/schema/migrations/mediawords-4405-4406.sql deleted file mode 100644 index 6c3a729a5f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4405-4406.sql +++ /dev/null @@ -1,436 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4404 and 4406. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4404, and you would like to upgrade both the Media Cloud and the --- database to be at version 4406, import this SQL file: --- --- psql mediacloud < mediawords-4404-4406.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE controversy_stories - ADD COLUMN valid_foreign_rss_story boolean DEFAULT false; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4406; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE SCHEMA stories_tags_map_media_sub_tables; - -SET search_path = public, pg_catalog; - -DROP VIEW media_no_dups; - -DROP VIEW stories_collected_in_past_day; - -ALTER TABLE database_variables - DROP CONSTRAINT database_variables_pkey; - -ALTER TABLE media - DROP CONSTRAINT media_self_dup; - -ALTER TABLE tags - DROP CONSTRAINT no_lead_or_trailing_whitspace; - -ALTER TABLE media_cluster_runs - DROP CONSTRAINT media_cluster_runs_state; - -ALTER TABLE media_cluster_maps - DROP CONSTRAINT media_cluster_maps_type; - -ALTER TABLE stories_tags_map - DROP CONSTRAINT stories_tags_map_tag; - -ALTER TABLE stories_tags_map - DROP CONSTRAINT stories_tags_map_story; - -ALTER TABLE stories_tags_map - DROP CONSTRAINT stories_tags_map_stories_id_fkey; - -ALTER TABLE stories_tags_map - DROP CONSTRAINT stories_tags_map_tags_id_fkey; - -ALTER TABLE controversy_links - DROP CONSTRAINT controversy_links_controversy_story_stories_id; - -DROP INDEX database_variables_name_key_index; - -DROP INDEX tags_tag_sets_id; - -DROP INDEX queries_description; - -DROP INDEX queries_hash; - -DROP INDEX queries_hash_version; - -DROP INDEX queries_md5_signature; - -DROP INDEX media_rss_full_text_detection_data_media_1305311081; - -DROP INDEX media_cluster_map_pole_similarities_map; - -DROP INDEX stories_guid_non_unique; - -DROP INDEX stories_guid_unique_temp; - -DROP INDEX story_sentence_words_dm; - -DROP INDEX weekly_words_topic; - -DROP INDEX top_500_weekly_words_media_null_dashboard; - -DROP INDEX top_500_weekly_words_dmds; - -DROP INDEX total_daily_words_date; - -DROP INDEX total_daily_words_date_dt; - -DROP INDEX top_500_weekly_author_words_publish_week; - -DROP INDEX query_story_searches_stories_map_qss; - -DROP INDEX story_similarities_1000_tiny_a_b; - -DROP INDEX story_similarities_1000_tiny_a_s; - -DROP INDEX story_similarities_1000_tiny_b_s; - -DROP INDEX controversy_merged_stories_map_source; - -DROP INDEX controversy_stories_sc; - -DROP INDEX controversy_links_scr; - -DROP INDEX controversy_seed_urls_controversy; - -DROP INDEX processed_stories_story; - -DROP INDEX cqssism_c; - -DROP INDEX cqssism_s; - -DROP TABLE word_cloud_topics; - -DROP TABLE ssw_queue; - -DROP TABLE story_similarities_1000_tiny_idf; - -DROP TABLE story_similarities_100_short; - -DROP TABLE tar_downloads_queue; - -DROP TABLE controversy_unmerged_media; - -DROP TABLE adhoc_momentum; - -DROP TABLE controversy_links_copy; - -DROP TABLE controversy_links_copy_20120920; - -DROP TABLE controversy_media_codes_20121020; - -DROP TABLE controversy_stories_20121018; - -DROP TABLE controversy_links_20121018; - -DROP TABLE controversy_stories_copy; - -DROP TABLE controversy_stories_copy_20120920; - -DROP TABLE controversy_links_distinct; - -DROP TABLE extractor_training_lines_corrupted_download_content; - -DROP TABLE hr_pilot_study_stories; - -DROP TABLE india_million; - -DROP TABLE ma_ms_queue; - -DROP TABLE pilot_story_sims; - -DROP TABLE pilot_story_sims_code; - -DROP TABLE pilot_study_stories; - -DROP TABLE questionable_downloads_rows; - -DROP TABLE ssw_dump; - -DROP TABLE stories_description_not_salvaged; - -DROP TABLE total_daily_media_words; - -DROP TABLE valid_trayvon_stories; - -DROP TABLE media_alexa_stats; - -DROP SEQUENCE database_variables_datebase_variables_id_seq; - -DROP SEQUENCE media_cluster_map_pole_simila_media_cluster_map_pole_simila_seq; - -DROP SEQUENCE total_top_500_weekly_words_total_top_500_words_id_seq; - -DROP SEQUENCE total_top_500_weekly_author_words_total_top_500_words_id_seq; - -DROP SEQUENCE story_similarities_story_similarities_id_seq1; - -DROP SEQUENCE story_similarities_story_similarities_id_seq2; - -DROP SEQUENCE story_similarities_100_short_story_similarities_id_seq; - -DROP SEQUENCE controversy_dates_controversy_dates_id_seq; - -DROP SEQUENCE controversy_seed_urls_controversy_seed_urls_id_seq; - -DROP SEQUENCE media_alexa_stats_media_alexa_stats_id_seq; - -ALTER TABLE database_variables - ALTER COLUMN database_variables_id TYPE serial primary key /* TYPE change - table: database_variables original: integer new: serial primary key */, - ALTER COLUMN database_variables_id DROP DEFAULT, - ALTER COLUMN database_variables_id DROP NOT NULL; - -ALTER TABLE media - ADD COLUMN use_pager boolean, - ADD COLUMN unpaged_stories int not null DEFAULT 0, - ALTER COLUMN dup_media_id TYPE int null references media on delete set /* TYPE change - table: media original: integer REFERENCES media(media_id) ON DELETE SET new: int null references media on delete set */; - -ALTER TABLE feeds - ADD COLUMN last_checksum text; - -ALTER TABLE media_rss_full_text_detection_data - DROP COLUMN avg_extracted_length, - ADD COLUMN avg_expected_length numeric, - ALTER COLUMN media_id TYPE int references media on delete cascade /* TYPE change - table: media_rss_full_text_detection_data original: integer new: int references media on delete cascade */; - -ALTER TABLE media_clusters - ALTER COLUMN media_clusters_id TYPE serial primary key /* TYPE change - table: media_clusters original: serial primary key new: serial primary key */, - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_clusters original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE media_cluster_map_pole_similarities - ALTER COLUMN media_cluster_map_pole_similarities_id TYPE serial primary key /* TYPE change - table: media_cluster_map_pole_similarities original: integer primary key new: serial primary key */, - ALTER COLUMN media_cluster_map_pole_similarities_id DROP DEFAULT, - ALTER COLUMN media_cluster_map_pole_similarities_id DROP NOT NULL, - ALTER COLUMN media_id TYPE int not null references media on delete cascade /* TYPE change - table: media_cluster_map_pole_similarities original: integer NOT NULL REFERENCES media(media_id) new: int not null references media on delete cascade */, - ALTER COLUMN queries_id TYPE int not null references queries on delete cascade /* TYPE change - table: media_cluster_map_pole_similarities original: integer NOT NULL REFERENCES queries(queries_id) new: int not null references queries on delete cascade */, - ALTER COLUMN similarity TYPE int /* TYPE change - table: media_cluster_map_pole_similarities original: integer new: int */, - ALTER COLUMN media_cluster_maps_id TYPE int not null references media_cluster_maps on delete cascade /* TYPE change - table: media_cluster_map_pole_similarities original: integer NOT NULL REFERENCES media_cluster_maps(media_cluster_maps_id) new: int not null references media_cluster_maps on delete cascade */; - -ALTER TABLE media_cluster_words - ALTER COLUMN media_cluster_words_id TYPE serial primary key /* TYPE change - table: media_cluster_words original: serial primary key new: serial primary key */, - ALTER COLUMN media_clusters_id TYPE int not null references media_clusters on delete cascade /* TYPE change - table: media_cluster_words original: int not null references media_clusters on delete cascade new: int not null references media_clusters on delete cascade */; - -ALTER TABLE media_cluster_links - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_cluster_links original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE media_cluster_zscores - ALTER COLUMN media_cluster_runs_id TYPE int not null references media_cluster_runs on delete cascade /* TYPE change - table: media_cluster_zscores original: int not null references media_cluster_runs on delete cascade new: int not null references media_cluster_runs on delete cascade */; - -ALTER TABLE stories - DROP COLUMN story_texts_id; - -ALTER TABLE ONLY downloads ALTER COLUMN "host" SET STATISTICS -1; - -ALTER TABLE stories_tags_map - ALTER COLUMN stories_id TYPE int not null references stories on delete cascade /* TYPE change - table: stories_tags_map original: int new: int not null references stories on delete cascade */, - ALTER COLUMN stories_id DROP NOT NULL, - ALTER COLUMN tags_id TYPE int not null references tags on delete cascade /* TYPE change - table: stories_tags_map original: int new: int not null references tags on delete cascade */, - ALTER COLUMN tags_id DROP NOT NULL; - -ALTER TABLE top_ten_tags_for_media - ALTER COLUMN media_id TYPE integer /* TYPE change - table: top_ten_tags_for_media original: integer NOT NULL REFERENCES media(media_id) ON DELETE CASCADE new: integer */, - ALTER COLUMN media_id SET NOT NULL, - ALTER COLUMN tags_id TYPE integer /* TYPE change - table: top_ten_tags_for_media original: integer NOT NULL REFERENCES tags(tags_id) new: integer */, - ALTER COLUMN tags_id SET NOT NULL, - ALTER COLUMN tag_sets_id TYPE integer /* TYPE change - table: top_ten_tags_for_media original: integer NOT NULL REFERENCES tag_sets(tag_sets_id) new: integer */, - ALTER COLUMN tag_sets_id SET NOT NULL; - -ALTER TABLE total_top_500_weekly_words - ALTER COLUMN total_top_500_weekly_words_id TYPE serial primary key /* TYPE change - table: total_top_500_weekly_words original: int primary key new: serial primary key */, - ALTER COLUMN total_top_500_weekly_words_id DROP DEFAULT, - ALTER COLUMN media_sets_id TYPE int not null references media_sets on delete cascade /* TYPE change - table: total_top_500_weekly_words original: int not null references media_sets(media_sets_id) on delete cascade new: int not null references media_sets on delete cascade */, - ALTER COLUMN dashboard_topics_id TYPE int null references dashboard_topics /* TYPE change - table: total_top_500_weekly_words original: int null references dashboard_topics(dashboard_topics_id) on delete cascade new: int null references dashboard_topics */; - -ALTER TABLE total_top_500_weekly_author_words - ALTER COLUMN total_top_500_weekly_author_words_id TYPE serial primary key /* TYPE change - table: total_top_500_weekly_author_words original: integer primary key new: serial primary key */, - ALTER COLUMN total_top_500_weekly_author_words_id DROP DEFAULT; - -ALTER TABLE query_story_searches_stories_map - ALTER COLUMN query_story_searches_id TYPE int references query_story_searches on delete cascade /* TYPE change - table: query_story_searches_stories_map original: int REFERENCES query_story_searches(query_story_searches_id) ON DELETE CASCADE new: int references query_story_searches on delete cascade */, - ALTER COLUMN stories_id TYPE int references stories on delete cascade /* TYPE change - table: query_story_searches_stories_map original: int REFERENCES stories(stories_id) ON DELETE CASCADE new: int references stories on delete cascade */; - -ALTER TABLE story_similarities - ALTER COLUMN story_similarities_id TYPE serial primary key /* TYPE change - table: story_similarities original: integer primary key new: serial primary key */, - ALTER COLUMN story_similarities_id DROP DEFAULT; - -ALTER TABLE controversy_dates - ALTER COLUMN controversy_dates_id TYPE serial primary key /* TYPE change - table: controversy_dates original: integer primary key new: serial primary key */, - ALTER COLUMN controversy_dates_id DROP DEFAULT, - ALTER COLUMN controversy_dates_id DROP NOT NULL, - ALTER COLUMN controversies_id TYPE int not null references controversies on delete cascade /* TYPE change - table: controversy_dates original: integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE new: int not null references controversies on delete cascade */; - -ALTER TABLE controversy_merged_stories_map - ALTER COLUMN source_stories_id TYPE int not null references stories on delete cascade /* TYPE change - table: controversy_merged_stories_map original: integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE new: int not null references stories on delete cascade */, - ALTER COLUMN target_stories_id TYPE int not null references stories on delete cascade /* TYPE change - table: controversy_merged_stories_map original: integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE new: int not null references stories on delete cascade */; - -ALTER TABLE controversy_stories - ADD COLUMN valid_foreign_rss_story boolean DEFAULT false, - ALTER COLUMN controversies_id TYPE int not null references controversies on delete cascade /* TYPE change - table: controversy_stories original: int not null references controversies(controversies_id) on delete cascade new: int not null references controversies on delete cascade */, - ALTER COLUMN stories_id TYPE int not null references stories on delete cascade /* TYPE change - table: controversy_stories original: int not null references stories(stories_id) on delete cascade new: int not null references stories on delete cascade */; - -ALTER TABLE controversy_seed_urls - ALTER COLUMN controversy_seed_urls_id TYPE serial primary key /* TYPE change - table: controversy_seed_urls original: integer primary key new: serial primary key */, - ALTER COLUMN controversy_seed_urls_id DROP DEFAULT, - ALTER COLUMN controversy_seed_urls_id DROP NOT NULL, - ALTER COLUMN controversies_id TYPE int not null references controversies on delete cascade /* TYPE change - table: controversy_seed_urls original: integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE new: int not null references controversies on delete cascade */, - ALTER COLUMN stories_id TYPE int references stories on delete cascade /* TYPE change - table: controversy_seed_urls original: integer REFERENCES stories(stories_id) ON DELETE CASCADE new: int references stories on delete cascade */, - ALTER COLUMN processed TYPE boolean not null /* TYPE change - table: controversy_seed_urls original: boolean new: boolean not null */, - ALTER COLUMN processed DROP NOT NULL; - -ALTER TABLE controversy_query_story_searches_imported_stories_map - ALTER COLUMN controversies_id TYPE int not null references controversies on delete cascade /* TYPE change - table: controversy_query_story_searches_imported_stories_map original: integer NOT NULL REFERENCES controversies(controversies_id) ON DELETE CASCADE new: int not null references controversies on delete cascade */, - ALTER COLUMN stories_id TYPE int not null references stories on delete cascade /* TYPE change - table: controversy_query_story_searches_imported_stories_map original: integer NOT NULL REFERENCES stories(stories_id) ON DELETE CASCADE new: int not null references stories on delete cascade */; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4406; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION download_relative_file_path_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - IF TG_OP = 'UPDATE' then - -- RAISE NOTICE 'UPDATE '; - - -- The second part is needed because of the way comparisons with null are handled. - path_change := ( OLD.path <> NEW.path ) AND ( ( OLD.path is not null) <> (NEW.path is not null) ) ; - -- RAISE NOTICE 'test result % ', path_change; - - IF path_change is null THEN - -- RAISE NOTICE 'Path change % != %', OLD.path, NEW.path; - NEW.relative_file_path = get_relative_file_path(NEW.path); - - IF NEW.relative_file_path = 'inline' THEN - NEW.file_status = 'inline'; - END IF; - ELSE - -- RAISE NOTICE 'NO path change % = %', OLD.path, NEW.path; - END IF; - ELSIF TG_OP = 'INSERT' then - NEW.relative_file_path = get_relative_file_path(NEW.path); - - IF NEW.relative_file_path = 'inline' THEN - NEW.file_status = 'inline'; - END IF; - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE media - ADD CONSTRAINT media_self_dup CHECK ( dup_media_id IS NULL OR dup_media_id <> media_id ); - -ALTER TABLE media_cluster_runs - ADD CONSTRAINT media_cluster_runs_state check (state in ('pending', 'executing', 'completed')); - -ALTER TABLE media_cluster_maps - ADD CONSTRAINT media_cluster_maps_type check( map_type in ('cluster', 'polar' )); - -ALTER TABLE controversy_links - ADD CONSTRAINT controversy_links_controversy_story_stories_id foreign key ( stories_id, controversies_id ) references controversy_stories ( stories_id, controversies_id ) - on delete cascade; - -CREATE INDEX tags_tag_sets_id ON tags (tag_sets_id); - -CREATE UNIQUE INDEX queries_hash_version ON queries (md5_signature, query_version); - -CREATE INDEX queries_md5_signature ON queries (md5_signature); - -CREATE INDEX media_rss_full_text_detection_data_media ON media_rss_full_text_detection_data (media_id); - -CREATE INDEX media_cluster_map_pole_similarities_map ON media_cluster_map_pole_similarities (media_cluster_maps_id); - -CREATE UNIQUE INDEX stories_guid ON stories (guid, media_id); - -CREATE INDEX story_sentence_words_dm ON story_sentence_words (publish_day, media_id); - -CREATE INDEX weekly_words_topic ON weekly_words (publish_week, dashboard_topics_id); - -CREATE INDEX top_500_weekly_words_media_null_dashboard ON top_500_weekly_words (publish_week,media_sets_id, dashboard_topics_id) - where dashboard_topics_id is null; - -CREATE INDEX top_500_weekly_words_dmds ON top_500_weekly_words using btree (publish_week, media_sets_id, dashboard_topics_id, stem); - -CREATE INDEX top_500_weekly_author_words_publish_week ON top_500_weekly_author_words (publish_week); - -CREATE INDEX story_similarities_a_b ON story_similarities ( stories_id_a, stories_id_b ); - -CREATE INDEX story_similarities_a_s ON story_similarities ( stories_id_a, similarity, publish_day_b ); - -CREATE INDEX story_similarities_b_s ON story_similarities ( stories_id_b, similarity, publish_day_a ); - -CREATE INDEX controversy_merged_stories_map_source ON controversy_merged_stories_map ( source_stories_id ); - -CREATE UNIQUE INDEX controversy_stories_sc ON controversy_stories ( stories_id, controversies_id ); - -CREATE UNIQUE INDEX controversy_links_scr ON controversy_links ( stories_id, controversies_id, ref_stories_id ); - -CREATE INDEX controversy_seed_urls_controversy ON controversy_seed_urls ( controversies_id ); - -CREATE INDEX processed_stories_story ON processed_stories ( stories_id ); - -CREATE INDEX cqssism_c ON controversy_query_story_searches_imported_stories_map ( controversies_id ); - -CREATE INDEX cqssism_s ON controversy_query_story_searches_imported_stories_map ( stories_id ); - -CREATE VIEW stories_collected_in_past_day AS - select * from stories where collect_date > now() - interval '1 day'; - - -- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4406-4407.sql b/apps/postgresql-server/schema/migrations/mediawords-4406-4407.sql deleted file mode 100644 index 938224f349..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4406-4407.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4406 and 4407. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4406, and you would like to upgrade both the Media Cloud and the --- database to be at version 4407, import this SQL file: --- --- psql mediacloud < mediawords-4406-4407.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE controversies - ALTER COLUMN query_story_searches_id TYPE int not null references query_story_searches /* TYPE change - table: controversies original: int new: int not null references query_story_searches */, - ALTER COLUMN query_story_searches_id DROP NOT NULL; - -ALTER TABLE controversy_seed_urls - ADD COLUMN assume_match boolean not null DEFAULT false; - -CREATE INDEX controversy_seed_urls_url ON controversy_seed_urls (url); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4407; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4407-4408.sql b/apps/postgresql-server/schema/migrations/mediawords-4407-4408.sql deleted file mode 100644 index 4d99368272..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4407-4408.sql +++ /dev/null @@ -1,313 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4407 and 4408. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4407, and you would like to upgrade both the Media Cloud and the --- database to be at version 4408, import this SQL file: --- --- psql mediacloud < mediawords-4407-4408.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP FUNCTION is_stop_stem("size" TEXT, stem TEXT); - -DROP INDEX relative_file_paths_old_format_to_verify; - -DROP TABLE stopwords_tiny; - -DROP TABLE stopword_stems_tiny; - -DROP TABLE stopwords_short; - -DROP TABLE stopword_stems_short; - -DROP TABLE stopwords_long; - -DROP TABLE stopword_stems_long; - -CREATE TABLE feedless_stories ( - stories_id integer, - media_id integer -); - -CREATE TABLE auth_users ( - users_id SERIAL PRIMARY KEY, - email TEXT UNIQUE NOT NULL, - password_hash TEXT NOT NULL CONSTRAINT password_hash_sha256 CHECK(LENGTH(password_hash) = 137), - full_name TEXT NOT NULL, - notes TEXT, - active BOOLEAN NOT NULL DEFAULT true, - password_reset_token_hash TEXT UNIQUE NULL CONSTRAINT password_reset_token_hash_sha256 CHECK(LENGTH(password_reset_token_hash) = 137 OR password_reset_token_hash IS NULL), - last_unsuccessful_login_attempt TIMESTAMP NOT NULL DEFAULT TIMESTAMP 'epoch' -); - -CREATE TABLE auth_roles ( - roles_id SERIAL PRIMARY KEY, - "role" TEXT UNIQUE NOT NULL CONSTRAINT role_name_can_not_contain_spaces CHECK(role NOT LIKE '% %'), - description TEXT NOT NULL -); - -CREATE TABLE auth_users_roles_map ( - auth_users_roles_map SERIAL PRIMARY KEY, - users_id INTEGER NOT NULL REFERENCES auth_users(users_id) - ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE, - roles_id INTEGER NOT NULL REFERENCES auth_roles(roles_id) - ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE -); - -CREATE TABLE media_edits ( - media_edits_id SERIAL PRIMARY KEY, - edit_timestamp TIMESTAMP NOT NULL DEFAULT LOCALTIMESTAMP, - edited_field VARCHAR(64) NOT NULL - CONSTRAINT edited_field_not_empty CHECK(LENGTH(edited_field) > 0), - old_value TEXT NOT NULL, - new_value TEXT NOT NULL, - reason TEXT NOT NULL - CONSTRAINT reason_not_empty CHECK(LENGTH(reason) > 0), - users_email TEXT NOT NULL REFERENCES auth_users(email) - ON DELETE NO ACTION ON UPDATE NO ACTION DEFERRABLE -); - -CREATE TABLE story_edits ( - story_edits_id SERIAL PRIMARY KEY, - stories_id INT NOT NULL REFERENCES stories(stories_id) - - ON DELETE NO ACTION ON UPDATE NO ACTION DEFERRABLE, - edit_timestamp TIMESTAMP NOT NULL DEFAULT LOCALTIMESTAMP, - edited_field VARCHAR(64) NOT NULL - CONSTRAINT edited_field_not_empty CHECK(LENGTH(edited_field) > 0), - old_value TEXT NOT NULL, - new_value TEXT NOT NULL, - reason TEXT NOT NULL - CONSTRAINT reason_not_empty CHECK(LENGTH(reason) > 0), - users_email TEXT NOT NULL REFERENCES auth_users(email) - ON DELETE NO ACTION ON UPDATE NO ACTION DEFERRABLE -); - --- Feed statuses that determine whether the feed will be fetched --- or skipped -CREATE TYPE feed_feed_status AS ENUM ( - -- Feed is active, being fetched - 'active', - -- Feed is (temporary) disabled (usually by hand), not being fetched - 'inactive', - -- Feed was moderated as the one that shouldn't be fetched, but is still kept around - -- to reduce the moderation queue next time the page is being scraped for feeds to find - -- new ones - 'skipped' -); - -ALTER TABLE feeds - ADD COLUMN feed_status feed_feed_status not null DEFAULT 'active'; - -ALTER TABLE dashboard_topics - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE stories - ADD COLUMN "language" varchar(3); - -ALTER TABLE story_sentences - ADD COLUMN "language" varchar(3); - -ALTER TABLE story_sentence_words - ADD COLUMN "language" varchar(3); - -ALTER TABLE daily_words - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE weekly_words - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE top_500_weekly_words - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE daily_country_counts - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE daily_author_words - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE weekly_author_words - ADD COLUMN "language" varchar(3) NOT NULL; - -ALTER TABLE top_500_weekly_author_words - ADD COLUMN "language" varchar(3) NOT NULL; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4408; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE auth_users_roles_map - ADD CONSTRAINT no_duplicate_entries UNIQUE (users_id, roles_id); - -CREATE INDEX stories_language ON stories (language); - -CREATE INDEX story_sentences_language ON story_sentences (language); - -CREATE INDEX feedless_stories_story ON feedless_stories USING btree (stories_id); - -CREATE INDEX auth_users_roles_map_users_id_roles_id ON auth_users_roles_map (users_id, roles_id); - - --- --- Manually added stopwords --- (output of MediaWords::DB::Schema::get_is_stop_stem_function_tables_and_definition()) --- - --- PostgreSQL sends notices about implicit keys that are being created, --- and the test suite takes them for warnings. -SET client_min_messages=WARNING; - --- "Full" stopwords -DROP TABLE IF EXISTS stopwords_tiny; -CREATE TABLE stopwords_tiny ( - stopwords_tiny_id SERIAL PRIMARY KEY, - stopword VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopwords_tiny_stopword - ON stopwords_tiny(stopword, language); - --- Stopword stems -DROP TABLE IF EXISTS stopword_stems_tiny; -CREATE TABLE stopword_stems_tiny ( - stopword_stems_tiny_id SERIAL PRIMARY KEY, - stopword_stem VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopword_stems_tiny_stopword_stem - ON stopword_stems_tiny(stopword_stem, language); - --- Reset the message level back to "notice". -SET client_min_messages=NOTICE; - -INSERT INTO stopwords_tiny (stopword, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('sådan', 'da'), ('nu', 'da'), ('anden', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('ville', 'da'), ('kunne', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('alle', 'da'), ('når', 'da'), ('dig', 'da'), ('over', 'da'), ('af', 'da'), ('vil', 'da'), ('men', 'da'), ('ned', 'da'), ('have', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('ind', 'da'), ('været', 'da'), ('også', 'da'), ('hendes', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('hos', 'da'), ('har', 'da'), ('dog', 'da'), ('mod', 'da'), ('der', 'da'), ('hans', 'da'), ('denne', 'da'), ('være', 'da'), ('og', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('et', 'da'), ('er', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('disse', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('en', 'da'), ('sin', 'da'), ('mange', 'da'), ('vor', 'da'), ('meget', 'da'), ('hende', 'da'), ('de', 'da'), ('deres', 'da'), ('din', 'da'), ('mine', 'da'), ('op', 'da'), ('hun', 'da'), ('noget', 'da'), ('sine', 'da'), ('ham', 'da'), ('ikke', 'da'), ('skal', 'da'), ('efter', 'da'), ('havde', 'da'), ('ad', 'da'), ('skulle', 'da'), ('at', 'da'), ('jo', 'da'), ('i', 'da'), ('alt', 'da'), ('under', 'da'), ('jer', 'da'), ('min', 'da'), ('mig', 'da'), ('nogle', 'da'), ('sig', 'da'), ('eller', 'da'), ('blive', 'da'), ('blev', 'da'), ('bliver', 'da'), ('mit', 'da'), ('dette', 'da');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('nu', 'da'), ('sådan', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('hav', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('vær', 'da'), ('når', 'da'), ('dig', 'da'), ('and', 'da'), ('bliv', 'da'), ('over', 'da'), ('ell', 'da'), ('af', 'da'), ('vil', 'da'), ('all', 'da'), ('men', 'da'), ('ned', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('und', 'da'), ('ind', 'da'), ('også', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('har', 'da'), ('hos', 'da'), ('dog', 'da'), ('ikk', 'da'), ('havd', 'da'), ('mod', 'da'), ('der', 'da'), ('dis', 'da'), ('og', 'da'), ('nogl', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('mang', 'da'), ('et', 'da'), ('er', 'da'), ('hend', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('meg', 'da'), ('en', 'da'), ('sin', 'da'), ('vor', 'da'), ('de', 'da'), ('din', 'da'), ('op', 'da'), ('hun', 'da'), ('skul', 'da'), ('ham', 'da'), ('skal', 'da'), ('ad', 'da'), ('kun', 'da'), ('at', 'da'), ('jo', 'da'), ('eft', 'da'), ('i', 'da'), ('alt', 'da'), ('mig', 'da'), ('jer', 'da'), ('min', 'da'), ('nog', 'da'), ('sig', 'da'), ('blev', 'da'), ('mit', 'da');INSERT INTO stopwords_tiny (stopword, language) VALUES ('einiger', 'de'), ('habe', 'de'), ('kann', 'de'), ('deinen', 'de'), ('da', 'de'), ('anderr', 'de'), ('meiner', 'de'), ('dann', 'de'), ('einer', 'de'), ('meines', 'de'), ('hat', 'de'), ('gewesen', 'de'), ('eure', 'de'), ('solche', 'de'), ('ihre', 'de'), ('mich', 'de'), ('vom', 'de'), ('sehr', 'de'), ('also', 'de'), ('über', 'de'), ('allem', 'de'), ('jener', 'de'), ('warst', 'de'), ('alle', 'de'), ('solchem', 'de'), ('eures', 'de'), ('zwischen', 'de'), ('jenen', 'de'), ('hatten', 'de'), ('jeder', 'de'), ('sonst', 'de'), ('ohne', 'de'), ('alles', 'de'), ('bei', 'de'), ('die', 'de'), ('dir', 'de'), ('welchem', 'de'), ('etwas', 'de'), ('will', 'de'), ('anderem', 'de'), ('solcher', 'de'), ('und', 'de'), ('das', 'de'), ('musste', 'de'), ('zur', 'de'), ('gegen', 'de'), ('man', 'de'), ('keines', 'de'), ('diese', 'de'), ('einige', 'de'), ('ander', 'de'), ('diesen', 'de'), ('seinen', 'de'), ('euren', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('allen', 'de'), ('eurer', 'de'), ('bin', 'de'), ('hinter', 'de'), ('um', 'de'), ('unsem', 'de'), ('ein', 'de'), ('weiter', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('manchen', 'de'), ('keinem', 'de'), ('ich', 'de'), ('daß', 'de'), ('jenem', 'de'), ('dieses', 'de'), ('wird', 'de'), ('dieselbe', 'de'), ('seines', 'de'), ('wirst', 'de'), ('dem', 'de'), ('solchen', 'de'), ('sind', 'de'), ('hier', 'de'), ('wo', 'de'), ('weg', 'de'), ('euer', 'de'), ('des', 'de'), ('diesem', 'de'), ('dies', 'de'), ('nichts', 'de'), ('deines', 'de'), ('ihrer', 'de'), ('war', 'de'), ('demselben', 'de'), ('meinen', 'de'), ('euch', 'de'), ('am', 'de'), ('nach', 'de'), ('unser', 'de'), ('anderm', 'de'), ('selbst', 'de'), ('während', 'de'), ('deiner', 'de'), ('unsen', 'de'), ('solches', 'de'), ('welche', 'de'), ('dasselbe', 'de'), ('zum', 'de'), ('werde', 'de'), ('ihnen', 'de'), ('eines', 'de'), ('meine', 'de'), ('mit', 'de'), ('ihm', 'de'), ('anders', 'de'), ('anderes', 'de'), ('ihr', 'de'), ('jene', 'de'), ('jenes', 'de'), ('wollen', 'de'), ('deinem', 'de'), ('ob', 'de'), ('du', 'de'), ('einigem', 'de'), ('einiges', 'de'), ('welchen', 'de'), ('einen', 'de'), ('ihn', 'de'), ('jede', 'de'), ('wie', 'de'), ('wenn', 'de'), ('denselben', 'de'), ('derer', 'de'), ('unses', 'de'), ('deine', 'de'), ('ist', 'de'), ('mancher', 'de'), ('meinem', 'de'), ('keinen', 'de'), ('dessen', 'de'), ('nicht', 'de'), ('unter', 'de'), ('kein', 'de'), ('bist', 'de'), ('doch', 'de'), ('anderen', 'de'), ('wir', 'de'), ('ihren', 'de'), ('keiner', 'de'), ('ihrem', 'de'), ('unse', 'de'), ('eurem', 'de'), ('sein', 'de'), ('dein', 'de'), ('hab', 'de'), ('welcher', 'de'), ('würden', 'de'), ('keine', 'de'), ('seinem', 'de'), ('haben', 'de'), ('auf', 'de'), ('manche', 'de'), ('desselben', 'de'), ('hin', 'de'), ('denn', 'de'), ('jedes', 'de'), ('den', 'de'), ('derselbe', 'de'), ('einem', 'de'), ('auch', 'de'), ('jeden', 'de'), ('sollte', 'de'), ('an', 'de'), ('der', 'de'), ('seiner', 'de'), ('derselben', 'de'), ('mein', 'de'), ('könnte', 'de'), ('durch', 'de'), ('es', 'de'), ('wieder', 'de'), ('seine', 'de'), ('machen', 'de'), ('würde', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('andere', 'de'), ('bis', 'de'), ('ihres', 'de'), ('muss', 'de'), ('zwar', 'de'), ('waren', 'de'), ('sich', 'de'), ('so', 'de'), ('hatte', 'de'), ('er', 'de'), ('manches', 'de'), ('einigen', 'de'), ('nur', 'de'), ('im', 'de'), ('jedem', 'de'), ('dich', 'de'), ('eine', 'de'), ('vor', 'de'), ('können', 'de'), ('aus', 'de'), ('aller', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('andern', 'de'), ('indem', 'de'), ('wollte', 'de'), ('welches', 'de'), ('manchem', 'de'), ('einig', 'de'), ('sie', 'de'), ('dieser', 'de'), ('noch', 'de'), ('für', 'de'), ('dieselben', 'de'), ('anderer', 'de'), ('werden', 'de'), ('nun', 'de'), ('von', 'de'), ('aber', 'de'), ('ins', 'de'), ('was', 'de'), ('sondern', 'de');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('wollt', 'de'), ('mach', 'de'), ('kann', 'de'), ('anderr', 'de'), ('da', 'de'), ('dann', 'de'), ('jen', 'de'), ('hat', 'de'), ('mich', 'de'), ('also', 'de'), ('sehr', 'de'), ('vom', 'de'), ('konnt', 'de'), ('warst', 'de'), ('hint', 'de'), ('and', 'de'), ('sonst', 'de'), ('bei', 'de'), ('all', 'de'), ('die', 'de'), ('demselb', 'de'), ('dir', 'de'), ('etwas', 'de'), ('will', 'de'), ('derselb', 'de'), ('ind', 'de'), ('und', 'de'), ('das', 'de'), ('solch', 'de'), ('zur', 'de'), ('ohn', 'de'), ('man', 'de'), ('musst', 'de'), ('dess', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('bin', 'de'), ('um', 'de'), ('ein', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('ich', 'de'), ('wird', 'de'), ('manch', 'de'), ('wirst', 'de'), ('wied', 'de'), ('dem', 'de'), ('sind', 'de'), ('woll', 'de'), ('hier', 'de'), ('wo', 'de'), ('des', 'de'), ('euer', 'de'), ('weg', 'de'), ('dies', 'de'), ('war', 'de'), ('denselb', 'de'), ('am', 'de'), ('euch', 'de'), ('nach', 'de'), ('anderm', 'de'), ('selb', 'de'), ('wahrend', 'de'), ('zum', 'de'), ('eur', 'de'), ('ihm', 'de'), ('mit', 'de'), ('hatt', 'de'), ('ihr', 'de'), ('ob', 'de'), ('du', 'de'), ('ihn', 'de'), ('wie', 'de'), ('wenn', 'de'), ('fur', 'de'), ('ist', 'de'), ('unt', 'de'), ('welch', 'de'), ('nicht', 'de'), ('kein', 'de'), ('sond', 'de'), ('bist', 'de'), ('doch', 'de'), ('wir', 'de'), ('uber', 'de'), ('sein', 'de'), ('hab', 'de'), ('dein', 'de'), ('auf', 'de'), ('zwisch', 'de'), ('hin', 'de'), ('denn', 'de'), ('sollt', 'de'), ('gewes', 'de'), ('desselb', 'de'), ('den', 'de'), ('weit', 'de'), ('auch', 'de'), ('an', 'de'), ('der', 'de'), ('konn', 'de'), ('mein', 'de'), ('durch', 'de'), ('es', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('bis', 'de'), ('muss', 'de'), ('zwar', 'de'), ('sich', 'de'), ('so', 'de'), ('er', 'de'), ('nur', 'de'), ('im', 'de'), ('dich', 'de'), ('vor', 'de'), ('aus', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('jed', 'de'), ('dieselb', 'de'), ('werd', 'de'), ('einig', 'de'), ('sie', 'de'), ('noch', 'de'), ('dass', 'de'), ('nun', 'de'), ('dasselb', 'de'), ('wurd', 'de'), ('von', 'de'), ('geg', 'de'), ('ins', 'de'), ('aber', 'de'), ('was', 'de');INSERT INTO stopwords_tiny (stopword, language) VALUES ('these', 'en'), ('you', 'en'), ('both', 'en'), ('which', 'en'), ('my', 'en'), ('didn''t', 'en'), ('if', 'en'), ('we''ll', 'en'), ('himself', 'en'), ('him', 'en'), ('own', 'en'), ('doesn''t', 'en'), ('he''ll', 'en'), ('each', 'en'), ('yours', 'en'), ('what', 'en'), ('them', 'en'), ('there''s', 'en'), ('your', 'en'), ('again', 'en'), ('but', 'en'), ('too', 'en'), ('and', 'en'), ('why''s', 'en'), ('over', 'en'), ('shan''t', 'en'), ('of', 'en'), ('here''s', 'en'), ('they''ll', 'en'), ('all', 'en'), ('is', 'en'), ('being', 'en'), ('she', 'en'), ('nor', 'en'), ('have', 'en'), ('when', 'en'), ('we''d', 'en'), ('it', 'en'), ('where', 'en'), ('who''s', 'en'), ('shouldn''t', 'en'), ('were', 'en'), ('i''ve', 'en'), ('a', 'en'), ('yourself', 'en'), ('you''ll', 'en'), ('you''d', 'en'), ('would', 'en'), ('no', 'en'), ('off', 'en'), ('hadn''t', 'en'), ('how''s', 'en'), ('in', 'en'), ('wouldn''t', 'en'), ('where''s', 'en'), ('i''ll', 'en'), ('very', 'en'), ('only', 'en'), ('ours', 'en'), ('me', 'en'), ('they''re', 'en'), ('by', 'en'), ('i''m', 'en'), ('can''t', 'en'), ('haven''t', 'en'), ('myself', 'en'), ('they', 'en'), ('you''re', 'en'), ('same', 'en'), ('let''s', 'en'), ('she''d', 'en'), ('after', 'en'), ('hers', 'en'), ('doing', 'en'), ('am', 'en'), ('his', 'en'), ('i', 'en'), ('under', 'en'), ('she''s', 'en'), ('hasn''t', 'en'), ('there', 'en'), ('through', 'en'), ('while', 'en'), ('weren''t', 'en'), ('as', 'en'), ('couldn''t', 'en'), ('themselves', 'en'), ('why', 'en'), ('because', 'en'), ('itself', 'en'), ('ourselves', 'en'), ('has', 'en'), ('cannot', 'en'), ('not', 'en'), ('that', 'en'), ('mustn''t', 'en'), ('on', 'en'), ('i''d', 'en'), ('our', 'en'), ('who', 'en'), ('further', 'en'), ('won''t', 'en'), ('its', 'en'), ('out', 'en'), ('aren''t', 'en'), ('theirs', 'en'), ('ought', 'en'), ('some', 'en'), ('with', 'en'), ('here', 'en'), ('you''ve', 'en'), ('did', 'en'), ('do', 'en'), ('into', 'en'), ('we', 'en'), ('to', 'en'), ('herself', 'en'), ('from', 'en'), ('when''s', 'en'), ('her', 'en'), ('below', 'en'), ('during', 'en'), ('they''d', 'en'), ('any', 'en'), ('isn''t', 'en'), ('then', 'en'), ('he''d', 'en'), ('more', 'en'), ('we''re', 'en'), ('an', 'en'), ('the', 'en'), ('against', 'en'), ('they''ve', 'en'), ('don''t', 'en'), ('it''s', 'en'), ('we''ve', 'en'), ('or', 'en'), ('could', 'en'), ('few', 'en'), ('does', 'en'), ('down', 'en'), ('about', 'en'), ('whom', 'en'), ('before', 'en'), ('this', 'en'), ('so', 'en'), ('once', 'en'), ('how', 'en'), ('for', 'en'), ('wasn''t', 'en'), ('those', 'en'), ('other', 'en'), ('be', 'en'), ('their', 'en'), ('such', 'en'), ('most', 'en'), ('he''s', 'en'), ('up', 'en'), ('are', 'en'), ('having', 'en'), ('above', 'en'), ('been', 'en'), ('should', 'en'), ('at', 'en'), ('that''s', 'en'), ('had', 'en'), ('yourselves', 'en'), ('what''s', 'en'), ('he', 'en'), ('she''ll', 'en'), ('between', 'en'), ('than', 'en'), ('until', 'en'), ('was', 'en');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('these', 'en'), ('you', 'en'), ('which', 'en'), ('both', 'en'), ('my', 'en'), ('didn''t', 'en'), ('we''ll', 'en'), ('if', 'en'), ('him', 'en'), ('himself', 'en'), ('abov', 'en'), ('doesn''t', 'en'), ('own', 'en'), ('he''ll', 'en'), ('each', 'en'), ('what', 'en'), ('them', 'en'), ('dure', 'en'), ('your', 'en'), ('but', 'en'), ('again', 'en'), ('and', 'en'), ('too', 'en'), ('over', 'en'), ('shan''t', 'en'), ('of', 'en'), ('they''ll', 'en'), ('all', 'en'), ('is', 'en'), ('she', 'en'), ('have', 'en'), ('nor', 'en'), ('we''d', 'en'), ('when', 'en'), ('it', 'en'), ('where', 'en'), ('shouldn''t', 'en'), ('we''v', 'en'), ('were', 'en'), ('a', 'en'), ('yourself', 'en'), ('you''ll', 'en'), ('you''d', 'en'), ('would', 'en'), ('no', 'en'), ('off', 'en'), ('hadn''t', 'en'), ('becaus', 'en'), ('in', 'en'), ('wouldn''t', 'en'), ('i''ll', 'en'), ('me', 'en'), ('by', 'en'), ('i''m', 'en'), ('can''t', 'en'), ('haven''t', 'en'), ('myself', 'en'), ('befor', 'en'), ('they', 'en'), ('doe', 'en'), ('same', 'en'), ('she''d', 'en'), ('after', 'en'), ('am', 'en'), ('his', 'en'), ('we''r', 'en'), ('i', 'en'), ('under', 'en'), ('they''v', 'en'), ('hasn''t', 'en'), ('through', 'en'), ('there', 'en'), ('weren''t', 'en'), ('while', 'en'), ('onli', 'en'), ('you''v', 'en'), ('couldn''t', 'en'), ('as', 'en'), ('itself', 'en'), ('has', 'en'), ('cannot', 'en'), ('not', 'en'), ('that', 'en'), ('mustn''t', 'en'), ('i''v', 'en'), ('you''r', 'en'), ('on', 'en'), ('i''d', 'en'), ('our', 'en'), ('who', 'en'), ('further', 'en'), ('won''t', 'en'), ('out', 'en'), ('aren''t', 'en'), ('veri', 'en'), ('ought', 'en'), ('some', 'en'), ('with', 'en'), ('here', 'en'), ('did', 'en'), ('do', 'en'), ('into', 'en'), ('we', 'en'), ('to', 'en'), ('herself', 'en'), ('from', 'en'), ('her', 'en'), ('below', 'en'), ('they''d', 'en'), ('they''r', 'en'), ('isn''t', 'en'), ('then', 'en'), ('he''d', 'en'), ('more', 'en'), ('an', 'en'), ('the', 'en'), ('against', 'en'), ('don''t', 'en'), ('let', 'en'), ('or', 'en'), ('could', 'en'), ('few', 'en'), ('down', 'en'), ('about', 'en'), ('whom', 'en'), ('this', 'en'), ('so', 'en'), ('themselv', 'en'), ('for', 'en'), ('how', 'en'), ('wasn''t', 'en'), ('whi', 'en'), ('those', 'en'), ('other', 'en'), ('be', 'en'), ('their', 'en'), ('such', 'en'), ('most', 'en'), ('are', 'en'), ('up', 'en'), ('onc', 'en'), ('been', 'en'), ('at', 'en'), ('should', 'en'), ('ourselv', 'en'), ('had', 'en'), ('between', 'en'), ('she''ll', 'en'), ('he', 'en'), ('than', 'en'), ('until', 'en'), ('was', 'en'), ('ani', 'en'), ('yourselv', 'en');INSERT INTO stopwords_tiny (stopword, language) VALUES ('estuvierais', 'es'), ('fueras', 'es'), ('fuiste', 'es'), ('para', 'es'), ('otro', 'es'), ('siendo', 'es'), ('estuviesen', 'es'), ('e', 'es'), ('tuvieron', 'es'), ('tendrás', 'es'), ('hubisteis', 'es'), ('tuvo', 'es'), ('eran', 'es'), ('eres', 'es'), ('está', 'es'), ('estaba', 'es'), ('ti', 'es'), ('seáis', 'es'), ('estáis', 'es'), ('tuvieran', 'es'), ('estuviste', 'es'), ('mi', 'es'), ('tiene', 'es'), ('por', 'es'), ('tuya', 'es'), ('cual', 'es'), ('habrá', 'es'), ('otros', 'es'), ('algunas', 'es'), ('contra', 'es'), ('tenías', 'es'), ('sido', 'es'), ('se', 'es'), ('mío', 'es'), ('estábamos', 'es'), ('les', 'es'), ('son', 'es'), ('tuvieseis', 'es'), ('a', 'es'), ('no', 'es'), ('ese', 'es'), ('hubiste', 'es'), ('tuyos', 'es'), ('tengo', 'es'), ('una', 'es'), ('más', 'es'), ('mías', 'es'), ('estaréis', 'es'), ('estuviese', 'es'), ('me', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('hayan', 'es'), ('estaríais', 'es'), ('que', 'es'), ('estabas', 'es'), ('hubimos', 'es'), ('nuestra', 'es'), ('seríamos', 'es'), ('eso', 'es'), ('tuve', 'es'), ('estas', 'es'), ('sobre', 'es'), ('habido', 'es'), ('fuimos', 'es'), ('vuestra', 'es'), ('también', 'es'), ('unos', 'es'), ('hemos', 'es'), ('esos', 'es'), ('vuestros', 'es'), ('estuviéramos', 'es'), ('sería', 'es'), ('o', 'es'), ('tenían', 'es'), ('has', 'es'), ('tenemos', 'es'), ('estuvieron', 'es'), ('estemos', 'es'), ('hay', 'es'), ('sí', 'es'), ('fuera', 'es'), ('tuvisteis', 'es'), ('estuvieseis', 'es'), ('habrían', 'es'), ('al', 'es'), ('fuéramos', 'es'), ('habidas', 'es'), ('teníais', 'es'), ('fuese', 'es'), ('habíamos', 'es'), ('estarán', 'es'), ('yo', 'es'), ('qué', 'es'), ('vosotras', 'es'), ('habrás', 'es'), ('hayamos', 'es'), ('estuviera', 'es'), ('suyas', 'es'), ('del', 'es'), ('habías', 'es'), ('estarás', 'es'), ('hayáis', 'es'), ('estará', 'es'), ('somos', 'es'), ('otra', 'es'), ('tendrías', 'es'), ('habían', 'es'), ('estuvieran', 'es'), ('es', 'es'), ('fuésemos', 'es'), ('habríamos', 'es'), ('otras', 'es'), ('estuvisteis', 'es'), ('vuestro', 'es'), ('esas', 'es'), ('tuviéramos', 'es'), ('hasta', 'es'), ('estad', 'es'), ('estaban', 'es'), ('mí', 'es'), ('míos', 'es'), ('tuviste', 'es'), ('estado', 'es'), ('en', 'es'), ('de', 'es'), ('un', 'es'), ('serían', 'es'), ('fuerais', 'es'), ('estadas', 'es'), ('teniendo', 'es'), ('estando', 'es'), ('como', 'es'), ('habrán', 'es'), ('estuviésemos', 'es'), ('fui', 'es'), ('seréis', 'es'), ('he', 'es'), ('estaríamos', 'es'), ('quienes', 'es'), ('esto', 'es'), ('los', 'es'), ('habida', 'es'), ('tuviésemos', 'es'), ('estar', 'es'), ('estuve', 'es'), ('sus', 'es'), ('tendrían', 'es'), ('tienen', 'es'), ('nuestros', 'es'), ('le', 'es'), ('estoy', 'es'), ('y', 'es'), ('nos', 'es'), ('estás', 'es'), ('hubieses', 'es'), ('sean', 'es'), ('tengáis', 'es'), ('estuvimos', 'es'), ('hayas', 'es'), ('nuestras', 'es'), ('pero', 'es'), ('porque', 'es'), ('fuesen', 'es'), ('tened', 'es'), ('esté', 'es'), ('seremos', 'es'), ('tuvieses', 'es'), ('mis', 'es'), ('sois', 'es'), ('habidos', 'es'), ('seré', 'es'), ('suyos', 'es'), ('hubieseis', 'es'), ('tendrán', 'es'), ('uno', 'es'), ('la', 'es'), ('serán', 'es'), ('estaremos', 'es'), ('con', 'es'), ('nosotras', 'es'), ('tenidas', 'es'), ('tuvierais', 'es'), ('todo', 'es'), ('estaría', 'es'), ('nosotros', 'es'), ('ellos', 'es'), ('ellas', 'es'), ('tuviesen', 'es'), ('han', 'es'), ('fue', 'es'), ('tendríais', 'es'), ('habréis', 'es'), ('tuviese', 'es'), ('cuando', 'es'), ('hubieran', 'es'), ('tenidos', 'es'), ('ya', 'es'), ('fueran', 'es'), ('teníamos', 'es'), ('os', 'es'), ('tenía', 'es'), ('tenéis', 'es'), ('durante', 'es'), ('tu', 'es'), ('habríais', 'es'), ('tenida', 'es'), ('era', 'es'), ('hubierais', 'es'), ('suyo', 'es'), ('fueron', 'es'), ('tuvieras', 'es'), ('tendremos', 'es'), ('estés', 'es'), ('estuvo', 'es'), ('sin', 'es'), ('tenido', 'es'), ('tuvimos', 'es'), ('tengas', 'es'), ('algunos', 'es'), ('este', 'es'), ('habrías', 'es'), ('habría', 'es'), ('ha', 'es'), ('habéis', 'es'), ('las', 'es'), ('había', 'es'), ('tienes', 'es'), ('estén', 'es'), ('nada', 'es'), ('habíais', 'es'), ('tú', 'es'), ('tuviera', 'es'), ('éramos', 'es'), ('tuyas', 'es'), ('hubieron', 'es'), ('tendría', 'es'), ('estuvieras', 'es'), ('antes', 'es'), ('su', 'es'), ('estabais', 'es'), ('suya', 'es'), ('mucho', 'es'), ('hubiésemos', 'es'), ('quien', 'es'), ('habré', 'es'), ('lo', 'es'), ('hubo', 'es'), ('soy', 'es'), ('todos', 'es'), ('tus', 'es'), ('estos', 'es'), ('vuestras', 'es'), ('hubiesen', 'es'), ('será', 'es'), ('tengamos', 'es'), ('poco', 'es'), ('tengan', 'es'), ('tenga', 'es'), ('estaré', 'es'), ('ni', 'es'), ('desde', 'es'), ('ella', 'es'), ('estada', 'es'), ('estarían', 'es'), ('haya', 'es'), ('sea', 'es'), ('tuyo', 'es'), ('entre', 'es'), ('tendréis', 'es'), ('vosotros', 'es'), ('muchos', 'es'), ('estéis', 'es'), ('eras', 'es'), ('donde', 'es'), ('tendríamos', 'es'), ('fueses', 'es'), ('estados', 'es'), ('algo', 'es'), ('hubiese', 'es'), ('seas', 'es'), ('estarías', 'es'), ('serías', 'es'), ('muy', 'es'), ('hubiera', 'es'), ('hubiéramos', 'es'), ('esta', 'es'), ('estuvieses', 'es'), ('tendrá', 'es'), ('erais', 'es'), ('mía', 'es'), ('estamos', 'es'), ('hube', 'es'), ('hubieras', 'es'), ('habiendo', 'es'), ('habremos', 'es'), ('seamos', 'es'), ('seríais', 'es'), ('él', 'es'), ('están', 'es'), ('ante', 'es'), ('esa', 'es'), ('te', 'es'), ('tendré', 'es'), ('serás', 'es'), ('nuestro', 'es'), ('el', 'es'), ('tanto', 'es');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('sus', 'es'), ('sid', 'es'), ('som', 'es'), ('esteis', 'es'), ('le', 'es'), ('tendras', 'es'), ('estoy', 'es'), ('otro', 'es'), ('nos', 'es'), ('y', 'es'), ('sean', 'es'), ('e', 'es'), ('qui', 'es'), ('porqu', 'es'), ('nad', 'es'), ('mis', 'es'), ('eran', 'es'), ('eres', 'es'), ('sois', 'es'), ('habran', 'es'), ('hem', 'es'), ('tien', 'es'), ('uno', 'es'), ('ti', 'es'), ('la', 'es'), ('con', 'es'), ('much', 'es'), ('estem', 'es'), ('mi', 'es'), ('fues', 'es'), ('ellas', 'es'), ('ellos', 'es'), ('por', 'es'), ('cual', 'es'), ('otros', 'es'), ('fue', 'es'), ('han', 'es'), ('vosotr', 'es'), ('esten', 'es'), ('se', 'es'), ('sobr', 'es'), ('com', 'es'), ('les', 'es'), ('son', 'es'), ('a', 'es'), ('ya', 'es'), ('dond', 'es'), ('no', 'es'), ('ese', 'es'), ('estam', 'es'), ('tuy', 'es'), ('os', 'es'), ('tu', 'es'), ('una', 'es'), ('seras', 'es'), ('estes', 'es'), ('era', 'es'), ('mas', 'es'), ('si', 'es'), ('fueron', 'es'), ('me', 'es'), ('sin', 'es'), ('mios', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('que', 'es'), ('desd', 'es'), ('este', 'es'), ('estaran', 'es'), ('ha', 'es'), ('eso', 'es'), ('par', 'es'), ('estas', 'es'), ('las', 'es'), ('estand', 'es'), ('unos', 'es'), ('mia', 'es'), ('poc', 'es'), ('esos', 'es'), ('estaras', 'es'), ('estuv', 'es'), ('tambien', 'es'), ('o', 'es'), ('su', 'es'), ('antes', 'es'), ('estabais', 'es'), ('tuv', 'es'), ('has', 'es'), ('durant', 'es'), ('fuer', 'es'), ('quien', 'es'), ('lo', 'es'), ('hay', 'es'), ('tendran', 'es'), ('soy', 'es'), ('nosotr', 'es'), ('eram', 'es'), ('estos', 'es'), ('tus', 'es'), ('hub', 'es'), ('siend', 'es'), ('seais', 'es'), ('al', 'es'), ('suy', 'es'), ('ten', 'es'), ('mio', 'es'), ('yo', 'es'), ('seam', 'es'), ('hab', 'es'), ('ni', 'es'), ('ser', 'es'), ('ella', 'es'), ('hast', 'es'), ('del', 'es'), ('sea', 'es'), ('algun', 'es'), ('entre', 'es'), ('tendr', 'es'), ('tant', 'es'), ('estab', 'es'), ('eras', 'es'), ('habr', 'es'), ('habras', 'es'), ('otra', 'es'), ('tod', 'es'), ('es', 'es'), ('otras', 'es'), ('algo', 'es'), ('cuand', 'es'), ('seas', 'es'), ('esas', 'es'), ('muy', 'es'), ('fuim', 'es'), ('estad', 'es'), ('fuist', 'es'), ('per', 'es'), ('esta', 'es'), ('en', 'es'), ('de', 'es'), ('erais', 'es'), ('un', 'es'), ('vuestr', 'es'), ('estan', 'es'), ('fuerais', 'es'), ('teng', 'es'), ('mias', 'es'), ('contr', 'es'), ('seran', 'es'), ('ante', 'es'), ('esa', 'es'), ('estais', 'es'), ('fui', 'es'), ('te', 'es'), ('he', 'es'), ('nuestr', 'es'), ('esto', 'es'), ('los', 'es'), ('el', 'es'), ('estar', 'es');INSERT INTO stopwords_tiny (stopword, language) VALUES ('ne', 'fi'), ('teillä', 'fi'), ('vaikka', 'fi'), ('jota', 'fi'), ('hänestä', 'fi'), ('yli', 'fi'), ('heidät', 'fi'), ('noille', 'fi'), ('tallä', 'fi'), ('joksi', 'fi'), ('joilta', 'fi'), ('tästä', 'fi'), ('teitä', 'fi'), ('siihen', 'fi'), ('tuoksi', 'fi'), ('niille', 'fi'), ('tässä', 'fi'), ('niissä', 'fi'), ('keissä', 'fi'), ('tuossa', 'fi'), ('hänellä', 'fi'), ('vaan', 'fi'), ('tuon', 'fi'), ('meihin', 'fi'), ('näinä', 'fi'), ('meiltä', 'fi'), ('ollut', 'fi'), ('näitä', 'fi'), ('jolta', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('heissä', 'fi'), ('teidän', 'fi'), ('tähän', 'fi'), ('tuona', 'fi'), ('meille', 'fi'), ('näille', 'fi'), ('sinä', 'fi'), ('näiksi', 'fi'), ('täksi', 'fi'), ('niillä', 'fi'), ('tänä', 'fi'), ('keneen', 'fi'), ('nämä', 'fi'), ('kanssa', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sille', 'fi'), ('mistä', 'fi'), ('noiksi', 'fi'), ('tälle', 'fi'), ('jotka', 'fi'), ('mutta', 'fi'), ('jonka', 'fi'), ('sinua', 'fi'), ('minulla', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('joiden', 'fi'), ('noita', 'fi'), ('olin', 'fi'), ('meidät', 'fi'), ('joka', 'fi'), ('niiden', 'fi'), ('tätä', 'fi'), ('näissä', 'fi'), ('mihin', 'fi'), ('meidän', 'fi'), ('kenessä', 'fi'), ('kenen', 'fi'), ('noissa', 'fi'), ('me', 'fi'), ('kenellä', 'fi'), ('keille', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('miltä', 'fi'), ('siinä', 'fi'), ('tuolta', 'fi'), ('tältä', 'fi'), ('kenenä', 'fi'), ('ei', 'fi'), ('keillä', 'fi'), ('minut', 'fi'), ('häntä', 'fi'), ('minä', 'fi'), ('keiltä', 'fi'), ('teidät', 'fi'), ('sinulta', 'fi'), ('koska', 'fi'), ('sitä', 'fi'), ('tuolle', 'fi'), ('nuo', 'fi'), ('joista', 'fi'), ('noilta', 'fi'), ('näiltä', 'fi'), ('olivat', 'fi'), ('olisivat', 'fi'), ('mukaan', 'fi'), ('näillä', 'fi'), ('heille', 'fi'), ('niitä', 'fi'), ('niistä', 'fi'), ('heillä', 'fi'), ('olette', 'fi'), ('olimme', 'fi'), ('kenet', 'fi'), ('että', 'fi'), ('joita', 'fi'), ('mille', 'fi'), ('näistä', 'fi'), ('olleet', 'fi'), ('sinulla', 'fi'), ('heistä', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('keneltä', 'fi'), ('olen', 'fi'), ('jossa', 'fi'), ('näiden', 'fi'), ('joissa', 'fi'), ('häneen', 'fi'), ('minkä', 'fi'), ('olisi', 'fi'), ('itse', 'fi'), ('olet', 'fi'), ('on', 'fi'), ('josta', 'fi'), ('sinuun', 'fi'), ('jolla', 'fi'), ('ovat', 'fi'), ('noin', 'fi'), ('joihin', 'fi'), ('mikä', 'fi'), ('heiltä', 'fi'), ('oli', 'fi'), ('minussa', 'fi'), ('miksi', 'fi'), ('niiksi', 'fi'), ('tämä', 'fi'), ('keihin', 'fi'), ('jos', 'fi'), ('olisitte', 'fi'), ('minulta', 'fi'), ('meistä', 'fi'), ('siitä', 'fi'), ('millä', 'fi'), ('sillä', 'fi'), ('ketkä', 'fi'), ('ketä', 'fi'), ('poikki', 'fi'), ('noiden', 'fi'), ('joilla', 'fi'), ('sinun', 'fi'), ('tuo', 'fi'), ('niiltä', 'fi'), ('hänessä', 'fi'), ('ole', 'fi'), ('tuotä', 'fi'), ('sinusta', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('kenestä', 'fi'), ('sinussa', 'fi'), ('eivät', 'fi'), ('teille', 'fi'), ('tuohon', 'fi'), ('hänen', 'fi'), ('näihin', 'fi'), ('keistä', 'fi'), ('teihin', 'fi'), ('olisin', 'fi'), ('olisimme', 'fi'), ('teistä', 'fi'), ('keneksi', 'fi'), ('ette', 'fi'), ('teissä', 'fi'), ('siltä', 'fi'), ('jolle', 'fi'), ('kuka', 'fi'), ('noista', 'fi'), ('jona', 'fi'), ('keiksi', 'fi'), ('teiltä', 'fi'), ('johon', 'fi'), ('olla', 'fi'), ('sinulle', 'fi'), ('keitä', 'fi'), ('häneltä', 'fi'), ('tuolla', 'fi'), ('nyt', 'fi'), ('noihin', 'fi'), ('hänet', 'fi'), ('heidän', 'fi'), ('noina', 'fi'), ('minun', 'fi'), ('et', 'fi'), ('heihin', 'fi'), ('siksi', 'fi'), ('noilla', 'fi'), ('tuosta', 'fi'), ('kenelle', 'fi'), ('meissä', 'fi'), ('minulle', 'fi'), ('en', 'fi'), ('niihin', 'fi'), ('joiksi', 'fi'), ('hänelle', 'fi'), ('mitkä', 'fi'), ('minua', 'fi'), ('olitte', 'fi'), ('kun', 'fi'), ('ja', 'fi'), ('sinut', 'fi'), ('mitä', 'fi'), ('niinä', 'fi'), ('minuun', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('joille', 'fi'), ('olemme', 'fi'), ('meillä', 'fi'), ('minusta', 'fi'), ('te', 'fi'), ('he', 'fi'), ('missä', 'fi'), ('emme', 'fi'), ('tämän', 'fi'), ('keiden', 'fi');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ne', 'fi'), ('jota', 'fi'), ('yli', 'fi'), ('ketk', 'fi'), ('teitä', 'fi'), ('vaik', 'fi'), ('täs', 'fi'), ('niide', 'fi'), ('näide', 'fi'), ('em', 'fi'), ('täl', 'fi'), ('joih', 'fi'), ('jol', 'fi'), ('tuon', 'fi'), ('vaan', 'fi'), ('mis', 'fi'), ('näinä', 'fi'), ('näitä', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('tuona', 'fi'), ('meil', 'fi'), ('sinä', 'fi'), ('tänä', 'fi'), ('nämä', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sinu', 'fi'), ('sil', 'fi'), ('joho', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('noita', 'fi'), ('joka', 'fi'), ('tätä', 'fi'), ('teidä', 'fi'), ('olisiv', 'fi'), ('me', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('siinä', 'fi'), ('tuos', 'fi'), ('joks', 'fi'), ('mitk', 'fi'), ('poik', 'fi'), ('ei', 'fi'), ('minä', 'fi'), ('heis', 'fi'), ('mut', 'fi'), ('keiks', 'fi'), ('sitä', 'fi'), ('noil', 'fi'), ('nuo', 'fi'), ('jotk', 'fi'), ('tuol', 'fi'), ('olu', 'fi'), ('niitä', 'fi'), ('heil', 'fi'), ('joita', 'fi'), ('noih', 'fi'), ('oliv', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('mink', 'fi'), ('näil', 'fi'), ('näih', 'fi'), ('eivä', 'fi'), ('olet', 'fi'), ('häne', 'fi'), ('on', 'fi'), ('joil', 'fi'), ('its', 'fi'), ('olis', 'fi'), ('noin', 'fi'), ('mikä', 'fi'), ('oli', 'fi'), ('tämä', 'fi'), ('tuoks', 'fi'), ('kan', 'fi'), ('ova', 'fi'), ('kene', 'fi'), ('jos', 'fi'), ('näiks', 'fi'), ('meis', 'fi'), ('siitä', 'fi'), ('tähä', 'fi'), ('keih', 'fi'), ('ketä', 'fi'), ('muka', 'fi'), ('tuo', 'fi'), ('ole', 'fi'), ('mih', 'fi'), ('tuotä', 'fi'), ('ol', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('meih', 'fi'), ('teis', 'fi'), ('teil', 'fi'), ('jonk', 'fi'), ('kuka', 'fi'), ('niih', 'fi'), ('jona', 'fi'), ('joiks', 'fi'), ('näis', 'fi'), ('tal', 'fi'), ('meidä', 'fi'), ('heih', 'fi'), ('keitä', 'fi'), ('miks', 'fi'), ('nyt', 'fi'), ('et', 'fi'), ('noina', 'fi'), ('joide', 'fi'), ('keil', 'fi'), ('täks', 'fi'), ('heidä', 'fi'), ('en', 'fi'), ('niil', 'fi'), ('mil', 'fi'), ('siks', 'fi'), ('noide', 'fi'), ('minu', 'fi'), ('keide', 'fi'), ('kun', 'fi'), ('teih', 'fi'), ('ja', 'fi'), ('niinä', 'fi'), ('mitä', 'fi'), ('tuoho', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('jois', 'fi'), ('he', 'fi'), ('te', 'fi'), ('siihe', 'fi'), ('nois', 'fi'), ('niiks', 'fi'), ('keis', 'fi'), ('kosk', 'fi'), ('noiks', 'fi'), ('niis', 'fi');INSERT INTO stopwords_tiny (stopword, language) VALUES ('ne', 'fr'), ('le', 'fr'), ('serais', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('ayons', 'fr'), ('eus', 'fr'), ('ta', 'fr'), ('mais', 'fr'), ('ayante', 'fr'), ('êtes', 'fr'), ('serions', 'fr'), ('qui', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('étantes', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('aurons', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('soyons', 'fr'), ('serez', 'fr'), ('pour', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('eussiez', 'fr'), ('ayez', 'fr'), ('aies', 'fr'), ('aurions', 'fr'), ('ayantes', 'fr'), ('se', 'fr'), ('eûtes', 'fr'), ('ai', 'fr'), ('fussiez', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('ayants', 'fr'), ('aux', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('soyez', 'fr'), ('seras', 'fr'), ('étants', 'fr'), ('ces', 'fr'), ('me', 'fr'), ('aurez', 'fr'), ('s', 'fr'), ('eussions', 'fr'), ('étiez', 'fr'), ('c', 'fr'), ('que', 'fr'), ('vos', 'fr'), ('eusse', 'fr'), ('des', 'fr'), ('étions', 'fr'), ('aurai', 'fr'), ('par', 'fr'), ('mes', 'fr'), ('soit', 'fr'), ('même', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('mon', 'fr'), ('votre', 'fr'), ('aviez', 'fr'), ('m', 'fr'), ('serai', 'fr'), ('as', 'fr'), ('aurait', 'fr'), ('étante', 'fr'), ('nous', 'fr'), ('une', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('étée', 'fr'), ('d', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('fusse', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('eussent', 'fr'), ('eusses', 'fr'), ('étais', 'fr'), ('sont', 'fr'), ('fussions', 'fr'), ('étés', 'fr'), ('dans', 'fr'), ('fûtes', 'fr'), ('eu', 'fr'), ('auront', 'fr'), ('fusses', 'fr'), ('auraient', 'fr'), ('avions', 'fr'), ('serons', 'fr'), ('eûmes', 'fr'), ('avais', 'fr'), ('fus', 'fr'), ('aura', 'fr'), ('lui', 'fr'), ('notre', 'fr'), ('eux', 'fr'), ('auras', 'fr'), ('sommes', 'fr'), ('ait', 'fr'), ('ton', 'fr'), ('sera', 'fr'), ('es', 'fr'), ('avez', 'fr'), ('je', 'fr'), ('j', 'fr'), ('étées', 'fr'), ('avons', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('sa', 'fr'), ('seriez', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('en', 'fr'), ('de', 'fr'), ('un', 'fr'), ('elle', 'fr'), ('aie', 'fr'), ('furent', 'fr'), ('pas', 'fr'), ('serait', 'fr'), ('ce', 'fr'), ('seraient', 'fr'), ('aurais', 'fr'), ('moi', 'fr'), ('n', 'fr'), ('ou', 'fr'), ('te', 'fr'), ('ses', 'fr'), ('tes', 'fr'), ('fûmes', 'fr'), ('auriez', 'fr'), ('l', 'fr'), ('eues', 'fr'), ('à', 'fr'), ('avec', 'fr');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ayon', 'fr'), ('ne', 'fr'), ('fuss', 'fr'), ('soyon', 'fr'), ('auron', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('le', 'fr'), ('d', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('ta', 'fr'), ('eus', 'fr'), ('mais', 'fr'), ('e', 'fr'), ('qui', 'fr'), ('seron', 'fr'), ('eussent', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('étais', 'fr'), ('ête', 'fr'), ('sont', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('fûm', 'fr'), ('euss', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('eu', 'fr'), ('fussion', 'fr'), ('auront', 'fr'), ('ser', 'fr'), ('pour', 'fr'), ('avon', 'fr'), ('soi', 'fr'), ('notr', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('avais', 'fr'), ('ayez', 'fr'), ('fus', 'fr'), ('se', 'fr'), ('lui', 'fr'), ('ai', 'fr'), ('eux', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('aurion', 'fr'), ('ton', 'fr'), ('ait', 'fr'), ('eûm', 'fr'), ('avez', 'fr'), ('dan', 'fr'), ('je', 'fr'), ('aux', 'fr'), ('j', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('sa', 'fr'), ('aur', 'fr'), ('étion', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('eussion', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('me', 'fr'), ('s', 'fr'), ('en', 'fr'), ('mêm', 'fr'), ('c', 'fr'), ('de', 'fr'), ('un', 'fr'), ('que', 'fr'), ('elle', 'fr'), ('vos', 'fr'), ('furent', 'fr'), ('aie', 'fr'), ('avi', 'fr'), ('pas', 'fr'), ('par', 'fr'), ('soit', 'fr'), ('ce', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('serion', 'fr'), ('mon', 'fr'), ('moi', 'fr'), ('ou', 'fr'), ('n', 'fr'), ('te', 'fr'), ('m', 'fr'), ('éti', 'fr'), ('avion', 'fr'), ('l', 'fr'), ('somm', 'fr'), ('as', 'fr'), ('à', 'fr'), ('avec', 'fr'), ('votr', 'fr'), ('une', 'fr'), ('nous', 'fr');INSERT INTO stopwords_tiny (stopword, language) VALUES ('szinte', 'hu'), ('ne', 'hu'), ('továbbá', 'hu'), ('lenni', 'hu'), ('ki', 'hu'), ('igen', 'hu'), ('való', 'hu'), ('voltak', 'hu'), ('ellen', 'hu'), ('mindent', 'hu'), ('illetve', 'hu'), ('cikkek', 'hu'), ('azzal', 'hu'), ('mindig', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('annak', 'hu'), ('úgy', 'hu'), ('számára', 'hu'), ('szerint', 'hu'), ('ismét', 'hu'), ('nem', 'hu'), ('azután', 'hu'), ('sem', 'hu'), ('mert', 'hu'), ('valamint', 'hu'), ('jobban', 'hu'), ('teljes', 'hu'), ('amelyek', 'hu'), ('vagyok', 'hu'), ('arról', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('mindenki', 'hu'), ('olyan', 'hu'), ('mi', 'hu'), ('között', 'hu'), ('egyes', 'hu'), ('maga', 'hu'), ('legalább', 'hu'), ('általában', 'hu'), ('vissza', 'hu'), ('õk', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('lehetett', 'hu'), ('azok', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('amelynek', 'hu'), ('ehhez', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('saját', 'hu'), ('hiszen', 'hu'), ('aki', 'hu'), ('mikor', 'hu'), ('újabb', 'hu'), ('itt', 'hu'), ('ami', 'hu'), ('ugyanis', 'hu'), ('egyre', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('kívül', 'hu'), ('valami', 'hu'), ('s', 'hu'), ('meg', 'hu'), ('ahhoz', 'hu'), ('azon', 'hu'), ('eddig', 'hu'), ('egyik', 'hu'), ('valaki', 'hu'), ('ill', 'hu'), ('ison', 'hu'), ('vagyis', 'hu'), ('pedig', 'hu'), ('keressünk', 'hu'), ('át', 'hu'), ('amelyekben', 'hu'), ('alatt', 'hu'), ('mellett', 'hu'), ('új', 'hu'), ('elõször', 'hu'), ('õket', 'hu'), ('én', 'hu'), ('elõtt', 'hu'), ('egyetlen', 'hu'), ('azonban', 'hu'), ('éppen', 'hu'), ('tovább', 'hu'), ('ott', 'hu'), ('közül', 'hu'), ('ebben', 'hu'), ('melyek', 'hu'), ('lenne', 'hu'), ('mit', 'hu'), ('magát', 'hu'), ('szemben', 'hu'), ('õ', 'hu'), ('ezzel', 'hu'), ('belül', 'hu'), ('lett', 'hu'), ('ahol', 'hu'), ('amikor', 'hu'), ('viszont', 'hu'), ('azért', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('semmi', 'hu'), ('kellett', 'hu'), ('ezt', 'hu'), ('több', 'hu'), ('nagyon', 'hu'), ('abban', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('ilyenkor', 'hu'), ('nagyobb', 'hu'), ('még', 'hu'), ('amely', 'hu'), ('ekkor', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('újra', 'hu'), ('neki', 'hu'), ('mivel', 'hu'), ('voltam', 'hu'), ('bár', 'hu'), ('benne', 'hu'), ('felé', 'hu'), ('nélkül', 'hu'), ('által', 'hu'), ('ezen', 'hu'), ('amelyet', 'hu'), ('elég', 'hu'), ('miért', 'hu'), ('talán', 'hu'), ('minden', 'hu'), ('amolyan', 'hu'), ('hogy', 'hu'), ('persze', 'hu'), ('egyéb', 'hu'), ('milyen', 'hu'), ('utána', 'hu'), ('sokkal', 'hu'), ('akkor', 'hu'), ('emilyen', 'hu'), ('mint', 'hu'), ('mely', 'hu'), ('ilyen', 'hu'), ('voltunk', 'hu'), ('nekem', 'hu'), ('amit', 'hu'), ('vele', 'hu'), ('míg', 'hu'), ('sok', 'hu'), ('keresztül', 'hu'), ('elsõ', 'hu'), ('cikkeket', 'hu'), ('mintha', 'hu'), ('hogyan', 'hu'), ('vannak', 'hu'), ('csak', 'hu'), ('után', 'hu'), ('nagy', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('be', 'hu'), ('lehet', 'hu'), ('most', 'hu'), ('fel', 'hu'), ('hanem', 'hu'), ('arra', 'hu'), ('másik', 'hu'), ('és', 'hu'), ('majd', 'hu'), ('aztán', 'hu'), ('amelyeket', 'hu'), ('volna', 'hu'), ('össze', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('ennek', 'hu'), ('tehát', 'hu'), ('rá', 'hu'), ('erre', 'hu'), ('sokat', 'hu'), ('néha', 'hu'), ('ezek', 'hu'), ('ezért', 'hu'), ('legyen', 'hu'), ('akik', 'hu'), ('el', 'hu'), ('van', 'hu'), ('egy', 'hu');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ne', 'hu'), ('eb', 'hu'), ('ki', 'hu'), ('való', 'hu'), ('lehetet', 'hu'), ('mos', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('úgy', 'hu'), ('nem', 'hu'), ('hisz', 'hu'), ('ak', 'hu'), ('sem', 'hu'), ('teljes', 'hu'), ('lenn', 'hu'), ('közöt', 'hu'), ('cik', 'hu'), ('is', 'hu'), ('ut', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('ek', 'hu'), ('viszon', 'hu'), ('maj', 'hu'), ('mi', 'hu'), ('vel', 'hu'), ('it', 'hu'), ('egyes', 'hu'), ('leh', 'hu'), ('val', 'hu'), ('legalább', 'hu'), ('õk', 'hu'), ('han', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('valamin', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('újabb', 'hu'), ('ugyanis', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('valam', 'hu'), ('azut', 'hu'), ('s', 'hu'), ('keress', 'hu'), ('meg', 'hu'), ('ab', 'hu'), ('azon', 'hu'), ('ill', 'hu'), ('mily', 'hu'), ('ped', 'hu'), ('vagyis', 'hu'), ('egyetl', 'hu'), ('át', 'hu'), ('elõször', 'hu'), ('új', 'hu'), ('am', 'hu'), ('én', 'hu'), ('kereszt', 'hu'), ('oly', 'hu'), ('tovább', 'hu'), ('ot', 'hu'), ('ah', 'hu'), ('köz', 'hu'), ('õ', 'hu'), ('szám', 'hu'), ('ahol', 'hu'), ('elõt', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('épp', 'hu'), ('szint', 'hu'), ('mellet', 'hu'), ('több', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('emily', 'hu'), ('nagyobb', 'hu'), ('bel', 'hu'), ('még', 'hu'), ('eh', 'hu'), ('amely', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('ell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('mer', 'hu'), ('szem', 'hu'), ('bár', 'hu'), ('által', 'hu'), ('alat', 'hu'), ('elég', 'hu'), ('vol', 'hu'), ('persz', 'hu'), ('amoly', 'hu'), ('minden', 'hu'), ('job', 'hu'), ('hogy', 'hu'), ('egyéb', 'hu'), ('legy', 'hu'), ('vissz', 'hu'), ('mely', 'hu'), ('an', 'hu'), ('ilyen', 'hu'), ('ism', 'hu'), ('tal', 'hu'), ('míg', 'hu'), ('let', 'hu'), ('teh', 'hu'), ('semm', 'hu'), ('néh', 'hu'), ('minth', 'hu'), ('sa', 'hu'), ('sok', 'hu'), ('er', 'hu'), ('elsõ', 'hu'), ('csak', 'hu'), ('en', 'hu'), ('nagy', 'hu'), ('után', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('ed', 'hu'), ('be', 'hu'), ('továb', 'hu'), ('voln', 'hu'), ('fel', 'hu'), ('mag', 'hu'), ('és', 'hu'), ('benn', 'hu'), ('kív', 'hu'), ('illetv', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('kellet', 'hu'), ('min', 'hu'), ('rá', 'hu'), ('ig', 'hu'), ('ar', 'hu'), ('nek', 'hu'), ('ily', 'hu'), ('nél', 'hu'), ('el', 'hu'), ('össz', 'hu'), ('van', 'hu'), ('egy', 'hu'), ('szerin', 'hu');INSERT INTO stopwords_tiny (stopword, language) VALUES ('ne', 'it'), ('sugli', 'it'), ('delle', 'it'), ('avevo', 'it'), ('avete', 'it'), ('avessimo', 'it'), ('stemmo', 'it'), ('e', 'it'), ('queste', 'it'), ('ebbero', 'it'), ('facevano', 'it'), ('noi', 'it'), ('alle', 'it'), ('avesti', 'it'), ('tutto', 'it'), ('aveva', 'it'), ('sarete', 'it'), ('sua', 'it'), ('all', 'it'), ('nello', 'it'), ('chi', 'it'), ('ti', 'it'), ('faceste', 'it'), ('mi', 'it'), ('tua', 'it'), ('vostro', 'it'), ('fosti', 'it'), ('se', 'it'), ('gli', 'it'), ('sulla', 'it'), ('quelli', 'it'), ('miei', 'it'), ('ai', 'it'), ('avevano', 'it'), ('a', 'it'), ('nostra', 'it'), ('eravamo', 'it'), ('ma', 'it'), ('in', 'it'), ('una', 'it'), ('facessero', 'it'), ('si', 'it'), ('stavamo', 'it'), ('staremmo', 'it'), ('saremo', 'it'), ('quanti', 'it'), ('avessi', 'it'), ('anche', 'it'), ('siamo', 'it'), ('li', 'it'), ('avreste', 'it'), ('staresti', 'it'), ('sia', 'it'), ('questi', 'it'), ('facemmo', 'it'), ('stavo', 'it'), ('coi', 'it'), ('avrete', 'it'), ('dei', 'it'), ('sulle', 'it'), ('stettero', 'it'), ('faremmo', 'it'), ('sue', 'it'), ('ebbe', 'it'), ('o', 'it'), ('fecero', 'it'), ('stetti', 'it'), ('farai', 'it'), ('quante', 'it'), ('stesse', 'it'), ('della', 'it'), ('saremmo', 'it'), ('vi', 'it'), ('avrai', 'it'), ('stessimo', 'it'), ('farebbero', 'it'), ('dov', 'it'), ('starei', 'it'), ('sullo', 'it'), ('starà', 'it'), ('avessero', 'it'), ('al', 'it'), ('dello', 'it'), ('starai', 'it'), ('avrò', 'it'), ('avevate', 'it'), ('eravate', 'it'), ('quale', 'it'), ('stai', 'it'), ('avuto', 'it'), ('sarò', 'it'), ('perché', 'it'), ('mio', 'it'), ('aveste', 'it'), ('dagli', 'it'), ('avuti', 'it'), ('del', 'it'), ('stanno', 'it'), ('farà', 'it'), ('eri', 'it'), ('facesse', 'it'), ('facevo', 'it'), ('nei', 'it'), ('fu', 'it'), ('farebbe', 'it'), ('farete', 'it'), ('facciate', 'it'), ('stia', 'it'), ('suoi', 'it'), ('quanta', 'it'), ('essendo', 'it'), ('abbia', 'it'), ('quella', 'it'), ('stiamo', 'it'), ('avevi', 'it'), ('sarai', 'it'), ('avrei', 'it'), ('ed', 'it'), ('un', 'it'), ('stessero', 'it'), ('avrebbero', 'it'), ('sei', 'it'), ('sul', 'it'), ('sarà', 'it'), ('sto', 'it'), ('questo', 'it'), ('fareste', 'it'), ('dall', 'it'), ('stavi', 'it'), ('più', 'it'), ('faranno', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('stava', 'it'), ('fossero', 'it'), ('dal', 'it'), ('sarei', 'it'), ('voi', 'it'), ('abbiate', 'it'), ('facciamo', 'it'), ('avremo', 'it'), ('sarebbe', 'it'), ('da', 'it'), ('degli', 'it'), ('le', 'it'), ('come', 'it'), ('mie', 'it'), ('vostre', 'it'), ('ci', 'it'), ('negl', 'it'), ('saresti', 'it'), ('foste', 'it'), ('ebbi', 'it'), ('faccia', 'it'), ('steste', 'it'), ('sta', 'it'), ('stesti', 'it'), ('dalla', 'it'), ('furono', 'it'), ('fanno', 'it'), ('saranno', 'it'), ('fece', 'it'), ('dallo', 'it'), ('quelle', 'it'), ('siate', 'it'), ('stette', 'it'), ('dell', 'it'), ('avevamo', 'it'), ('sull', 'it'), ('uno', 'it'), ('la', 'it'), ('con', 'it'), ('vostri', 'it'), ('dagl', 'it'), ('starebbero', 'it'), ('stessi', 'it'), ('hai', 'it'), ('faccio', 'it'), ('farò', 'it'), ('il', 'it'), ('nostro', 'it'), ('ho', 'it'), ('facciano', 'it'), ('avrà', 'it'), ('loro', 'it'), ('tu', 'it'), ('cui', 'it'), ('era', 'it'), ('starebbe', 'it'), ('fossi', 'it'), ('facendo', 'it'), ('hanno', 'it'), ('c', 'it'), ('facessi', 'it'), ('alla', 'it'), ('vostra', 'it'), ('sarebbero', 'it'), ('ha', 'it'), ('dalle', 'it'), ('feci', 'it'), ('tue', 'it'), ('tra', 'it'), ('nell', 'it'), ('i', 'it'), ('faresti', 'it'), ('mia', 'it'), ('stiate', 'it'), ('nel', 'it'), ('avesse', 'it'), ('faceva', 'it'), ('degl', 'it'), ('sono', 'it'), ('su', 'it'), ('sareste', 'it'), ('quanto', 'it'), ('contro', 'it'), ('stareste', 'it'), ('lei', 'it'), ('faremo', 'it'), ('lo', 'it'), ('nostri', 'it'), ('avuta', 'it'), ('facevamo', 'it'), ('siete', 'it'), ('questa', 'it'), ('avemmo', 'it'), ('facevate', 'it'), ('sui', 'it'), ('facesti', 'it'), ('fai', 'it'), ('avranno', 'it'), ('starete', 'it'), ('dove', 'it'), ('di', 'it'), ('tuo', 'it'), ('sugl', 'it'), ('abbiano', 'it'), ('avresti', 'it'), ('stiano', 'it'), ('starò', 'it'), ('siano', 'it'), ('nelle', 'it'), ('quello', 'it'), ('fummo', 'it'), ('farei', 'it'), ('lui', 'it'), ('fossimo', 'it'), ('avute', 'it'), ('non', 'it'), ('fosse', 'it'), ('ero', 'it'), ('io', 'it'), ('avrebbe', 'it'), ('per', 'it'), ('nella', 'it'), ('nostre', 'it'), ('staremo', 'it'), ('tuoi', 'it'), ('facevi', 'it'), ('facessimo', 'it'), ('stavate', 'it'), ('dai', 'it'), ('stavano', 'it'), ('è', 'it'), ('erano', 'it'), ('ad', 'it'), ('agl', 'it'), ('col', 'it'), ('stando', 'it'), ('negli', 'it'), ('avremmo', 'it'), ('l', 'it'), ('abbiamo', 'it'), ('tutti', 'it'), ('allo', 'it'), ('avendo', 'it'), ('staranno', 'it');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ne', 'it'), ('facess', 'it'), ('le', 'it'), ('da', 'it'), ('stessim', 'it'), ('stav', 'it'), ('avrebb', 'it'), ('far', 'it'), ('mie', 'it'), ('hann', 'it'), ('stiam', 'it'), ('tutt', 'it'), ('ci', 'it'), ('negl', 'it'), ('avessim', 'it'), ('e', 'it'), ('ebbi', 'it'), ('stett', 'it'), ('avre', 'it'), ('sta', 'it'), ('fann', 'it'), ('noi', 'it'), ('alle', 'it'), ('abbi', 'it'), ('avest', 'it'), ('essend', 'it'), ('avra', 'it'), ('eran', 'it'), ('quest', 'it'), ('sua', 'it'), ('stest', 'it'), ('dell', 'it'), ('all', 'it'), ('farann', 'it'), ('uno', 'it'), ('fecer', 'it'), ('nostr', 'it'), ('chi', 'it'), ('ti', 'it'), ('sull', 'it'), ('la', 'it'), ('avesser', 'it'), ('con', 'it'), ('farem', 'it'), ('mi', 'it'), ('tua', 'it'), ('dagl', 'it'), ('perc', 'it'), ('fare', 'it'), ('siat', 'it'), ('hai', 'it'), ('starann', 'it'), ('starem', 'it'), ('sara', 'it'), ('se', 'it'), ('sar', 'it'), ('gli', 'it'), ('ai', 'it'), ('sarest', 'it'), ('sarem', 'it'), ('com', 'it'), ('il', 'it'), ('son', 'it'), ('a', 'it'), ('farebb', 'it'), ('avrann', 'it'), ('ho', 'it'), ('avrà', 'it'), ('avrem', 'it'), ('tu', 'it'), ('ma', 'it'), ('in', 'it'), ('cui', 'it'), ('una', 'it'), ('stess', 'it'), ('era', 'it'), ('stand', 'it'), ('si', 'it'), ('star', 'it'), ('c', 'it'), ('anche', 'it'), ('stavam', 'it'), ('li', 'it'), ('alla', 'it'), ('sia', 'it'), ('siam', 'it'), ('stare', 'it'), ('ha', 'it'), ('avevam', 'it'), ('starest', 'it'), ('tue', 'it'), ('coi', 'it'), ('dei', 'it'), ('lor', 'it'), ('fac', 'it'), ('i', 'it'), ('tra', 'it'), ('nell', 'it'), ('avrest', 'it'), ('sue', 'it'), ('mia', 'it'), ('nel', 'it'), ('ebbe', 'it'), ('degl', 'it'), ('siet', 'it'), ('avet', 'it'), ('o', 'it'), ('su', 'it'), ('facessim', 'it'), ('stemm', 'it'), ('sare', 'it'), ('lei', 'it'), ('stetter', 'it'), ('facc', 'it'), ('lo', 'it'), ('vi', 'it'), ('erav', 'it'), ('sian', 'it'), ('fur', 'it'), ('dov', 'it'), ('avret', 'it'), ('al', 'it'), ('avrò', 'it'), ('sui', 'it'), ('fai', 'it'), ('fossim', 'it'), ('sarann', 'it'), ('stesser', 'it'), ('di', 'it'), ('vostr', 'it'), ('tuo', 'it'), ('mio', 'it'), ('avev', 'it'), ('stara', 'it'), ('avend', 'it'), ('sugl', 'it'), ('sarebber', 'it'), ('quant', 'it'), ('avemm', 'it'), ('del', 'it'), ('eri', 'it'), ('lui', 'it'), ('non', 'it'), ('fost', 'it'), ('nei', 'it'), ('avremm', 'it'), ('fu', 'it'), ('starebb', 'it'), ('eravam', 'it'), ('avrebber', 'it'), ('abbiam', 'it'), ('fara', 'it'), ('sarebb', 'it'), ('ero', 'it'), ('facest', 'it'), ('ebber', 'it'), ('quell', 'it'), ('io', 'it'), ('qual', 'it'), ('per', 'it'), ('ed', 'it'), ('un', 'it'), ('stann', 'it'), ('sti', 'it'), ('sei', 'it'), ('starebber', 'it'), ('sul', 'it'), ('sto', 'it'), ('dai', 'it'), ('farebber', 'it'), ('è', 'it'), ('fosser', 'it'), ('fec', 'it'), ('contr', 'it'), ('dall', 'it'), ('agl', 'it'), ('ad', 'it'), ('col', 'it'), ('avess', 'it'), ('più', 'it'), ('foss', 'it'), ('fumm', 'it'), ('farest', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('l', 'it'), ('dal', 'it'), ('voi', 'it'), ('allo', 'it'), ('avut', 'it');INSERT INTO stopwords_tiny (stopword, language) VALUES ('su', 'lt'), ('ne', 'lt'), ('į', 'lt'), ('pasakė', 'lt'), ('kurių', 'lt'), ('d', 'lt'), ('lietuva', 'lt'), ('kai', 'lt'), ('arba', 'lt'), ('po', 'lt'), ('kurios', 'lt'), ('kam', 'lt'), ('darbo', 'lt'), ('daugiau', 'lt'), ('niekada', 'lt'), ('tas', 'lt'), ('metus', 'lt'), ('kovo', 'lt'), ('negali', 'lt'), ('kaip', 'lt'), ('kada', 'lt'), ('už', 'lt'), ('jos', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('savo', 'lt'), ('labai', 'lt'), ('tik', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('pat', 'lt'), ('negu', 'lt'), ('gali', 'lt'), ('čia', 'lt'), ('turi', 'lt'), ('bet', 'lt'), ('tuo', 'lt'), ('mažas', 'lt'), ('bei', 'lt'), ('tai', 'lt'), ('prašau', 'lt'), ('to', 'lt'), ('reikia', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jau', 'lt'), ('metu', 'lt'), ('lietuvos', 'lt'), ('jog', 'lt'), ('mažiau', 'lt'), ('val', 'lt'), ('lietuvoje', 'lt'), ('jei', 'lt'), ('jį', 'lt'), ('man', 'lt'), ('turėjo', 'lt'), ('nes', 'lt'), ('vienas', 'lt'), ('nei', 'lt'), ('daugiausia', 'lt'), ('mūsų', 'lt'), ('tarp', 'lt'), ('a', 'lt'), ('tačiau', 'lt'), ('net', 'lt'), ('šalia', 'lt'), ('niekas', 'lt'), ('mažai', 'lt'), ('todėl', 'lt'), ('yra', 'lt'), ('jie', 'lt'), ('pasak', 'lt'), ('per', 'lt'), ('dar', 'lt'), ('apie', 'lt'), ('kuris', 'lt'), ('žmonių', 'lt'), ('šalies', 'lt'), ('metais', 'lt'), ('ir', 'lt'), ('be', 'lt'), ('gauna', 'lt'), ('dėl', 'lt'), ('taip', 'lt'), ('kurie', 'lt'), ('dieną', 'lt'), ('jam', 'lt'), ('jų', 'lt'), ('kad', 'lt'), ('visi', 'lt'), ('r', 'lt'), ('bus', 'lt'), ('klausti', 'lt'), ('mano', 'lt'), ('mes', 'lt'), ('sakė', 'lt'), ('būti', 'lt'), ('dažnai', 'lt'), ('šios', 'lt'), ('jis', 'lt'), ('nuo', 'lt'), ('nors', 'lt'), ('iki', 'lt'), ('buvo', 'lt'), ('kitas', 'lt'), ('metų', 'lt'), ('kiekvienas', 'lt'), ('tikrai', 'lt'), ('iš', 'lt'), ('būtų', 'lt'), ('jo', 'lt'), ('tavo', 'lt'), ('nėra', 'lt'), ('prie', 'lt'), ('sakyti', 'lt'), ('m', 'lt'), ('daryti', 'lt'), ('kartą', 'lt'), ('ar', 'lt'), ('kas', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('klausimas', 'lt'), ('o', 'lt'), ('gauti', 'lt');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('', 'lt'), ('d', 'lt'), ('vis', 'lt'), ('neg', 'lt'), ('kam', 'lt'), ('tikr', 'lt'), ('klaus', 'lt'), ('tur', 'lt'), ('kaip', 'lt'), ('už', 'lt'), ('gau', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('b', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('tik', 'lt'), ('pat', 'lt'), ('bet', 'lt'), ('nor', 'lt'), ('dažn', 'lt'), ('lie', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jog', 'lt'), ('val', 'lt'), ('būt', 'lt'), ('buv', 'lt'), ('gal', 'lt'), ('man', 'lt'), ('negal', 'lt'), ('lab', 'lt'), ('bū', 'lt'), ('mūs', 'lt'), ('darb', 'lt'), ('tarp', 'lt'), ('met', 'lt'), ('š', 'lt'), ('maž', 'lt'), ('net', 'lt'), ('niek', 'lt'), ('j', 'lt'), ('praša', 'lt'), ('kuri', 'lt'), ('k', 'lt'), ('todėl', 'lt'), ('pri', 'lt'), ('nėr', 'lt'), ('pasak', 'lt'), ('t', 'lt'), ('re', 'lt'), ('v', 'lt'), ('per', 'lt'), ('s', 'lt'), ('dar', 'lt'), ('šal', 'lt'), ('ir', 'lt'), ('dėl', 'lt'), ('arb', 'lt'), ('taip', 'lt'), ('kart', 'lt'), ('kiekv', 'lt'), ('jam', 'lt'), ('kad', 'lt'), ('gaun', 'lt'), ('yr', 'lt'), ('r', 'lt'), ('kov', 'lt'), ('api', 'lt'), ('ja', 'lt'), ('niekad', 'lt'), ('iš', 'lt'), ('n', 'lt'), ('klausim', 'lt'), ('m', 'lt'), ('ar', 'lt'), ('p', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('kit', 'lt'), ('sak', 'lt'), ('žmon', 'lt'), ('tat', 'lt');INSERT INTO stopwords_tiny (stopword, language) VALUES ('want', 'nl'), ('deze', 'nl'), ('wil', 'nl'), ('kon', 'nl'), ('nu', 'nl'), ('heeft', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('wie', 'nl'), ('zijn', 'nl'), ('wat', 'nl'), ('niets', 'nl'), ('kunnen', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('iemand', 'nl'), ('doen', 'nl'), ('worden', 'nl'), ('door', 'nl'), ('haar', 'nl'), ('ge', 'nl'), ('tegen', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('veel', 'nl'), ('of', 'nl'), ('zij', 'nl'), ('uit', 'nl'), ('alles', 'nl'), ('toen', 'nl'), ('voor', 'nl'), ('hem', 'nl'), ('die', 'nl'), ('is', 'nl'), ('iets', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('daar', 'nl'), ('zelf', 'nl'), ('ze', 'nl'), ('hebben', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('andere', 'nl'), ('in', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('waren', 'nl'), ('geen', 'nl'), ('wezen', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('naar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('reeds', 'nl'), ('meer', 'nl'), ('en', 'nl'), ('heb', 'nl'), ('de', 'nl'), ('hier', 'nl'), ('bij', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('maar', 'nl'), ('op', 'nl'), ('eens', 'nl'), ('hun', 'nl'), ('toch', 'nl'), ('mij', 'nl'), ('geweest', 'nl'), ('tot', 'nl'), ('werd', 'nl'), ('ja', 'nl'), ('dat', 'nl'), ('moet', 'nl'), ('hij', 'nl'), ('ook', 'nl'), ('ben', 'nl'), ('had', 'nl'), ('te', 'nl'), ('hoe', 'nl'), ('nog', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('wordt', 'nl'), ('ik', 'nl'), ('was', 'nl'), ('van', 'nl');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('want', 'nl'), ('wil', 'nl'), ('kunn', 'nl'), ('kon', 'nl'), ('dor', 'nl'), ('heeft', 'nl'), ('nu', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('zijn', 'nl'), ('wie', 'nl'), ('wat', 'nl'), ('iet', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('doen', 'nl'), ('iemand', 'nl'), ('ge', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('of', 'nl'), ('mer', 'nl'), ('uit', 'nl'), ('zij', 'nl'), ('hem', 'nl'), ('toen', 'nl'), ('alles', 'nl'), ('is', 'nl'), ('die', 'nl'), ('hebb', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('zelf', 'nl'), ('vel', 'nl'), ('ze', 'nl'), ('nar', 'nl'), ('har', 'nl'), ('ander', 'nl'), ('dez', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('teg', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('in', 'nl'), ('red', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('mar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('dar', 'nl'), ('heb', 'nl'), ('en', 'nl'), ('vor', 'nl'), ('de', 'nl'), ('word', 'nl'), ('bij', 'nl'), ('hier', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('op', 'nl'), ('hun', 'nl'), ('mij', 'nl'), ('toch', 'nl'), ('war', 'nl'), ('tot', 'nl'), ('geweest', 'nl'), ('dat', 'nl'), ('ja', 'nl'), ('werd', 'nl'), ('ook', 'nl'), ('hij', 'nl'), ('moet', 'nl'), ('gen', 'nl'), ('had', 'nl'), ('ben', 'nl'), ('te', 'nl'), ('nog', 'nl'), ('hoe', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('ik', 'nl'), ('wordt', 'nl'), ('was', 'nl'), ('van', 'nl'), ('wez', 'nl');INSERT INTO stopwords_tiny (stopword, language) VALUES ('fra', 'no'), ('som', 'no'), ('da', 'no'), ('ingen', 'no'), ('kvi', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('bare', 'no'), ('ville', 'no'), ('det', 'no'), ('alle', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('honom', 'no'), ('hoss', 'no'), ('ut', 'no'), ('somme', 'no'), ('men', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('vors', 'no'), ('han', 'no'), ('noka', 'no'), ('man', 'no'), ('hjå', 'no'), ('har', 'no'), ('deim', 'no'), ('eg', 'no'), ('samme', 'no'), ('elles', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('hennar', 'no'), ('seg', 'no'), ('vort', 'no'), ('inkje', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('mine', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('deira', 'no'), ('sånn', 'no'), ('å', 'no'), ('skal', 'no'), ('dei', 'no'), ('nå', 'no'), ('nokor', 'no'), ('hvordan', 'no'), ('i', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('dykkar', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('dette', 'no'), ('verte', 'no'), ('du', 'no'), ('vere', 'no'), ('vart', 'no'), ('deires', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('begge', 'no'), ('kunne', 'no'), ('med', 'no'), ('kan', 'no'), ('varte', 'no'), ('somt', 'no'), ('før', 'no'), ('di', 'no'), ('inni', 'no'), ('siden', 'no'), ('noko', 'no'), ('ikkje', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('også', 'no'), ('dykk', 'no'), ('ble', 'no'), ('etter', 'no'), ('den', 'no'), ('vore', 'no'), ('hvilke', 'no'), ('nokon', 'no'), ('deg', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('hans', 'no'), ('denne', 'no'), ('ingi', 'no'), ('være', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('hadde', 'no'), ('vår', 'no'), ('så', 'no'), ('nokre', 'no'), ('hvor', 'no'), ('både', 'no'), ('so', 'no'), ('et', 'no'), ('hvilken', 'no'), ('henne', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('dere', 'no'), ('disse', 'no'), ('jeg', 'no'), ('en', 'no'), ('mange', 'no'), ('de', 'no'), ('meget', 'no'), ('deres', 'no'), ('din', 'no'), ('hun', 'no'), ('medan', 'no'), ('uten', 'no'), ('mykje', 'no'), ('hennes', 'no'), ('noen', 'no'), ('ikke', 'no'), ('sine', 'no'), ('kun', 'no'), ('skulle', 'no'), ('ja', 'no'), ('at', 'no'), ('hossen', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('inn', 'no'), ('eller', 'no'), ('hvorfor', 'no'), ('mot', 'no');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('sid', 'no'), ('fra', 'no'), ('som', 'no'), ('kunn', 'no'), ('da', 'no'), ('dett', 'no'), ('bar', 'no'), ('kvi', 'no'), ('mykj', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('det', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('henn', 'no'), ('honom', 'no'), ('nok', 'no'), ('all', 'no'), ('hoss', 'no'), ('ut', 'no'), ('men', 'no'), ('skull', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('hadd', 'no'), ('han', 'no'), ('man', 'no'), ('har', 'no'), ('hjå', 'no'), ('deir', 'no'), ('deim', 'no'), ('nokr', 'no'), ('eg', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('seg', 'no'), ('vort', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('sånn', 'no'), ('skal', 'no'), ('å', 'no'), ('dei', 'no'), ('nå', 'no'), ('i', 'no'), ('hvordan', 'no'), ('nokor', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('ikkj', 'no'), ('du', 'no'), ('vart', 'no'), ('begg', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('hvilk', 'no'), ('med', 'no'), ('kan', 'no'), ('somt', 'no'), ('vær', 'no'), ('båd', 'no'), ('før', 'no'), ('di', 'no'), ('ell', 'no'), ('inni', 'no'), ('noko', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('dykk', 'no'), ('ver', 'no'), ('også', 'no'), ('ble', 'no'), ('denn', 'no'), ('vill', 'no'), ('den', 'no'), ('ikk', 'no'), ('nokon', 'no'), ('deg', 'no'), ('samm', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('inkj', 'no'), ('ingi', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('vår', 'no'), ('så', 'no'), ('hvor', 'no'), ('mang', 'no'), ('et', 'no'), ('so', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('ing', 'no'), ('jeg', 'no'), ('en', 'no'), ('vor', 'no'), ('de', 'no'), ('vert', 'no'), ('din', 'no'), ('hun', 'no'), ('uten', 'no'), ('medan', 'no'), ('noen', 'no'), ('diss', 'no'), ('kun', 'no'), ('ja', 'no'), ('at', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('hvorfor', 'no'), ('inn', 'no'), ('mot', 'no'), ('somm', 'no');INSERT INTO stopwords_tiny (stopword, language) VALUES ('terá', 'pt'), ('havemos', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('da', 'pt'), ('houvéramos', 'pt'), ('para', 'pt'), ('nosso', 'pt'), ('pelos', 'pt'), ('nos', 'pt'), ('são', 'pt'), ('há', 'pt'), ('tenham', 'pt'), ('delas', 'pt'), ('mais', 'pt'), ('e', 'pt'), ('fôramos', 'pt'), ('estávamos', 'pt'), ('isso', 'pt'), ('nem', 'pt'), ('formos', 'pt'), ('hão', 'pt'), ('pelas', 'pt'), ('em', 'pt'), ('não', 'pt'), ('estejam', 'pt'), ('tive', 'pt'), ('seremos', 'pt'), ('sem', 'pt'), ('houvera', 'pt'), ('está', 'pt'), ('houver', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('aquilo', 'pt'), ('houveremos', 'pt'), ('fossem', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('terão', 'pt'), ('por', 'pt'), ('das', 'pt'), ('vocês', 'pt'), ('essa', 'pt'), ('estivera', 'pt'), ('essas', 'pt'), ('houvermos', 'pt'), ('nossa', 'pt'), ('muito', 'pt'), ('tivéramos', 'pt'), ('seria', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('se', 'pt'), ('dos', 'pt'), ('hajamos', 'pt'), ('com', 'pt'), ('na', 'pt'), ('tivéssemos', 'pt'), ('a', 'pt'), ('foram', 'pt'), ('estão', 'pt'), ('um', 'pt'), ('no', 'pt'), ('tenhamos', 'pt'), ('os', 'pt'), ('hajam', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('aquelas', 'pt'), ('aqueles', 'pt'), ('tivemos', 'pt'), ('estes', 'pt'), ('teriam', 'pt'), ('era', 'pt'), ('esteve', 'pt'), ('nossas', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('serei', 'pt'), ('terei', 'pt'), ('tem', 'pt'), ('teve', 'pt'), ('houverá', 'pt'), ('tivesse', 'pt'), ('tenha', 'pt'), ('fôssemos', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('serão', 'pt'), ('este', 'pt'), ('seríamos', 'pt'), ('estou', 'pt'), ('estas', 'pt'), ('só', 'pt'), ('lhe', 'pt'), ('houveria', 'pt'), ('houveram', 'pt'), ('pela', 'pt'), ('tivessem', 'pt'), ('nossos', 'pt'), ('teu', 'pt'), ('tiver', 'pt'), ('lhes', 'pt'), ('tivermos', 'pt'), ('éramos', 'pt'), ('estivéramos', 'pt'), ('meu', 'pt'), ('sejamos', 'pt'), ('estiver', 'pt'), ('estiveram', 'pt'), ('as', 'pt'), ('houveriam', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('houvesse', 'pt'), ('houverei', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quando', 'pt'), ('sejam', 'pt'), ('hei', 'pt'), ('houvessem', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('pelo', 'pt'), ('minhas', 'pt'), ('fora', 'pt'), ('houveríamos', 'pt'), ('será', 'pt'), ('esse', 'pt'), ('houvéssemos', 'pt'), ('temos', 'pt'), ('num', 'pt'), ('estivessem', 'pt'), ('do', 'pt'), ('tinha', 'pt'), ('aquela', 'pt'), ('já', 'pt'), ('fomos', 'pt'), ('teria', 'pt'), ('eu', 'pt'), ('tenho', 'pt'), ('numa', 'pt'), ('elas', 'pt'), ('deles', 'pt'), ('haja', 'pt'), ('esteja', 'pt'), ('estejamos', 'pt'), ('entre', 'pt'), ('estivemos', 'pt'), ('somos', 'pt'), ('houverão', 'pt'), ('teríamos', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('tivera', 'pt'), ('fosse', 'pt'), ('tiverem', 'pt'), ('dela', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('houvemos', 'pt'), ('teus', 'pt'), ('estive', 'pt'), ('tínhamos', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('mesmo', 'pt'), ('ela', 'pt'), ('teremos', 'pt'), ('seja', 'pt'), ('de', 'pt'), ('estivesse', 'pt'), ('dele', 'pt'), ('estamos', 'pt'), ('houve', 'pt'), ('nas', 'pt'), ('estavam', 'pt'), ('tinham', 'pt'), ('uma', 'pt'), ('aquele', 'pt'), ('como', 'pt'), ('estava', 'pt'), ('forem', 'pt'), ('esses', 'pt'), ('depois', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('estivéssemos', 'pt'), ('estiverem', 'pt'), ('houverem', 'pt'), ('minha', 'pt'), ('à', 'pt'), ('estivermos', 'pt'), ('tiveram', 'pt'), ('seriam', 'pt'), ('você', 'pt');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('tev', 'pt'), ('som', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('form', 'pt'), ('da', 'pt'), ('nos', 'pt'), ('estivéss', 'pt'), ('há', 'pt'), ('mesm', 'pt'), ('terã', 'pt'), ('mais', 'pt'), ('tenh', 'pt'), ('e', 'pt'), ('haj', 'pt'), ('isso', 'pt'), ('hav', 'pt'), ('nem', 'pt'), ('em', 'pt'), ('aquel', 'pt'), ('sem', 'pt'), ('está', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('minh', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('por', 'pt'), ('das', 'pt'), ('tiv', 'pt'), ('essa', 'pt'), ('vocês', 'pt'), ('serã', 'pt'), ('essas', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('muit', 'pt'), ('dos', 'pt'), ('se', 'pt'), ('na', 'pt'), ('com', 'pt'), ('a', 'pt'), ('um', 'pt'), ('no', 'pt'), ('estam', 'pt'), ('estej', 'pt'), ('os', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('tính', 'pt'), ('estes', 'pt'), ('era', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('tem', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('este', 'pt'), ('só', 'pt'), ('estas', 'pt'), ('estou', 'pt'), ('par', 'pt'), ('lhe', 'pt'), ('estáv', 'pt'), ('teu', 'pt'), ('estav', 'pt'), ('lhes', 'pt'), ('estã', 'pt'), ('meu', 'pt'), ('as', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('sej', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quand', 'pt'), ('tinh', 'pt'), ('hei', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('esse', 'pt'), ('num', 'pt'), ('do', 'pt'), ('noss', 'pt'), ('já', 'pt'), ('eu', 'pt'), ('elas', 'pt'), ('ter', 'pt'), ('ser', 'pt'), ('del', 'pt'), ('sã', 'pt'), ('hã', 'pt'), ('éram', 'pt'), ('entre', 'pt'), ('tivéss', 'pt'), ('depo', 'pt'), ('houv', 'pt'), ('fom', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('pel', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('teus', 'pt'), ('estev', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('ela', 'pt'), ('de', 'pt'), ('nas', 'pt'), ('uma', 'pt'), ('foss', 'pt'), ('fôss', 'pt'), ('nã', 'pt'), ('esses', 'pt'), ('fôr', 'pt'), ('aquil', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('voc', 'pt'), ('estiv', 'pt'), ('à', 'pt'), ('houvéss', 'pt');INSERT INTO stopwords_tiny (stopword, language) VALUES ('încă', 'ro'), ('toţi', 'ro'), ('ne', 'ro'), ('multă', 'ro'), ('înseşi', 'ro'), ('oricât', 'ro'), ('totuşi', 'ro'), ('puţini', 'ro'), ('înşine', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('veţi', 'ro'), ('unii', 'ro'), ('altcineva', 'ro'), ('aceiaşi', 'ro'), ('atât', 'ro'), ('parte', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('primii', 'ro'), ('doime', 'ro'), ('sine ', 'ro'), ('oi', 'ro'), ('tău', 'ro'), ('câtă', 'ro'), ('unei', 'ro'), ('mare', 'ro'), ('însăşi', 'ro'), ('celorlalte', 'ro'), ('oricăruia', 'ro'), ('celei', 'ro'), ('mi', 'ro'), ('acelaşi', 'ro'), ('ălora', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('cine', 'ro'), ('puţină', 'ro'), ('primelor', 'ro'), ('alea', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('suntem', 'ro'), ('acelui', 'ro'), ('însele', 'ro'), ('una', 'ro'), ('însămi', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('foarte', 'ro'), ('acea', 'ro'), ('mine', 'ro'), ('puţine', 'ro'), ('şapte', 'ro'), ('multe', 'ro'), ('atâţi', 'ro'), ('care', 'ro'), ('cei', 'ro'), ('oricâtă', 'ro'), ('am', 'ro'), ('acelei', 'ro'), ('lor', 'ro'), ('dintre', 'ro'), ('celorlalţi', 'ro'), ('alţii', 'ro'), ('alt', 'ro'), ('orice', 'ro'), ('vreunii', 'ro'), ('aceasta', 'ro'), ('alta', 'ro'), ('aceşti', 'ro'), ('eşti', 'ro'), ('amândoi', 'ro'), ('noastră', 'ro'), ('meu', 'ro'), ('unul', 'ro'), ('fiţi', 'ro'), ('fiind', 'ro'), ('ambele', 'ro'), ('altora', 'ro'), ('celuilalt', 'ro'), ('aceea', 'ro'), ('spre', 'ro'), ('o', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('fiecărei', 'ro'), ('vreuna', 'ro'), ('bine', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('vreunele', 'ro'), ('alte', 'ro'), ('celelalte', 'ro'), ('eram', 'ro'), ('celălalt', 'ro'), ('există', 'ro'), ('amândurora', 'ro'), ('al', 'ro'), ('numai', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('aceştia', 'ro'), ('spune', 'ro'), ('cutare', 'ro'), ('fiu', 'ro'), ('câţi', 'ro'), ('mult', 'ro'), ('altcuiva ', 'ro'), ('oricui', 'ro'), ('ea', 'ro'), ('cinci', 'ro'), ('aceloraşi', 'ro'), ('vreunuia', 'ro'), ('mei', 'ro'), ('deşi', 'ro'), ('mele', 'ro'), ('primul', 'ro'), ('însevă', 'ro'), ('vreuneia', 'ro'), ('acelor', 'ro'), ('câte', 'ro'), ('oricăreia', 'ro'), ('ca', 'ro'), ('oricâtor', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('primilor', 'ro'), ('oricărora', 'ro'), ('aceia', 'ro'), ('câteva', 'ro'), ('fost', 'ro'), ('vreunul', 'ro'), ('vreo', 'ro'), ('ăstora', 'ro'), ('despre', 'ro'), ('după', 'ro'), ('însăţi', 'ro'), ('fii', 'ro'), ('destui', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('nimeni', 'ro'), ('noştri', 'ro'), ('oricine', 'ro'), ('aceleaşi', 'ro'), ('oţi', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('ăia', 'ro'), ('vor', 'ro'), ('fără', 'ro'), ('de', 'ro'), ('atâtora', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('întâiul', 'ro'), ('ăştia', 'ro'), ('faţă', 'ro'), ('ambii', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('sale', 'ro'), ('nişte', 'ro'), ('trei', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('atunci', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('că', 'ro'), ('eraţi', 'ro'), ('vreunora', 'ro'), ('cele', 'ro'), ('ambelor', 'ro'), ('ar', 'ro'), ('atâtor', 'ro'), ('cineva', 'ro'), ('iar', 'ro'), ('deci', 'ro'), ('îndoit', 'ro'), ('voi', 'ro'), ('avut', 'ro'), ('însuşi', 'ro'), ('cărei', 'ro'), ('până', 'ro'), ('aceluiaşi', 'ro'), ('căror ', 'ro'), ('cât', 'ro'), ('noua', 'ro'), ('altceva', 'ro'), ('le', 'ro'), ('astea', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('oricâte', 'ro'), ('ci', 'ro'), ('cel', 'ro'), ('acele', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('sunteţi', 'ro'), ('ăla', 'ro'), ('alteia', 'ro'), ('câtorva', 'ro'), ('ambilor', 'ro'), ('acesteia', 'ro'), ('asupra', 'ro'), ('putea', 'ro'), ('pentru', 'ro'), ('va', 'ro'), ('aveaţi', 'ro'), ('când', 'ro'), ('aveai', 'ro'), ('aproape', 'ro'), ('la', 'ro'), ('celui', 'ro'), ('aceleia', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('oricărui', 'ro'), ('puţin', 'ro'), ('ăsta', 'ro'), ('acei', 'ro'), ('întâia', 'ro'), ('oricâţi', 'ro'), ('acestea', 'ro'), ('dacă', 'ro'), ('acestei', 'ro'), ('însumi', 'ro'), ('treime', 'ro'), ('oricare', 'ro'), ('altuia', 'ro'), ('unde', 'ro'), ('altei', 'ro'), ('înşivă', 'ro'), ('altele', 'ro'), ('fiecare', 'ro'), ('ăluia', 'ro'), ('erai', 'ro'), ('îşi', 'ro'), ('aceeaşi', 'ro'), ('fiecăreia', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('acesta', 'ro'), ('primului', 'ro'), ('s', 'ro'), ('această', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('primei', 'ro'), ('trebuie', 'ro'), ('primele', 'ro'), ('este', 'ro'), ('înşişi', 'ro'), ('însutit', 'ro'), ('patru', 'ro'), ('nimănui', 'ro'), ('ei', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('au', 'ro'), ('voastre', 'ro'), ('şase', 'ro'), ('noastre', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('doar', 'ro'), ('ăsteia', 'ro'), ('oarecare', 'ro'), ('atâta', 'ro'), ('astfel', 'ro'), ('oricâtora', 'ro'), ('întreit', 'ro'), ('acelea', 'ro'), ('poate', 'ro'), ('chiar', 'ro'), ('unuia', 'ro'), ('nimic', 'ro'), ('acela', 'ro'), ('atâţia', 'ro'), ('dumneavoastră', 'ro'), ('către', 'ro'), ('fiecăruia', 'ro'), ('ţie', 'ro'), ('destule', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('ceva', 'ro'), ('sutime', 'ro'), ('voastră ', 'ro'), ('cealaltă', 'ro'), ('aceste', 'ro'), ('aceleiaşi', 'ro'), ('face', 'ro'), ('aş', 'ro'), ('toată', 'ro'), ('cu', 'ro'), ('ăstuia', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('mulţi', 'ro'), ('cum', 'ro'), ('peste', 'ro'), ('acelora', 'ro'), ('cărui', 'ro'), ('vreunui', 'ro'), ('nouă', 'ro'), ('lui', 'ro'), ('voştri', 'ro'), ('unele', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('uneia', 'ro'), ('unora ', 'ro'), ('prima', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('atâtea', 'ro'), ('zece', 'ro'), ('aşadar', 'ro'), ('ieri', 'ro'), ('tăi', 'ro'), ('destulă', 'ro'), ('or', 'ro'), ('acestor', 'ro'), ('acel', 'ro'), ('avea', 'ro'), ('câtor', 'ro'), ('câtva', 'ro'), ('v', 'ro'), ('dar', 'ro'), ('totul', 'ro'), ('însuţi', 'ro'), ('câţiva', 'ro'), ('spus', 'ro'), ('doua', 'ro'), ('câtora', 'ro'), ('din', 'ro'), ('săi', 'ro'), ('însene', 'ro'), ('acestuia', 'ro'), ('altor', 'ro'), ('acestora', 'ro'), ('ţi', 'ro'), ('are', 'ro'), ('ceilalţi', 'ro'), ('ce', 'ro'), ('amândouă', 'ro'), ('unu', 'ro'), ('aia', 'ro'), ('vouă', 'ro'), ('oricăror', 'ro'), ('sieşi', 'ro'), ('nici', 'ro'), ('printre', 'ro'), ('decât', 'ro'), ('avem', 'ro'), ('toate', 'ro'), ('ale ', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('aveţi', 'ro'), ('celeilalte', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('aveam', 'ro'), ('tuturor', 'ro'), ('aţi', 'ro'), ('tale', 'ro'), ('tine', 'ro'), ('lângă', 'ro'), ('el', 'ro'), ('altul', 'ro'), ('ăleia', 'ro');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ne', 'ro'), ('încă', 'ro'), ('oricât', 'ro'), ('put', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('nou', 'ro'), ('atât', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('şas', 'ro'), ('sine ', 'ro'), ('tău', 'ro'), ('oi', 'ro'), ('oricin', 'ro'), ('dou', 'ro'), ('unei', 'ro'), ('aproap', 'ro'), ('amândou', 'ro'), ('celeilalt', 'ro'), ('mi', 'ro'), ('sal', 'ro'), ('mel', 'ro'), ('part', 'ro'), ('poat', 'ro'), ('oricăr', 'ro'), ('aceleaş', 'ro'), ('noastr', 'ro'), ('asupr', 'ro'), ('dup', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('bin', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('prim', 'ro'), ('cev', 'ro'), ('acelui', 'ro'), ('celu', 'ro'), ('înşin', 'ro'), ('una', 'ro'), ('toţ', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('altu', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('însut', 'ro'), ('ambe', 'ro'), ('ălui', 'ro'), ('dumneavoastr', 'ro'), ('cei', 'ro'), ('acelei', 'ro'), ('am', 'ro'), ('lor', 'ro'), ('celelalt', 'ro'), ('ace', 'ro'), ('alt', 'ro'), ('car', 'ro'), ('alta', 'ro'), ('eşti', 'ro'), ('spun', 'ro'), ('meu', 'ro'), ('foart', 'ro'), ('căru', 'ro'), ('fiind', 'ro'), ('o', 'ro'), ('spre', 'ro'), ('celuilalt', 'ro'), ('căre', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('altcinev', 'ro'), ('fiecărei', 'ro'), ('noştr', 'ro'), ('însăş', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('alte', 'ro'), ('însăm', 'ro'), ('eram', 'ro'), ('aceşt', 'ro'), ('celălalt', 'ro'), ('toat', 'ro'), ('al', 'ro'), ('înşiş', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('celorlalţ', 'ro'), ('fiu', 'ro'), ('mult', 'ro'), ('câţiv', 'ro'), ('altcuiva ', 'ro'), ('câtev', 'ro'), ('vou', 'ro'), ('ea', 'ro'), ('numa', 'ro'), ('tin', 'ro'), ('mei', 'ro'), ('celorlalt', 'ro'), ('acelor', 'ro'), ('ca', 'ro'), ('ăşti', 'ro'), ('oricâtor', 'ro'), ('însum', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('şapt', 'ro'), ('deş', 'ro'), ('fost', 'ro'), ('cin', 'ro'), ('aceast', 'ro'), ('vreo', 'ro'), ('fii', 'ro'), ('tal', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('înşiv', 'ro'), ('ăstui', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('oţi', 'ro'), ('ăia', 'ro'), ('însuţ', 'ro'), ('vor', 'ro'), ('de', 'ro'), ('un', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('însăţ', 'ro'), ('oric', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('aceeaş', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('ăstor', 'ro'), ('că', 'ro'), ('eraţ', 'ro'), ('cele', 'ro'), ('min', 'ro'), ('înseş', 'ro'), ('ar', 'ro'), ('cutar', 'ro'), ('atâtor', 'ro'), ('iar', 'ro'), ('voi', 'ro'), ('îndoit', 'ro'), ('avut', 'ro'), ('cât', 'ro'), ('căror ', 'ro'), ('le', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('cel', 'ro'), ('ci', 'ro'), ('acele', 'ro'), ('prime', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('aveţ', 'ro'), ('ăla', 'ro'), ('oricăre', 'ro'), ('tre', 'ro'), ('câţ', 'ro'), ('pentru', 'ro'), ('nic', 'ro'), ('va', 'ro'), ('când', 'ro'), ('la', 'ro'), ('aste', 'ro'), ('oricărui', 'ro'), ('veţ', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('puţin', 'ro'), ('amândo', 'ro'), ('acei', 'ro'), ('ăsta', 'ro'), ('totuş', 'ro'), ('exist', 'ro'), ('zec', 'ro'), ('ale', 'ro'), ('acestei', 'ro'), ('înse', 'ro'), ('atunc', 'ro'), ('ier', 'ro'), ('altei', 'ro'), ('unde', 'ro'), ('pân', 'ro'), ('ambi', 'ro'), ('îşi', 'ro'), ('uni', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('s', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('ave', 'ro'), ('însen', 'ro'), ('cinc', 'ro'), ('este', 'ro'), ('oarec', 'ro'), ('câtorv', 'ro'), ('patru', 'ro'), ('ei', 'ro'), ('vreune', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('despr', 'ro'), ('sutim', 'ro'), ('au', 'ro'), ('fac', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('cealalt', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('altcev', 'ro'), ('ăstei', 'ro'), ('doar', 'ro'), ('nişt', 'ro'), ('întâi', 'ro'), ('astfel', 'ro'), ('cinev', 'ro'), ('întreit', 'ro'), ('ălor', 'ro'), ('une', 'ro'), ('chiar', 'ro'), ('fiec', 'ro'), ('nimic', 'ro'), ('făr', 'ro'), ('treim', 'ro'), ('pest', 'ro'), ('dintr', 'ro'), ('ţie', 'ro'), ('ceilalţ', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('dest', 'ro'), ('voastră ', 'ro'), ('cătr', 'ro'), ('dec', 'ro'), ('aceste', 'ro'), ('aş', 'ro'), ('cu', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('câtv', 'ro'), ('cum', 'ro'), ('lâng', 'ro'), ('mulţ', 'ro'), ('vreunui', 'ro'), ('lui', 'ro'), ('fiţ', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('unora ', 'ro'), ('printr', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('aşadar', 'ro'), ('tăi', 'ro'), ('ălei', 'ro'), ('acestor', 'ro'), ('or', 'ro'), ('acel', 'ro'), ('nimăn', 'ro'), ('sieş', 'ro'), ('câtor', 'ro'), ('avea', 'ro'), ('mar', 'ro'), ('atâţ', 'ro'), ('v', 'ro'), ('nimen', 'ro'), ('faţ', 'ro'), ('dar', 'ro'), ('însev', 'ro'), ('spus', 'ro'), ('voştr', 'ro'), ('din', 'ro'), ('voastr', 'ro'), ('săi', 'ro'), ('altor', 'ro'), ('ţi', 'ro'), ('fiecăr', 'ro'), ('trebui', 'ro'), ('are', 'ro'), ('doim', 'ro'), ('ce', 'ro'), ('aia', 'ro'), ('fiecăre', 'ro'), ('unu', 'ro'), ('oricăror', 'ro'), ('decât', 'ro'), ('ale ', 'ro'), ('avem', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('dac', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('tuturor', 'ro'), ('aveam', 'ro'), ('aţi', 'ro'), ('el', 'ro'), ('însuş', 'ro');INSERT INTO stopwords_tiny (stopword, language) VALUES ('вот', 'ru'), ('при', 'ru'), ('из', 'ru'), ('пока', 'ru'), ('есть', 'ru'), ('также', 'ru'), ('как', 'ru'), ('всего', 'ru'), ('от', 'ru'), ('только', 'ru'), ('для', 'ru'), ('Как', 'ru'), ('у', 'ru'), ('дело', 'ru'), ('эти', 'ru'), ('уже', 'ru'), ('быть', 'ru'), ('еще', 'ru'), ('лишь', 'ru'), ('к', 'ru'), ('жизни', 'ru'), ('деньги', 'ru'), ('теперь', 'ru'), ('год', 'ru'), ('него', 'ru'), ('когда', 'ru'), ('о', 'ru'), ('ли', 'ru'), ('будет', 'ru'), ('через', 'ru'), ('больше', 'ru'), ('чем', 'ru'), ('все', 'ru'), ('года', 'ru'), ('за', 'ru'), ('им', 'ru'), ('со', 'ru'), ('там', 'ru'), ('Однако', 'ru'), ('коп', 'ru'), ('этот', 'ru'), ('Если', 'ru'), ('Я', 'ru'), ('по', 'ru'), ('под', 'ru'), ('сегодня', 'ru'), ('мы', 'ru'), ('нас', 'ru'), ('они', 'ru'), ('ни', 'ru'), ('году', 'ru'), ('лет', 'ru'), ('в', 'ru'), ('было', 'ru'), ('По', 'ru'), ('С', 'ru'), ('своих', 'ru'), ('кто', 'ru'), ('том', 'ru'), ('бы', 'ru'), ('В', 'ru'), ('даже', 'ru'), ('человек', 'ru'), ('это', 'ru'), ('мне', 'ru'), ('если', 'ru'), ('этой', 'ru'), ('не', 'ru'), ('У', 'ru'), ('того', 'ru'), ('свою', 'ru'), ('раз', 'ru'), ('А', 'ru'), ('который', 'ru'), ('Он', 'ru'), ('нет', 'ru'), ('руб', 'ru'), ('сейчас', 'ru'), ('просто', 'ru'), ('этом', 'ru'), ('между', 'ru'), ('время', 'ru'), ('после', 'ru'), ('об', 'ru'), ('этого', 'ru'), ('были', 'ru'), ('их', 'ru'), ('ему', 'ru'), ('же', 'ru'), ('И', 'ru'), ('Но', 'ru'), ('себя', 'ru'), ('Что', 'ru'), ('более', 'ru'), ('которые', 'ru'), ('так', 'ru'), ('может', 'ru'), ('но', 'ru'), ('чтобы', 'ru'), ('на', 'ru'), ('и', 'ru'), ('что', 'ru'), ('он', 'ru'), ('тех', 'ru'), ('ее', 'ru'), ('она', 'ru'), ('К', 'ru'), ('себе', 'ru'), ('На', 'ru'), ('один', 'ru'), ('своей', 'ru'), ('можно', 'ru'), ('до', 'ru'), ('то', 'ru'), ('Это', 'ru'), ('его', 'ru'), ('тем', 'ru'), ('два', 'ru'), ('свои', 'ru'), ('Не', 'ru'), ('а', 'ru'), ('без', 'ru'), ('был', 'ru'), ('хотя', 'ru'), ('очень', 'ru'), ('во', 'ru'), ('процентов', 'ru'), ('где', 'ru'), ('была', 'ru'), ('с', 'ru'), ('Иллюстрация', 'ru'), ('я', 'ru'), ('день', 'ru'), ('здесь', 'ru'), ('или', 'ru'), ('будут', 'ru'), ('несколько', 'ru'), ('которых', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('три', 'ru');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('сейчас', 'ru'), ('руб', 'ru'), ('при', 'ru'), ('вот', 'ru'), ('из', 'ru'), ('тепер', 'ru'), ('как', 'ru'), ('от', 'ru'), ('е', 'ru'), ('об', 'ru'), ('для', 'ru'), ('межд', 'ru'), ('у', 'ru'), ('их', 'ru'), ('тольк', 'ru'), ('ил', 'ru'), ('же', 'ru'), ('дел', 'ru'), ('к', 'ru'), ('жизн', 'ru'), ('деньг', 'ru'), ('год', 'ru'), ('о', 'ru'), ('ли', 'ru'), ('может', 'ru'), ('так', 'ru'), ('будет', 'ru'), ('но', 'ru'), ('через', 'ru'), ('хот', 'ru'), ('на', 'ru'), ('и', 'ru'), ('что', 'ru'), ('сво', 'ru'), ('он', 'ru'), ('чем', 'ru'), ('все', 'ru'), ('тех', 'ru'), ('за', 'ru'), ('ест', 'ru'), ('даж', 'ru'), ('им', 'ru'), ('со', 'ru'), ('эт', 'ru'), ('зде', 'ru'), ('там', 'ru'), ('один', 'ru'), ('нег', 'ru'), ('этот', 'ru'), ('коп', 'ru'), ('уж', 'ru'), ('всег', 'ru'), ('то', 'ru'), ('до', 'ru'), ('ег', 'ru'), ('врем', 'ru'), ('пок', 'ru'), ('по', 'ru'), ('под', 'ru'), ('мы', 'ru'), ('прост', 'ru'), ('нас', 'ru'), ('нескольк', 'ru'), ('ни', 'ru'), ('очен', 'ru'), ('два', 'ru'), ('тем', 'ru'), ('быт', 'ru'), ('лет', 'ru'), ('в', 'ru'), ('когд', 'ru'), ('лиш', 'ru'), ('а', 'ru'), ('кто', 'ru'), ('том', 'ru'), ('иллюстрац', 'ru'), ('бы', 'ru'), ('без', 'ru'), ('себ', 'ru'), ('однак', 'ru'), ('человек', 'ru'), ('был', 'ru'), ('во', 'ru'), ('мне', 'ru'), ('сегодн', 'ru'), ('где', 'ru'), ('тог', 'ru'), ('ден', 'ru'), ('с', 'ru'), ('не', 'ru'), ('я', 'ru'), ('есл', 'ru'), ('чтоб', 'ru'), ('будут', 'ru'), ('такж', 'ru'), ('посл', 'ru'), ('ещ', 'ru'), ('раз', 'ru'), ('ем', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('больш', 'ru'), ('процент', 'ru'), ('бол', 'ru'), ('можн', 'ru'), ('три', 'ru'), ('нет', 'ru'), ('котор', 'ru');INSERT INTO stopwords_tiny (stopword, language) VALUES ('som', 'sv'), ('du', 'sv'), ('att', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('sådan', 'sv'), ('utan', 'sv'), ('nu', 'sv'), ('sådana', 'sv'), ('ingen', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('samma', 'sv'), ('bli', 'sv'), ('vars', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('med', 'sv'), ('ert', 'sv'), ('vilken', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('detta', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('ut', 'sv'), ('inom', 'sv'), ('någon', 'sv'), ('men', 'sv'), ('inte', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('vara', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('mina', 'sv'), ('till', 'sv'), ('ett', 'sv'), ('sitta', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('när', 'sv'), ('man', 'sv'), ('den', 'sv'), ('dess', 'sv'), ('har', 'sv'), ('sådant', 'sv'), ('hade', 'sv'), ('hans', 'sv'), ('vid', 'sv'), ('dessa', 'sv'), ('dina', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('vilka', 'sv'), ('deras', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('henne', 'sv'), ('blivit', 'sv'), ('varje', 'sv'), ('vilkas', 'sv'), ('om', 'sv'), ('dem', 'sv'), ('en', 'sv'), ('sin', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('alla', 'sv'), ('kunde', 'sv'), ('våra', 'sv'), ('hennes', 'sv'), ('åt', 'sv'), ('något', 'sv'), ('ha', 'sv'), ('vem', 'sv'), ('efter', 'sv'), ('skulle', 'sv'), ('icke', 'sv'), ('i', 'sv'), ('över', 'sv'), ('än', 'sv'), ('sina', 'sv'), ('denna', 'sv'), ('mitt', 'sv'), ('under', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('några', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('eller', 'sv'), ('mot', 'sv'), ('blev', 'sv'), ('är', 'sv');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('som', 'sv'), ('att', 'sv'), ('du', 'sv'), ('dett', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('nu', 'sv'), ('utan', 'sv'), ('sådan', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('bli', 'sv'), ('med', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('ert', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('ell', 'sv'), ('henn', 'sv'), ('int', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('all', 'sv'), ('ut', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('inom', 'sv'), ('men', 'sv'), ('någon', 'sv'), ('skull', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('vilk', 'sv'), ('till', 'sv'), ('und', 'sv'), ('ett', 'sv'), ('denn', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('den', 'sv'), ('man', 'sv'), ('när', 'sv'), ('har', 'sv'), ('dess', 'sv'), ('sådant', 'sv'), ('varj', 'sv'), ('samm', 'sv'), ('der', 'sv'), ('vid', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('kund', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('blivit', 'sv'), ('om', 'sv'), ('ing', 'sv'), ('någr', 'sv'), ('dem', 'sv'), ('sin', 'sv'), ('en', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('ick', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('ha', 'sv'), ('något', 'sv'), ('åt', 'sv'), ('vem', 'sv'), ('eft', 'sv'), ('än', 'sv'), ('över', 'sv'), ('i', 'sv'), ('had', 'sv'), ('mitt', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('sitt', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('mot', 'sv'), ('är', 'sv'), ('blev', 'sv');INSERT INTO stopwords_tiny (stopword, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('filan', 'tr'), ('böylece', 'tr'), ('biz', 'tr'), ('hepsi', 'tr'), ('senin', 'tr'), ('hepsine', 'tr'), ('hiçbirini', 'tr'), ('hangisi', 'tr'), ('e', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('bazısı', 'tr'), ('þey', 'tr'), ('kendi', 'tr'), ('dokuz', 'tr'), ('siz', 'tr'), ('kendine', 'tr'), ('diğeri', 'tr'), ('bu', 'tr'), ('seni', 'tr'), ('hem', 'tr'), ('onu', 'tr'), ('kimi', 'tr'), ('bizim', 'tr'), ('defa', 'tr'), ('burada', 'tr'), ('mi', 'tr'), ('hiç kimse', 'tr'), ('biri', 'tr'), ('bütün', 'tr'), ('ve', 'tr'), ('altmýþ', 'tr'), ('son', 'tr'), ('onun', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('dört', 'tr'), ('milyon', 'tr'), ('u', 'tr'), ('yerine', 'tr'), ('k', 'tr'), ('orada', 'tr'), ('beş', 'tr'), ('mý', 'tr'), ('buna', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('þuna', 'tr'), ('dahi', 'tr'), ('içinde', 'tr'), ('çoğuna', 'tr'), ('kaç', 'tr'), ('en gibi', 'tr'), ('böyle', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('beni', 'tr'), ('kadar', 'tr'), ('şunda', 'tr'), ('zaten', 'tr'), ('dolayı', 'tr'), ('o', 'tr'), ('bunun', 'tr'), ('hangi', 'tr'), ('şeye', 'tr'), ('d', 'tr'), ('on', 'tr'), ('herkese', 'tr'), ('yüz', 'tr'), ('ya da', 'tr'), ('bazıları', 'tr'), ('herkes', 'tr'), ('nereden', 'tr'), ('bile', 'tr'), ('doksan', 'tr'), ('hatta', 'tr'), ('bana', 'tr'), ('onlari', 'tr'), ('z', 'tr'), ('artık', 'tr'), ('şimdi', 'tr'), ('bazen', 'tr'), ('onlara', 'tr'), ('fakat', 'tr'), ('size', 'tr'), ('birşey', 'tr'), ('var', 'tr'), ('neden', 'tr'), ('hep', 'tr'), ('nereye', 'tr'), ('bunu', 'tr'), ('bunda', 'tr'), ('ğ', 'tr'), ('birkez', 'tr'), ('oysa', 'tr'), ('birkaç', 'tr'), ('yine', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('t', 'tr'), ('bizden', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('onların', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('birçoğu', 'tr'), ('madem', 'tr'), ('yedi', 'tr'), ('onlar', 'tr'), ('ötürü', 'tr'), ('þeyi', 'tr'), ('herkesi', 'tr'), ('bir', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('yoksa', 'tr'), ('yetmiþ', 'tr'), ('þeyler', 'tr'), ('birþeyi', 'tr'), ('birisi', 'tr'), ('n', 'tr'), ('şeyi', 'tr'), ('p', 'tr'), ('INSERmi', 'tr'), ('hiç', 'tr'), ('diye', 'tr'), ('kimin', 'tr'), ('hepsini', 'tr'), ('da', 'tr'), ('y', 'tr'), ('önce', 'tr'), ('g', 'tr'), ('her biri', 'tr'), ('nerede', 'tr'), ('tabi', 'tr'), ('ne zaman', 'tr'), ('bazı', 'tr'), ('daha', 'tr'), ('sizden', 'tr'), ('benim', 'tr'), ('değil', 'tr'), ('benden', 'tr'), ('katrilyon', 'tr'), ('acaba', 'tr'), ('sizi', 'tr'), ('ç', 'tr'), ('kendini', 'tr'), ('belki', 'tr'), ('elli', 'tr'), ('trilyon', 'tr'), ('şundan', 'tr'), ('ya', 'tr'), ('şeyden', 'tr'), ('asla', 'tr'), ('ü', 'tr'), ('tüm', 'tr'), ('mı', 'tr'), ('tümü', 'tr'), ('çünkü', 'tr'), ('ama', 'tr'), ('işte', 'tr'), ('birkaçı', 'tr'), ('s', 'tr'), ('rağmen', 'tr'), ('c', 'tr'), ('üzere', 'tr'), ('altı', 'tr'), ('şunu', 'tr'), ('diğerleri', 'tr'), ('bize', 'tr'), ('veyahut', 'tr'), ('şunun', 'tr'), ('senden', 'tr'), ('nasýl', 'tr'), ('üç', 'tr'), ('i', 'tr'), ('m', 'tr'), ('gibi', 'tr'), ('hiçbiri', 'tr'), ('ile', 'tr'), ('sen', 'tr'), ('niçin', 'tr'), ('sizin', 'tr'), ('yirmi', 'tr'), ('gene', 'tr'), ('kim', 'tr'), ('az', 'tr'), ('felan', 'tr'), ('altý', 'tr'), ('b', 'tr'), ('nesi', 'tr'), ('nerde', 'tr'), ('kimisi', 'tr'), ('þu', 'tr'), ('birşeyi', 'tr'), ('elbette', 'tr'), ('şöyle', 'tr'), ('milyar', 'tr'), ('oysaki', 'tr'), ('şunlar', 'tr'), ('kime', 'tr'), ('bazý', 'tr'), ('mu', 'tr'), ('her', 'tr'), ('öbürü', 'tr'), ('bundan', 'tr'), ('þundan', 'tr'), ('onlardan', 'tr'), ('seksen', 'tr'), ('kimden', 'tr'), ('bizi', 'tr'), ('birçokları', 'tr'), ('henüz', 'tr'), ('diğer', 'tr'), ('þunu', 'tr'), ('birçok', 'tr'), ('ş', 'tr'), ('ı', 'tr'), ('şeyler', 'tr'), ('neyse', 'tr'), ('tamam', 'tr'), ('sekiz', 'tr'), ('öyle', 'tr'), ('hani', 'tr'), ('onlarýn', 'tr'), ('ise', 'tr'), ('birþey', 'tr'), ('niye', 'tr'), ('v', 'tr'), ('ondan', 'tr'), ('zira', 'tr'), ('hiçbirine', 'tr'), ('ne kadar', 'tr'), ('çoğu', 'tr'), ('r', 'tr'), ('þunda', 'tr'), ('onu otuz', 'tr'), ('h', 'tr'), ('falan', 'tr'), ('sonra', 'tr'), ('iki', 'tr'), ('nedir', 'tr'), ('ona', 'tr'), ('f', 'tr'), ('demek', 'tr'), ('sana', 'tr'), ('þeyden', 'tr'), ('çoğunu', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('aslında', 'tr'), ('yani', 'tr'), ('şuna', 'tr'), ('için', 'tr');INSERT INTO stopword_stems_tiny (stopword_stem, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('biz', 'tr'), ('da', 'tr'), ('hepsi', 'tr'), ('y', 'tr'), ('hat', 'tr'), ('g', 'tr'), ('tama', 'tr'), ('ner', 'tr'), ('tabi', 'tr'), ('iş', 'tr'), ('e', 'tr'), ('bazı', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('kendi', 'tr'), ('þey', 'tr'), ('en gip', 'tr'), ('siz', 'tr'), ('ke', 'tr'), ('þe', 'tr'), ('değil', 'tr'), ('üzer', 'tr'), ('ge', 'tr'), ('bu', 'tr'), ('şun', 'tr'), ('hem', 'tr'), ('is', 'tr'), ('dok', 'tr'), ('defa', 'tr'), ('hiçbir', 'tr'), ('mi', 'tr'), ('ç', 'tr'), ('heps', 'tr'), ('belki', 'tr'), ('se', 'tr'), ('öbür', 'tr'), ('elli', 'tr'), ('ve', 'tr'), ('il', 'tr'), ('altmýþ', 'tr'), ('fala', 'tr'), ('sek', 'tr'), ('son', 'tr'), ('niç', 'tr'), ('baze', 'tr'), ('ba', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('zate', 'tr'), ('ya', 'tr'), ('dört', 'tr'), ('yer', 'tr'), ('ü', 'tr'), ('u', 'tr'), ('tüm', 'tr'), ('k', 'tr'), ('mı', 'tr'), ('böyleç', 'tr'), ('beş', 'tr'), ('katrilyo', 'tr'), ('mý', 'tr'), ('çünkü', 'tr'), ('biris', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('s', 'tr'), ('ya ', 'tr'), ('dahi', 'tr'), ('dah', 'tr'), ('c', 'tr'), ('bazıs', 'tr'), ('bura', 'tr'), ('kaç', 'tr'), ('kimis', 'tr'), ('iç', 'tr'), ('veyahut', 'tr'), ('birþe', 'tr'), ('am', 'tr'), ('üç', 'tr'), ('nasýl', 'tr'), ('milyo', 'tr'), ('i', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('þ', 'tr'), ('kadar', 'tr'), ('m', 'tr'), ('ik', 'tr'), ('as', 'tr'), ('onu ot', 'tr'), ('sen', 'tr'), ('acap', 'tr'), ('ne zama', 'tr'), ('o', 'tr'), ('hangi', 'tr'), ('yirmi', 'tr'), ('d', 'tr'), ('kim', 'tr'), ('on', 'tr'), ('yüz', 'tr'), ('az', 'tr'), ('herkes', 'tr'), ('gip', 'tr'), ('ötür', 'tr'), ('altý', 'tr'), ('onlarý', 'tr'), ('al', 'tr'), ('onlari', 'tr'), ('hangis', 'tr'), ('b', 'tr'), ('büt', 'tr'), ('z', 'tr'), ('bil', 'tr'), ('trilyo', 'tr'), ('di', 'tr'), ('þu', 'tr'), ('fakat', 'tr'), ('milyar', 'tr'), ('birşey', 'tr'), ('oysaki', 'tr'), ('var', 'tr'), ('mu', 'tr'), ('bazý', 'tr'), ('ni', 'tr'), ('her', 'tr'), ('hep', 'tr'), ('fila', 'tr'), ('nes', 'tr'), ('asl', 'tr'), ('ğ', 'tr'), ('rağme', 'tr'), ('birkez', 'tr'), ('made', 'tr'), ('bö', 'tr'), ('diğer', 'tr'), ('ora', 'tr'), ('birkaç', 'tr'), ('birçok', 'tr'), ('şö', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('ş', 'tr'), ('insermi', 'tr'), ('ı', 'tr'), ('sa', 'tr'), ('hiç k', 'tr'), ('her bir', 'tr'), ('t', 'tr'), ('hani', 'tr'), ('birþey', 'tr'), ('v', 'tr'), ('elbet', 'tr'), ('zira', 'tr'), ('yok', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('be', 'tr'), ('ne kadar', 'tr'), ('yedi', 'tr'), ('dola', 'tr'), ('r', 'tr'), ('şe', 'tr'), ('bir', 'tr'), ('sonra', 'tr'), ('h', 'tr'), ('birşe', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('nere', 'tr'), ('demek', 'tr'), ('f', 'tr'), ('yetmiþ', 'tr'), ('n', 'tr'), ('baz', 'tr'), ('he', 'tr'), ('ar', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('p', 'tr'), ('hiç', 'tr'), ('yani', 'tr'), ('fela', 'tr'); - --- PostgreSQL sends notices about implicit keys that are being created, --- and the test suite takes them for warnings. -SET client_min_messages=WARNING; - --- "Full" stopwords -DROP TABLE IF EXISTS stopwords_short; -CREATE TABLE stopwords_short ( - stopwords_short_id SERIAL PRIMARY KEY, - stopword VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopwords_short_stopword - ON stopwords_short(stopword, language); - --- Stopword stems -DROP TABLE IF EXISTS stopword_stems_short; -CREATE TABLE stopword_stems_short ( - stopword_stems_short_id SERIAL PRIMARY KEY, - stopword_stem VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopword_stems_short_stopword_stem - ON stopword_stems_short(stopword_stem, language); - --- Reset the message level back to "notice". -SET client_min_messages=NOTICE; - -INSERT INTO stopwords_short (stopword, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('sådan', 'da'), ('nu', 'da'), ('anden', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('ville', 'da'), ('kunne', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('alle', 'da'), ('når', 'da'), ('dig', 'da'), ('over', 'da'), ('af', 'da'), ('vil', 'da'), ('men', 'da'), ('ned', 'da'), ('have', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('ind', 'da'), ('været', 'da'), ('også', 'da'), ('hendes', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('hos', 'da'), ('har', 'da'), ('dog', 'da'), ('mod', 'da'), ('der', 'da'), ('hans', 'da'), ('denne', 'da'), ('være', 'da'), ('og', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('et', 'da'), ('er', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('disse', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('en', 'da'), ('sin', 'da'), ('mange', 'da'), ('vor', 'da'), ('meget', 'da'), ('hende', 'da'), ('de', 'da'), ('deres', 'da'), ('din', 'da'), ('mine', 'da'), ('op', 'da'), ('hun', 'da'), ('noget', 'da'), ('sine', 'da'), ('ham', 'da'), ('ikke', 'da'), ('skal', 'da'), ('efter', 'da'), ('havde', 'da'), ('ad', 'da'), ('skulle', 'da'), ('at', 'da'), ('jo', 'da'), ('i', 'da'), ('alt', 'da'), ('under', 'da'), ('jer', 'da'), ('min', 'da'), ('mig', 'da'), ('nogle', 'da'), ('sig', 'da'), ('eller', 'da'), ('blive', 'da'), ('blev', 'da'), ('bliver', 'da'), ('mit', 'da'), ('dette', 'da');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('nu', 'da'), ('sådan', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('hav', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('vær', 'da'), ('når', 'da'), ('dig', 'da'), ('and', 'da'), ('bliv', 'da'), ('over', 'da'), ('ell', 'da'), ('af', 'da'), ('vil', 'da'), ('all', 'da'), ('men', 'da'), ('ned', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('und', 'da'), ('ind', 'da'), ('også', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('har', 'da'), ('hos', 'da'), ('dog', 'da'), ('ikk', 'da'), ('havd', 'da'), ('mod', 'da'), ('der', 'da'), ('dis', 'da'), ('og', 'da'), ('nogl', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('mang', 'da'), ('et', 'da'), ('er', 'da'), ('hend', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('meg', 'da'), ('en', 'da'), ('sin', 'da'), ('vor', 'da'), ('de', 'da'), ('din', 'da'), ('op', 'da'), ('hun', 'da'), ('skul', 'da'), ('ham', 'da'), ('skal', 'da'), ('ad', 'da'), ('kun', 'da'), ('at', 'da'), ('jo', 'da'), ('eft', 'da'), ('i', 'da'), ('alt', 'da'), ('mig', 'da'), ('jer', 'da'), ('min', 'da'), ('nog', 'da'), ('sig', 'da'), ('blev', 'da'), ('mit', 'da');INSERT INTO stopwords_short (stopword, language) VALUES ('einiger', 'de'), ('habe', 'de'), ('kann', 'de'), ('deinen', 'de'), ('da', 'de'), ('anderr', 'de'), ('meiner', 'de'), ('dann', 'de'), ('einer', 'de'), ('meines', 'de'), ('hat', 'de'), ('gewesen', 'de'), ('eure', 'de'), ('solche', 'de'), ('ihre', 'de'), ('mich', 'de'), ('vom', 'de'), ('sehr', 'de'), ('also', 'de'), ('über', 'de'), ('allem', 'de'), ('jener', 'de'), ('warst', 'de'), ('alle', 'de'), ('solchem', 'de'), ('eures', 'de'), ('zwischen', 'de'), ('jenen', 'de'), ('hatten', 'de'), ('jeder', 'de'), ('sonst', 'de'), ('ohne', 'de'), ('alles', 'de'), ('bei', 'de'), ('die', 'de'), ('dir', 'de'), ('welchem', 'de'), ('etwas', 'de'), ('will', 'de'), ('anderem', 'de'), ('solcher', 'de'), ('und', 'de'), ('das', 'de'), ('musste', 'de'), ('zur', 'de'), ('gegen', 'de'), ('man', 'de'), ('keines', 'de'), ('diese', 'de'), ('einige', 'de'), ('ander', 'de'), ('diesen', 'de'), ('seinen', 'de'), ('euren', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('allen', 'de'), ('eurer', 'de'), ('bin', 'de'), ('hinter', 'de'), ('um', 'de'), ('unsem', 'de'), ('ein', 'de'), ('weiter', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('manchen', 'de'), ('keinem', 'de'), ('ich', 'de'), ('daß', 'de'), ('jenem', 'de'), ('dieses', 'de'), ('wird', 'de'), ('dieselbe', 'de'), ('seines', 'de'), ('wirst', 'de'), ('dem', 'de'), ('solchen', 'de'), ('sind', 'de'), ('hier', 'de'), ('wo', 'de'), ('weg', 'de'), ('euer', 'de'), ('des', 'de'), ('diesem', 'de'), ('dies', 'de'), ('nichts', 'de'), ('deines', 'de'), ('ihrer', 'de'), ('war', 'de'), ('demselben', 'de'), ('meinen', 'de'), ('euch', 'de'), ('am', 'de'), ('nach', 'de'), ('unser', 'de'), ('anderm', 'de'), ('selbst', 'de'), ('während', 'de'), ('deiner', 'de'), ('unsen', 'de'), ('solches', 'de'), ('welche', 'de'), ('dasselbe', 'de'), ('zum', 'de'), ('werde', 'de'), ('ihnen', 'de'), ('eines', 'de'), ('meine', 'de'), ('mit', 'de'), ('ihm', 'de'), ('anders', 'de'), ('anderes', 'de'), ('ihr', 'de'), ('jene', 'de'), ('jenes', 'de'), ('wollen', 'de'), ('deinem', 'de'), ('ob', 'de'), ('du', 'de'), ('einigem', 'de'), ('einiges', 'de'), ('welchen', 'de'), ('einen', 'de'), ('ihn', 'de'), ('jede', 'de'), ('wie', 'de'), ('wenn', 'de'), ('denselben', 'de'), ('derer', 'de'), ('unses', 'de'), ('deine', 'de'), ('ist', 'de'), ('mancher', 'de'), ('meinem', 'de'), ('keinen', 'de'), ('dessen', 'de'), ('nicht', 'de'), ('unter', 'de'), ('kein', 'de'), ('bist', 'de'), ('doch', 'de'), ('anderen', 'de'), ('wir', 'de'), ('ihren', 'de'), ('keiner', 'de'), ('ihrem', 'de'), ('unse', 'de'), ('eurem', 'de'), ('sein', 'de'), ('dein', 'de'), ('hab', 'de'), ('welcher', 'de'), ('würden', 'de'), ('keine', 'de'), ('seinem', 'de'), ('haben', 'de'), ('auf', 'de'), ('manche', 'de'), ('desselben', 'de'), ('hin', 'de'), ('denn', 'de'), ('jedes', 'de'), ('den', 'de'), ('derselbe', 'de'), ('einem', 'de'), ('auch', 'de'), ('jeden', 'de'), ('sollte', 'de'), ('an', 'de'), ('der', 'de'), ('seiner', 'de'), ('derselben', 'de'), ('mein', 'de'), ('könnte', 'de'), ('durch', 'de'), ('es', 'de'), ('wieder', 'de'), ('seine', 'de'), ('machen', 'de'), ('würde', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('andere', 'de'), ('bis', 'de'), ('ihres', 'de'), ('muss', 'de'), ('zwar', 'de'), ('waren', 'de'), ('sich', 'de'), ('so', 'de'), ('hatte', 'de'), ('er', 'de'), ('manches', 'de'), ('einigen', 'de'), ('nur', 'de'), ('im', 'de'), ('jedem', 'de'), ('dich', 'de'), ('eine', 'de'), ('vor', 'de'), ('können', 'de'), ('aus', 'de'), ('aller', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('andern', 'de'), ('indem', 'de'), ('wollte', 'de'), ('welches', 'de'), ('manchem', 'de'), ('einig', 'de'), ('sie', 'de'), ('dieser', 'de'), ('noch', 'de'), ('für', 'de'), ('dieselben', 'de'), ('anderer', 'de'), ('werden', 'de'), ('nun', 'de'), ('von', 'de'), ('aber', 'de'), ('ins', 'de'), ('was', 'de'), ('sondern', 'de');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('wollt', 'de'), ('mach', 'de'), ('kann', 'de'), ('anderr', 'de'), ('da', 'de'), ('dann', 'de'), ('jen', 'de'), ('hat', 'de'), ('mich', 'de'), ('also', 'de'), ('sehr', 'de'), ('vom', 'de'), ('konnt', 'de'), ('warst', 'de'), ('hint', 'de'), ('and', 'de'), ('sonst', 'de'), ('bei', 'de'), ('all', 'de'), ('die', 'de'), ('demselb', 'de'), ('dir', 'de'), ('etwas', 'de'), ('will', 'de'), ('derselb', 'de'), ('ind', 'de'), ('und', 'de'), ('das', 'de'), ('solch', 'de'), ('zur', 'de'), ('ohn', 'de'), ('man', 'de'), ('musst', 'de'), ('dess', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('bin', 'de'), ('um', 'de'), ('ein', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('ich', 'de'), ('wird', 'de'), ('manch', 'de'), ('wirst', 'de'), ('wied', 'de'), ('dem', 'de'), ('sind', 'de'), ('woll', 'de'), ('hier', 'de'), ('wo', 'de'), ('des', 'de'), ('euer', 'de'), ('weg', 'de'), ('dies', 'de'), ('war', 'de'), ('denselb', 'de'), ('am', 'de'), ('euch', 'de'), ('nach', 'de'), ('anderm', 'de'), ('selb', 'de'), ('wahrend', 'de'), ('zum', 'de'), ('eur', 'de'), ('ihm', 'de'), ('mit', 'de'), ('hatt', 'de'), ('ihr', 'de'), ('ob', 'de'), ('du', 'de'), ('ihn', 'de'), ('wie', 'de'), ('wenn', 'de'), ('fur', 'de'), ('ist', 'de'), ('unt', 'de'), ('welch', 'de'), ('nicht', 'de'), ('kein', 'de'), ('sond', 'de'), ('bist', 'de'), ('doch', 'de'), ('wir', 'de'), ('uber', 'de'), ('sein', 'de'), ('hab', 'de'), ('dein', 'de'), ('auf', 'de'), ('zwisch', 'de'), ('hin', 'de'), ('denn', 'de'), ('sollt', 'de'), ('gewes', 'de'), ('desselb', 'de'), ('den', 'de'), ('weit', 'de'), ('auch', 'de'), ('an', 'de'), ('der', 'de'), ('konn', 'de'), ('mein', 'de'), ('durch', 'de'), ('es', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('bis', 'de'), ('muss', 'de'), ('zwar', 'de'), ('sich', 'de'), ('so', 'de'), ('er', 'de'), ('nur', 'de'), ('im', 'de'), ('dich', 'de'), ('vor', 'de'), ('aus', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('jed', 'de'), ('dieselb', 'de'), ('werd', 'de'), ('einig', 'de'), ('sie', 'de'), ('noch', 'de'), ('dass', 'de'), ('nun', 'de'), ('dasselb', 'de'), ('wurd', 'de'), ('von', 'de'), ('geg', 'de'), ('ins', 'de'), ('aber', 'de'), ('was', 'de');INSERT INTO stopwords_short (stopword, language) VALUES ('shine', 'en'), ('vary', 'en'), ('form', 'en'), ('shore', 'en'), ('baby', 'en'), ('many', 'en'), ('buy', 'en'), ('million', 'en'), ('hat', 'en'), ('fat', 'en'), ('january', 'en'), ('stead', 'en'), ('slave', 'en'), ('give', 'en'), ('life', 'en'), ('serve', 'en'), ('thousand', 'en'), ('soldier', 'en'), ('what', 'en'), ('subject', 'en'), ('them', 'en'), ('light', 'en'), ('village', 'en'), ('moment', 'en'), ('captain', 'en'), ('gray', 'en'), ('happen', 'en'), ('again', 'en'), ('twenty', 'en'), ('molecule', 'en'), ('figure', 'en'), ('energy', 'en'), ('system', 'en'), ('story', 'en'), ('wall', 'en'), ('of', 'en'), ('month', 'en'), ('true', 'en'), ('rule', 'en'), ('still', 'en'), ('fraction', 'en'), ('corn', 'en'), ('seed', 'en'), ('wind', 'en'), ('she', 'en'), ('cover', 'en'), ('winter', 'en'), ('december', 'en'), ('will', 'en'), ('nor', 'en'), ('press', 'en'), ('much', 'en'), ('lie', 'en'), ('act', 'en'), ('when', 'en'), ('skin', 'en'), ('imagine', 'en'), ('throw', 'en'), ('chief', 'en'), ('part', 'en'), ('exact', 'en'), ('every', 'en'), ('board', 'en'), ('exercise', 'en'), ('can', 'en'), ('wild', 'en'), ('where', 'en'), ('magnet', 'en'), ('land', 'en'), ('ground', 'en'), ('man', 'en'), ('provide', 'en'), ('wait', 'en'), ('settle', 'en'), ('doctor', 'en'), ('rain', 'en'), ('left', 'en'), ('favor', 'en'), ('way', 'en'), ('experiment', 'en'), ('neighbor', 'en'), ('a', 'en'), ('coast', 'en'), ('bit', 'en'), ('fact', 'en'), ('july', 'en'), ('wrong', 'en'), ('in', 'en'), ('quite', 'en'), ('cause', 'en'), ('stand', 'en'), ('knew', 'en'), ('very', 'en'), ('only', 'en'), ('crop', 'en'), ('may', 'en'), ('me', 'en'), ('by', 'en'), ('moon', 'en'), ('thank', 'en'), ('inch', 'en'), ('believe', 'en'), ('type', 'en'), ('dark', 'en'), ('fair', 'en'), ('cool', 'en'), ('care', 'en'), ('isn', 'en'), ('problem', 'en'), ('share', 'en'), ('cow', 'en'), ('check', 'en'), ('beauty', 'en'), ('death', 'en'), ('sky', 'en'), ('spoke', 'en'), ('fruit', 'en'), ('note', 'en'), ('nature', 'en'), ('there', 'en'), ('heat', 'en'), ('eye', 'en'), ('numeral', 'en'), ('dry', 'en'), ('quick', 'en'), ('fast', 'en'), ('slow', 'en'), ('yes', 'en'), ('market', 'en'), ('fight', 'en'), ('stretch', 'en'), ('soil', 'en'), ('warm', 'en'), ('on', 'en'), ('map', 'en'), ('gone', 'en'), ('matter', 'en'), ('shall', 'en'), ('ran', 'en'), ('race', 'en'), ('dictionary', 'en'), ('study', 'en'), ('sign', 'en'), ('south', 'en'), ('colony', 'en'), ('base', 'en'), ('trouble', 'en'), ('arm', 'en'), ('change', 'en'), ('strong', 'en'), ('quart', 'en'), ('match', 'en'), ('people', 'en'), ('lift', 'en'), ('top', 'en'), ('walk', 'en'), ('november', 'en'), ('poem', 'en'), ('hard', 'en'), ('family', 'en'), ('dress', 'en'), ('number', 'en'), ('front', 'en'), ('visit', 'en'), ('meet', 'en'), ('seven', 'en'), ('case', 'en'), ('certain', 'en'), ('good', 'en'), ('climb', 'en'), ('proper', 'en'), ('occur', 'en'), ('mouth', 'en'), ('vowel', 'en'), ('master', 'en'), ('syllable', 'en'), ('west', 'en'), ('remember', 'en'), ('ago', 'en'), ('fish', 'en'), ('silver', 'en'), ('now', 'en'), ('little', 'en'), ('then', 'en'), ('ear', 'en'), ('shoe', 'en'), ('surface', 'en'), ('test', 'en'), ('produce', 'en'), ('hear', 'en'), ('truck', 'en'), ('slip', 'en'), ('segment', 'en'), ('roll', 'en'), ('state', 'en'), ('blow', 'en'), ('step', 'en'), ('bear', 'en'), ('mass', 'en'), ('atom', 'en'), ('pitch', 'en'), ('few', 'en'), ('down', 'en'), ('about', 'en'), ('sand', 'en'), ('column', 'en'), ('told', 'en'), ('show', 'en'), ('win', 'en'), ('swim', 'en'), ('rub', 'en'), ('how', 'en'), ('ring', 'en'), ('steam', 'en'), ('event', 'en'), ('bell', 'en'), ('york', 'en'), ('those', 'en'), ('prove', 'en'), ('heard', 'en'), ('their', 'en'), ('scale', 'en'), ('best', 'en'), ('suit', 'en'), ('wheel', 'en'), ('flat', 'en'), ('wash', 'en'), ('direct', 'en'), ('chart', 'en'), ('even', 'en'), ('wing', 'en'), ('finish', 'en'), ('up', 'en'), ('oxygen', 'en'), ('famous', 'en'), ('crease', 'en'), ('dead', 'en'), ('king', 'en'), ('should', 'en'), ('say', 'en'), ('mountain', 'en'), ('together', 'en'), ('log', 'en'), ('machine', 'en'), ('plane', 'en'), ('original', 'en'), ('he', 'en'), ('sail', 'en'), ('score', 'en'), ('fresh', 'en'), ('question', 'en'), ('blood', 'en'), ('gather', 'en'), ('point', 'en'), ('sat', 'en'), ('hand', 'en'), ('animal', 'en'), ('cold', 'en'), ('which', 'en'), ('fly', 'en'), ('come', 'en'), ('stone', 'en'), ('far', 'en'), ('rose', 'en'), ('river', 'en'), ('liquid', 'en'), ('least', 'en'), ('bar', 'en'), ('less', 'en'), ('suffix', 'en'), ('keep', 'en'), ('fine', 'en'), ('better', 'en'), ('flow', 'en'), ('own', 'en'), ('sound', 'en'), ('also', 'en'), ('egg', 'en'), ('deep', 'en'), ('rope', 'en'), ('equate', 'en'), ('interest', 'en'), ('chick', 'en'), ('east', 'en'), ('supply', 'en'), ('sun', 'en'), ('poor', 'en'), ('final', 'en'), ('dance', 'en'), ('pull', 'en'), ('continent', 'en'), ('possible', 'en'), ('but', 'en'), ('too', 'en'), ('and', 'en'), ('small', 'en'), ('over', 'en'), ('tall', 'en'), ('six', 'en'), ('divide', 'en'), ('kept', 'en'), ('surprise', 'en'), ('spend', 'en'), ('is', 'en'), ('found', 'en'), ('coat', 'en'), ('men', 'en'), ('color', 'en'), ('watch', 'en'), ('dear', 'en'), ('think', 'en'), ('paper', 'en'), ('station', 'en'), ('anger', 'en'), ('art', 'en'), ('last', 'en'), ('danger', 'en'), ('build', 'en'), ('length', 'en'), ('food', 'en'), ('often', 'en'), ('game', 'en'), ('metal', 'en'), ('were', 'en'), ('next', 'en'), ('law', 'en'), ('pay', 'en'), ('square', 'en'), ('off', 'en'), ('thus', 'en'), ('horse', 'en'), ('like', 'en'), ('turn', 'en'), ('similar', 'en'), ('got', 'en'), ('past', 'en'), ('collect', 'en'), ('iron', 'en'), ('camp', 'en'), ('bright', 'en'), ('cost', 'en'), ('value', 'en'), ('toward', 'en'), ('circle', 'en'), ('seat', 'en'), ('summer', 'en'), ('ask', 'en'), ('evening', 'en'), ('contain', 'en'), ('receive', 'en'), ('push', 'en'), ('word', 'en'), ('hold', 'en'), ('gave', 'en'), ('thick', 'en'), ('fear', 'en'), ('especially', 'en'), ('position', 'en'), ('include', 'en'), ('felt', 'en'), ('same', 'en'), ('indicate', 'en'), ('hit', 'en'), ('deal', 'en'), ('science', 'en'), ('run', 'en'), ('city', 'en'), ('cut', 'en'), ('free', 'en'), ('plant', 'en'), ('person', 'en'), ('rich', 'en'), ('low', 'en'), ('his', 'en'), ('shell', 'en'), ('company', 'en'), ('crowd', 'en'), ('operate', 'en'), ('make', 'en'), ('under', 'en'), ('answer', 'en'), ('industry', 'en'), ('through', 'en'), ('oil', 'en'), ('while', 'en'), ('big', 'en'), ('please', 'en'), ('august', 'en'), ('nothing', 'en'), ('happy', 'en'), ('allow', 'en'), ('hour', 'en'), ('rise', 'en'), ('move', 'en'), ('held', 'en'), ('table', 'en'), ('clear', 'en'), ('that', 'en'), ('take', 'en'), ('milk', 'en'), ('choose', 'en'), ('green', 'en'), ('our', 'en'), ('triangle', 'en'), ('tool', 'en'), ('sent', 'en'), ('yellow', 'en'), ('property', 'en'), ('line', 'en'), ('middle', 'en'), ('grew', 'en'), ('mount', 'en'), ('farm', 'en'), ('dollar', 'en'), ('shout', 'en'), ('join', 'en'), ('lake', 'en'), ('minute', 'en'), ('some', 'en'), ('with', 'en'), ('result', 'en'), ('here', 'en'), ('eat', 'en'), ('brother', 'en'), ('born', 'en'), ('we', 'en'), ('hasn', 'en'), ('solution', 'en'), ('notice', 'en'), ('face', 'en'), ('to', 'en'), ('group', 'en'), ('cat', 'en'), ('gentle', 'en'), ('student', 'en'), ('motion', 'en'), ('school', 'en'), ('home', 'en'), ('teach', 'en'), ('set', 'en'), ('real', 'en'), ('house', 'en'), ('country', 'en'), ('depend', 'en'), ('ease', 'en'), ('wire', 'en'), ('spot', 'en'), ('radio', 'en'), ('job', 'en'), ('spread', 'en'), ('large', 'en'), ('unit', 'en'), ('consonant', 'en'), ('try', 'en'), ('sense', 'en'), ('ocean', 'en'), ('glass', 'en'), ('an', 'en'), ('made', 'en'), ('against', 'en'), ('bed', 'en'), ('page', 'en'), ('yet', 'en'), ('need', 'en'), ('neck', 'en'), ('fell', 'en'), ('aren', 'en'), ('white', 'en'), ('earth', 'en'), ('or', 'en'), ('wonder', 'en'), ('soon', 'en'), ('could', 'en'), ('rock', 'en'), ('read', 'en'), ('music', 'en'), ('does', 'en'), ('field', 'en'), ('once', 'en'), ('broke', 'en'), ('enemy', 'en'), ('grand', 'en'), ('jump', 'en'), ('else', 'en'), ('port', 'en'), ('process', 'en'), ('end', 'en'), ('paint', 'en'), ('miss', 'en'), ('engine', 'en'), ('such', 'en'), ('separate', 'en'), ('shape', 'en'), ('busy', 'en'), ('nation', 'en'), ('reach', 'en'), ('window', 'en'), ('solve', 'en'), ('degree', 'en'), ('young', 'en'), ('are', 'en'), ('center', 'en'), ('insect', 'en'), ('bring', 'en'), ('long', 'en'), ('great', 'en'), ('current', 'en'), ('laugh', 'en'), ('begin', 'en'), ('edge', 'en'), ('connect', 'en'), ('sight', 'en'), ('particular', 'en'), ('hunt', 'en'), ('guess', 'en'), ('arrange', 'en'), ('feed', 'en'), ('example', 'en'), ('between', 'en'), ('yard', 'en'), ('than', 'en'), ('determine', 'en'), ('space', 'en'), ('force', 'en'), ('reply', 'en'), ('june', 'en'), ('just', 'en'), ('measure', 'en'), ('book', 'en'), ('present', 'en'), ('catch', 'en'), ('difficult', 'en'), ('sleep', 'en'), ('these', 'en'), ('melody', 'en'), ('you', 'en'), ('both', 'en'), ('floor', 'en'), ('put', 'en'), ('my', 'en'), ('print', 'en'), ('fun', 'en'), ('heavy', 'en'), ('took', 'en'), ('see', 'en'), ('north', 'en'), ('verb', 'en'), ('didn', 'en'), ('power', 'en'), ('pound', 'en'), ('pose', 'en'), ('expect', 'en'), ('discuss', 'en'), ('symbol', 'en'), ('history', 'en'), ('substance', 'en'), ('period', 'en'), ('night', 'en'), ('fit', 'en'), ('letter', 'en'), ('claim', 'en'), ('old', 'en'), ('though', 'en'), ('follow', 'en'), ('several', 'en'), ('piece', 'en'), ('rest', 'en'), ('time', 'en'), ('round', 'en'), ('girl', 'en'), ('all', 'en'), ('woman', 'en'), ('weather', 'en'), ('speed', 'en'), ('said', 'en'), ('pass', 'en'), ('afraid', 'en'), ('draw', 'en'), ('common', 'en'), ('either', 'en'), ('branch', 'en'), ('paragraph', 'en'), ('thought', 'en'), ('son', 'en'), ('human', 'en'), ('lead', 'en'), ('chair', 'en'), ('would', 'en'), ('spell', 'en'), ('offer', 'en'), ('no', 'en'), ('chord', 'en'), ('kill', 'en'), ('copy', 'en'), ('record', 'en'), ('practice', 'en'), ('might', 'en'), ('order', 'en'), ('red', 'en'), ('sing', 'en'), ('locate', 'en'), ('salt', 'en'), ('valley', 'en'), ('parent', 'en'), ('correct', 'en'), ('total', 'en'), ('cent', 'en'), ('teeth', 'en'), ('feel', 'en'), ('mine', 'en'), ('mind', 'en'), ('support', 'en'), ('general', 'en'), ('women', 'en'), ('object', 'en'), ('add', 'en'), ('invent', 'en'), ('whose', 'en'), ('century', 'en'), ('after', 'en'), ('am', 'en'), ('get', 'en'), ('phrase', 'en'), ('nine', 'en'), ('cell', 'en'), ('control', 'en'), ('huge', 'en'), ('mile', 'en'), ('quiet', 'en'), ('bat', 'en'), ('don', 'en'), ('five', 'en'), ('heart', 'en'), ('cry', 'en'), ('back', 'en'), ('car', 'en'), ('water', 'en'), ('go', 'en'), ('pattern', 'en'), ('hundred', 'en'), ('as', 'en'), ('clock', 'en'), ('why', 'en'), ('product', 'en'), ('february', 'en'), ('complete', 'en'), ('tree', 'en'), ('know', 'en'), ('third', 'en'), ('represent', 'en'), ('has', 'en'), ('string', 'en'), ('consider', 'en'), ('boy', 'en'), ('day', 'en'), ('break', 'en'), ('list', 'en'), ('out', 'en'), ('course', 'en'), ('govern', 'en'), ('second', 'en'), ('reason', 'en'), ('wrote', 'en'), ('element', 'en'), ('skill', 'en'), ('black', 'en'), ('tell', 'en'), ('agree', 'en'), ('seem', 'en'), ('came', 'en'), ('repeat', 'en'), ('love', 'en'), ('wave', 'en'), ('leave', 'en'), ('behind', 'en'), ('leg', 'en'), ('must', 'en'), ('october', 'en'), ('wood', 'en'), ('section', 'en'), ('protect', 'en'), ('size', 'en'), ('body', 'en'), ('design', 'en'), ('bad', 'en'), ('kind', 'en'), ('fall', 'en'), ('able', 'en'), ('create', 'en'), ('during', 'en'), ('sure', 'en'), ('work', 'en'), ('path', 'en'), ('bread', 'en'), ('instrument', 'en'), ('bottom', 'en'), ('single', 'en'), ('hole', 'en'), ('observe', 'en'), ('side', 'en'), ('pretty', 'en'), ('subtract', 'en'), ('probable', 'en'), ('hot', 'en'), ('speak', 'en'), ('street', 'en'), ('since', 'en'), ('self', 'en'), ('multiply', 'en'), ('town', 'en'), ('sell', 'en'), ('whole', 'en'), ('cloud', 'en'), ('went', 'en'), ('new', 'en'), ('flower', 'en'), ('drink', 'en'), ('sentence', 'en'), ('dad', 'en'), ('character', 'en'), ('enough', 'en'), ('raise', 'en'), ('enter', 'en'), ('rather', 'en'), ('other', 'en'), ('whether', 'en'), ('I', 'en'), ('continue', 'en'), ('modern', 'en'), ('lady', 'en'), ('three', 'en'), ('ever', 'en'), ('send', 'en'), ('clothe', 'en'), ('band', 'en'), ('gun', 'en'), ('age', 'en'), ('term', 'en'), ('short', 'en'), ('develop', 'en'), ('plain', 'en'), ('gold', 'en'), ('april', 'en'), ('use', 'en'), ('been', 'en'), ('call', 'en'), ('except', 'en'), ('sugar', 'en'), ('loud', 'en'), ('week', 'en'), ('apple', 'en'), ('noun', 'en'), ('quotient', 'en'), ('blog', 'en'), ('excite', 'en'), ('mix', 'en'), ('smell', 'en'), ('until', 'en'), ('was', 'en'), ('chance', 'en'), ('comment', 'en'), ('nose', 'en'), ('capital', 'en'), ('want', 'en'), ('compare', 'en'), ('plural', 'en'), ('instant', 'en'), ('late', 'en'), ('two', 'en'), ('steel', 'en'), ('if', 'en'), ('key', 'en'), ('save', 'en'), ('view', 'en'), ('foot', 'en'), ('him', 'en'), ('wide', 'en'), ('plan', 'en'), ('temperature', 'en'), ('always', 'en'), ('lost', 'en'), ('each', 'en'), ('count', 'en'), ('finger', 'en'), ('carry', 'en'), ('glad', 'en'), ('tone', 'en'), ('name', 'en'), ('ship', 'en'), ('sit', 'en'), ('open', 'en'), ('appear', 'en'), ('wish', 'en'), ('doesn', 'en'), ('your', 'en'), ('door', 'en'), ('tube', 'en'), ('touch', 'en'), ('noon', 'en'), ('usual', 'en'), ('tire', 'en'), ('start', 'en'), ('box', 'en'), ('learn', 'en'), ('written', 'en'), ('populate', 'en'), ('gas', 'en'), ('bone', 'en'), ('area', 'en'), ('forest', 'en'), ('hope', 'en'), ('die', 'en'), ('have', 'en'), ('cook', 'en'), ('distant', 'en'), ('it', 'en'), ('sheet', 'en'), ('organ', 'en'), ('voice', 'en'), ('right', 'en'), ('talk', 'en'), ('differ', 'en'), ('charge', 'en'), ('dog', 'en'), ('necessary', 'en'), ('require', 'en'), ('led', 'en'), ('electric', 'en'), ('post', 'en'), ('soft', 'en'), ('natural', 'en'), ('special', 'en'), ('march', 'en'), ('ride', 'en'), ('wouldn', 'en'), ('weight', 'en'), ('tail', 'en'), ('write', 'en'), ('safe', 'en'), ('head', 'en'), ('star', 'en'), ('sister', 'en'), ('stream', 'en'), ('guide', 'en'), ('describe', 'en'), ('full', 'en'), ('arrive', 'en'), ('opposite', 'en'), ('card', 'en'), ('burn', 'en'), ('father', 'en'), ('ice', 'en'), ('ready', 'en'), ('smile', 'en'), ('four', 'en'), ('half', 'en'), ('they', 'en'), ('lone', 'en'), ('lot', 'en'), ('money', 'en'), ('war', 'en'), ('cotton', 'en'), ('close', 'en'), ('fill', 'en'), ('fire', 'en'), ('us', 'en'), ('select', 'en'), ('boat', 'en'), ('store', 'en'), ('trip', 'en'), ('track', 'en'), ('mean', 'en'), ('well', 'en'), ('office', 'en'), ('brought', 'en'), ('wear', 'en'), ('shop', 'en'), ('began', 'en'), ('thing', 'en'), ('block', 'en'), ('meat', 'en'), ('eight', 'en'), ('condition', 'en'), ('stick', 'en'), ('forward', 'en'), ('experience', 'en'), ('grow', 'en'), ('bird', 'en'), ('double', 'en'), ('mark', 'en'), ('tiny', 'en'), ('who', 'en'), ('travel', 'en'), ('listen', 'en'), ('row', 'en'), ('sharp', 'en'), ('ball', 'en'), ('beat', 'en'), ('bought', 'en'), ('brown', 'en'), ('caught', 'en'), ('region', 'en'), ('grass', 'en'), ('rail', 'en'), ('bank', 'en'), ('level', 'en'), ('one', 'en'), ('simple', 'en'), ('train', 'en'), ('season', 'en'), ('friend', 'en'), ('division', 'en'), ('did', 'en'), ('range', 'en'), ('drop', 'en'), ('do', 'en'), ('find', 'en'), ('ten', 'en'), ('shoulder', 'en'), ('garden', 'en'), ('oh', 'en'), ('from', 'en'), ('stood', 'en'), ('planet', 'en'), ('root', 'en'), ('method', 'en'), ('hair', 'en'), ('prepare', 'en'), ('her', 'en'), ('feet', 'en'), ('language', 'en'), ('suggest', 'en'), ('effect', 'en'), ('tie', 'en'), ('sea', 'en'), ('any', 'en'), ('party', 'en'), ('stay', 'en'), ('strange', 'en'), ('perhaps', 'en'), ('live', 'en'), ('decimal', 'en'), ('more', 'en'), ('year', 'en'), ('decide', 'en'), ('material', 'en'), ('the', 'en'), ('hill', 'en'), ('search', 'en'), ('done', 'en'), ('cross', 'en'), ('island', 'en'), ('sudden', 'en'), ('speech', 'en'), ('spring', 'en'), ('pair', 'en'), ('dream', 'en'), ('let', 'en'), ('this', 'en'), ('before', 'en'), ('success', 'en'), ('so', 'en'), ('wife', 'en'), ('lay', 'en'), ('song', 'en'), ('for', 'en'), ('meant', 'en'), ('pick', 'en'), ('duck', 'en'), ('clean', 'en'), ('near', 'en'), ('among', 'en'), ('main', 'en'), ('morning', 'en'), ('be', 'en'), ('idea', 'en'), ('corner', 'en'), ('class', 'en'), ('team', 'en'), ('major', 'en'), ('september', 'en'), ('basic', 'en'), ('most', 'en'), ('world', 'en'), ('place', 'en'), ('room', 'en'), ('never', 'en'), ('look', 'en'), ('high', 'en'), ('saw', 'en'), ('straight', 'en'), ('drive', 'en'), ('blue', 'en'), ('early', 'en'), ('above', 'en'), ('at', 'en'), ('fig', 'en'), ('help', 'en'), ('silent', 'en'), ('picture', 'en'), ('equal', 'en'), ('joy', 'en'), ('desert', 'en'), ('had', 'en'), ('first', 'en'), ('stop', 'en'), ('thin', 'en'), ('trade', 'en'), ('play', 'en'), ('mother', 'en'), ('snow', 'en'), ('hurry', 'en'), ('broad', 'en'), ('shouldn', 'en'), ('road', 'en'), ('air', 'en'), ('noise', 'en');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('shine', 'en'), ('tini', 'en'), ('form', 'en'), ('shore', 'en'), ('buy', 'en'), ('fat', 'en'), ('hat', 'en'), ('million', 'en'), ('slave', 'en'), ('stead', 'en'), ('give', 'en'), ('life', 'en'), ('sens', 'en'), ('soldier', 'en'), ('thousand', 'en'), ('what', 'en'), ('subject', 'en'), ('them', 'en'), ('light', 'en'), ('perhap', 'en'), ('captain', 'en'), ('moment', 'en'), ('arriv', 'en'), ('gray', 'en'), ('again', 'en'), ('happen', 'en'), ('tabl', 'en'), ('system', 'en'), ('month', 'en'), ('of', 'en'), ('wall', 'en'), ('rule', 'en'), ('true', 'en'), ('corn', 'en'), ('fraction', 'en'), ('still', 'en'), ('serv', 'en'), ('seed', 'en'), ('she', 'en'), ('wind', 'en'), ('cover', 'en'), ('winter', 'en'), ('nor', 'en'), ('will', 'en'), ('juli', 'en'), ('press', 'en'), ('lie', 'en'), ('much', 'en'), ('act', 'en'), ('skin', 'en'), ('when', 'en'), ('throw', 'en'), ('chief', 'en'), ('exact', 'en'), ('part', 'en'), ('board', 'en'), ('can', 'en'), ('magnet', 'en'), ('where', 'en'), ('wild', 'en'), ('ground', 'en'), ('land', 'en'), ('man', 'en'), ('doctor', 'en'), ('wait', 'en'), ('eas', 'en'), ('favor', 'en'), ('left', 'en'), ('rain', 'en'), ('way', 'en'), ('a', 'en'), ('neighbor', 'en'), ('bit', 'en'), ('coast', 'en'), ('fact', 'en'), ('wrong', 'en'), ('in', 'en'), ('stand', 'en'), ('knew', 'en'), ('crop', 'en'), ('may', 'en'), ('me', 'en'), ('by', 'en'), ('moon', 'en'), ('thank', 'en'), ('inch', 'en'), ('type', 'en'), ('dark', 'en'), ('fair', 'en'), ('pleas', 'en'), ('alway', 'en'), ('care', 'en'), ('cool', 'en'), ('isn', 'en'), ('problem', 'en'), ('creat', 'en'), ('forc', 'en'), ('check', 'en'), ('cow', 'en'), ('share', 'en'), ('describ', 'en'), ('death', 'en'), ('sky', 'en'), ('complet', 'en'), ('twenti', 'en'), ('edg', 'en'), ('fruit', 'en'), ('spoke', 'en'), ('note', 'en'), ('heat', 'en'), ('there', 'en'), ('singl', 'en'), ('eye', 'en'), ('fast', 'en'), ('quick', 'en'), ('slow', 'en'), ('market', 'en'), ('yes', 'en'), ('fight', 'en'), ('separ', 'en'), ('soil', 'en'), ('stretch', 'en'), ('warm', 'en'), ('exercis', 'en'), ('strang', 'en'), ('languag', 'en'), ('gone', 'en'), ('map', 'en'), ('on', 'en'), ('matter', 'en'), ('shall', 'en'), ('gentl', 'en'), ('race', 'en'), ('ran', 'en'), ('sign', 'en'), ('base', 'en'), ('south', 'en'), ('arm', 'en'), ('carri', 'en'), ('quart', 'en'), ('strong', 'en'), ('match', 'en'), ('chang', 'en'), ('lift', 'en'), ('top', 'en'), ('walk', 'en'), ('valu', 'en'), ('novemb', 'en'), ('poem', 'en'), ('temperatur', 'en'), ('dress', 'en'), ('hard', 'en'), ('front', 'en'), ('number', 'en'), ('meet', 'en'), ('visit', 'en'), ('case', 'en'), ('seven', 'en'), ('exampl', 'en'), ('certain', 'en'), ('charg', 'en'), ('climb', 'en'), ('good', 'en'), ('occur', 'en'), ('proper', 'en'), ('mouth', 'en'), ('master', 'en'), ('vowel', 'en'), ('west', 'en'), ('ago', 'en'), ('fish', 'en'), ('silver', 'en'), ('now', 'en'), ('ear', 'en'), ('then', 'en'), ('shoe', 'en'), ('test', 'en'), ('vari', 'en'), ('hear', 'en'), ('octob', 'en'), ('segment', 'en'), ('slip', 'en'), ('truck', 'en'), ('roll', 'en'), ('blow', 'en'), ('state', 'en'), ('morn', 'en'), ('bear', 'en'), ('step', 'en'), ('mani', 'en'), ('atom', 'en'), ('mass', 'en'), ('few', 'en'), ('pitch', 'en'), ('about', 'en'), ('down', 'en'), ('column', 'en'), ('sand', 'en'), ('creas', 'en'), ('show', 'en'), ('told', 'en'), ('rub', 'en'), ('swim', 'en'), ('win', 'en'), ('how', 'en'), ('ring', 'en'), ('event', 'en'), ('steam', 'en'), ('bell', 'en'), ('continu', 'en'), ('heard', 'en'), ('prove', 'en'), ('those', 'en'), ('york', 'en'), ('their', 'en'), ('scale', 'en'), ('best', 'en'), ('suit', 'en'), ('flat', 'en'), ('wheel', 'en'), ('chart', 'en'), ('direct', 'en'), ('wash', 'en'), ('even', 'en'), ('finish', 'en'), ('wing', 'en'), ('up', 'en'), ('famous', 'en'), ('oxygen', 'en'), ('onc', 'en'), ('posit', 'en'), ('dead', 'en'), ('king', 'en'), ('mountain', 'en'), ('say', 'en'), ('should', 'en'), ('decim', 'en'), ('log', 'en'), ('pretti', 'en'), ('divis', 'en'), ('plane', 'en'), ('he', 'en'), ('sail', 'en'), ('fresh', 'en'), ('score', 'en'), ('energi', 'en'), ('condit', 'en'), ('blood', 'en'), ('question', 'en'), ('gather', 'en'), ('point', 'en'), ('hand', 'en'), ('sat', 'en'), ('cold', 'en'), ('which', 'en'), ('come', 'en'), ('stone', 'en'), ('necessari', 'en'), ('far', 'en'), ('rose', 'en'), ('bar', 'en'), ('least', 'en'), ('liquid', 'en'), ('river', 'en'), ('less', 'en'), ('measur', 'en'), ('better', 'en'), ('fine', 'en'), ('keep', 'en'), ('suffix', 'en'), ('flow', 'en'), ('own', 'en'), ('egg', 'en'), ('also', 'en'), ('sound', 'en'), ('deep', 'en'), ('parti', 'en'), ('rope', 'en'), ('interest', 'en'), ('east', 'en'), ('chick', 'en'), ('final', 'en'), ('poor', 'en'), ('sun', 'en'), ('pull', 'en'), ('but', 'en'), ('and', 'en'), ('too', 'en'), ('chanc', 'en'), ('small', 'en'), ('electr', 'en'), ('over', 'en'), ('requir', 'en'), ('six', 'en'), ('tall', 'en'), ('dri', 'en'), ('kept', 'en'), ('found', 'en'), ('is', 'en'), ('spend', 'en'), ('coat', 'en'), ('color', 'en'), ('men', 'en'), ('dear', 'en'), ('watch', 'en'), ('think', 'en'), ('paper', 'en'), ('station', 'en'), ('art', 'en'), ('anger', 'en'), ('last', 'en'), ('danger', 'en'), ('build', 'en'), ('notic', 'en'), ('food', 'en'), ('length', 'en'), ('sinc', 'en'), ('often', 'en'), ('game', 'en'), ('metal', 'en'), ('were', 'en'), ('villag', 'en'), ('law', 'en'), ('next', 'en'), ('pay', 'en'), ('off', 'en'), ('thus', 'en'), ('like', 'en'), ('similar', 'en'), ('turn', 'en'), ('got', 'en'), ('cloth', 'en'), ('collect', 'en'), ('past', 'en'), ('iron', 'en'), ('camp', 'en'), ('ladi', 'en'), ('cost', 'en'), ('bright', 'en'), ('toward', 'en'), ('seat', 'en'), ('summer', 'en'), ('ask', 'en'), ('contain', 'en'), ('push', 'en'), ('word', 'en'), ('gave', 'en'), ('hold', 'en'), ('thick', 'en'), ('befor', 'en'), ('fear', 'en'), ('arrang', 'en'), ('felt', 'en'), ('hit', 'en'), ('same', 'en'), ('deal', 'en'), ('cut', 'en'), ('run', 'en'), ('free', 'en'), ('person', 'en'), ('plant', 'en'), ('natur', 'en'), ('his', 'en'), ('low', 'en'), ('rich', 'en'), ('shell', 'en'), ('agre', 'en'), ('crowd', 'en'), ('make', 'en'), ('triangl', 'en'), ('under', 'en'), ('answer', 'en'), ('cri', 'en'), ('oil', 'en'), ('through', 'en'), ('while', 'en'), ('onli', 'en'), ('big', 'en'), ('august', 'en'), ('allow', 'en'), ('degre', 'en'), ('hour', 'en'), ('held', 'en'), ('move', 'en'), ('rise', 'en'), ('syllabl', 'en'), ('clear', 'en'), ('that', 'en'), ('take', 'en'), ('milk', 'en'), ('green', 'en'), ('our', 'en'), ('copi', 'en'), ('tool', 'en'), ('excit', 'en'), ('sent', 'en'), ('danc', 'en'), ('line', 'en'), ('yellow', 'en'), ('grew', 'en'), ('mount', 'en'), ('materi', 'en'), ('dollar', 'en'), ('farm', 'en'), ('join', 'en'), ('shout', 'en'), ('probabl', 'en'), ('lake', 'en'), ('veri', 'en'), ('some', 'en'), ('with', 'en'), ('result', 'en'), ('eat', 'en'), ('here', 'en'), ('brother', 'en'), ('born', 'en'), ('hasn', 'en'), ('we', 'en'), ('face', 'en'), ('cat', 'en'), ('group', 'en'), ('to', 'en'), ('possibl', 'en'), ('student', 'en'), ('motion', 'en'), ('school', 'en'), ('home', 'en'), ('real', 'en'), ('set', 'en'), ('teach', 'en'), ('depend', 'en'), ('wire', 'en'), ('simpl', 'en'), ('job', 'en'), ('radio', 'en'), ('spot', 'en'), ('spread', 'en'), ('unit', 'en'), ('solv', 'en'), ('an', 'en'), ('glass', 'en'), ('ocean', 'en'), ('circl', 'en'), ('against', 'en'), ('made', 'en'), ('engin', 'en'), ('bed', 'en'), ('page', 'en'), ('yet', 'en'), ('neck', 'en'), ('need', 'en'), ('aren', 'en'), ('fell', 'en'), ('earth', 'en'), ('white', 'en'), ('or', 'en'), ('oper', 'en'), ('could', 'en'), ('soon', 'en'), ('wonder', 'en'), ('coloni', 'en'), ('music', 'en'), ('read', 'en'), ('rock', 'en'), ('field', 'en'), ('compar', 'en'), ('broke', 'en'), ('grand', 'en'), ('decemb', 'en'), ('origin', 'en'), ('jump', 'en'), ('januari', 'en'), ('port', 'en'), ('end', 'en'), ('process', 'en'), ('paint', 'en'), ('miss', 'en'), ('such', 'en'), ('shape', 'en'), ('surpris', 'en'), ('nation', 'en'), ('reach', 'en'), ('window', 'en'), ('are', 'en'), ('young', 'en'), ('center', 'en'), ('countri', 'en'), ('voic', 'en'), ('beauti', 'en'), ('bring', 'en'), ('insect', 'en'), ('current', 'en'), ('great', 'en'), ('long', 'en'), ('includ', 'en'), ('begin', 'en'), ('laugh', 'en'), ('connect', 'en'), ('sight', 'en'), ('guess', 'en'), ('hunt', 'en'), ('particular', 'en'), ('feed', 'en'), ('between', 'en'), ('yard', 'en'), ('than', 'en'), ('space', 'en'), ('solut', 'en'), ('just', 'en'), ('june', 'en'), ('book', 'en'), ('present', 'en'), ('difficult', 'en'), ('catch', 'en'), ('sleep', 'en'), ('stori', 'en'), ('these', 'en'), ('you', 'en'), ('both', 'en'), ('floor', 'en'), ('my', 'en'), ('put', 'en'), ('fun', 'en'), ('print', 'en'), ('see', 'en'), ('took', 'en'), ('offic', 'en'), ('north', 'en'), ('didn', 'en'), ('verb', 'en'), ('pound', 'en'), ('power', 'en'), ('expect', 'en'), ('pose', 'en'), ('discuss', 'en'), ('symbol', 'en'), ('fit', 'en'), ('night', 'en'), ('period', 'en'), ('letter', 'en'), ('settl', 'en'), ('claim', 'en'), ('old', 'en'), ('happi', 'en'), ('dure', 'en'), ('though', 'en'), ('follow', 'en'), ('rest', 'en'), ('time', 'en'), ('round', 'en'), ('girl', 'en'), ('all', 'en'), ('woman', 'en'), ('speed', 'en'), ('weather', 'en'), ('pass', 'en'), ('said', 'en'), ('draw', 'en'), ('afraid', 'en'), ('either', 'en'), ('common', 'en'), ('branch', 'en'), ('pictur', 'en'), ('locat', 'en'), ('heavi', 'en'), ('paragraph', 'en'), ('thought', 'en'), ('bodi', 'en'), ('lead', 'en'), ('human', 'en'), ('son', 'en'), ('chair', 'en'), ('offer', 'en'), ('spell', 'en'), ('would', 'en'), ('chord', 'en'), ('no', 'en'), ('kill', 'en'), ('earli', 'en'), ('septemb', 'en'), ('might', 'en'), ('record', 'en'), ('order', 'en'), ('red', 'en'), ('sing', 'en'), ('sentenc', 'en'), ('salt', 'en'), ('correct', 'en'), ('parent', 'en'), ('valley', 'en'), ('figur', 'en'), ('total', 'en'), ('cent', 'en'), ('feel', 'en'), ('teeth', 'en'), ('mind', 'en'), ('mine', 'en'), ('general', 'en'), ('support', 'en'), ('object', 'en'), ('women', 'en'), ('add', 'en'), ('minut', 'en'), ('invent', 'en'), ('doe', 'en'), ('whose', 'en'), ('peopl', 'en'), ('after', 'en'), ('numer', 'en'), ('am', 'en'), ('get', 'en'), ('guid', 'en'), ('cell', 'en'), ('nine', 'en'), ('phrase', 'en'), ('mile', 'en'), ('huge', 'en'), ('control', 'en'), ('bat', 'en'), ('quiet', 'en'), ('heart', 'en'), ('five', 'en'), ('don', 'en'), ('observ', 'en'), ('car', 'en'), ('back', 'en'), ('water', 'en'), ('go', 'en'), ('conson', 'en'), ('pattern', 'en'), ('as', 'en'), ('clock', 'en'), ('believ', 'en'), ('product', 'en'), ('readi', 'en'), ('surfac', 'en'), ('know', 'en'), ('tree', 'en'), ('industri', 'en'), ('has', 'en'), ('third', 'en'), ('string', 'en'), ('dictionari', 'en'), ('boy', 'en'), ('day', 'en'), ('list', 'en'), ('break', 'en'), ('govern', 'en'), ('out', 'en'), ('second', 'en'), ('reason', 'en'), ('wrote', 'en'), ('element', 'en'), ('especi', 'en'), ('black', 'en'), ('skill', 'en'), ('practic', 'en'), ('tell', 'en'), ('came', 'en'), ('seem', 'en'), ('melodi', 'en'), ('love', 'en'), ('repeat', 'en'), ('wave', 'en'), ('leg', 'en'), ('behind', 'en'), ('must', 'en'), ('wood', 'en'), ('tri', 'en'), ('protect', 'en'), ('section', 'en'), ('size', 'en'), ('design', 'en'), ('kind', 'en'), ('bad', 'en'), ('fall', 'en'), ('choos', 'en'), ('cours', 'en'), ('citi', 'en'), ('charact', 'en'), ('sure', 'en'), ('famili', 'en'), ('instrument', 'en'), ('bread', 'en'), ('path', 'en'), ('work', 'en'), ('bottom', 'en'), ('caus', 'en'), ('hole', 'en'), ('side', 'en'), ('subtract', 'en'), ('divid', 'en'), ('hot', 'en'), ('speak', 'en'), ('everi', 'en'), ('street', 'en'), ('prepar', 'en'), ('self', 'en'), ('town', 'en'), ('produc', 'en'), ('sell', 'en'), ('whole', 'en'), ('cloud', 'en'), ('went', 'en'), ('flower', 'en'), ('new', 'en'), ('drink', 'en'), ('dad', 'en'), ('enough', 'en'), ('whi', 'en'), ('enter', 'en'), ('rather', 'en'), ('other', 'en'), ('whether', 'en'), ('suppli', 'en'), ('modern', 'en'), ('popul', 'en'), ('ever', 'en'), ('three', 'en'), ('februari', 'en'), ('send', 'en'), ('band', 'en'), ('gun', 'en'), ('age', 'en'), ('term', 'en'), ('develop', 'en'), ('short', 'en'), ('gold', 'en'), ('plain', 'en'), ('use', 'en'), ('april', 'en'), ('been', 'en'), ('hurri', 'en'), ('except', 'en'), ('call', 'en'), ('sugar', 'en'), ('loud', 'en'), ('quit', 'en'), ('week', 'en'), ('noun', 'en'), ('properti', 'en'), ('hous', 'en'), ('nois', 'en'), ('blog', 'en'), ('quotient', 'en'), ('sever', 'en'), ('mix', 'en'), ('smell', 'en'), ('until', 'en'), ('ani', 'en'), ('comment', 'en'), ('was', 'en'), ('nose', 'en'), ('substanc', 'en'), ('want', 'en'), ('plural', 'en'), ('busi', 'en'), ('equat', 'en'), ('late', 'en'), ('instant', 'en'), ('steel', 'en'), ('two', 'en'), ('key', 'en'), ('if', 'en'), ('save', 'en'), ('compani', 'en'), ('him', 'en'), ('foot', 'en'), ('view', 'en'), ('abov', 'en'), ('repres', 'en'), ('middl', 'en'), ('plan', 'en'), ('wide', 'en'), ('lost', 'en'), ('finger', 'en'), ('count', 'en'), ('each', 'en'), ('glad', 'en'), ('tone', 'en'), ('name', 'en'), ('ship', 'en'), ('appear', 'en'), ('open', 'en'), ('sit', 'en'), ('doesn', 'en'), ('wish', 'en'), ('door', 'en'), ('your', 'en'), ('tube', 'en'), ('scienc', 'en'), ('noon', 'en'), ('touch', 'en'), ('tire', 'en'), ('usual', 'en'), ('learn', 'en'), ('box', 'en'), ('start', 'en'), ('written', 'en'), ('troubl', 'en'), ('gas', 'en'), ('area', 'en'), ('bone', 'en'), ('hope', 'en'), ('forest', 'en'), ('larg', 'en'), ('die', 'en'), ('squar', 'en'), ('decid', 'en'), ('have', 'en'), ('littl', 'en'), ('cook', 'en'), ('experi', 'en'), ('it', 'en'), ('distant', 'en'), ('leav', 'en'), ('sheet', 'en'), ('organ', 'en'), ('right', 'en'), ('differ', 'en'), ('talk', 'en'), ('dog', 'en'), ('els', 'en'), ('led', 'en'), ('post', 'en'), ('soft', 'en'), ('hors', 'en'), ('special', 'en'), ('ride', 'en'), ('march', 'en'), ('wouldn', 'en'), ('tail', 'en'), ('weight', 'en'), ('write', 'en'), ('safe', 'en'), ('rais', 'en'), ('head', 'en'), ('machin', 'en'), ('sister', 'en'), ('star', 'en'), ('stream', 'en'), ('studi', 'en'), ('full', 'en'), ('doubl', 'en'), ('burn', 'en'), ('card', 'en'), ('father', 'en'), ('ice', 'en'), ('consid', 'en'), ('opposit', 'en'), ('indic', 'en'), ('four', 'en'), ('smile', 'en'), ('half', 'en'), ('lot', 'en'), ('lone', 'en'), ('they', 'en'), ('provid', 'en'), ('money', 'en'), ('cotton', 'en'), ('war', 'en'), ('fill', 'en'), ('close', 'en'), ('fire', 'en'), ('us', 'en'), ('select', 'en'), ('i', 'en'), ('fli', 'en'), ('boat', 'en'), ('store', 'en'), ('track', 'en'), ('trip', 'en'), ('mean', 'en'), ('anim', 'en'), ('brought', 'en'), ('well', 'en'), ('began', 'en'), ('shop', 'en'), ('wear', 'en'), ('meat', 'en'), ('block', 'en'), ('thing', 'en'), ('eight', 'en'), ('stick', 'en'), ('forward', 'en'), ('grow', 'en'), ('hundr', 'en'), ('bird', 'en'), ('mark', 'en'), ('appl', 'en'), ('listen', 'en'), ('travel', 'en'), ('who', 'en'), ('rememb', 'en'), ('row', 'en'), ('brown', 'en'), ('bought', 'en'), ('beat', 'en'), ('ball', 'en'), ('sharp', 'en'), ('caught', 'en'), ('grass', 'en'), ('region', 'en'), ('repli', 'en'), ('bank', 'en'), ('rail', 'en'), ('level', 'en'), ('one', 'en'), ('train', 'en'), ('abl', 'en'), ('season', 'en'), ('friend', 'en'), ('centuri', 'en'), ('receiv', 'en'), ('did', 'en'), ('do', 'en'), ('drop', 'en'), ('find', 'en'), ('ten', 'en'), ('garden', 'en'), ('shoulder', 'en'), ('contin', 'en'), ('oh', 'en'), ('from', 'en'), ('planet', 'en'), ('stood', 'en'), ('method', 'en'), ('root', 'en'), ('hair', 'en'), ('her', 'en'), ('determin', 'en'), ('feet', 'en'), ('effect', 'en'), ('suggest', 'en'), ('sea', 'en'), ('tie', 'en'), ('stay', 'en'), ('enemi', 'en'), ('live', 'en'), ('babi', 'en'), ('more', 'en'), ('year', 'en'), ('hill', 'en'), ('the', 'en'), ('done', 'en'), ('search', 'en'), ('island', 'en'), ('cross', 'en'), ('sudden', 'en'), ('speech', 'en'), ('spring', 'en'), ('let', 'en'), ('dream', 'en'), ('pair', 'en'), ('this', 'en'), ('so', 'en'), ('success', 'en'), ('capit', 'en'), ('lay', 'en'), ('wife', 'en'), ('for', 'en'), ('song', 'en'), ('meant', 'en'), ('molecul', 'en'), ('duck', 'en'), ('pick', 'en'), ('clean', 'en'), ('togeth', 'en'), ('main', 'en'), ('among', 'en'), ('near', 'en'), ('noth', 'en'), ('be', 'en'), ('idea', 'en'), ('class', 'en'), ('corner', 'en'), ('major', 'en'), ('team', 'en'), ('basic', 'en'), ('most', 'en'), ('imagin', 'en'), ('place', 'en'), ('world', 'en'), ('room', 'en'), ('look', 'en'), ('never', 'en'), ('high', 'en'), ('saw', 'en'), ('blue', 'en'), ('drive', 'en'), ('straight', 'en'), ('help', 'en'), ('fig', 'en'), ('at', 'en'), ('joy', 'en'), ('equal', 'en'), ('silent', 'en'), ('piec', 'en'), ('had', 'en'), ('desert', 'en'), ('first', 'en'), ('stop', 'en'), ('thin', 'en'), ('rang', 'en'), ('mother', 'en'), ('play', 'en'), ('trade', 'en'), ('snow', 'en'), ('histori', 'en'), ('broad', 'en'), ('air', 'en'), ('road', 'en'), ('shouldn', 'en'), ('multipli', 'en');INSERT INTO stopwords_short (stopword, language) VALUES ('estuvierais', 'es'), ('fueras', 'es'), ('fuiste', 'es'), ('para', 'es'), ('otro', 'es'), ('siendo', 'es'), ('estuviesen', 'es'), ('e', 'es'), ('tuvieron', 'es'), ('tendrás', 'es'), ('hubisteis', 'es'), ('tuvo', 'es'), ('eran', 'es'), ('eres', 'es'), ('está', 'es'), ('estaba', 'es'), ('ti', 'es'), ('seáis', 'es'), ('estáis', 'es'), ('tuvieran', 'es'), ('estuviste', 'es'), ('mi', 'es'), ('tiene', 'es'), ('por', 'es'), ('tuya', 'es'), ('cual', 'es'), ('habrá', 'es'), ('otros', 'es'), ('algunas', 'es'), ('contra', 'es'), ('tenías', 'es'), ('sido', 'es'), ('se', 'es'), ('mío', 'es'), ('estábamos', 'es'), ('les', 'es'), ('son', 'es'), ('tuvieseis', 'es'), ('a', 'es'), ('no', 'es'), ('ese', 'es'), ('hubiste', 'es'), ('tuyos', 'es'), ('tengo', 'es'), ('una', 'es'), ('más', 'es'), ('mías', 'es'), ('estaréis', 'es'), ('estuviese', 'es'), ('me', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('hayan', 'es'), ('estaríais', 'es'), ('que', 'es'), ('estabas', 'es'), ('hubimos', 'es'), ('nuestra', 'es'), ('seríamos', 'es'), ('eso', 'es'), ('tuve', 'es'), ('estas', 'es'), ('sobre', 'es'), ('habido', 'es'), ('fuimos', 'es'), ('vuestra', 'es'), ('también', 'es'), ('unos', 'es'), ('hemos', 'es'), ('esos', 'es'), ('vuestros', 'es'), ('estuviéramos', 'es'), ('sería', 'es'), ('o', 'es'), ('tenían', 'es'), ('has', 'es'), ('tenemos', 'es'), ('estuvieron', 'es'), ('estemos', 'es'), ('hay', 'es'), ('sí', 'es'), ('fuera', 'es'), ('tuvisteis', 'es'), ('estuvieseis', 'es'), ('habrían', 'es'), ('al', 'es'), ('fuéramos', 'es'), ('habidas', 'es'), ('teníais', 'es'), ('fuese', 'es'), ('habíamos', 'es'), ('estarán', 'es'), ('yo', 'es'), ('qué', 'es'), ('vosotras', 'es'), ('habrás', 'es'), ('hayamos', 'es'), ('estuviera', 'es'), ('suyas', 'es'), ('del', 'es'), ('habías', 'es'), ('estarás', 'es'), ('hayáis', 'es'), ('estará', 'es'), ('somos', 'es'), ('otra', 'es'), ('tendrías', 'es'), ('habían', 'es'), ('estuvieran', 'es'), ('es', 'es'), ('fuésemos', 'es'), ('habríamos', 'es'), ('otras', 'es'), ('estuvisteis', 'es'), ('vuestro', 'es'), ('esas', 'es'), ('tuviéramos', 'es'), ('hasta', 'es'), ('estad', 'es'), ('estaban', 'es'), ('mí', 'es'), ('míos', 'es'), ('tuviste', 'es'), ('estado', 'es'), ('en', 'es'), ('de', 'es'), ('un', 'es'), ('serían', 'es'), ('fuerais', 'es'), ('estadas', 'es'), ('teniendo', 'es'), ('estando', 'es'), ('como', 'es'), ('habrán', 'es'), ('estuviésemos', 'es'), ('fui', 'es'), ('seréis', 'es'), ('he', 'es'), ('estaríamos', 'es'), ('quienes', 'es'), ('esto', 'es'), ('los', 'es'), ('habida', 'es'), ('tuviésemos', 'es'), ('estar', 'es'), ('estuve', 'es'), ('sus', 'es'), ('tendrían', 'es'), ('tienen', 'es'), ('nuestros', 'es'), ('le', 'es'), ('estoy', 'es'), ('y', 'es'), ('nos', 'es'), ('estás', 'es'), ('hubieses', 'es'), ('sean', 'es'), ('tengáis', 'es'), ('estuvimos', 'es'), ('hayas', 'es'), ('nuestras', 'es'), ('pero', 'es'), ('porque', 'es'), ('fuesen', 'es'), ('tened', 'es'), ('esté', 'es'), ('seremos', 'es'), ('tuvieses', 'es'), ('mis', 'es'), ('sois', 'es'), ('habidos', 'es'), ('seré', 'es'), ('suyos', 'es'), ('hubieseis', 'es'), ('tendrán', 'es'), ('uno', 'es'), ('la', 'es'), ('serán', 'es'), ('estaremos', 'es'), ('con', 'es'), ('nosotras', 'es'), ('tenidas', 'es'), ('tuvierais', 'es'), ('todo', 'es'), ('estaría', 'es'), ('nosotros', 'es'), ('ellos', 'es'), ('ellas', 'es'), ('tuviesen', 'es'), ('han', 'es'), ('fue', 'es'), ('tendríais', 'es'), ('habréis', 'es'), ('tuviese', 'es'), ('cuando', 'es'), ('hubieran', 'es'), ('tenidos', 'es'), ('ya', 'es'), ('fueran', 'es'), ('teníamos', 'es'), ('os', 'es'), ('tenía', 'es'), ('tenéis', 'es'), ('durante', 'es'), ('tu', 'es'), ('habríais', 'es'), ('tenida', 'es'), ('era', 'es'), ('hubierais', 'es'), ('suyo', 'es'), ('fueron', 'es'), ('tuvieras', 'es'), ('tendremos', 'es'), ('estés', 'es'), ('estuvo', 'es'), ('sin', 'es'), ('tenido', 'es'), ('tuvimos', 'es'), ('tengas', 'es'), ('algunos', 'es'), ('este', 'es'), ('habrías', 'es'), ('habría', 'es'), ('ha', 'es'), ('habéis', 'es'), ('las', 'es'), ('había', 'es'), ('tienes', 'es'), ('estén', 'es'), ('nada', 'es'), ('habíais', 'es'), ('tú', 'es'), ('tuviera', 'es'), ('éramos', 'es'), ('tuyas', 'es'), ('hubieron', 'es'), ('tendría', 'es'), ('estuvieras', 'es'), ('antes', 'es'), ('su', 'es'), ('estabais', 'es'), ('suya', 'es'), ('mucho', 'es'), ('hubiésemos', 'es'), ('quien', 'es'), ('habré', 'es'), ('lo', 'es'), ('hubo', 'es'), ('soy', 'es'), ('todos', 'es'), ('tus', 'es'), ('estos', 'es'), ('vuestras', 'es'), ('hubiesen', 'es'), ('será', 'es'), ('tengamos', 'es'), ('poco', 'es'), ('tengan', 'es'), ('tenga', 'es'), ('estaré', 'es'), ('ni', 'es'), ('desde', 'es'), ('ella', 'es'), ('estada', 'es'), ('estarían', 'es'), ('haya', 'es'), ('sea', 'es'), ('tuyo', 'es'), ('entre', 'es'), ('tendréis', 'es'), ('vosotros', 'es'), ('muchos', 'es'), ('estéis', 'es'), ('eras', 'es'), ('donde', 'es'), ('tendríamos', 'es'), ('fueses', 'es'), ('estados', 'es'), ('algo', 'es'), ('hubiese', 'es'), ('seas', 'es'), ('estarías', 'es'), ('serías', 'es'), ('muy', 'es'), ('hubiera', 'es'), ('hubiéramos', 'es'), ('esta', 'es'), ('estuvieses', 'es'), ('tendrá', 'es'), ('erais', 'es'), ('mía', 'es'), ('estamos', 'es'), ('hube', 'es'), ('hubieras', 'es'), ('habiendo', 'es'), ('habremos', 'es'), ('seamos', 'es'), ('seríais', 'es'), ('él', 'es'), ('están', 'es'), ('ante', 'es'), ('esa', 'es'), ('te', 'es'), ('tendré', 'es'), ('serás', 'es'), ('nuestro', 'es'), ('el', 'es'), ('tanto', 'es');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('sus', 'es'), ('sid', 'es'), ('som', 'es'), ('esteis', 'es'), ('le', 'es'), ('tendras', 'es'), ('estoy', 'es'), ('otro', 'es'), ('nos', 'es'), ('y', 'es'), ('sean', 'es'), ('e', 'es'), ('qui', 'es'), ('porqu', 'es'), ('nad', 'es'), ('mis', 'es'), ('eran', 'es'), ('eres', 'es'), ('sois', 'es'), ('habran', 'es'), ('hem', 'es'), ('tien', 'es'), ('uno', 'es'), ('ti', 'es'), ('la', 'es'), ('con', 'es'), ('much', 'es'), ('estem', 'es'), ('mi', 'es'), ('fues', 'es'), ('ellas', 'es'), ('ellos', 'es'), ('por', 'es'), ('cual', 'es'), ('otros', 'es'), ('fue', 'es'), ('han', 'es'), ('vosotr', 'es'), ('esten', 'es'), ('se', 'es'), ('sobr', 'es'), ('com', 'es'), ('les', 'es'), ('son', 'es'), ('a', 'es'), ('ya', 'es'), ('dond', 'es'), ('no', 'es'), ('ese', 'es'), ('estam', 'es'), ('tuy', 'es'), ('os', 'es'), ('tu', 'es'), ('una', 'es'), ('seras', 'es'), ('estes', 'es'), ('era', 'es'), ('mas', 'es'), ('si', 'es'), ('fueron', 'es'), ('me', 'es'), ('sin', 'es'), ('mios', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('que', 'es'), ('desd', 'es'), ('este', 'es'), ('estaran', 'es'), ('ha', 'es'), ('eso', 'es'), ('par', 'es'), ('estas', 'es'), ('las', 'es'), ('estand', 'es'), ('unos', 'es'), ('mia', 'es'), ('poc', 'es'), ('esos', 'es'), ('estaras', 'es'), ('estuv', 'es'), ('tambien', 'es'), ('o', 'es'), ('su', 'es'), ('antes', 'es'), ('estabais', 'es'), ('tuv', 'es'), ('has', 'es'), ('durant', 'es'), ('fuer', 'es'), ('quien', 'es'), ('lo', 'es'), ('hay', 'es'), ('tendran', 'es'), ('soy', 'es'), ('nosotr', 'es'), ('eram', 'es'), ('estos', 'es'), ('tus', 'es'), ('hub', 'es'), ('siend', 'es'), ('seais', 'es'), ('al', 'es'), ('suy', 'es'), ('ten', 'es'), ('mio', 'es'), ('yo', 'es'), ('seam', 'es'), ('hab', 'es'), ('ni', 'es'), ('ser', 'es'), ('ella', 'es'), ('hast', 'es'), ('del', 'es'), ('sea', 'es'), ('algun', 'es'), ('entre', 'es'), ('tendr', 'es'), ('tant', 'es'), ('estab', 'es'), ('eras', 'es'), ('habr', 'es'), ('habras', 'es'), ('otra', 'es'), ('tod', 'es'), ('es', 'es'), ('otras', 'es'), ('algo', 'es'), ('cuand', 'es'), ('seas', 'es'), ('esas', 'es'), ('muy', 'es'), ('fuim', 'es'), ('estad', 'es'), ('fuist', 'es'), ('per', 'es'), ('esta', 'es'), ('en', 'es'), ('de', 'es'), ('erais', 'es'), ('un', 'es'), ('vuestr', 'es'), ('estan', 'es'), ('fuerais', 'es'), ('teng', 'es'), ('mias', 'es'), ('contr', 'es'), ('seran', 'es'), ('ante', 'es'), ('esa', 'es'), ('estais', 'es'), ('fui', 'es'), ('te', 'es'), ('he', 'es'), ('nuestr', 'es'), ('esto', 'es'), ('los', 'es'), ('el', 'es'), ('estar', 'es');INSERT INTO stopwords_short (stopword, language) VALUES ('ne', 'fi'), ('teillä', 'fi'), ('vaikka', 'fi'), ('jota', 'fi'), ('hänestä', 'fi'), ('yli', 'fi'), ('heidät', 'fi'), ('noille', 'fi'), ('tallä', 'fi'), ('joksi', 'fi'), ('joilta', 'fi'), ('tästä', 'fi'), ('teitä', 'fi'), ('siihen', 'fi'), ('tuoksi', 'fi'), ('niille', 'fi'), ('tässä', 'fi'), ('niissä', 'fi'), ('keissä', 'fi'), ('tuossa', 'fi'), ('hänellä', 'fi'), ('vaan', 'fi'), ('tuon', 'fi'), ('meihin', 'fi'), ('näinä', 'fi'), ('meiltä', 'fi'), ('ollut', 'fi'), ('näitä', 'fi'), ('jolta', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('heissä', 'fi'), ('teidän', 'fi'), ('tähän', 'fi'), ('tuona', 'fi'), ('meille', 'fi'), ('näille', 'fi'), ('sinä', 'fi'), ('näiksi', 'fi'), ('täksi', 'fi'), ('niillä', 'fi'), ('tänä', 'fi'), ('keneen', 'fi'), ('nämä', 'fi'), ('kanssa', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sille', 'fi'), ('mistä', 'fi'), ('noiksi', 'fi'), ('tälle', 'fi'), ('jotka', 'fi'), ('mutta', 'fi'), ('jonka', 'fi'), ('sinua', 'fi'), ('minulla', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('joiden', 'fi'), ('noita', 'fi'), ('olin', 'fi'), ('meidät', 'fi'), ('joka', 'fi'), ('niiden', 'fi'), ('tätä', 'fi'), ('näissä', 'fi'), ('mihin', 'fi'), ('meidän', 'fi'), ('kenessä', 'fi'), ('kenen', 'fi'), ('noissa', 'fi'), ('me', 'fi'), ('kenellä', 'fi'), ('keille', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('miltä', 'fi'), ('siinä', 'fi'), ('tuolta', 'fi'), ('tältä', 'fi'), ('kenenä', 'fi'), ('ei', 'fi'), ('keillä', 'fi'), ('minut', 'fi'), ('häntä', 'fi'), ('minä', 'fi'), ('keiltä', 'fi'), ('teidät', 'fi'), ('sinulta', 'fi'), ('koska', 'fi'), ('sitä', 'fi'), ('tuolle', 'fi'), ('nuo', 'fi'), ('joista', 'fi'), ('noilta', 'fi'), ('näiltä', 'fi'), ('olivat', 'fi'), ('olisivat', 'fi'), ('mukaan', 'fi'), ('näillä', 'fi'), ('heille', 'fi'), ('niitä', 'fi'), ('niistä', 'fi'), ('heillä', 'fi'), ('olette', 'fi'), ('olimme', 'fi'), ('kenet', 'fi'), ('että', 'fi'), ('joita', 'fi'), ('mille', 'fi'), ('näistä', 'fi'), ('olleet', 'fi'), ('sinulla', 'fi'), ('heistä', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('keneltä', 'fi'), ('olen', 'fi'), ('jossa', 'fi'), ('näiden', 'fi'), ('joissa', 'fi'), ('häneen', 'fi'), ('minkä', 'fi'), ('olisi', 'fi'), ('itse', 'fi'), ('olet', 'fi'), ('on', 'fi'), ('josta', 'fi'), ('sinuun', 'fi'), ('jolla', 'fi'), ('ovat', 'fi'), ('noin', 'fi'), ('joihin', 'fi'), ('mikä', 'fi'), ('heiltä', 'fi'), ('oli', 'fi'), ('minussa', 'fi'), ('miksi', 'fi'), ('niiksi', 'fi'), ('tämä', 'fi'), ('keihin', 'fi'), ('jos', 'fi'), ('olisitte', 'fi'), ('minulta', 'fi'), ('meistä', 'fi'), ('siitä', 'fi'), ('millä', 'fi'), ('sillä', 'fi'), ('ketkä', 'fi'), ('ketä', 'fi'), ('poikki', 'fi'), ('noiden', 'fi'), ('joilla', 'fi'), ('sinun', 'fi'), ('tuo', 'fi'), ('niiltä', 'fi'), ('hänessä', 'fi'), ('ole', 'fi'), ('tuotä', 'fi'), ('sinusta', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('kenestä', 'fi'), ('sinussa', 'fi'), ('eivät', 'fi'), ('teille', 'fi'), ('tuohon', 'fi'), ('hänen', 'fi'), ('näihin', 'fi'), ('keistä', 'fi'), ('teihin', 'fi'), ('olisin', 'fi'), ('olisimme', 'fi'), ('teistä', 'fi'), ('keneksi', 'fi'), ('ette', 'fi'), ('teissä', 'fi'), ('siltä', 'fi'), ('jolle', 'fi'), ('kuka', 'fi'), ('noista', 'fi'), ('jona', 'fi'), ('keiksi', 'fi'), ('teiltä', 'fi'), ('johon', 'fi'), ('olla', 'fi'), ('sinulle', 'fi'), ('keitä', 'fi'), ('häneltä', 'fi'), ('tuolla', 'fi'), ('nyt', 'fi'), ('noihin', 'fi'), ('hänet', 'fi'), ('heidän', 'fi'), ('noina', 'fi'), ('minun', 'fi'), ('et', 'fi'), ('heihin', 'fi'), ('siksi', 'fi'), ('noilla', 'fi'), ('tuosta', 'fi'), ('kenelle', 'fi'), ('meissä', 'fi'), ('minulle', 'fi'), ('en', 'fi'), ('niihin', 'fi'), ('joiksi', 'fi'), ('hänelle', 'fi'), ('mitkä', 'fi'), ('minua', 'fi'), ('olitte', 'fi'), ('kun', 'fi'), ('ja', 'fi'), ('sinut', 'fi'), ('mitä', 'fi'), ('niinä', 'fi'), ('minuun', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('joille', 'fi'), ('olemme', 'fi'), ('meillä', 'fi'), ('minusta', 'fi'), ('te', 'fi'), ('he', 'fi'), ('missä', 'fi'), ('emme', 'fi'), ('tämän', 'fi'), ('keiden', 'fi');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ne', 'fi'), ('jota', 'fi'), ('yli', 'fi'), ('ketk', 'fi'), ('teitä', 'fi'), ('vaik', 'fi'), ('täs', 'fi'), ('niide', 'fi'), ('näide', 'fi'), ('em', 'fi'), ('täl', 'fi'), ('joih', 'fi'), ('jol', 'fi'), ('tuon', 'fi'), ('vaan', 'fi'), ('mis', 'fi'), ('näinä', 'fi'), ('näitä', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('tuona', 'fi'), ('meil', 'fi'), ('sinä', 'fi'), ('tänä', 'fi'), ('nämä', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sinu', 'fi'), ('sil', 'fi'), ('joho', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('noita', 'fi'), ('joka', 'fi'), ('tätä', 'fi'), ('teidä', 'fi'), ('olisiv', 'fi'), ('me', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('siinä', 'fi'), ('tuos', 'fi'), ('joks', 'fi'), ('mitk', 'fi'), ('poik', 'fi'), ('ei', 'fi'), ('minä', 'fi'), ('heis', 'fi'), ('mut', 'fi'), ('keiks', 'fi'), ('sitä', 'fi'), ('noil', 'fi'), ('nuo', 'fi'), ('jotk', 'fi'), ('tuol', 'fi'), ('olu', 'fi'), ('niitä', 'fi'), ('heil', 'fi'), ('joita', 'fi'), ('noih', 'fi'), ('oliv', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('mink', 'fi'), ('näil', 'fi'), ('näih', 'fi'), ('eivä', 'fi'), ('olet', 'fi'), ('häne', 'fi'), ('on', 'fi'), ('joil', 'fi'), ('its', 'fi'), ('olis', 'fi'), ('noin', 'fi'), ('mikä', 'fi'), ('oli', 'fi'), ('tämä', 'fi'), ('tuoks', 'fi'), ('kan', 'fi'), ('ova', 'fi'), ('kene', 'fi'), ('jos', 'fi'), ('näiks', 'fi'), ('meis', 'fi'), ('siitä', 'fi'), ('tähä', 'fi'), ('keih', 'fi'), ('ketä', 'fi'), ('muka', 'fi'), ('tuo', 'fi'), ('ole', 'fi'), ('mih', 'fi'), ('tuotä', 'fi'), ('ol', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('meih', 'fi'), ('teis', 'fi'), ('teil', 'fi'), ('jonk', 'fi'), ('kuka', 'fi'), ('niih', 'fi'), ('jona', 'fi'), ('joiks', 'fi'), ('näis', 'fi'), ('tal', 'fi'), ('meidä', 'fi'), ('heih', 'fi'), ('keitä', 'fi'), ('miks', 'fi'), ('nyt', 'fi'), ('et', 'fi'), ('noina', 'fi'), ('joide', 'fi'), ('keil', 'fi'), ('täks', 'fi'), ('heidä', 'fi'), ('en', 'fi'), ('niil', 'fi'), ('mil', 'fi'), ('siks', 'fi'), ('noide', 'fi'), ('minu', 'fi'), ('keide', 'fi'), ('kun', 'fi'), ('teih', 'fi'), ('ja', 'fi'), ('niinä', 'fi'), ('mitä', 'fi'), ('tuoho', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('jois', 'fi'), ('he', 'fi'), ('te', 'fi'), ('siihe', 'fi'), ('nois', 'fi'), ('niiks', 'fi'), ('keis', 'fi'), ('kosk', 'fi'), ('noiks', 'fi'), ('niis', 'fi');INSERT INTO stopwords_short (stopword, language) VALUES ('ne', 'fr'), ('le', 'fr'), ('serais', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('ayons', 'fr'), ('eus', 'fr'), ('ta', 'fr'), ('mais', 'fr'), ('ayante', 'fr'), ('êtes', 'fr'), ('serions', 'fr'), ('qui', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('étantes', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('aurons', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('soyons', 'fr'), ('serez', 'fr'), ('pour', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('eussiez', 'fr'), ('ayez', 'fr'), ('aies', 'fr'), ('aurions', 'fr'), ('ayantes', 'fr'), ('se', 'fr'), ('eûtes', 'fr'), ('ai', 'fr'), ('fussiez', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('ayants', 'fr'), ('aux', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('soyez', 'fr'), ('seras', 'fr'), ('étants', 'fr'), ('ces', 'fr'), ('me', 'fr'), ('aurez', 'fr'), ('s', 'fr'), ('eussions', 'fr'), ('étiez', 'fr'), ('c', 'fr'), ('que', 'fr'), ('vos', 'fr'), ('eusse', 'fr'), ('des', 'fr'), ('étions', 'fr'), ('aurai', 'fr'), ('par', 'fr'), ('mes', 'fr'), ('soit', 'fr'), ('même', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('mon', 'fr'), ('votre', 'fr'), ('aviez', 'fr'), ('m', 'fr'), ('serai', 'fr'), ('as', 'fr'), ('aurait', 'fr'), ('étante', 'fr'), ('nous', 'fr'), ('une', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('étée', 'fr'), ('d', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('fusse', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('eussent', 'fr'), ('eusses', 'fr'), ('étais', 'fr'), ('sont', 'fr'), ('fussions', 'fr'), ('étés', 'fr'), ('dans', 'fr'), ('fûtes', 'fr'), ('eu', 'fr'), ('auront', 'fr'), ('fusses', 'fr'), ('auraient', 'fr'), ('avions', 'fr'), ('serons', 'fr'), ('eûmes', 'fr'), ('avais', 'fr'), ('fus', 'fr'), ('aura', 'fr'), ('lui', 'fr'), ('notre', 'fr'), ('eux', 'fr'), ('auras', 'fr'), ('sommes', 'fr'), ('ait', 'fr'), ('ton', 'fr'), ('sera', 'fr'), ('es', 'fr'), ('avez', 'fr'), ('je', 'fr'), ('j', 'fr'), ('étées', 'fr'), ('avons', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('sa', 'fr'), ('seriez', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('en', 'fr'), ('de', 'fr'), ('un', 'fr'), ('elle', 'fr'), ('aie', 'fr'), ('furent', 'fr'), ('pas', 'fr'), ('serait', 'fr'), ('ce', 'fr'), ('seraient', 'fr'), ('aurais', 'fr'), ('moi', 'fr'), ('n', 'fr'), ('ou', 'fr'), ('te', 'fr'), ('ses', 'fr'), ('tes', 'fr'), ('fûmes', 'fr'), ('auriez', 'fr'), ('l', 'fr'), ('eues', 'fr'), ('à', 'fr'), ('avec', 'fr');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ayon', 'fr'), ('ne', 'fr'), ('fuss', 'fr'), ('soyon', 'fr'), ('auron', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('le', 'fr'), ('d', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('ta', 'fr'), ('eus', 'fr'), ('mais', 'fr'), ('e', 'fr'), ('qui', 'fr'), ('seron', 'fr'), ('eussent', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('étais', 'fr'), ('ête', 'fr'), ('sont', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('fûm', 'fr'), ('euss', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('eu', 'fr'), ('fussion', 'fr'), ('auront', 'fr'), ('ser', 'fr'), ('pour', 'fr'), ('avon', 'fr'), ('soi', 'fr'), ('notr', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('avais', 'fr'), ('ayez', 'fr'), ('fus', 'fr'), ('se', 'fr'), ('lui', 'fr'), ('ai', 'fr'), ('eux', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('aurion', 'fr'), ('ton', 'fr'), ('ait', 'fr'), ('eûm', 'fr'), ('avez', 'fr'), ('dan', 'fr'), ('je', 'fr'), ('aux', 'fr'), ('j', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('sa', 'fr'), ('aur', 'fr'), ('étion', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('eussion', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('me', 'fr'), ('s', 'fr'), ('en', 'fr'), ('mêm', 'fr'), ('c', 'fr'), ('de', 'fr'), ('un', 'fr'), ('que', 'fr'), ('elle', 'fr'), ('vos', 'fr'), ('furent', 'fr'), ('aie', 'fr'), ('avi', 'fr'), ('pas', 'fr'), ('par', 'fr'), ('soit', 'fr'), ('ce', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('serion', 'fr'), ('mon', 'fr'), ('moi', 'fr'), ('ou', 'fr'), ('n', 'fr'), ('te', 'fr'), ('m', 'fr'), ('éti', 'fr'), ('avion', 'fr'), ('l', 'fr'), ('somm', 'fr'), ('as', 'fr'), ('à', 'fr'), ('avec', 'fr'), ('votr', 'fr'), ('une', 'fr'), ('nous', 'fr');INSERT INTO stopwords_short (stopword, language) VALUES ('szinte', 'hu'), ('ne', 'hu'), ('továbbá', 'hu'), ('lenni', 'hu'), ('ki', 'hu'), ('igen', 'hu'), ('való', 'hu'), ('voltak', 'hu'), ('ellen', 'hu'), ('mindent', 'hu'), ('illetve', 'hu'), ('cikkek', 'hu'), ('azzal', 'hu'), ('mindig', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('annak', 'hu'), ('úgy', 'hu'), ('számára', 'hu'), ('szerint', 'hu'), ('ismét', 'hu'), ('nem', 'hu'), ('azután', 'hu'), ('sem', 'hu'), ('mert', 'hu'), ('valamint', 'hu'), ('jobban', 'hu'), ('teljes', 'hu'), ('amelyek', 'hu'), ('vagyok', 'hu'), ('arról', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('mindenki', 'hu'), ('olyan', 'hu'), ('mi', 'hu'), ('között', 'hu'), ('egyes', 'hu'), ('maga', 'hu'), ('legalább', 'hu'), ('általában', 'hu'), ('vissza', 'hu'), ('õk', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('lehetett', 'hu'), ('azok', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('amelynek', 'hu'), ('ehhez', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('saját', 'hu'), ('hiszen', 'hu'), ('aki', 'hu'), ('mikor', 'hu'), ('újabb', 'hu'), ('itt', 'hu'), ('ami', 'hu'), ('ugyanis', 'hu'), ('egyre', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('kívül', 'hu'), ('valami', 'hu'), ('s', 'hu'), ('meg', 'hu'), ('ahhoz', 'hu'), ('azon', 'hu'), ('eddig', 'hu'), ('egyik', 'hu'), ('valaki', 'hu'), ('ill', 'hu'), ('ison', 'hu'), ('vagyis', 'hu'), ('pedig', 'hu'), ('keressünk', 'hu'), ('át', 'hu'), ('amelyekben', 'hu'), ('alatt', 'hu'), ('mellett', 'hu'), ('új', 'hu'), ('elõször', 'hu'), ('õket', 'hu'), ('én', 'hu'), ('elõtt', 'hu'), ('egyetlen', 'hu'), ('azonban', 'hu'), ('éppen', 'hu'), ('tovább', 'hu'), ('ott', 'hu'), ('közül', 'hu'), ('ebben', 'hu'), ('melyek', 'hu'), ('lenne', 'hu'), ('mit', 'hu'), ('magát', 'hu'), ('szemben', 'hu'), ('õ', 'hu'), ('ezzel', 'hu'), ('belül', 'hu'), ('lett', 'hu'), ('ahol', 'hu'), ('amikor', 'hu'), ('viszont', 'hu'), ('azért', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('semmi', 'hu'), ('kellett', 'hu'), ('ezt', 'hu'), ('több', 'hu'), ('nagyon', 'hu'), ('abban', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('ilyenkor', 'hu'), ('nagyobb', 'hu'), ('még', 'hu'), ('amely', 'hu'), ('ekkor', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('újra', 'hu'), ('neki', 'hu'), ('mivel', 'hu'), ('voltam', 'hu'), ('bár', 'hu'), ('benne', 'hu'), ('felé', 'hu'), ('nélkül', 'hu'), ('által', 'hu'), ('ezen', 'hu'), ('amelyet', 'hu'), ('elég', 'hu'), ('miért', 'hu'), ('talán', 'hu'), ('minden', 'hu'), ('amolyan', 'hu'), ('hogy', 'hu'), ('persze', 'hu'), ('egyéb', 'hu'), ('milyen', 'hu'), ('utána', 'hu'), ('sokkal', 'hu'), ('akkor', 'hu'), ('emilyen', 'hu'), ('mint', 'hu'), ('mely', 'hu'), ('ilyen', 'hu'), ('voltunk', 'hu'), ('nekem', 'hu'), ('amit', 'hu'), ('vele', 'hu'), ('míg', 'hu'), ('sok', 'hu'), ('keresztül', 'hu'), ('elsõ', 'hu'), ('cikkeket', 'hu'), ('mintha', 'hu'), ('hogyan', 'hu'), ('vannak', 'hu'), ('csak', 'hu'), ('után', 'hu'), ('nagy', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('be', 'hu'), ('lehet', 'hu'), ('most', 'hu'), ('fel', 'hu'), ('hanem', 'hu'), ('arra', 'hu'), ('másik', 'hu'), ('és', 'hu'), ('majd', 'hu'), ('aztán', 'hu'), ('amelyeket', 'hu'), ('volna', 'hu'), ('össze', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('ennek', 'hu'), ('tehát', 'hu'), ('rá', 'hu'), ('erre', 'hu'), ('sokat', 'hu'), ('néha', 'hu'), ('ezek', 'hu'), ('ezért', 'hu'), ('legyen', 'hu'), ('akik', 'hu'), ('el', 'hu'), ('van', 'hu'), ('egy', 'hu');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ne', 'hu'), ('eb', 'hu'), ('ki', 'hu'), ('való', 'hu'), ('lehetet', 'hu'), ('mos', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('úgy', 'hu'), ('nem', 'hu'), ('hisz', 'hu'), ('ak', 'hu'), ('sem', 'hu'), ('teljes', 'hu'), ('lenn', 'hu'), ('közöt', 'hu'), ('cik', 'hu'), ('is', 'hu'), ('ut', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('ek', 'hu'), ('viszon', 'hu'), ('maj', 'hu'), ('mi', 'hu'), ('vel', 'hu'), ('it', 'hu'), ('egyes', 'hu'), ('leh', 'hu'), ('val', 'hu'), ('legalább', 'hu'), ('õk', 'hu'), ('han', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('valamin', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('újabb', 'hu'), ('ugyanis', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('valam', 'hu'), ('azut', 'hu'), ('s', 'hu'), ('keress', 'hu'), ('meg', 'hu'), ('ab', 'hu'), ('azon', 'hu'), ('ill', 'hu'), ('mily', 'hu'), ('ped', 'hu'), ('vagyis', 'hu'), ('egyetl', 'hu'), ('át', 'hu'), ('elõször', 'hu'), ('új', 'hu'), ('am', 'hu'), ('én', 'hu'), ('kereszt', 'hu'), ('oly', 'hu'), ('tovább', 'hu'), ('ot', 'hu'), ('ah', 'hu'), ('köz', 'hu'), ('õ', 'hu'), ('szám', 'hu'), ('ahol', 'hu'), ('elõt', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('épp', 'hu'), ('szint', 'hu'), ('mellet', 'hu'), ('több', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('emily', 'hu'), ('nagyobb', 'hu'), ('bel', 'hu'), ('még', 'hu'), ('eh', 'hu'), ('amely', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('ell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('mer', 'hu'), ('szem', 'hu'), ('bár', 'hu'), ('által', 'hu'), ('alat', 'hu'), ('elég', 'hu'), ('vol', 'hu'), ('persz', 'hu'), ('amoly', 'hu'), ('minden', 'hu'), ('job', 'hu'), ('hogy', 'hu'), ('egyéb', 'hu'), ('legy', 'hu'), ('vissz', 'hu'), ('mely', 'hu'), ('an', 'hu'), ('ilyen', 'hu'), ('ism', 'hu'), ('tal', 'hu'), ('míg', 'hu'), ('let', 'hu'), ('teh', 'hu'), ('semm', 'hu'), ('néh', 'hu'), ('minth', 'hu'), ('sa', 'hu'), ('sok', 'hu'), ('er', 'hu'), ('elsõ', 'hu'), ('csak', 'hu'), ('en', 'hu'), ('nagy', 'hu'), ('után', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('ed', 'hu'), ('be', 'hu'), ('továb', 'hu'), ('voln', 'hu'), ('fel', 'hu'), ('mag', 'hu'), ('és', 'hu'), ('benn', 'hu'), ('kív', 'hu'), ('illetv', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('kellet', 'hu'), ('min', 'hu'), ('rá', 'hu'), ('ig', 'hu'), ('ar', 'hu'), ('nek', 'hu'), ('ily', 'hu'), ('nél', 'hu'), ('el', 'hu'), ('össz', 'hu'), ('van', 'hu'), ('egy', 'hu'), ('szerin', 'hu');INSERT INTO stopwords_short (stopword, language) VALUES ('ne', 'it'), ('sugli', 'it'), ('delle', 'it'), ('avevo', 'it'), ('avete', 'it'), ('avessimo', 'it'), ('stemmo', 'it'), ('e', 'it'), ('queste', 'it'), ('ebbero', 'it'), ('facevano', 'it'), ('noi', 'it'), ('alle', 'it'), ('avesti', 'it'), ('tutto', 'it'), ('aveva', 'it'), ('sarete', 'it'), ('sua', 'it'), ('all', 'it'), ('nello', 'it'), ('chi', 'it'), ('ti', 'it'), ('faceste', 'it'), ('mi', 'it'), ('tua', 'it'), ('vostro', 'it'), ('fosti', 'it'), ('se', 'it'), ('gli', 'it'), ('sulla', 'it'), ('quelli', 'it'), ('miei', 'it'), ('ai', 'it'), ('avevano', 'it'), ('a', 'it'), ('nostra', 'it'), ('eravamo', 'it'), ('ma', 'it'), ('in', 'it'), ('una', 'it'), ('facessero', 'it'), ('si', 'it'), ('stavamo', 'it'), ('staremmo', 'it'), ('saremo', 'it'), ('quanti', 'it'), ('avessi', 'it'), ('anche', 'it'), ('siamo', 'it'), ('li', 'it'), ('avreste', 'it'), ('staresti', 'it'), ('sia', 'it'), ('questi', 'it'), ('facemmo', 'it'), ('stavo', 'it'), ('coi', 'it'), ('avrete', 'it'), ('dei', 'it'), ('sulle', 'it'), ('stettero', 'it'), ('faremmo', 'it'), ('sue', 'it'), ('ebbe', 'it'), ('o', 'it'), ('fecero', 'it'), ('stetti', 'it'), ('farai', 'it'), ('quante', 'it'), ('stesse', 'it'), ('della', 'it'), ('saremmo', 'it'), ('vi', 'it'), ('avrai', 'it'), ('stessimo', 'it'), ('farebbero', 'it'), ('dov', 'it'), ('starei', 'it'), ('sullo', 'it'), ('starà', 'it'), ('avessero', 'it'), ('al', 'it'), ('dello', 'it'), ('starai', 'it'), ('avrò', 'it'), ('avevate', 'it'), ('eravate', 'it'), ('quale', 'it'), ('stai', 'it'), ('avuto', 'it'), ('sarò', 'it'), ('perché', 'it'), ('mio', 'it'), ('aveste', 'it'), ('dagli', 'it'), ('avuti', 'it'), ('del', 'it'), ('stanno', 'it'), ('farà', 'it'), ('eri', 'it'), ('facesse', 'it'), ('facevo', 'it'), ('nei', 'it'), ('fu', 'it'), ('farebbe', 'it'), ('farete', 'it'), ('facciate', 'it'), ('stia', 'it'), ('suoi', 'it'), ('quanta', 'it'), ('essendo', 'it'), ('abbia', 'it'), ('quella', 'it'), ('stiamo', 'it'), ('avevi', 'it'), ('sarai', 'it'), ('avrei', 'it'), ('ed', 'it'), ('un', 'it'), ('stessero', 'it'), ('avrebbero', 'it'), ('sei', 'it'), ('sul', 'it'), ('sarà', 'it'), ('sto', 'it'), ('questo', 'it'), ('fareste', 'it'), ('dall', 'it'), ('stavi', 'it'), ('più', 'it'), ('faranno', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('stava', 'it'), ('fossero', 'it'), ('dal', 'it'), ('sarei', 'it'), ('voi', 'it'), ('abbiate', 'it'), ('facciamo', 'it'), ('avremo', 'it'), ('sarebbe', 'it'), ('da', 'it'), ('degli', 'it'), ('le', 'it'), ('come', 'it'), ('mie', 'it'), ('vostre', 'it'), ('ci', 'it'), ('negl', 'it'), ('saresti', 'it'), ('foste', 'it'), ('ebbi', 'it'), ('faccia', 'it'), ('steste', 'it'), ('sta', 'it'), ('stesti', 'it'), ('dalla', 'it'), ('furono', 'it'), ('fanno', 'it'), ('saranno', 'it'), ('fece', 'it'), ('dallo', 'it'), ('quelle', 'it'), ('siate', 'it'), ('stette', 'it'), ('dell', 'it'), ('avevamo', 'it'), ('sull', 'it'), ('uno', 'it'), ('la', 'it'), ('con', 'it'), ('vostri', 'it'), ('dagl', 'it'), ('starebbero', 'it'), ('stessi', 'it'), ('hai', 'it'), ('faccio', 'it'), ('farò', 'it'), ('il', 'it'), ('nostro', 'it'), ('ho', 'it'), ('facciano', 'it'), ('avrà', 'it'), ('loro', 'it'), ('tu', 'it'), ('cui', 'it'), ('era', 'it'), ('starebbe', 'it'), ('fossi', 'it'), ('facendo', 'it'), ('hanno', 'it'), ('c', 'it'), ('facessi', 'it'), ('alla', 'it'), ('vostra', 'it'), ('sarebbero', 'it'), ('ha', 'it'), ('dalle', 'it'), ('feci', 'it'), ('tue', 'it'), ('tra', 'it'), ('nell', 'it'), ('i', 'it'), ('faresti', 'it'), ('mia', 'it'), ('stiate', 'it'), ('nel', 'it'), ('avesse', 'it'), ('faceva', 'it'), ('degl', 'it'), ('sono', 'it'), ('su', 'it'), ('sareste', 'it'), ('quanto', 'it'), ('contro', 'it'), ('stareste', 'it'), ('lei', 'it'), ('faremo', 'it'), ('lo', 'it'), ('nostri', 'it'), ('avuta', 'it'), ('facevamo', 'it'), ('siete', 'it'), ('questa', 'it'), ('avemmo', 'it'), ('facevate', 'it'), ('sui', 'it'), ('facesti', 'it'), ('fai', 'it'), ('avranno', 'it'), ('starete', 'it'), ('dove', 'it'), ('di', 'it'), ('tuo', 'it'), ('sugl', 'it'), ('abbiano', 'it'), ('avresti', 'it'), ('stiano', 'it'), ('starò', 'it'), ('siano', 'it'), ('nelle', 'it'), ('quello', 'it'), ('fummo', 'it'), ('farei', 'it'), ('lui', 'it'), ('fossimo', 'it'), ('avute', 'it'), ('non', 'it'), ('fosse', 'it'), ('ero', 'it'), ('io', 'it'), ('avrebbe', 'it'), ('per', 'it'), ('nella', 'it'), ('nostre', 'it'), ('staremo', 'it'), ('tuoi', 'it'), ('facevi', 'it'), ('facessimo', 'it'), ('stavate', 'it'), ('dai', 'it'), ('stavano', 'it'), ('è', 'it'), ('erano', 'it'), ('ad', 'it'), ('agl', 'it'), ('col', 'it'), ('stando', 'it'), ('negli', 'it'), ('avremmo', 'it'), ('l', 'it'), ('abbiamo', 'it'), ('tutti', 'it'), ('allo', 'it'), ('avendo', 'it'), ('staranno', 'it');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ne', 'it'), ('facess', 'it'), ('le', 'it'), ('da', 'it'), ('stessim', 'it'), ('stav', 'it'), ('avrebb', 'it'), ('far', 'it'), ('mie', 'it'), ('hann', 'it'), ('stiam', 'it'), ('tutt', 'it'), ('ci', 'it'), ('negl', 'it'), ('avessim', 'it'), ('e', 'it'), ('ebbi', 'it'), ('stett', 'it'), ('avre', 'it'), ('sta', 'it'), ('fann', 'it'), ('noi', 'it'), ('alle', 'it'), ('abbi', 'it'), ('avest', 'it'), ('essend', 'it'), ('avra', 'it'), ('eran', 'it'), ('quest', 'it'), ('sua', 'it'), ('stest', 'it'), ('dell', 'it'), ('all', 'it'), ('farann', 'it'), ('uno', 'it'), ('fecer', 'it'), ('nostr', 'it'), ('chi', 'it'), ('ti', 'it'), ('sull', 'it'), ('la', 'it'), ('avesser', 'it'), ('con', 'it'), ('farem', 'it'), ('mi', 'it'), ('tua', 'it'), ('dagl', 'it'), ('perc', 'it'), ('fare', 'it'), ('siat', 'it'), ('hai', 'it'), ('starann', 'it'), ('starem', 'it'), ('sara', 'it'), ('se', 'it'), ('sar', 'it'), ('gli', 'it'), ('ai', 'it'), ('sarest', 'it'), ('sarem', 'it'), ('com', 'it'), ('il', 'it'), ('son', 'it'), ('a', 'it'), ('farebb', 'it'), ('avrann', 'it'), ('ho', 'it'), ('avrà', 'it'), ('avrem', 'it'), ('tu', 'it'), ('ma', 'it'), ('in', 'it'), ('cui', 'it'), ('una', 'it'), ('stess', 'it'), ('era', 'it'), ('stand', 'it'), ('si', 'it'), ('star', 'it'), ('c', 'it'), ('anche', 'it'), ('stavam', 'it'), ('li', 'it'), ('alla', 'it'), ('sia', 'it'), ('siam', 'it'), ('stare', 'it'), ('ha', 'it'), ('avevam', 'it'), ('starest', 'it'), ('tue', 'it'), ('coi', 'it'), ('dei', 'it'), ('lor', 'it'), ('fac', 'it'), ('i', 'it'), ('tra', 'it'), ('nell', 'it'), ('avrest', 'it'), ('sue', 'it'), ('mia', 'it'), ('nel', 'it'), ('ebbe', 'it'), ('degl', 'it'), ('siet', 'it'), ('avet', 'it'), ('o', 'it'), ('su', 'it'), ('facessim', 'it'), ('stemm', 'it'), ('sare', 'it'), ('lei', 'it'), ('stetter', 'it'), ('facc', 'it'), ('lo', 'it'), ('vi', 'it'), ('erav', 'it'), ('sian', 'it'), ('fur', 'it'), ('dov', 'it'), ('avret', 'it'), ('al', 'it'), ('avrò', 'it'), ('sui', 'it'), ('fai', 'it'), ('fossim', 'it'), ('sarann', 'it'), ('stesser', 'it'), ('di', 'it'), ('vostr', 'it'), ('tuo', 'it'), ('mio', 'it'), ('avev', 'it'), ('stara', 'it'), ('avend', 'it'), ('sugl', 'it'), ('sarebber', 'it'), ('quant', 'it'), ('avemm', 'it'), ('del', 'it'), ('eri', 'it'), ('lui', 'it'), ('non', 'it'), ('fost', 'it'), ('nei', 'it'), ('avremm', 'it'), ('fu', 'it'), ('starebb', 'it'), ('eravam', 'it'), ('avrebber', 'it'), ('abbiam', 'it'), ('fara', 'it'), ('sarebb', 'it'), ('ero', 'it'), ('facest', 'it'), ('ebber', 'it'), ('quell', 'it'), ('io', 'it'), ('qual', 'it'), ('per', 'it'), ('ed', 'it'), ('un', 'it'), ('stann', 'it'), ('sti', 'it'), ('sei', 'it'), ('starebber', 'it'), ('sul', 'it'), ('sto', 'it'), ('dai', 'it'), ('farebber', 'it'), ('è', 'it'), ('fosser', 'it'), ('fec', 'it'), ('contr', 'it'), ('dall', 'it'), ('agl', 'it'), ('ad', 'it'), ('col', 'it'), ('avess', 'it'), ('più', 'it'), ('foss', 'it'), ('fumm', 'it'), ('farest', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('l', 'it'), ('dal', 'it'), ('voi', 'it'), ('allo', 'it'), ('avut', 'it');INSERT INTO stopwords_short (stopword, language) VALUES ('su', 'lt'), ('ne', 'lt'), ('į', 'lt'), ('pasakė', 'lt'), ('kurių', 'lt'), ('d', 'lt'), ('lietuva', 'lt'), ('kai', 'lt'), ('arba', 'lt'), ('po', 'lt'), ('kurios', 'lt'), ('kam', 'lt'), ('darbo', 'lt'), ('daugiau', 'lt'), ('niekada', 'lt'), ('tas', 'lt'), ('metus', 'lt'), ('kovo', 'lt'), ('negali', 'lt'), ('kaip', 'lt'), ('kada', 'lt'), ('už', 'lt'), ('jos', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('savo', 'lt'), ('labai', 'lt'), ('tik', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('pat', 'lt'), ('negu', 'lt'), ('gali', 'lt'), ('čia', 'lt'), ('turi', 'lt'), ('bet', 'lt'), ('tuo', 'lt'), ('mažas', 'lt'), ('bei', 'lt'), ('tai', 'lt'), ('prašau', 'lt'), ('to', 'lt'), ('reikia', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jau', 'lt'), ('metu', 'lt'), ('lietuvos', 'lt'), ('jog', 'lt'), ('mažiau', 'lt'), ('val', 'lt'), ('lietuvoje', 'lt'), ('jei', 'lt'), ('jį', 'lt'), ('man', 'lt'), ('turėjo', 'lt'), ('nes', 'lt'), ('vienas', 'lt'), ('nei', 'lt'), ('daugiausia', 'lt'), ('mūsų', 'lt'), ('tarp', 'lt'), ('a', 'lt'), ('tačiau', 'lt'), ('net', 'lt'), ('šalia', 'lt'), ('niekas', 'lt'), ('mažai', 'lt'), ('todėl', 'lt'), ('yra', 'lt'), ('jie', 'lt'), ('pasak', 'lt'), ('per', 'lt'), ('dar', 'lt'), ('apie', 'lt'), ('kuris', 'lt'), ('žmonių', 'lt'), ('šalies', 'lt'), ('metais', 'lt'), ('ir', 'lt'), ('be', 'lt'), ('gauna', 'lt'), ('dėl', 'lt'), ('taip', 'lt'), ('kurie', 'lt'), ('dieną', 'lt'), ('jam', 'lt'), ('jų', 'lt'), ('kad', 'lt'), ('visi', 'lt'), ('r', 'lt'), ('bus', 'lt'), ('klausti', 'lt'), ('mano', 'lt'), ('mes', 'lt'), ('sakė', 'lt'), ('būti', 'lt'), ('dažnai', 'lt'), ('šios', 'lt'), ('jis', 'lt'), ('nuo', 'lt'), ('nors', 'lt'), ('iki', 'lt'), ('buvo', 'lt'), ('kitas', 'lt'), ('metų', 'lt'), ('kiekvienas', 'lt'), ('tikrai', 'lt'), ('iš', 'lt'), ('būtų', 'lt'), ('jo', 'lt'), ('tavo', 'lt'), ('nėra', 'lt'), ('prie', 'lt'), ('sakyti', 'lt'), ('m', 'lt'), ('daryti', 'lt'), ('kartą', 'lt'), ('ar', 'lt'), ('kas', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('klausimas', 'lt'), ('o', 'lt'), ('gauti', 'lt');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('', 'lt'), ('d', 'lt'), ('vis', 'lt'), ('neg', 'lt'), ('kam', 'lt'), ('tikr', 'lt'), ('klaus', 'lt'), ('tur', 'lt'), ('kaip', 'lt'), ('už', 'lt'), ('gau', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('b', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('tik', 'lt'), ('pat', 'lt'), ('bet', 'lt'), ('nor', 'lt'), ('dažn', 'lt'), ('lie', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jog', 'lt'), ('val', 'lt'), ('būt', 'lt'), ('buv', 'lt'), ('gal', 'lt'), ('man', 'lt'), ('negal', 'lt'), ('lab', 'lt'), ('bū', 'lt'), ('mūs', 'lt'), ('darb', 'lt'), ('tarp', 'lt'), ('met', 'lt'), ('š', 'lt'), ('maž', 'lt'), ('net', 'lt'), ('niek', 'lt'), ('j', 'lt'), ('praša', 'lt'), ('kuri', 'lt'), ('k', 'lt'), ('todėl', 'lt'), ('pri', 'lt'), ('nėr', 'lt'), ('pasak', 'lt'), ('t', 'lt'), ('re', 'lt'), ('v', 'lt'), ('per', 'lt'), ('s', 'lt'), ('dar', 'lt'), ('šal', 'lt'), ('ir', 'lt'), ('dėl', 'lt'), ('arb', 'lt'), ('taip', 'lt'), ('kart', 'lt'), ('kiekv', 'lt'), ('jam', 'lt'), ('kad', 'lt'), ('gaun', 'lt'), ('yr', 'lt'), ('r', 'lt'), ('kov', 'lt'), ('api', 'lt'), ('ja', 'lt'), ('niekad', 'lt'), ('iš', 'lt'), ('n', 'lt'), ('klausim', 'lt'), ('m', 'lt'), ('ar', 'lt'), ('p', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('kit', 'lt'), ('sak', 'lt'), ('žmon', 'lt'), ('tat', 'lt');INSERT INTO stopwords_short (stopword, language) VALUES ('want', 'nl'), ('deze', 'nl'), ('wil', 'nl'), ('kon', 'nl'), ('nu', 'nl'), ('heeft', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('wie', 'nl'), ('zijn', 'nl'), ('wat', 'nl'), ('niets', 'nl'), ('kunnen', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('iemand', 'nl'), ('doen', 'nl'), ('worden', 'nl'), ('door', 'nl'), ('haar', 'nl'), ('ge', 'nl'), ('tegen', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('veel', 'nl'), ('of', 'nl'), ('zij', 'nl'), ('uit', 'nl'), ('alles', 'nl'), ('toen', 'nl'), ('voor', 'nl'), ('hem', 'nl'), ('die', 'nl'), ('is', 'nl'), ('iets', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('daar', 'nl'), ('zelf', 'nl'), ('ze', 'nl'), ('hebben', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('andere', 'nl'), ('in', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('waren', 'nl'), ('geen', 'nl'), ('wezen', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('naar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('reeds', 'nl'), ('meer', 'nl'), ('en', 'nl'), ('heb', 'nl'), ('de', 'nl'), ('hier', 'nl'), ('bij', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('maar', 'nl'), ('op', 'nl'), ('eens', 'nl'), ('hun', 'nl'), ('toch', 'nl'), ('mij', 'nl'), ('geweest', 'nl'), ('tot', 'nl'), ('werd', 'nl'), ('ja', 'nl'), ('dat', 'nl'), ('moet', 'nl'), ('hij', 'nl'), ('ook', 'nl'), ('ben', 'nl'), ('had', 'nl'), ('te', 'nl'), ('hoe', 'nl'), ('nog', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('wordt', 'nl'), ('ik', 'nl'), ('was', 'nl'), ('van', 'nl');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('want', 'nl'), ('wil', 'nl'), ('kunn', 'nl'), ('kon', 'nl'), ('dor', 'nl'), ('heeft', 'nl'), ('nu', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('zijn', 'nl'), ('wie', 'nl'), ('wat', 'nl'), ('iet', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('doen', 'nl'), ('iemand', 'nl'), ('ge', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('of', 'nl'), ('mer', 'nl'), ('uit', 'nl'), ('zij', 'nl'), ('hem', 'nl'), ('toen', 'nl'), ('alles', 'nl'), ('is', 'nl'), ('die', 'nl'), ('hebb', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('zelf', 'nl'), ('vel', 'nl'), ('ze', 'nl'), ('nar', 'nl'), ('har', 'nl'), ('ander', 'nl'), ('dez', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('teg', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('in', 'nl'), ('red', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('mar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('dar', 'nl'), ('heb', 'nl'), ('en', 'nl'), ('vor', 'nl'), ('de', 'nl'), ('word', 'nl'), ('bij', 'nl'), ('hier', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('op', 'nl'), ('hun', 'nl'), ('mij', 'nl'), ('toch', 'nl'), ('war', 'nl'), ('tot', 'nl'), ('geweest', 'nl'), ('dat', 'nl'), ('ja', 'nl'), ('werd', 'nl'), ('ook', 'nl'), ('hij', 'nl'), ('moet', 'nl'), ('gen', 'nl'), ('had', 'nl'), ('ben', 'nl'), ('te', 'nl'), ('nog', 'nl'), ('hoe', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('ik', 'nl'), ('wordt', 'nl'), ('was', 'nl'), ('van', 'nl'), ('wez', 'nl');INSERT INTO stopwords_short (stopword, language) VALUES ('fra', 'no'), ('som', 'no'), ('da', 'no'), ('ingen', 'no'), ('kvi', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('bare', 'no'), ('ville', 'no'), ('det', 'no'), ('alle', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('honom', 'no'), ('hoss', 'no'), ('ut', 'no'), ('somme', 'no'), ('men', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('vors', 'no'), ('han', 'no'), ('noka', 'no'), ('man', 'no'), ('hjå', 'no'), ('har', 'no'), ('deim', 'no'), ('eg', 'no'), ('samme', 'no'), ('elles', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('hennar', 'no'), ('seg', 'no'), ('vort', 'no'), ('inkje', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('mine', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('deira', 'no'), ('sånn', 'no'), ('å', 'no'), ('skal', 'no'), ('dei', 'no'), ('nå', 'no'), ('nokor', 'no'), ('hvordan', 'no'), ('i', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('dykkar', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('dette', 'no'), ('verte', 'no'), ('du', 'no'), ('vere', 'no'), ('vart', 'no'), ('deires', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('begge', 'no'), ('kunne', 'no'), ('med', 'no'), ('kan', 'no'), ('varte', 'no'), ('somt', 'no'), ('før', 'no'), ('di', 'no'), ('inni', 'no'), ('siden', 'no'), ('noko', 'no'), ('ikkje', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('også', 'no'), ('dykk', 'no'), ('ble', 'no'), ('etter', 'no'), ('den', 'no'), ('vore', 'no'), ('hvilke', 'no'), ('nokon', 'no'), ('deg', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('hans', 'no'), ('denne', 'no'), ('ingi', 'no'), ('være', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('hadde', 'no'), ('vår', 'no'), ('så', 'no'), ('nokre', 'no'), ('hvor', 'no'), ('både', 'no'), ('so', 'no'), ('et', 'no'), ('hvilken', 'no'), ('henne', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('dere', 'no'), ('disse', 'no'), ('jeg', 'no'), ('en', 'no'), ('mange', 'no'), ('de', 'no'), ('meget', 'no'), ('deres', 'no'), ('din', 'no'), ('hun', 'no'), ('medan', 'no'), ('uten', 'no'), ('mykje', 'no'), ('hennes', 'no'), ('noen', 'no'), ('ikke', 'no'), ('sine', 'no'), ('kun', 'no'), ('skulle', 'no'), ('ja', 'no'), ('at', 'no'), ('hossen', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('inn', 'no'), ('eller', 'no'), ('hvorfor', 'no'), ('mot', 'no');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('sid', 'no'), ('fra', 'no'), ('som', 'no'), ('kunn', 'no'), ('da', 'no'), ('dett', 'no'), ('bar', 'no'), ('kvi', 'no'), ('mykj', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('det', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('henn', 'no'), ('honom', 'no'), ('nok', 'no'), ('all', 'no'), ('hoss', 'no'), ('ut', 'no'), ('men', 'no'), ('skull', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('hadd', 'no'), ('han', 'no'), ('man', 'no'), ('har', 'no'), ('hjå', 'no'), ('deir', 'no'), ('deim', 'no'), ('nokr', 'no'), ('eg', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('seg', 'no'), ('vort', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('sånn', 'no'), ('skal', 'no'), ('å', 'no'), ('dei', 'no'), ('nå', 'no'), ('i', 'no'), ('hvordan', 'no'), ('nokor', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('ikkj', 'no'), ('du', 'no'), ('vart', 'no'), ('begg', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('hvilk', 'no'), ('med', 'no'), ('kan', 'no'), ('somt', 'no'), ('vær', 'no'), ('båd', 'no'), ('før', 'no'), ('di', 'no'), ('ell', 'no'), ('inni', 'no'), ('noko', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('dykk', 'no'), ('ver', 'no'), ('også', 'no'), ('ble', 'no'), ('denn', 'no'), ('vill', 'no'), ('den', 'no'), ('ikk', 'no'), ('nokon', 'no'), ('deg', 'no'), ('samm', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('inkj', 'no'), ('ingi', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('vår', 'no'), ('så', 'no'), ('hvor', 'no'), ('mang', 'no'), ('et', 'no'), ('so', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('ing', 'no'), ('jeg', 'no'), ('en', 'no'), ('vor', 'no'), ('de', 'no'), ('vert', 'no'), ('din', 'no'), ('hun', 'no'), ('uten', 'no'), ('medan', 'no'), ('noen', 'no'), ('diss', 'no'), ('kun', 'no'), ('ja', 'no'), ('at', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('hvorfor', 'no'), ('inn', 'no'), ('mot', 'no'), ('somm', 'no');INSERT INTO stopwords_short (stopword, language) VALUES ('terá', 'pt'), ('havemos', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('da', 'pt'), ('houvéramos', 'pt'), ('para', 'pt'), ('nosso', 'pt'), ('pelos', 'pt'), ('nos', 'pt'), ('são', 'pt'), ('há', 'pt'), ('tenham', 'pt'), ('delas', 'pt'), ('mais', 'pt'), ('e', 'pt'), ('fôramos', 'pt'), ('estávamos', 'pt'), ('isso', 'pt'), ('nem', 'pt'), ('formos', 'pt'), ('hão', 'pt'), ('pelas', 'pt'), ('em', 'pt'), ('não', 'pt'), ('estejam', 'pt'), ('tive', 'pt'), ('seremos', 'pt'), ('sem', 'pt'), ('houvera', 'pt'), ('está', 'pt'), ('houver', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('aquilo', 'pt'), ('houveremos', 'pt'), ('fossem', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('terão', 'pt'), ('por', 'pt'), ('das', 'pt'), ('vocês', 'pt'), ('essa', 'pt'), ('estivera', 'pt'), ('essas', 'pt'), ('houvermos', 'pt'), ('nossa', 'pt'), ('muito', 'pt'), ('tivéramos', 'pt'), ('seria', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('se', 'pt'), ('dos', 'pt'), ('hajamos', 'pt'), ('com', 'pt'), ('na', 'pt'), ('tivéssemos', 'pt'), ('a', 'pt'), ('foram', 'pt'), ('estão', 'pt'), ('um', 'pt'), ('no', 'pt'), ('tenhamos', 'pt'), ('os', 'pt'), ('hajam', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('aquelas', 'pt'), ('aqueles', 'pt'), ('tivemos', 'pt'), ('estes', 'pt'), ('teriam', 'pt'), ('era', 'pt'), ('esteve', 'pt'), ('nossas', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('serei', 'pt'), ('terei', 'pt'), ('tem', 'pt'), ('teve', 'pt'), ('houverá', 'pt'), ('tivesse', 'pt'), ('tenha', 'pt'), ('fôssemos', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('serão', 'pt'), ('este', 'pt'), ('seríamos', 'pt'), ('estou', 'pt'), ('estas', 'pt'), ('só', 'pt'), ('lhe', 'pt'), ('houveria', 'pt'), ('houveram', 'pt'), ('pela', 'pt'), ('tivessem', 'pt'), ('nossos', 'pt'), ('teu', 'pt'), ('tiver', 'pt'), ('lhes', 'pt'), ('tivermos', 'pt'), ('éramos', 'pt'), ('estivéramos', 'pt'), ('meu', 'pt'), ('sejamos', 'pt'), ('estiver', 'pt'), ('estiveram', 'pt'), ('as', 'pt'), ('houveriam', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('houvesse', 'pt'), ('houverei', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quando', 'pt'), ('sejam', 'pt'), ('hei', 'pt'), ('houvessem', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('pelo', 'pt'), ('minhas', 'pt'), ('fora', 'pt'), ('houveríamos', 'pt'), ('será', 'pt'), ('esse', 'pt'), ('houvéssemos', 'pt'), ('temos', 'pt'), ('num', 'pt'), ('estivessem', 'pt'), ('do', 'pt'), ('tinha', 'pt'), ('aquela', 'pt'), ('já', 'pt'), ('fomos', 'pt'), ('teria', 'pt'), ('eu', 'pt'), ('tenho', 'pt'), ('numa', 'pt'), ('elas', 'pt'), ('deles', 'pt'), ('haja', 'pt'), ('esteja', 'pt'), ('estejamos', 'pt'), ('entre', 'pt'), ('estivemos', 'pt'), ('somos', 'pt'), ('houverão', 'pt'), ('teríamos', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('tivera', 'pt'), ('fosse', 'pt'), ('tiverem', 'pt'), ('dela', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('houvemos', 'pt'), ('teus', 'pt'), ('estive', 'pt'), ('tínhamos', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('mesmo', 'pt'), ('ela', 'pt'), ('teremos', 'pt'), ('seja', 'pt'), ('de', 'pt'), ('estivesse', 'pt'), ('dele', 'pt'), ('estamos', 'pt'), ('houve', 'pt'), ('nas', 'pt'), ('estavam', 'pt'), ('tinham', 'pt'), ('uma', 'pt'), ('aquele', 'pt'), ('como', 'pt'), ('estava', 'pt'), ('forem', 'pt'), ('esses', 'pt'), ('depois', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('estivéssemos', 'pt'), ('estiverem', 'pt'), ('houverem', 'pt'), ('minha', 'pt'), ('à', 'pt'), ('estivermos', 'pt'), ('tiveram', 'pt'), ('seriam', 'pt'), ('você', 'pt');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('tev', 'pt'), ('som', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('form', 'pt'), ('da', 'pt'), ('nos', 'pt'), ('estivéss', 'pt'), ('há', 'pt'), ('mesm', 'pt'), ('terã', 'pt'), ('mais', 'pt'), ('tenh', 'pt'), ('e', 'pt'), ('haj', 'pt'), ('isso', 'pt'), ('hav', 'pt'), ('nem', 'pt'), ('em', 'pt'), ('aquel', 'pt'), ('sem', 'pt'), ('está', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('minh', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('por', 'pt'), ('das', 'pt'), ('tiv', 'pt'), ('essa', 'pt'), ('vocês', 'pt'), ('serã', 'pt'), ('essas', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('muit', 'pt'), ('dos', 'pt'), ('se', 'pt'), ('na', 'pt'), ('com', 'pt'), ('a', 'pt'), ('um', 'pt'), ('no', 'pt'), ('estam', 'pt'), ('estej', 'pt'), ('os', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('tính', 'pt'), ('estes', 'pt'), ('era', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('tem', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('este', 'pt'), ('só', 'pt'), ('estas', 'pt'), ('estou', 'pt'), ('par', 'pt'), ('lhe', 'pt'), ('estáv', 'pt'), ('teu', 'pt'), ('estav', 'pt'), ('lhes', 'pt'), ('estã', 'pt'), ('meu', 'pt'), ('as', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('sej', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quand', 'pt'), ('tinh', 'pt'), ('hei', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('esse', 'pt'), ('num', 'pt'), ('do', 'pt'), ('noss', 'pt'), ('já', 'pt'), ('eu', 'pt'), ('elas', 'pt'), ('ter', 'pt'), ('ser', 'pt'), ('del', 'pt'), ('sã', 'pt'), ('hã', 'pt'), ('éram', 'pt'), ('entre', 'pt'), ('tivéss', 'pt'), ('depo', 'pt'), ('houv', 'pt'), ('fom', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('pel', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('teus', 'pt'), ('estev', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('ela', 'pt'), ('de', 'pt'), ('nas', 'pt'), ('uma', 'pt'), ('foss', 'pt'), ('fôss', 'pt'), ('nã', 'pt'), ('esses', 'pt'), ('fôr', 'pt'), ('aquil', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('voc', 'pt'), ('estiv', 'pt'), ('à', 'pt'), ('houvéss', 'pt');INSERT INTO stopwords_short (stopword, language) VALUES ('încă', 'ro'), ('toţi', 'ro'), ('ne', 'ro'), ('multă', 'ro'), ('înseşi', 'ro'), ('oricât', 'ro'), ('totuşi', 'ro'), ('puţini', 'ro'), ('înşine', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('veţi', 'ro'), ('unii', 'ro'), ('altcineva', 'ro'), ('aceiaşi', 'ro'), ('atât', 'ro'), ('parte', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('primii', 'ro'), ('doime', 'ro'), ('sine ', 'ro'), ('oi', 'ro'), ('tău', 'ro'), ('câtă', 'ro'), ('unei', 'ro'), ('mare', 'ro'), ('însăşi', 'ro'), ('celorlalte', 'ro'), ('oricăruia', 'ro'), ('celei', 'ro'), ('mi', 'ro'), ('acelaşi', 'ro'), ('ălora', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('cine', 'ro'), ('puţină', 'ro'), ('primelor', 'ro'), ('alea', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('suntem', 'ro'), ('acelui', 'ro'), ('însele', 'ro'), ('una', 'ro'), ('însămi', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('foarte', 'ro'), ('acea', 'ro'), ('mine', 'ro'), ('puţine', 'ro'), ('şapte', 'ro'), ('multe', 'ro'), ('atâţi', 'ro'), ('care', 'ro'), ('cei', 'ro'), ('oricâtă', 'ro'), ('am', 'ro'), ('acelei', 'ro'), ('lor', 'ro'), ('dintre', 'ro'), ('celorlalţi', 'ro'), ('alţii', 'ro'), ('alt', 'ro'), ('orice', 'ro'), ('vreunii', 'ro'), ('aceasta', 'ro'), ('alta', 'ro'), ('aceşti', 'ro'), ('eşti', 'ro'), ('amândoi', 'ro'), ('noastră', 'ro'), ('meu', 'ro'), ('unul', 'ro'), ('fiţi', 'ro'), ('fiind', 'ro'), ('ambele', 'ro'), ('altora', 'ro'), ('celuilalt', 'ro'), ('aceea', 'ro'), ('spre', 'ro'), ('o', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('fiecărei', 'ro'), ('vreuna', 'ro'), ('bine', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('vreunele', 'ro'), ('alte', 'ro'), ('celelalte', 'ro'), ('eram', 'ro'), ('celălalt', 'ro'), ('există', 'ro'), ('amândurora', 'ro'), ('al', 'ro'), ('numai', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('aceştia', 'ro'), ('spune', 'ro'), ('cutare', 'ro'), ('fiu', 'ro'), ('câţi', 'ro'), ('mult', 'ro'), ('altcuiva ', 'ro'), ('oricui', 'ro'), ('ea', 'ro'), ('cinci', 'ro'), ('aceloraşi', 'ro'), ('vreunuia', 'ro'), ('mei', 'ro'), ('deşi', 'ro'), ('mele', 'ro'), ('primul', 'ro'), ('însevă', 'ro'), ('vreuneia', 'ro'), ('acelor', 'ro'), ('câte', 'ro'), ('oricăreia', 'ro'), ('ca', 'ro'), ('oricâtor', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('primilor', 'ro'), ('oricărora', 'ro'), ('aceia', 'ro'), ('câteva', 'ro'), ('fost', 'ro'), ('vreunul', 'ro'), ('vreo', 'ro'), ('ăstora', 'ro'), ('despre', 'ro'), ('după', 'ro'), ('însăţi', 'ro'), ('fii', 'ro'), ('destui', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('nimeni', 'ro'), ('noştri', 'ro'), ('oricine', 'ro'), ('aceleaşi', 'ro'), ('oţi', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('ăia', 'ro'), ('vor', 'ro'), ('fără', 'ro'), ('de', 'ro'), ('atâtora', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('întâiul', 'ro'), ('ăştia', 'ro'), ('faţă', 'ro'), ('ambii', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('sale', 'ro'), ('nişte', 'ro'), ('trei', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('atunci', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('că', 'ro'), ('eraţi', 'ro'), ('vreunora', 'ro'), ('cele', 'ro'), ('ambelor', 'ro'), ('ar', 'ro'), ('atâtor', 'ro'), ('cineva', 'ro'), ('iar', 'ro'), ('deci', 'ro'), ('îndoit', 'ro'), ('voi', 'ro'), ('avut', 'ro'), ('însuşi', 'ro'), ('cărei', 'ro'), ('până', 'ro'), ('aceluiaşi', 'ro'), ('căror ', 'ro'), ('cât', 'ro'), ('noua', 'ro'), ('altceva', 'ro'), ('le', 'ro'), ('astea', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('oricâte', 'ro'), ('ci', 'ro'), ('cel', 'ro'), ('acele', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('sunteţi', 'ro'), ('ăla', 'ro'), ('alteia', 'ro'), ('câtorva', 'ro'), ('ambilor', 'ro'), ('acesteia', 'ro'), ('asupra', 'ro'), ('putea', 'ro'), ('pentru', 'ro'), ('va', 'ro'), ('aveaţi', 'ro'), ('când', 'ro'), ('aveai', 'ro'), ('aproape', 'ro'), ('la', 'ro'), ('celui', 'ro'), ('aceleia', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('oricărui', 'ro'), ('puţin', 'ro'), ('ăsta', 'ro'), ('acei', 'ro'), ('întâia', 'ro'), ('oricâţi', 'ro'), ('acestea', 'ro'), ('dacă', 'ro'), ('acestei', 'ro'), ('însumi', 'ro'), ('treime', 'ro'), ('oricare', 'ro'), ('altuia', 'ro'), ('unde', 'ro'), ('altei', 'ro'), ('înşivă', 'ro'), ('altele', 'ro'), ('fiecare', 'ro'), ('ăluia', 'ro'), ('erai', 'ro'), ('îşi', 'ro'), ('aceeaşi', 'ro'), ('fiecăreia', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('acesta', 'ro'), ('primului', 'ro'), ('s', 'ro'), ('această', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('primei', 'ro'), ('trebuie', 'ro'), ('primele', 'ro'), ('este', 'ro'), ('înşişi', 'ro'), ('însutit', 'ro'), ('patru', 'ro'), ('nimănui', 'ro'), ('ei', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('au', 'ro'), ('voastre', 'ro'), ('şase', 'ro'), ('noastre', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('doar', 'ro'), ('ăsteia', 'ro'), ('oarecare', 'ro'), ('atâta', 'ro'), ('astfel', 'ro'), ('oricâtora', 'ro'), ('întreit', 'ro'), ('acelea', 'ro'), ('poate', 'ro'), ('chiar', 'ro'), ('unuia', 'ro'), ('nimic', 'ro'), ('acela', 'ro'), ('atâţia', 'ro'), ('dumneavoastră', 'ro'), ('către', 'ro'), ('fiecăruia', 'ro'), ('ţie', 'ro'), ('destule', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('ceva', 'ro'), ('sutime', 'ro'), ('voastră ', 'ro'), ('cealaltă', 'ro'), ('aceste', 'ro'), ('aceleiaşi', 'ro'), ('face', 'ro'), ('aş', 'ro'), ('toată', 'ro'), ('cu', 'ro'), ('ăstuia', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('mulţi', 'ro'), ('cum', 'ro'), ('peste', 'ro'), ('acelora', 'ro'), ('cărui', 'ro'), ('vreunui', 'ro'), ('nouă', 'ro'), ('lui', 'ro'), ('voştri', 'ro'), ('unele', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('uneia', 'ro'), ('unora ', 'ro'), ('prima', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('atâtea', 'ro'), ('zece', 'ro'), ('aşadar', 'ro'), ('ieri', 'ro'), ('tăi', 'ro'), ('destulă', 'ro'), ('or', 'ro'), ('acestor', 'ro'), ('acel', 'ro'), ('avea', 'ro'), ('câtor', 'ro'), ('câtva', 'ro'), ('v', 'ro'), ('dar', 'ro'), ('totul', 'ro'), ('însuţi', 'ro'), ('câţiva', 'ro'), ('spus', 'ro'), ('doua', 'ro'), ('câtora', 'ro'), ('din', 'ro'), ('săi', 'ro'), ('însene', 'ro'), ('acestuia', 'ro'), ('altor', 'ro'), ('acestora', 'ro'), ('ţi', 'ro'), ('are', 'ro'), ('ceilalţi', 'ro'), ('ce', 'ro'), ('amândouă', 'ro'), ('unu', 'ro'), ('aia', 'ro'), ('vouă', 'ro'), ('oricăror', 'ro'), ('sieşi', 'ro'), ('nici', 'ro'), ('printre', 'ro'), ('decât', 'ro'), ('avem', 'ro'), ('toate', 'ro'), ('ale ', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('aveţi', 'ro'), ('celeilalte', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('aveam', 'ro'), ('tuturor', 'ro'), ('aţi', 'ro'), ('tale', 'ro'), ('tine', 'ro'), ('lângă', 'ro'), ('el', 'ro'), ('altul', 'ro'), ('ăleia', 'ro');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ne', 'ro'), ('încă', 'ro'), ('oricât', 'ro'), ('put', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('nou', 'ro'), ('atât', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('şas', 'ro'), ('sine ', 'ro'), ('tău', 'ro'), ('oi', 'ro'), ('oricin', 'ro'), ('dou', 'ro'), ('unei', 'ro'), ('aproap', 'ro'), ('amândou', 'ro'), ('celeilalt', 'ro'), ('mi', 'ro'), ('sal', 'ro'), ('mel', 'ro'), ('part', 'ro'), ('poat', 'ro'), ('oricăr', 'ro'), ('aceleaş', 'ro'), ('noastr', 'ro'), ('asupr', 'ro'), ('dup', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('bin', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('prim', 'ro'), ('cev', 'ro'), ('acelui', 'ro'), ('celu', 'ro'), ('înşin', 'ro'), ('una', 'ro'), ('toţ', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('altu', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('însut', 'ro'), ('ambe', 'ro'), ('ălui', 'ro'), ('dumneavoastr', 'ro'), ('cei', 'ro'), ('acelei', 'ro'), ('am', 'ro'), ('lor', 'ro'), ('celelalt', 'ro'), ('ace', 'ro'), ('alt', 'ro'), ('car', 'ro'), ('alta', 'ro'), ('eşti', 'ro'), ('spun', 'ro'), ('meu', 'ro'), ('foart', 'ro'), ('căru', 'ro'), ('fiind', 'ro'), ('o', 'ro'), ('spre', 'ro'), ('celuilalt', 'ro'), ('căre', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('altcinev', 'ro'), ('fiecărei', 'ro'), ('noştr', 'ro'), ('însăş', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('alte', 'ro'), ('însăm', 'ro'), ('eram', 'ro'), ('aceşt', 'ro'), ('celălalt', 'ro'), ('toat', 'ro'), ('al', 'ro'), ('înşiş', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('celorlalţ', 'ro'), ('fiu', 'ro'), ('mult', 'ro'), ('câţiv', 'ro'), ('altcuiva ', 'ro'), ('câtev', 'ro'), ('vou', 'ro'), ('ea', 'ro'), ('numa', 'ro'), ('tin', 'ro'), ('mei', 'ro'), ('celorlalt', 'ro'), ('acelor', 'ro'), ('ca', 'ro'), ('ăşti', 'ro'), ('oricâtor', 'ro'), ('însum', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('şapt', 'ro'), ('deş', 'ro'), ('fost', 'ro'), ('cin', 'ro'), ('aceast', 'ro'), ('vreo', 'ro'), ('fii', 'ro'), ('tal', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('înşiv', 'ro'), ('ăstui', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('oţi', 'ro'), ('ăia', 'ro'), ('însuţ', 'ro'), ('vor', 'ro'), ('de', 'ro'), ('un', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('însăţ', 'ro'), ('oric', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('aceeaş', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('ăstor', 'ro'), ('că', 'ro'), ('eraţ', 'ro'), ('cele', 'ro'), ('min', 'ro'), ('înseş', 'ro'), ('ar', 'ro'), ('cutar', 'ro'), ('atâtor', 'ro'), ('iar', 'ro'), ('voi', 'ro'), ('îndoit', 'ro'), ('avut', 'ro'), ('cât', 'ro'), ('căror ', 'ro'), ('le', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('cel', 'ro'), ('ci', 'ro'), ('acele', 'ro'), ('prime', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('aveţ', 'ro'), ('ăla', 'ro'), ('oricăre', 'ro'), ('tre', 'ro'), ('câţ', 'ro'), ('pentru', 'ro'), ('nic', 'ro'), ('va', 'ro'), ('când', 'ro'), ('la', 'ro'), ('aste', 'ro'), ('oricărui', 'ro'), ('veţ', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('puţin', 'ro'), ('amândo', 'ro'), ('acei', 'ro'), ('ăsta', 'ro'), ('totuş', 'ro'), ('exist', 'ro'), ('zec', 'ro'), ('ale', 'ro'), ('acestei', 'ro'), ('înse', 'ro'), ('atunc', 'ro'), ('ier', 'ro'), ('altei', 'ro'), ('unde', 'ro'), ('pân', 'ro'), ('ambi', 'ro'), ('îşi', 'ro'), ('uni', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('s', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('ave', 'ro'), ('însen', 'ro'), ('cinc', 'ro'), ('este', 'ro'), ('oarec', 'ro'), ('câtorv', 'ro'), ('patru', 'ro'), ('ei', 'ro'), ('vreune', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('despr', 'ro'), ('sutim', 'ro'), ('au', 'ro'), ('fac', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('cealalt', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('altcev', 'ro'), ('ăstei', 'ro'), ('doar', 'ro'), ('nişt', 'ro'), ('întâi', 'ro'), ('astfel', 'ro'), ('cinev', 'ro'), ('întreit', 'ro'), ('ălor', 'ro'), ('une', 'ro'), ('chiar', 'ro'), ('fiec', 'ro'), ('nimic', 'ro'), ('făr', 'ro'), ('treim', 'ro'), ('pest', 'ro'), ('dintr', 'ro'), ('ţie', 'ro'), ('ceilalţ', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('dest', 'ro'), ('voastră ', 'ro'), ('cătr', 'ro'), ('dec', 'ro'), ('aceste', 'ro'), ('aş', 'ro'), ('cu', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('câtv', 'ro'), ('cum', 'ro'), ('lâng', 'ro'), ('mulţ', 'ro'), ('vreunui', 'ro'), ('lui', 'ro'), ('fiţ', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('unora ', 'ro'), ('printr', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('aşadar', 'ro'), ('tăi', 'ro'), ('ălei', 'ro'), ('acestor', 'ro'), ('or', 'ro'), ('acel', 'ro'), ('nimăn', 'ro'), ('sieş', 'ro'), ('câtor', 'ro'), ('avea', 'ro'), ('mar', 'ro'), ('atâţ', 'ro'), ('v', 'ro'), ('nimen', 'ro'), ('faţ', 'ro'), ('dar', 'ro'), ('însev', 'ro'), ('spus', 'ro'), ('voştr', 'ro'), ('din', 'ro'), ('voastr', 'ro'), ('săi', 'ro'), ('altor', 'ro'), ('ţi', 'ro'), ('fiecăr', 'ro'), ('trebui', 'ro'), ('are', 'ro'), ('doim', 'ro'), ('ce', 'ro'), ('aia', 'ro'), ('fiecăre', 'ro'), ('unu', 'ro'), ('oricăror', 'ro'), ('decât', 'ro'), ('ale ', 'ro'), ('avem', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('dac', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('tuturor', 'ro'), ('aveam', 'ro'), ('aţi', 'ro'), ('el', 'ro'), ('însuş', 'ro');INSERT INTO stopwords_short (stopword, language) VALUES ('sid', 'ru'), ('группа', 'ru'), ('каким', 'ru'), ('при', 'ru'), ('кстате', 'ru'), ('знаю', 'ru'), ('мог', 'ru'), ('как', 'ru'), ('помощи', 'ru'), ('для', 'ru'), ('над', 'ru'), ('дело', 'ru'), ('завода', 'ru'), ('нельзя', 'ru'), ('место', 'ru'), ('последний', 'ru'), ('последнее', 'ru'), ('проект', 'ru'), ('вообще', 'ru'), ('еще', 'ru'), ('дать', 'ru'), ('хорошо', 'ru'), ('века', 'ru'), ('лишь', 'ru'), ('Чтобы', 'ru'), ('жизни', 'ru'), ('недавно', 'ru'), ('него', 'ru'), ('будет', 'ru'), ('каждая', 'ru'), ('Пресс', 'ru'), ('года', 'ru'), ('срок', 'ru'), ('НЕ', 'ru'), ('новой', 'ru'), ('количество', 'ru'), ('февраля', 'ru'), ('руководитель', 'ru'), ('главы', 'ru'), ('закон', 'ru'), ('похоже', 'ru'), ('одну', 'ru'), ('жителей', 'ru'), ('семьи', 'ru'), ('коп', 'ru'), ('Если', 'ru'), ('Я', 'ru'), ('мая', 'ru'), ('ранее', 'ru'), ('естественно', 'ru'), ('быстро', 'ru'), ('меня', 'ru'), ('представителей', 'ru'), ('первой', 'ru'), ('http', 'ru'), ('дом', 'ru'), ('они', 'ru'), ('новостей', 'ru'), ('вице', 'ru'), ('Когда', 'ru'), ('понять', 'ru'), ('становится', 'ru'), ('кто', 'ru'), ('корреспонденту', 'ru'), ('имеют', 'ru'), ('В', 'ru'), ('начальник', 'ru'), ('которой', 'ru'), ('неделю', 'ru'), ('это', 'ru'), ('мне', 'ru'), ('части', 'ru'), ('если', 'ru'), ('дня', 'ru'), ('этой', 'ru'), ('livejournal', 'ru'), ('не', 'ru'), ('часть', 'ru'), ('заявление', 'ru'), ('того', 'ru'), ('председателя', 'ru'), ('период', 'ru'), ('друга', 'ru'), ('нему', 'ru'), ('рук', 'ru'), ('А', 'ru'), ('каждого', 'ru'), ('премьера', 'ru'), ('доме', 'ru'), ('мере', 'ru'), ('свой', 'ru'), ('история', 'ru'), ('который', 'ru'), ('раньше', 'ru'), ('деле', 'ru'), ('Мы', 'ru'), ('главный', 'ru'), ('получили', 'ru'), ('работе', 'ru'), ('решили', 'ru'), ('нужно', 'ru'), ('ведь', 'ru'), ('лучше', 'ru'), ('делаем', 'ru'), ('лично', 'ru'), ('перед', 'ru'), ('прямо', 'ru'), ('около', 'ru'), ('места', 'ru'), ('первых', 'ru'), ('игры', 'ru'), ('об', 'ru'), ('уровне', 'ru'), ('главным', 'ru'), ('Причем', 'ru'), ('Тем', 'ru'), ('свет', 'ru'), ('их', 'ru'), ('хочет', 'ru'), ('стал', 'ru'), ('целом', 'ru'), ('И', 'ru'), ('чего', 'ru'), ('необходимо', 'ru'), ('одной', 'ru'), ('представители', 'ru'), ('П', 'ru'), ('т', 'ru'), ('более', 'ru'), ('про', 'ru'), ('вы', 'ru'), ('главного', 'ru'), ('октября', 'ru'), ('массовой', 'ru'), ('обычно', 'ru'), ('апрель', 'ru'), ('сообщил', 'ru'), ('чтобы', 'ru'), ('второй', 'ru'), ('одна', 'ru'), ('и', 'ru'), ('интересы', 'ru'), ('он', 'ru'), ('могли', 'ru'), ('никаких', 'ru'), ('лица', 'ru'), ('Ну', 'ru'), ('ее', 'ru'), ('совершенно', 'ru'), ('руководителей', 'ru'), ('процесс', 'ru'), ('Борис', 'ru'), ('самый', 'ru'), ('руках', 'ru'), ('Кроме', 'ru'), ('едва', 'ru'), ('жить', 'ru'), ('человека', 'ru'), ('районе', 'ru'), ('которым', 'ru'), ('возможность', 'ru'), ('уж', 'ru'), ('возможности', 'ru'), ('выборах', 'ru'), ('От', 'ru'), ('сообщили', 'ru'), ('точнее', 'ru'), ('отдела', 'ru'), ('начале', 'ru'), ('столько', 'ru'), ('комментариев', 'ru'), ('его', 'ru'), ('свои', 'ru'), ('партия', 'ru'), ('новые', 'ru'), ('суббота', 'ru'), ('Об', 'ru'), ('прежде', 'ru'), ('трех', 'ru'), ('без', 'ru'), ('Тогда', 'ru'), ('О', 'ru'), ('трудно', 'ru'), ('найти', 'ru'), ('руки', 'ru'), ('кстати', 'ru'), ('хотя', 'ru'), ('ним', 'ru'), ('adriver', 'ru'), ('вопрос', 'ru'), ('Среди', 'ru'), ('декабря', 'ru'), ('amp', 'ru'), ('где', 'ru'), ('была', 'ru'), ('правда', 'ru'), ('вместе', 'ru'), ('Иллюстрация', 'ru'), ('власть', 'ru'), ('я', 'ru'), ('утверждают', 'ru'), ('Г', 'ru'), ('нибудь', 'ru'), ('против', 'ru'), ('месяца', 'ru'), ('Во', 'ru'), ('значит', 'ru'), ('встречи', 'ru'), ('несколько', 'ru'), ('никогда', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('Уже', 'ru'), ('остается', 'ru'), ('своими', 'ru'), ('Вот', 'ru'), ('оказались', 'ru'), ('Ю', 'ru'), ('скорее', 'ru'), ('практически', 'ru'), ('конечно', 'ru'), ('команды', 'ru'), ('почта', 'ru'), ('взять', 'ru'), ('многие', 'ru'), ('иметь', 'ru'), ('самой', 'ru'), ('городе', 'ru'), ('любая', 'ru'), ('из', 'ru'), ('Эти', 'ru'), ('Ассошиэйтед', 'ru'), ('пока', 'ru'), ('Нет', 'ru'), ('получил', 'ru'), ('Теперь', 'ru'), ('есть', 'ru'), ('также', 'ru'), ('от', 'ru'), ('всего', 'ru'), ('только', 'ru'), ('разных', 'ru'), ('одновременно', 'ru'), ('самое', 'ru'), ('одном', 'ru'), ('эта', 'ru'), ('этому', 'ru'), ('у', 'ru'), ('путь', 'ru'), ('Дело', 'ru'), ('должна', 'ru'), ('фонда', 'ru'), ('мировой', 'ru'), ('cgi', 'ru'), ('уже', 'ru'), ('быть', 'ru'), ('факт', 'ru'), ('жизнь', 'ru'), ('своим', 'ru'), ('Л', 'ru'), ('сборной', 'ru'), ('июля', 'ru'), ('утверждает', 'ru'), ('пять', 'ru'), ('год', 'ru'), ('теперь', 'ru'), ('потому', 'ru'), ('о', 'ru'), ('Д', 'ru'), ('сотрудников', 'ru'), ('говоря', 'ru'), ('той', 'ru'), ('якобы', 'ru'), ('условия', 'ru'), ('сами', 'ru'), ('миллион', 'ru'), ('сути', 'ru'), ('больше', 'ru'), ('все', 'ru'), ('удалось', 'ru'), ('Есть', 'ru'), ('политики', 'ru'), ('событий', 'ru'), ('им', 'ru'), ('своего', 'ru'), ('Через', 'ru'), ('именно', 'ru'), ('одного', 'ru'), ('стало', 'ru'), ('женщины', 'ru'), ('местных', 'ru'), ('главе', 'ru'), ('Она', 'ru'), ('г', 'ru'), ('которую', 'ru'), ('особенно', 'ru'), ('многих', 'ru'), ('Однако', 'ru'), ('Сергей', 'ru'), ('х', 'ru'), ('являетесь', 'ru'), ('наша', 'ru'), ('этот', 'ru'), ('равно', 'ru'), ('Еще', 'ru'), ('работает', 'ru'), ('число', 'ru'), ('каких', 'ru'), ('После', 'ru'), ('производства', 'ru'), ('под', 'ru'), ('связи', 'ru'), ('вас', 'ru'), ('сегодня', 'ru'), ('собой', 'ru'), ('Ведь', 'ru'), ('году', 'ru'), ('было', 'ru'), ('в', 'ru'), ('сих', 'ru'), ('всегда', 'ru'), ('говорится', 'ru'), ('С', 'ru'), ('наши', 'ru'), ('своему', 'ru'), ('другой', 'ru'), ('бы', 'ru'), ('первый', 'ru'), ('даже', 'ru'), ('качестве', 'ru'), ('подобная', 'ru'), ('нем', 'ru'), ('такое', 'ru'), ('лидер', 'ru'), ('оказался', 'ru'), ('миллиарда', 'ru'), ('несмотря', 'ru'), ('слишком', 'ru'), ('придется', 'ru'), ('Кстати', 'ru'), ('У', 'ru'), ('пост', 'ru'), ('мире', 'ru'), ('первого', 'ru'), ('ссылка', 'ru'), ('сделаем', 'ru'), ('времена', 'ru'), ('течение', 'ru'), ('слова', 'ru'), ('Е', 'ru'), ('Правда', 'ru'), ('другое', 'ru'), ('которому', 'ru'), ('участие', 'ru'), ('сейчас', 'ru'), ('руб', 'ru'), ('просто', 'ru'), ('да', 'ru'), ('руководство', 'ru'), ('развития', 'ru'), ('видимо', 'ru'), ('нынешнего', 'ru'), ('после', 'ru'), ('сентября', 'ru'), ('весьма', 'ru'), ('наших', 'ru'), ('двух', 'ru'), ('резко', 'ru'), ('например', 'ru'), ('января', 'ru'), ('шесть', 'ru'), ('вопросы', 'ru'), ('должен', 'ru'), ('Таким', 'ru'), ('долго', 'ru'), ('были', 'ru'), ('тоже', 'ru'), ('вся', 'ru'), ('предприятия', 'ru'), ('рядом', 'ru'), ('же', 'ru'), ('среди', 'ru'), ('дел', 'ru'), ('четыре', 'ru'), ('й', 'ru'), ('себя', 'ru'), ('rnd', 'ru'), ('пришлось', 'ru'), ('комитета', 'ru'), ('те', 'ru'), ('находившегося', 'ru'), ('тебе', 'ru'), ('состоянии', 'ru'), ('но', 'ru'), ('любой', 'ru'), ('говорят', 'ru'), ('на', 'ru'), ('что', 'ru'), ('говорил', 'ru'), ('силы', 'ru'), ('права', 'ru'), ('будто', 'ru'), ('включая', 'ru'), ('тех', 'ru'), ('считать', 'ru'), ('документы', 'ru'), ('кого', 'ru'), ('ты', 'ru'), ('себе', 'ru'), ('Кто', 'ru'), ('На', 'ru'), ('тысячи', 'ru'), ('кроме', 'ru'), ('Александр', 'ru'), ('Рейтер', 'ru'), ('земли', 'ru'), ('конца', 'ru'), ('то', 'ru'), ('числе', 'ru'), ('работать', 'ru'), ('говорить', 'ru'), ('Это', 'ru'), ('людям', 'ru'), ('центре', 'ru'), ('два', 'ru'), ('тем', 'ru'), ('Можно', 'ru'), ('дома', 'ru'), ('системы', 'ru'), ('каждый', 'ru'), ('какой', 'ru'), ('случае', 'ru'), ('средства', 'ru'), ('всё', 'ru'), ('ваш', 'ru'), ('дал', 'ru'), ('производство', 'ru'), ('стать', 'ru'), ('а', 'ru'), ('почти', 'ru'), ('первую', 'ru'), ('глава', 'ru'), ('роль', 'ru'), ('такая', 'ru'), ('столь', 'ru'), ('Александра', 'ru'), ('оказалась', 'ru'), ('НА', 'ru'), ('ответ', 'ru'), ('процентов', 'ru'), ('деятельности', 'ru'), ('часто', 'ru'), ('делам', 'ru'), ('никак', 'ru'), ('сторону', 'ru'), ('ныне', 'ru'), ('ходе', 'ru'), ('оно', 'ru'), ('деятельность', 'ru'), ('день', 'ru'), ('link', 'ru'), ('касается', 'ru'), ('градусов', 'ru'), ('прокуратуры', 'ru'), ('много', 'ru'), ('новых', 'ru'), ('отличие', 'ru'), ('Сергея', 'ru'), ('либо', 'ru'), ('таких', 'ru'), ('Сейчас', 'ru'), ('такие', 'ru'), ('три', 'ru'), ('времени', 'ru'), ('нее', 'ru'), ('Андрей', 'ru'), ('знает', 'ru'), ('Владимира', 'ru'), ('решения', 'ru'), ('выяснилось', 'ru'), ('пресс', 'ru'), ('марта', 'ru'), ('позиции', 'ru'), ('работу', 'ru'), ('е', 'ru'), ('снова', 'ru'), ('внимание', 'ru'), ('Как', 'ru'), ('весь', 'ru'), ('примеру', 'ru'), ('эти', 'ru'), ('сказать', 'ru'), ('люди', 'ru'), ('движения', 'ru'), ('выше', 'ru'), ('нам', 'ru'), ('к', 'ru'), ('интервью', 'ru'), ('территории', 'ru'), ('таки', 'ru'), ('ей', 'ru'), ('метра', 'ru'), ('деньги', 'ru'), ('Фото', 'ru'), ('отношении', 'ru'), ('последние', 'ru'), ('когда', 'ru'), ('вдруг', 'ru'), ('всю', 'ru'), ('Вы', 'ru'), ('Конечно', 'ru'), ('этим', 'ru'), ('Их', 'ru'), ('Нью', 'ru'), ('степени', 'ru'), ('апреля', 'ru'), ('большинство', 'ru'), ('дела', 'ru'), ('за', 'ru'), ('десять', 'ru'), ('должно', 'ru'), ('километров', 'ru'), ('крупных', 'ru'), ('там', 'ru'), ('назад', 'ru'), ('наш', 'ru'), ('площади', 'ru'), ('bin', 'ru'), ('Там', 'ru'), ('д', 'ru'), ('по', 'ru'), ('наиболее', 'ru'), ('могла', 'ru'), ('квартиры', 'ru'), ('мы', 'ru'), ('нас', 'ru'), ('явно', 'ru'), ('работа', 'ru'), ('женщин', 'ru'), ('ни', 'ru'), ('вряд', 'ru'), ('лет', 'ru'), ('столице', 'ru'), ('зрения', 'ru'), ('час', 'ru'), ('своих', 'ru'), ('операции', 'ru'), ('тот', 'ru'), ('сказал', 'ru'), ('невозможно', 'ru'), ('За', 'ru'), ('Известия', 'ru'), ('иначе', 'ru'), ('Поэтому', 'ru'), ('процента', 'ru'), ('www', 'ru'), ('ряд', 'ru'), ('Ни', 'ru'), ('минут', 'ru'), ('недели', 'ru'), ('опять', 'ru'), ('вовсе', 'ru'), ('специалистов', 'ru'), ('полностью', 'ru'), ('То', 'ru'), ('свою', 'ru'), ('этих', 'ru'), ('считает', 'ru'), ('своем', 'ru'), ('мене', 'ru'), ('фирмы', 'ru'), ('результате', 'ru'), ('кому', 'ru'), ('очередь', 'ru'), ('довольно', 'ru'), ('новый', 'ru'), ('Он', 'ru'), ('делу', 'ru'), ('Известиям', 'ru'), ('примерно', 'ru'), ('между', 'ru'), ('заместитель', 'ru'), ('знают', 'ru'), ('миллиона', 'ru'), ('время', 'ru'), ('которая', 'ru'), ('годы', 'ru'), ('всей', 'ru'), ('премьер', 'ru'), ('Хотя', 'ru'), ('края', 'ru'), ('До', 'ru'), ('тому', 'ru'), ('Грозном', 'ru'), ('комиссии', 'ru'), ('центр', 'ru'), ('ему', 'ru'), ('людей', 'ru'), ('дней', 'ru'), ('Но', 'ru'), ('куда', 'ru'), ('силу', 'ru'), ('вполне', 'ru'), ('ничего', 'ru'), ('которые', 'ru'), ('ситуация', 'ru'), ('Из', 'ru'), ('может', 'ru'), ('так', 'ru'), ('сотрудники', 'ru'), ('делать', 'ru'), ('html', 'ru'), ('далеко', 'ru'), ('легко', 'ru'), ('проблем', 'ru'), ('группы', 'ru'), ('вокруг', 'ru'), ('которыми', 'ru'), ('виде', 'ru'), ('августа', 'ru'), ('конце', 'ru'), ('Более', 'ru'), ('К', 'ru'), ('работавшую', 'ru'), ('style', 'ru'), ('впервые', 'ru'), ('самым', 'ru'), ('Виктор', 'ru'), ('района', 'ru'), ('действий', 'ru'), ('поскольку', 'ru'), ('можно', 'ru'), ('месяцев', 'ru'), ('события', 'ru'), ('действительно', 'ru'), ('стала', 'ru'), ('Алексей', 'ru'), ('Один', 'ru'), ('метров', 'ru'), ('Все', 'ru'), ('Впрочем', 'ru'), ('сама', 'ru'), ('некоторого', 'ru'), ('тогда', 'ru'), ('письмо', 'ru'), ('информацию', 'ru'), ('счет', 'ru'), ('сделать', 'ru'), ('дает', 'ru'), ('Его', 'ru'), ('войск', 'ru'), ('Пока', 'ru'), ('которого', 'ru'), ('пор', 'ru'), ('существует', 'ru'), ('вроде', 'ru'), ('немало', 'ru'), ('бывшего', 'ru'), ('эту', 'ru'), ('первая', 'ru'), ('Эта', 'ru'), ('смерти', 'ru'), ('ТАСС', 'ru'), ('членов', 'ru'), ('частности', 'ru'), ('специалисты', 'ru'), ('Этот', 'ru'), ('которых', 'ru'), ('При', 'ru'), ('стате', 'ru'), ('народа', 'ru'), ('могут', 'ru'), ('некоторых', 'ru'), ('данным', 'ru'), ('говорит', 'ru'), ('лиц', 'ru'), ('сто', 'ru'), ('Может', 'ru'), ('сколько', 'ru'), ('считают', 'ru'), ('вот', 'ru'), ('предприятий', 'ru'), ('которое', 'ru'), ('средств', 'ru'), ('Многие', 'ru'), ('итоге', 'ru'), ('дальше', 'ru'), ('тонн', 'ru'), ('является', 'ru'), ('пяти', 'ru'), ('проблемы', 'ru'), ('Потом', 'ru'), ('следует', 'ru'), ('раза', 'ru'), ('Почему', 'ru'), ('сообща', 'ru'), ('quot', 'ru'), ('друг', 'ru'), ('ИТАР', 'ru'), ('Они', 'ru'), ('Сегодня', 'ru'), ('основном', 'ru'), ('известно', 'ru'), ('такой', 'ru'), ('Между', 'ru'), ('ли', 'ru'), ('чаще', 'ru'), ('должны', 'ru'), ('большой', 'ru'), ('вторая', 'ru'), ('Да', 'ru'), ('через', 'ru'), ('точки', 'ru'), ('прежнему', 'ru'), ('самые', 'ru'), ('го', 'ru'), ('Даже', 'ru'), ('часов', 'ru'), ('чем', 'ru'), ('решил', 'ru'), ('директора', 'ru'), ('взгляд', 'ru'), ('часа', 'ru'), ('со', 'ru'), ('ситуацию', 'ru'), ('принять', 'ru'), ('правило', 'ru'), ('Б', 'ru'), ('уровень', 'ru'), ('помощь', 'ru'), ('поводу', 'ru'), ('собственности', 'ru'), ('четырех', 'ru'), ('первые', 'ru'), ('самом', 'ru'), ('одно', 'ru'), ('тысяч', 'ru'), ('сразу', 'ru'), ('Николай', 'ru'), ('органы', 'ru'), ('фирм', 'ru'), ('порядке', 'ru'), ('ясно', 'ru'), ('таким', 'ru'), ('М', 'ru'), ('опыт', 'ru'), ('однако', 'ru'), ('href', 'ru'), ('произошло', 'ru'), ('блог', 'ru'), ('почему', 'ru'), ('По', 'ru'), ('суда', 'ru'), ('том', 'ru'), ('всему', 'ru'), ('постоянно', 'ru'), ('человек', 'ru'), ('посколько', 'ru'), ('свое', 'ru'), ('тут', 'ru'), ('наконец', 'ru'), ('ними', 'ru'), ('м', 'ru'), ('прошлого', 'ru'), ('мало', 'ru'), ('центра', 'ru'), ('пятая', 'ru'), ('помощью', 'ru'), ('воды', 'ru'), ('приходится', 'ru'), ('две', 'ru'), ('прошла', 'ru'), ('кажется', 'ru'), ('чуть', 'ru'), ('последних', 'ru'), ('образом', 'ru'), ('раз', 'ru'), ('имени', 'ru'), ('компания', 'ru'), ('условиях', 'ru'), ('поэтому', 'ru'), ('Анатолий', 'ru'), ('прошлом', 'ru'), ('нет', 'ru'), ('туда', 'ru'), ('АО', 'ru'), ('стали', 'ru'), ('другом', 'ru'), ('этом', 'ru'), ('труда', 'ru'), ('меньше', 'ru'), ('нашей', 'ru'), ('отношения', 'ru'), ('самого', 'ru'), ('воскресение', 'ru'), ('некоторые', 'ru'), ('возможно', 'ru'), ('этого', 'ru'), ('какие', 'ru'), ('Михаил', 'ru'), ('котором', 'ru'), ('цены', 'ru'), ('нового', 'ru'), ('вновь', 'ru'), ('нашего', 'ru'), ('участников', 'ru'), ('сумму', 'ru'), ('проблема', 'ru'), ('Что', 'ru'), ('провести', 'ru'), ('июня', 'ru'), ('имеет', 'ru'), ('нескольких', 'ru'), ('главное', 'ru'), ('словам', 'ru'), ('дни', 'ru'), ('первым', 'ru'), ('вести', 'ru'), ('никто', 'ru'), ('Н', 'ru'), ('имя', 'ru'), ('ней', 'ru'), ('создать', 'ru'), ('города', 'ru'), ('продукции', 'ru'), ('месяц', 'ru'), ('потом', 'ru'), ('Так', 'ru'), ('бывший', 'ru'), ('зам', 'ru'), ('она', 'ru'), ('действия', 'ru'), ('днях', 'ru'), ('вместо', 'ru'), ('положение', 'ru'), ('сам', 'ru'), ('идет', 'ru'), ('находится', 'ru'), ('всем', 'ru'), ('области', 'ru'), ('один', 'ru'), ('Только', 'ru'), ('самых', 'ru'), ('театра', 'ru'), ('своей', 'ru'), ('давно', 'ru'), ('меры', 'ru'), ('Для', 'ru'), ('месте', 'ru'), ('Юрий', 'ru'), ('совсем', 'ru'), ('речь', 'ru'), ('одним', 'ru'), ('до', 'ru'), ('момент', 'ru'), ('начальника', 'ru'), ('вам', 'ru'), ('другие', 'ru'), ('затем', 'ru'), ('случай', 'ru'), ('пути', 'ru'), ('менее', 'ru'), ('заявил', 'ru'), ('право', 'ru'), ('стоит', 'ru'), ('очередной', 'ru'), ('Не', 'ru'), ('получить', 'ru'), ('конференции', 'ru'), ('станет', 'ru'), ('система', 'ru'), ('был', 'ru'), ('мой', 'ru'), ('оказалось', 'ru'), ('очень', 'ru'), ('скажем', 'ru'), ('во', 'ru'), ('такого', 'ru'), ('акций', 'ru'), ('фирма', 'ru'), ('ноября', 'ru'), ('с', 'ru'), ('хоть', 'ru'), ('Здесь', 'ru'), ('другим', 'ru'), ('полтора', 'ru'), ('других', 'ru'), ('здесь', 'ru'), ('команда', 'ru'), ('сделал', 'ru'), ('или', 'ru'), ('будут', 'ru'), ('достаточно', 'ru'), ('Без', 'ru'), ('Известий', 'ru'), ('надо', 'ru'), ('рода', 'ru'), ('стороны', 'ru'), ('город', 'ru'), ('происходит', 'ru'), ('Мне', 'ru'), ('Именно', 'ru'), ('начала', 'ru'), ('летний', 'ru'), ('ситуации', 'ru');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('sid', 'ru'), ('при', 'ru'), ('мог', 'ru'), ('как', 'ru'), ('деятельн', 'ru'), ('пришл', 'ru'), ('для', 'ru'), ('над', 'ru'), ('ма', 'ru'), ('проект', 'ru'), ('вперв', 'ru'), ('легк', 'ru'), ('систем', 'ru'), ('реш', 'ru'), ('будет', 'ru'), ('взят', 'ru'), ('поч', 'ru'), ('воскресен', 'ru'), ('равн', 'ru'), ('срок', 'ru'), ('теб', 'ru'), ('юр', 'ru'), ('закон', 'ru'), ('отношен', 'ru'), ('коп', 'ru'), ('туд', 'ru'), ('резк', 'ru'), ('недел', 'ru'), ('письм', 'ru'), ('площад', 'ru'), ('http', 'ru'), ('дом', 'ru'), ('вест', 'ru'), ('впроч', 'ru'), ('уда', 'ru'), ('кто', 'ru'), ('сил', 'ru'), ('имеют', 'ru'), ('себ', 'ru'), ('оп', 'ru'), ('начальник', 'ru'), ('прот', 'ru'), ('могл', 'ru'), ('мне', 'ru'), ('слишк', 'ru'), ('фот', 'ru'), ('дня', 'ru'), ('livejournal', 'ru'), ('не', 'ru'), ('народ', 'ru'), ('полтор', 'ru'), ('имет', 'ru'), ('период', 'ru'), ('известн', 'ru'), ('рук', 'ru'), ('отлич', 'ru'), ('одновремен', 'ru'), ('акц', 'ru'), ('врод', 'ru'), ('должн', 'ru'), ('перед', 'ru'), ('част', 'ru'), ('август', 'ru'), ('наход', 'ru'), ('основн', 'ru'), ('об', 'ru'), ('межд', 'ru'), ('внов', 'ru'), ('большинств', 'ru'), ('свет', 'ru'), ('их', 'ru'), ('хочет', 'ru'), ('стал', 'ru'), ('род', 'ru'), ('след', 'ru'), ('т', 'ru'), ('про', 'ru'), ('вы', 'ru'), ('тогд', 'ru'), ('кром', 'ru'), ('сут', 'ru'), ('и', 'ru'), ('реч', 'ru'), ('скаж', 'ru'), ('он', 'ru'), ('смерт', 'ru'), ('стольк', 'ru'), ('процесс', 'ru'), ('истор', 'ru'), ('земл', 'ru'), ('нег', 'ru'), ('конференц', 'ru'), ('уж', 'ru'), ('ве', 'ru'), ('завод', 'ru'), ('тасс', 'ru'), ('представител', 'ru'), ('очеред', 'ru'), ('получ', 'ru'), ('пут', 'ru'), ('произошл', 'ru'), ('территор', 'ru'), ('без', 'ru'), ('трех', 'ru'), ('найт', 'ru'), ('информац', 'ru'), ('ним', 'ru'), ('вопрос', 'ru'), ('adriver', 'ru'), ('сегодн', 'ru'), ('снов', 'ru'), ('amp', 'ru'), ('где', 'ru'), ('я', 'ru'), ('слов', 'ru'), ('такж', 'ru'), ('позиц', 'ru'), ('рол', 'ru'), ('ещ', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('качеств', 'ru'), ('рейтер', 'ru'), ('сдела', 'ru'), ('каса', 'ru'), ('постоя', 'ru'), ('корреспондент', 'ru'), ('окол', 'ru'), ('из', 'ru'), ('январ', 'ru'), ('некотор', 'ru'), ('наибол', 'ru'), ('от', 'ru'), ('анатол', 'ru'), ('б', 'ru'), ('зрен', 'ru'), ('у', 'ru'), ('cgi', 'ru'), ('факт', 'ru'), ('жизн', 'ru'), ('вовс', 'ru'), ('сторон', 'ru'), ('год', 'ru'), ('о', 'ru'), ('личн', 'ru'), ('оста', 'ru'), ('люб', 'ru'), ('помощ', 'ru'), ('район', 'ru'), ('сво', 'ru'), ('практическ', 'ru'), ('миллион', 'ru'), ('все', 'ru'), ('им', 'ru'), ('хорош', 'ru'), ('октябр', 'ru'), ('говор', 'ru'), ('г', 'ru'), ('кра', 'ru'), ('х', 'ru'), ('несмотр', 'ru'), ('этот', 'ru'), ('под', 'ru'), ('поскольк', 'ru'), ('нужн', 'ru'), ('вас', 'ru'), ('созда', 'ru'), ('театр', 'ru'), ('ком', 'ru'), ('движен', 'ru'), ('в', 'ru'), ('мер', 'ru'), ('поня', 'ru'), ('когд', 'ru'), ('сих', 'ru'), ('дне', 'ru'), ('фонд', 'ru'), ('дальш', 'ru'), ('работ', 'ru'), ('собствен', 'ru'), ('бы', 'ru'), ('куд', 'ru'), ('чащ', 'ru'), ('прав', 'ru'), ('нем', 'ru'), ('лидер', 'ru'), ('нич', 'ru'), ('решен', 'ru'), ('главн', 'ru'), ('прежн', 'ru'), ('руководств', 'ru'), ('трудн', 'ru'), ('март', 'ru'), ('пост', 'ru'), ('област', 'ru'), ('деся', 'ru'), ('соб', 'ru'), ('видим', 'ru'), ('нача', 'ru'), ('почт', 'ru'), ('вниман', 'ru'), ('руб', 'ru'), ('сейчас', 'ru'), ('да', 'ru'), ('п', 'ru'), ('существ', 'ru'), ('двух', 'ru'), ('оказа', 'ru'), ('например', 'ru'), ('новост', 'ru'), ('зна', 'ru'), ('комисс', 'ru'), ('документ', 'ru'), ('частност', 'ru'), ('вся', 'ru'), ('утвержда', 'ru'), ('же', 'ru'), ('дел', 'ru'), ('й', 'ru'), ('rnd', 'ru'), ('те', 'ru'), ('но', 'ru'), ('на', 'ru'), ('что', 'ru'), ('выш', 'ru'), ('производств', 'ru'), ('борис', 'ru'), ('тех', 'ru'), ('довольн', 'ru'), ('комментар', 'ru'), ('быстр', 'ru'), ('нын', 'ru'), ('сумм', 'ru'), ('ты', 'ru'), ('приня', 'ru'), ('инач', 'ru'), ('декабр', 'ru'), ('дан', 'ru'), ('жит', 'ru'), ('всег', 'ru'), ('то', 'ru'), ('мал', 'ru'), ('грозн', 'ru'), ('выясн', 'ru'), ('орга', 'ru'), ('два', 'ru'), ('тем', 'ru'), ('сред', 'ru'), ('ваш', 'ru'), ('всё', 'ru'), ('количеств', 'ru'), ('результат', 'ru'), ('дал', 'ru'), ('а', 'ru'), ('кажет', 'ru'), ('специалист', 'ru'), ('ответ', 'ru'), ('ден', 'ru'), ('никак', 'ru'), ('сентябр', 'ru'), ('есл', 'ru'), ('придет', 'ru'), ('чтоб', 'ru'), ('link', 'ru'), ('начал', 'ru'), ('посл', 'ru'), ('раньш', 'ru'), ('век', 'ru'), ('можн', 'ru'), ('глав', 'ru'), ('три', 'ru'), ('вед', 'ru'), ('якоб', 'ru'), ('знает', 'ru'), ('тепер', 'ru'), ('пресс', 'ru'), ('е', 'ru'), ('суд', 'ru'), ('сообщ', 'ru'), ('команд', 'ru'), ('четыр', 'ru'), ('нема', 'ru'), ('всегд', 'ru'), ('никогд', 'ru'), ('тольк', 'ru'), ('нам', 'ru'), ('к', 'ru'), ('втор', 'ru'), ('вдруг', 'ru'), ('всю', 'ru'), ('участник', 'ru'), ('хот', 'ru'), ('либ', 'ru'), ('дела', 'ru'), ('за', 'ru'), ('игр', 'ru'), ('точн', 'ru'), ('андр', 'ru'), ('заявлен', 'ru'), ('там', 'ru'), ('назад', 'ru'), ('июл', 'ru'), ('наш', 'ru'), ('очередн', 'ru'), ('политик', 'ru'), ('ситуац', 'ru'), ('bin', 'ru'), ('ег', 'ru'), ('д', 'ru'), ('по', 'ru'), ('мы', 'ru'), ('прост', 'ru'), ('нас', 'ru'), ('нескольк', 'ru'), ('женщин', 'ru'), ('работа', 'ru'), ('ни', 'ru'), ('вряд', 'ru'), ('лет', 'ru'), ('пят', 'ru'), ('час', 'ru'), ('крупн', 'ru'), ('тот', 'ru'), ('вид', 'ru'), ('однак', 'ru'), ('шест', 'ru'), ('мо', 'ru'), ('вод', 'ru'), ('квартир', 'ru'), ('www', 'ru'), ('заяв', 'ru'), ('ряд', 'ru'), ('стат', 'ru'), ('минут', 'ru'), ('недавн', 'ru'), ('достаточн', 'ru'), ('тог', 'ru'), ('степен', 'ru'), ('труд', 'ru'), ('услов', 'ru'), ('образ', 'ru'), ('миров', 'ru'), ('счита', 'ru'), ('пот', 'ru'), ('долж', 'ru'), ('метр', 'ru'), ('знают', 'ru'), ('уровен', 'ru'), ('приход', 'ru'), ('премьер', 'ru'), ('встреч', 'ru'), ('перв', 'ru'), ('возможн', 'ru'), ('центр', 'ru'), ('вмест', 'ru'), ('тон', 'ru'), ('так', 'ru'), ('может', 'ru'), ('миллиард', 'ru'), ('виц', 'ru'), ('html', 'ru'), ('времен', 'ru'), ('проблем', 'ru'), ('конечн', 'ru'), ('вокруг', 'ru'), ('явля', 'ru'), ('опя', 'ru'), ('эт', 'ru'), ('style', 'ru'), ('зде', 'ru'), ('особен', 'ru'), ('чут', 'ru'), ('подобн', 'ru'), ('заместител', 'ru'), ('власт', 'ru'), ('цел', 'ru'), ('пок', 'ru'), ('примерн', 'ru'), ('естествен', 'ru'), ('нельз', 'ru'), ('быт', 'ru'), ('мест', 'ru'), ('сем', 'ru'), ('сраз', 'ru'), ('лиш', 'ru'), ('счет', 'ru'), ('иллюстрац', 'ru'), ('развит', 'ru'), ('дает', 'ru'), ('войск', 'ru'), ('повод', 'ru'), ('необходим', 'ru'), ('пор', 'ru'), ('связ', 'ru'), ('никола', 'ru'), ('полност', 'ru'), ('уровн', 'ru'), ('ког', 'ru'), ('пример', 'ru'), ('могут', 'ru'), ('ссылк', 'ru'), ('алекс', 'ru'), ('лиц', 'ru'), ('котор', 'ru'), ('прокуратур', 'ru'), ('сто', 'ru'), ('лучш', 'ru'), ('давн', 'ru'), ('вот', 'ru'), ('включ', 'ru'), ('скор', 'ru'), ('председател', 'ru'), ('ноябр', 'ru'), ('средств', 'ru'), ('прошл', 'ru'), ('выбор', 'ru'), ('скольк', 'ru'), ('директор', 'ru'), ('едв', 'ru'), ('нынешн', 'ru'), ('разн', 'ru'), ('друг', 'ru'), ('quot', 'ru'), ('точк', 'ru'), ('деньг', 'ru'), ('ли', 'ru'), ('через', 'ru'), ('александр', 'ru'), ('последн', 'ru'), ('километр', 'ru'), ('член', 'ru'), ('го', 'ru'), ('знач', 'ru'), ('чем', 'ru'), ('весьм', 'ru'), ('станов', 'ru'), ('ест', 'ru'), ('даж', 'ru'), ('взгляд', 'ru'), ('со', 'ru'), ('нов', 'ru'), ('никт', 'ru'), ('компан', 'ru'), ('кажд', 'ru'), ('бывш', 'ru'), ('серге', 'ru'), ('сказа', 'ru'), ('числ', 'ru'), ('четырех', 'ru'), ('тысяч', 'ru'), ('происход', 'ru'), ('врем', 'ru'), ('фирм', 'ru'), ('прям', 'ru'), ('предприят', 'ru'), ('совершен', 'ru'), ('href', 'ru'), ('очен', 'ru'), ('блог', 'ru'), ('имен', 'ru'), ('долг', 'ru'), ('вполн', 'ru'), ('том', 'ru'), ('мен', 'ru'), ('человек', 'ru'), ('обычн', 'ru'), ('мног', 'ru'), ('н', 'ru'), ('столиц', 'ru'), ('тут', 'ru'), ('наконец', 'ru'), ('м', 'ru'), ('жител', 'ru'), ('конц', 'ru'), ('итог', 'ru'), ('ну', 'ru'), ('интерв', 'ru'), ('две', 'ru'), ('действ', 'ru'), ('сборн', 'ru'), ('градус', 'ru'), ('нью', 'ru'), ('чег', 'ru'), ('раз', 'ru'), ('положен', 'ru'), ('ем', 'ru'), ('бол', 'ru'), ('стол', 'ru'), ('июн', 'ru'), ('будт', 'ru'), ('прич', 'ru'), ('ю', 'ru'), ('нибуд', 'ru'), ('нет', 'ru'), ('массов', 'ru'), ('пробл', 'ru'), ('итар', 'ru'), ('ассошиэйтед', 'ru'), ('поэт', 'ru'), ('провест', 'ru'), ('порядк', 'ru'), ('совс', 'ru'), ('прежд', 'ru'), ('случа', 'ru'), ('ил', 'ru'), ('местн', 'ru'), ('имеет', 'ru'), ('дни', 'ru'), ('состоян', 'ru'), ('ран', 'ru'), ('л', 'ru'), ('течен', 'ru'), ('месяц', 'ru'), ('феврал', 'ru'), ('люд', 'ru'), ('продукц', 'ru'), ('зам', 'ru'), ('днях', 'ru'), ('мир', 'ru'), ('извест', 'ru'), ('сам', 'ru'), ('интерес', 'ru'), ('идет', 'ru'), ('один', 'ru'), ('всем', 'ru'), ('меньш', 'ru'), ('ход', 'ru'), ('апрел', 'ru'), ('явн', 'ru'), ('момент', 'ru'), ('одн', 'ru'), ('вообщ', 'ru'), ('до', 'ru'), ('вам', 'ru'), ('отдел', 'ru'), ('серг', 'ru'), ('тож', 'ru'), ('далек', 'ru'), ('комитет', 'ru'), ('сотрудник', 'ru'), ('летн', 'ru'), ('руководител', 'ru'), ('операц', 'ru'), ('похож', 'ru'), ('виктор', 'ru'), ('станет', 'ru'), ('кстат', 'ru'), ('цен', 'ru'), ('правд', 'ru'), ('был', 'ru'), ('во', 'ru'), ('суббот', 'ru'), ('с', 'ru'), ('дат', 'ru'), ('действительн', 'ru'), ('миха', 'ru'), ('событ', 'ru'), ('участ', 'ru'), ('будут', 'ru'), ('невозможн', 'ru'), ('ясн', 'ru'), ('владимир', 'ru'), ('зат', 'ru'), ('парт', 'ru'), ('город', 'ru'), ('групп', 'ru'), ('больш', 'ru'), ('процент', 'ru');INSERT INTO stopwords_short (stopword, language) VALUES ('som', 'sv'), ('du', 'sv'), ('att', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('sådan', 'sv'), ('utan', 'sv'), ('nu', 'sv'), ('sådana', 'sv'), ('ingen', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('samma', 'sv'), ('bli', 'sv'), ('vars', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('med', 'sv'), ('ert', 'sv'), ('vilken', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('detta', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('ut', 'sv'), ('inom', 'sv'), ('någon', 'sv'), ('men', 'sv'), ('inte', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('vara', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('mina', 'sv'), ('till', 'sv'), ('ett', 'sv'), ('sitta', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('när', 'sv'), ('man', 'sv'), ('den', 'sv'), ('dess', 'sv'), ('har', 'sv'), ('sådant', 'sv'), ('hade', 'sv'), ('hans', 'sv'), ('vid', 'sv'), ('dessa', 'sv'), ('dina', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('vilka', 'sv'), ('deras', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('henne', 'sv'), ('blivit', 'sv'), ('varje', 'sv'), ('vilkas', 'sv'), ('om', 'sv'), ('dem', 'sv'), ('en', 'sv'), ('sin', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('alla', 'sv'), ('kunde', 'sv'), ('våra', 'sv'), ('hennes', 'sv'), ('åt', 'sv'), ('något', 'sv'), ('ha', 'sv'), ('vem', 'sv'), ('efter', 'sv'), ('skulle', 'sv'), ('icke', 'sv'), ('i', 'sv'), ('över', 'sv'), ('än', 'sv'), ('sina', 'sv'), ('denna', 'sv'), ('mitt', 'sv'), ('under', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('några', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('eller', 'sv'), ('mot', 'sv'), ('blev', 'sv'), ('är', 'sv');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('som', 'sv'), ('att', 'sv'), ('du', 'sv'), ('dett', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('nu', 'sv'), ('utan', 'sv'), ('sådan', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('bli', 'sv'), ('med', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('ert', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('ell', 'sv'), ('henn', 'sv'), ('int', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('all', 'sv'), ('ut', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('inom', 'sv'), ('men', 'sv'), ('någon', 'sv'), ('skull', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('vilk', 'sv'), ('till', 'sv'), ('und', 'sv'), ('ett', 'sv'), ('denn', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('den', 'sv'), ('man', 'sv'), ('när', 'sv'), ('har', 'sv'), ('dess', 'sv'), ('sådant', 'sv'), ('varj', 'sv'), ('samm', 'sv'), ('der', 'sv'), ('vid', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('kund', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('blivit', 'sv'), ('om', 'sv'), ('ing', 'sv'), ('någr', 'sv'), ('dem', 'sv'), ('sin', 'sv'), ('en', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('ick', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('ha', 'sv'), ('något', 'sv'), ('åt', 'sv'), ('vem', 'sv'), ('eft', 'sv'), ('än', 'sv'), ('över', 'sv'), ('i', 'sv'), ('had', 'sv'), ('mitt', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('sitt', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('mot', 'sv'), ('är', 'sv'), ('blev', 'sv');INSERT INTO stopwords_short (stopword, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('filan', 'tr'), ('böylece', 'tr'), ('biz', 'tr'), ('hepsi', 'tr'), ('senin', 'tr'), ('hepsine', 'tr'), ('hiçbirini', 'tr'), ('hangisi', 'tr'), ('e', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('bazısı', 'tr'), ('þey', 'tr'), ('kendi', 'tr'), ('dokuz', 'tr'), ('siz', 'tr'), ('kendine', 'tr'), ('diğeri', 'tr'), ('bu', 'tr'), ('seni', 'tr'), ('hem', 'tr'), ('onu', 'tr'), ('kimi', 'tr'), ('bizim', 'tr'), ('defa', 'tr'), ('burada', 'tr'), ('mi', 'tr'), ('hiç kimse', 'tr'), ('biri', 'tr'), ('bütün', 'tr'), ('ve', 'tr'), ('altmýþ', 'tr'), ('son', 'tr'), ('onun', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('dört', 'tr'), ('milyon', 'tr'), ('u', 'tr'), ('yerine', 'tr'), ('k', 'tr'), ('orada', 'tr'), ('beş', 'tr'), ('mý', 'tr'), ('buna', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('þuna', 'tr'), ('dahi', 'tr'), ('içinde', 'tr'), ('çoğuna', 'tr'), ('kaç', 'tr'), ('en gibi', 'tr'), ('böyle', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('beni', 'tr'), ('kadar', 'tr'), ('şunda', 'tr'), ('zaten', 'tr'), ('dolayı', 'tr'), ('o', 'tr'), ('bunun', 'tr'), ('hangi', 'tr'), ('şeye', 'tr'), ('d', 'tr'), ('on', 'tr'), ('herkese', 'tr'), ('yüz', 'tr'), ('ya da', 'tr'), ('bazıları', 'tr'), ('herkes', 'tr'), ('nereden', 'tr'), ('bile', 'tr'), ('doksan', 'tr'), ('hatta', 'tr'), ('bana', 'tr'), ('onlari', 'tr'), ('z', 'tr'), ('artık', 'tr'), ('şimdi', 'tr'), ('bazen', 'tr'), ('onlara', 'tr'), ('fakat', 'tr'), ('size', 'tr'), ('birşey', 'tr'), ('var', 'tr'), ('neden', 'tr'), ('hep', 'tr'), ('nereye', 'tr'), ('bunu', 'tr'), ('bunda', 'tr'), ('ğ', 'tr'), ('birkez', 'tr'), ('oysa', 'tr'), ('birkaç', 'tr'), ('yine', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('t', 'tr'), ('bizden', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('onların', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('birçoğu', 'tr'), ('madem', 'tr'), ('yedi', 'tr'), ('onlar', 'tr'), ('ötürü', 'tr'), ('þeyi', 'tr'), ('herkesi', 'tr'), ('bir', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('yoksa', 'tr'), ('yetmiþ', 'tr'), ('þeyler', 'tr'), ('birþeyi', 'tr'), ('birisi', 'tr'), ('n', 'tr'), ('şeyi', 'tr'), ('p', 'tr'), ('INSERmi', 'tr'), ('hiç', 'tr'), ('diye', 'tr'), ('kimin', 'tr'), ('hepsini', 'tr'), ('da', 'tr'), ('y', 'tr'), ('önce', 'tr'), ('g', 'tr'), ('her biri', 'tr'), ('nerede', 'tr'), ('tabi', 'tr'), ('ne zaman', 'tr'), ('bazı', 'tr'), ('daha', 'tr'), ('sizden', 'tr'), ('benim', 'tr'), ('değil', 'tr'), ('benden', 'tr'), ('katrilyon', 'tr'), ('acaba', 'tr'), ('sizi', 'tr'), ('ç', 'tr'), ('kendini', 'tr'), ('belki', 'tr'), ('elli', 'tr'), ('trilyon', 'tr'), ('şundan', 'tr'), ('ya', 'tr'), ('şeyden', 'tr'), ('asla', 'tr'), ('ü', 'tr'), ('tüm', 'tr'), ('mı', 'tr'), ('tümü', 'tr'), ('çünkü', 'tr'), ('ama', 'tr'), ('işte', 'tr'), ('birkaçı', 'tr'), ('s', 'tr'), ('rağmen', 'tr'), ('c', 'tr'), ('üzere', 'tr'), ('altı', 'tr'), ('şunu', 'tr'), ('diğerleri', 'tr'), ('bize', 'tr'), ('veyahut', 'tr'), ('şunun', 'tr'), ('senden', 'tr'), ('nasýl', 'tr'), ('üç', 'tr'), ('i', 'tr'), ('m', 'tr'), ('gibi', 'tr'), ('hiçbiri', 'tr'), ('ile', 'tr'), ('sen', 'tr'), ('niçin', 'tr'), ('sizin', 'tr'), ('yirmi', 'tr'), ('gene', 'tr'), ('kim', 'tr'), ('az', 'tr'), ('felan', 'tr'), ('altý', 'tr'), ('b', 'tr'), ('nesi', 'tr'), ('nerde', 'tr'), ('kimisi', 'tr'), ('þu', 'tr'), ('birşeyi', 'tr'), ('elbette', 'tr'), ('şöyle', 'tr'), ('milyar', 'tr'), ('oysaki', 'tr'), ('şunlar', 'tr'), ('kime', 'tr'), ('bazý', 'tr'), ('mu', 'tr'), ('her', 'tr'), ('öbürü', 'tr'), ('bundan', 'tr'), ('þundan', 'tr'), ('onlardan', 'tr'), ('seksen', 'tr'), ('kimden', 'tr'), ('bizi', 'tr'), ('birçokları', 'tr'), ('henüz', 'tr'), ('diğer', 'tr'), ('þunu', 'tr'), ('birçok', 'tr'), ('ş', 'tr'), ('ı', 'tr'), ('şeyler', 'tr'), ('neyse', 'tr'), ('tamam', 'tr'), ('sekiz', 'tr'), ('öyle', 'tr'), ('hani', 'tr'), ('onlarýn', 'tr'), ('ise', 'tr'), ('birþey', 'tr'), ('niye', 'tr'), ('v', 'tr'), ('ondan', 'tr'), ('zira', 'tr'), ('hiçbirine', 'tr'), ('ne kadar', 'tr'), ('çoğu', 'tr'), ('r', 'tr'), ('þunda', 'tr'), ('onu otuz', 'tr'), ('h', 'tr'), ('falan', 'tr'), ('sonra', 'tr'), ('iki', 'tr'), ('nedir', 'tr'), ('ona', 'tr'), ('f', 'tr'), ('demek', 'tr'), ('sana', 'tr'), ('þeyden', 'tr'), ('çoğunu', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('aslında', 'tr'), ('yani', 'tr'), ('şuna', 'tr'), ('için', 'tr');INSERT INTO stopword_stems_short (stopword_stem, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('biz', 'tr'), ('da', 'tr'), ('hepsi', 'tr'), ('y', 'tr'), ('hat', 'tr'), ('g', 'tr'), ('tama', 'tr'), ('ner', 'tr'), ('tabi', 'tr'), ('iş', 'tr'), ('e', 'tr'), ('bazı', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('kendi', 'tr'), ('þey', 'tr'), ('en gip', 'tr'), ('siz', 'tr'), ('ke', 'tr'), ('þe', 'tr'), ('değil', 'tr'), ('üzer', 'tr'), ('ge', 'tr'), ('bu', 'tr'), ('şun', 'tr'), ('hem', 'tr'), ('is', 'tr'), ('dok', 'tr'), ('defa', 'tr'), ('hiçbir', 'tr'), ('mi', 'tr'), ('ç', 'tr'), ('heps', 'tr'), ('belki', 'tr'), ('se', 'tr'), ('öbür', 'tr'), ('elli', 'tr'), ('ve', 'tr'), ('il', 'tr'), ('altmýþ', 'tr'), ('fala', 'tr'), ('sek', 'tr'), ('son', 'tr'), ('niç', 'tr'), ('baze', 'tr'), ('ba', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('zate', 'tr'), ('ya', 'tr'), ('dört', 'tr'), ('yer', 'tr'), ('ü', 'tr'), ('u', 'tr'), ('tüm', 'tr'), ('k', 'tr'), ('mı', 'tr'), ('böyleç', 'tr'), ('beş', 'tr'), ('katrilyo', 'tr'), ('mý', 'tr'), ('çünkü', 'tr'), ('biris', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('s', 'tr'), ('ya ', 'tr'), ('dahi', 'tr'), ('dah', 'tr'), ('c', 'tr'), ('bazıs', 'tr'), ('bura', 'tr'), ('kaç', 'tr'), ('kimis', 'tr'), ('iç', 'tr'), ('veyahut', 'tr'), ('birþe', 'tr'), ('am', 'tr'), ('üç', 'tr'), ('nasýl', 'tr'), ('milyo', 'tr'), ('i', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('þ', 'tr'), ('kadar', 'tr'), ('m', 'tr'), ('ik', 'tr'), ('as', 'tr'), ('onu ot', 'tr'), ('sen', 'tr'), ('acap', 'tr'), ('ne zama', 'tr'), ('o', 'tr'), ('hangi', 'tr'), ('yirmi', 'tr'), ('d', 'tr'), ('kim', 'tr'), ('on', 'tr'), ('yüz', 'tr'), ('az', 'tr'), ('herkes', 'tr'), ('gip', 'tr'), ('ötür', 'tr'), ('altý', 'tr'), ('onlarý', 'tr'), ('al', 'tr'), ('onlari', 'tr'), ('hangis', 'tr'), ('b', 'tr'), ('büt', 'tr'), ('z', 'tr'), ('bil', 'tr'), ('trilyo', 'tr'), ('di', 'tr'), ('þu', 'tr'), ('fakat', 'tr'), ('milyar', 'tr'), ('birşey', 'tr'), ('oysaki', 'tr'), ('var', 'tr'), ('mu', 'tr'), ('bazý', 'tr'), ('ni', 'tr'), ('her', 'tr'), ('hep', 'tr'), ('fila', 'tr'), ('nes', 'tr'), ('asl', 'tr'), ('ğ', 'tr'), ('rağme', 'tr'), ('birkez', 'tr'), ('made', 'tr'), ('bö', 'tr'), ('diğer', 'tr'), ('ora', 'tr'), ('birkaç', 'tr'), ('birçok', 'tr'), ('şö', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('ş', 'tr'), ('insermi', 'tr'), ('ı', 'tr'), ('sa', 'tr'), ('hiç k', 'tr'), ('her bir', 'tr'), ('t', 'tr'), ('hani', 'tr'), ('birþey', 'tr'), ('v', 'tr'), ('elbet', 'tr'), ('zira', 'tr'), ('yok', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('be', 'tr'), ('ne kadar', 'tr'), ('yedi', 'tr'), ('dola', 'tr'), ('r', 'tr'), ('şe', 'tr'), ('bir', 'tr'), ('sonra', 'tr'), ('h', 'tr'), ('birşe', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('nere', 'tr'), ('demek', 'tr'), ('f', 'tr'), ('yetmiþ', 'tr'), ('n', 'tr'), ('baz', 'tr'), ('he', 'tr'), ('ar', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('p', 'tr'), ('hiç', 'tr'), ('yani', 'tr'), ('fela', 'tr'); - --- PostgreSQL sends notices about implicit keys that are being created, --- and the test suite takes them for warnings. -SET client_min_messages=WARNING; - --- "Full" stopwords -DROP TABLE IF EXISTS stopwords_long; -CREATE TABLE stopwords_long ( - stopwords_long_id SERIAL PRIMARY KEY, - stopword VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopwords_long_stopword - ON stopwords_long(stopword, language); - --- Stopword stems -DROP TABLE IF EXISTS stopword_stems_long; -CREATE TABLE stopword_stems_long ( - stopword_stems_long_id SERIAL PRIMARY KEY, - stopword_stem VARCHAR(256) NOT NULL, - language VARCHAR(3) NOT NULL /* 2- or 3-character ISO 690 language code */ -) WITH (OIDS=FALSE); -CREATE UNIQUE INDEX stopword_stems_long_stopword_stem - ON stopword_stems_long(stopword_stem, language); - --- Reset the message level back to "notice". -SET client_min_messages=NOTICE; - -INSERT INTO stopwords_long (stopword, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('sådan', 'da'), ('nu', 'da'), ('anden', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('ville', 'da'), ('kunne', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('alle', 'da'), ('når', 'da'), ('dig', 'da'), ('over', 'da'), ('af', 'da'), ('vil', 'da'), ('men', 'da'), ('ned', 'da'), ('have', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('ind', 'da'), ('været', 'da'), ('også', 'da'), ('hendes', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('hos', 'da'), ('har', 'da'), ('dog', 'da'), ('mod', 'da'), ('der', 'da'), ('hans', 'da'), ('denne', 'da'), ('være', 'da'), ('og', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('et', 'da'), ('er', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('disse', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('en', 'da'), ('sin', 'da'), ('mange', 'da'), ('vor', 'da'), ('meget', 'da'), ('hende', 'da'), ('de', 'da'), ('deres', 'da'), ('din', 'da'), ('mine', 'da'), ('op', 'da'), ('hun', 'da'), ('noget', 'da'), ('sine', 'da'), ('ham', 'da'), ('ikke', 'da'), ('skal', 'da'), ('efter', 'da'), ('havde', 'da'), ('ad', 'da'), ('skulle', 'da'), ('at', 'da'), ('jo', 'da'), ('i', 'da'), ('alt', 'da'), ('under', 'da'), ('jer', 'da'), ('min', 'da'), ('mig', 'da'), ('nogle', 'da'), ('sig', 'da'), ('eller', 'da'), ('blive', 'da'), ('blev', 'da'), ('bliver', 'da'), ('mit', 'da'), ('dette', 'da');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('fra', 'da'), ('som', 'da'), ('du', 'da'), ('da', 'da'), ('nu', 'da'), ('sådan', 'da'), ('vi', 'da'), ('til', 'da'), ('selv', 'da'), ('hav', 'da'), ('det', 'da'), ('med', 'da'), ('sit', 'da'), ('vær', 'da'), ('når', 'da'), ('dig', 'da'), ('and', 'da'), ('bliv', 'da'), ('over', 'da'), ('ell', 'da'), ('af', 'da'), ('vil', 'da'), ('all', 'da'), ('men', 'da'), ('ned', 'da'), ('var', 'da'), ('på', 'da'), ('her', 'da'), ('und', 'da'), ('ind', 'da'), ('også', 'da'), ('han', 'da'), ('man', 'da'), ('den', 'da'), ('har', 'da'), ('hos', 'da'), ('dog', 'da'), ('ikk', 'da'), ('havd', 'da'), ('mod', 'da'), ('der', 'da'), ('dis', 'da'), ('og', 'da'), ('nogl', 'da'), ('os', 'da'), ('hvis', 'da'), ('hvor', 'da'), ('mang', 'da'), ('et', 'da'), ('er', 'da'), ('hend', 'da'), ('hvad', 'da'), ('for', 'da'), ('ud', 'da'), ('om', 'da'), ('thi', 'da'), ('end', 'da'), ('jeg', 'da'), ('dem', 'da'), ('meg', 'da'), ('en', 'da'), ('sin', 'da'), ('vor', 'da'), ('de', 'da'), ('din', 'da'), ('op', 'da'), ('hun', 'da'), ('skul', 'da'), ('ham', 'da'), ('skal', 'da'), ('ad', 'da'), ('kun', 'da'), ('at', 'da'), ('jo', 'da'), ('eft', 'da'), ('i', 'da'), ('alt', 'da'), ('mig', 'da'), ('jer', 'da'), ('min', 'da'), ('nog', 'da'), ('sig', 'da'), ('blev', 'da'), ('mit', 'da');INSERT INTO stopwords_long (stopword, language) VALUES ('einiger', 'de'), ('habe', 'de'), ('kann', 'de'), ('deinen', 'de'), ('da', 'de'), ('anderr', 'de'), ('meiner', 'de'), ('dann', 'de'), ('einer', 'de'), ('meines', 'de'), ('hat', 'de'), ('gewesen', 'de'), ('eure', 'de'), ('solche', 'de'), ('ihre', 'de'), ('mich', 'de'), ('vom', 'de'), ('sehr', 'de'), ('also', 'de'), ('über', 'de'), ('allem', 'de'), ('jener', 'de'), ('warst', 'de'), ('alle', 'de'), ('solchem', 'de'), ('eures', 'de'), ('zwischen', 'de'), ('jenen', 'de'), ('hatten', 'de'), ('jeder', 'de'), ('sonst', 'de'), ('ohne', 'de'), ('alles', 'de'), ('bei', 'de'), ('die', 'de'), ('dir', 'de'), ('welchem', 'de'), ('etwas', 'de'), ('will', 'de'), ('anderem', 'de'), ('solcher', 'de'), ('und', 'de'), ('das', 'de'), ('musste', 'de'), ('zur', 'de'), ('gegen', 'de'), ('man', 'de'), ('keines', 'de'), ('diese', 'de'), ('einige', 'de'), ('ander', 'de'), ('diesen', 'de'), ('seinen', 'de'), ('euren', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('allen', 'de'), ('eurer', 'de'), ('bin', 'de'), ('hinter', 'de'), ('um', 'de'), ('unsem', 'de'), ('ein', 'de'), ('weiter', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('manchen', 'de'), ('keinem', 'de'), ('ich', 'de'), ('daß', 'de'), ('jenem', 'de'), ('dieses', 'de'), ('wird', 'de'), ('dieselbe', 'de'), ('seines', 'de'), ('wirst', 'de'), ('dem', 'de'), ('solchen', 'de'), ('sind', 'de'), ('hier', 'de'), ('wo', 'de'), ('weg', 'de'), ('euer', 'de'), ('des', 'de'), ('diesem', 'de'), ('dies', 'de'), ('nichts', 'de'), ('deines', 'de'), ('ihrer', 'de'), ('war', 'de'), ('demselben', 'de'), ('meinen', 'de'), ('euch', 'de'), ('am', 'de'), ('nach', 'de'), ('unser', 'de'), ('anderm', 'de'), ('selbst', 'de'), ('während', 'de'), ('deiner', 'de'), ('unsen', 'de'), ('solches', 'de'), ('welche', 'de'), ('dasselbe', 'de'), ('zum', 'de'), ('werde', 'de'), ('ihnen', 'de'), ('eines', 'de'), ('meine', 'de'), ('mit', 'de'), ('ihm', 'de'), ('anders', 'de'), ('anderes', 'de'), ('ihr', 'de'), ('jene', 'de'), ('jenes', 'de'), ('wollen', 'de'), ('deinem', 'de'), ('ob', 'de'), ('du', 'de'), ('einigem', 'de'), ('einiges', 'de'), ('welchen', 'de'), ('einen', 'de'), ('ihn', 'de'), ('jede', 'de'), ('wie', 'de'), ('wenn', 'de'), ('denselben', 'de'), ('derer', 'de'), ('unses', 'de'), ('deine', 'de'), ('ist', 'de'), ('mancher', 'de'), ('meinem', 'de'), ('keinen', 'de'), ('dessen', 'de'), ('nicht', 'de'), ('unter', 'de'), ('kein', 'de'), ('bist', 'de'), ('doch', 'de'), ('anderen', 'de'), ('wir', 'de'), ('ihren', 'de'), ('keiner', 'de'), ('ihrem', 'de'), ('unse', 'de'), ('eurem', 'de'), ('sein', 'de'), ('dein', 'de'), ('hab', 'de'), ('welcher', 'de'), ('würden', 'de'), ('keine', 'de'), ('seinem', 'de'), ('haben', 'de'), ('auf', 'de'), ('manche', 'de'), ('desselben', 'de'), ('hin', 'de'), ('denn', 'de'), ('jedes', 'de'), ('den', 'de'), ('derselbe', 'de'), ('einem', 'de'), ('auch', 'de'), ('jeden', 'de'), ('sollte', 'de'), ('an', 'de'), ('der', 'de'), ('seiner', 'de'), ('derselben', 'de'), ('mein', 'de'), ('könnte', 'de'), ('durch', 'de'), ('es', 'de'), ('wieder', 'de'), ('seine', 'de'), ('machen', 'de'), ('würde', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('andere', 'de'), ('bis', 'de'), ('ihres', 'de'), ('muss', 'de'), ('zwar', 'de'), ('waren', 'de'), ('sich', 'de'), ('so', 'de'), ('hatte', 'de'), ('er', 'de'), ('manches', 'de'), ('einigen', 'de'), ('nur', 'de'), ('im', 'de'), ('jedem', 'de'), ('dich', 'de'), ('eine', 'de'), ('vor', 'de'), ('können', 'de'), ('aus', 'de'), ('aller', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('andern', 'de'), ('indem', 'de'), ('wollte', 'de'), ('welches', 'de'), ('manchem', 'de'), ('einig', 'de'), ('sie', 'de'), ('dieser', 'de'), ('noch', 'de'), ('für', 'de'), ('dieselben', 'de'), ('anderer', 'de'), ('werden', 'de'), ('nun', 'de'), ('von', 'de'), ('aber', 'de'), ('ins', 'de'), ('was', 'de'), ('sondern', 'de');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('wollt', 'de'), ('mach', 'de'), ('kann', 'de'), ('anderr', 'de'), ('da', 'de'), ('dann', 'de'), ('jen', 'de'), ('hat', 'de'), ('mich', 'de'), ('also', 'de'), ('sehr', 'de'), ('vom', 'de'), ('konnt', 'de'), ('warst', 'de'), ('hint', 'de'), ('and', 'de'), ('sonst', 'de'), ('bei', 'de'), ('all', 'de'), ('die', 'de'), ('demselb', 'de'), ('dir', 'de'), ('etwas', 'de'), ('will', 'de'), ('derselb', 'de'), ('ind', 'de'), ('und', 'de'), ('das', 'de'), ('solch', 'de'), ('zur', 'de'), ('ohn', 'de'), ('man', 'de'), ('musst', 'de'), ('dess', 'de'), ('viel', 'de'), ('jetzt', 'de'), ('bin', 'de'), ('um', 'de'), ('ein', 'de'), ('einmal', 'de'), ('damit', 'de'), ('in', 'de'), ('soll', 'de'), ('ich', 'de'), ('wird', 'de'), ('manch', 'de'), ('wirst', 'de'), ('wied', 'de'), ('dem', 'de'), ('sind', 'de'), ('woll', 'de'), ('hier', 'de'), ('wo', 'de'), ('des', 'de'), ('euer', 'de'), ('weg', 'de'), ('dies', 'de'), ('war', 'de'), ('denselb', 'de'), ('am', 'de'), ('euch', 'de'), ('nach', 'de'), ('anderm', 'de'), ('selb', 'de'), ('wahrend', 'de'), ('zum', 'de'), ('eur', 'de'), ('ihm', 'de'), ('mit', 'de'), ('hatt', 'de'), ('ihr', 'de'), ('ob', 'de'), ('du', 'de'), ('ihn', 'de'), ('wie', 'de'), ('wenn', 'de'), ('fur', 'de'), ('ist', 'de'), ('unt', 'de'), ('welch', 'de'), ('nicht', 'de'), ('kein', 'de'), ('sond', 'de'), ('bist', 'de'), ('doch', 'de'), ('wir', 'de'), ('uber', 'de'), ('sein', 'de'), ('hab', 'de'), ('dein', 'de'), ('auf', 'de'), ('zwisch', 'de'), ('hin', 'de'), ('denn', 'de'), ('sollt', 'de'), ('gewes', 'de'), ('desselb', 'de'), ('den', 'de'), ('weit', 'de'), ('auch', 'de'), ('an', 'de'), ('der', 'de'), ('konn', 'de'), ('mein', 'de'), ('durch', 'de'), ('es', 'de'), ('oder', 'de'), ('dazu', 'de'), ('dort', 'de'), ('bis', 'de'), ('muss', 'de'), ('zwar', 'de'), ('sich', 'de'), ('so', 'de'), ('er', 'de'), ('nur', 'de'), ('im', 'de'), ('dich', 'de'), ('vor', 'de'), ('aus', 'de'), ('zu', 'de'), ('uns', 'de'), ('als', 'de'), ('mir', 'de'), ('weil', 'de'), ('jed', 'de'), ('dieselb', 'de'), ('werd', 'de'), ('einig', 'de'), ('sie', 'de'), ('noch', 'de'), ('dass', 'de'), ('nun', 'de'), ('dasselb', 'de'), ('wurd', 'de'), ('von', 'de'), ('geg', 'de'), ('ins', 'de'), ('aber', 'de'), ('was', 'de');INSERT INTO stopwords_long (stopword, language) VALUES ('issue', 'en'), ('supplied', 'en'), ('dropped', 'en'), ('expenditures', 'en'), ('distinguished', 'en'), ('calm', 'en'), ('taken', 'en'), ('nearest', 'en'), ('yelled', 'en'), ('load', 'en'), ('what', 'en'), ('married', 'en'), ('accurately', 'en'), ('rigid', 'en'), ('gray', 'en'), ('experts', 'en'), ('medicine', 'en'), ('conclusions', 'en'), ('twenty', 'en'), ('shown', 'en'), ('declared', 'en'), ('trust', 'en'), ('corn', 'en'), ('jan', 'en'), ('electronic', 'en'), ('contribute', 'en'), ('interview', 'en'), ('bay', 'en'), ('steadily', 'en'), ('going', 'en'), ('businesses', 'en'), ('artistic', 'en'), ('condemned', 'en'), ('exercise', 'en'), ('can', 'en'), ('wild', 'en'), ('wait', 'en'), ('contributions', 'en'), ('seeing', 'en'), ('favor', 'en'), ('rarely', 'en'), ('dates', 'en'), ('merchant', 'en'), ('fluid', 'en'), ('bench', 'en'), ('arc', 'en'), ('address', 'en'), ('helps', 'en'), ('crop', 'en'), ('me', 'en'), ('by', 'en'), ('favorite', 'en'), ('fair', 'en'), ('tragic', 'en'), ('surfaces', 'en'), ('interesting', 'en'), ('project', 'en'), ('industrial', 'en'), ('cool', 'en'), ('estimate', 'en'), ('share', 'en'), ('surrender', 'en'), ('death', 'en'), ('magazine', 'en'), ('farther', 'en'), ('characterized', 'en'), ('proof', 'en'), ('indirect', 'en'), ('verbal', 'en'), ('giving', 'en'), ('everyone', 'en'), ('loaded', 'en'), ('lies', 'en'), ('commodities', 'en'), ('rate', 'en'), ('gone', 'en'), ('tables', 'en'), ('matter', 'en'), ('invited', 'en'), ('owned', 'en'), ('ran', 'en'), ('trouble', 'en'), ('importance', 'en'), ('crown', 'en'), ('density', 'en'), ('reprint', 'en'), ('sensitivity', 'en'), ('people', 'en'), ('lift', 'en'), ('imposed', 'en'), ('identify', 'en'), ('divine', 'en'), ('guest', 'en'), ('treated', 'en'), ('conditions', 'en'), ('source', 'en'), ('protection', 'en'), ('processes', 'en'), ('local', 'en'), ('failed', 'en'), ('sink', 'en'), ('outcome', 'en'), ('striking', 'en'), ('really', 'en'), ('vowel', 'en'), ('fish', 'en'), ('powder', 'en'), ('allowing', 'en'), ('ear', 'en'), ('reader', 'en'), ('item', 'en'), ('hear', 'en'), ('truck', 'en'), ('express', 'en'), ('sovereignty', 'en'), ('suspicion', 'en'), ('whom', 'en'), ('show', 'en'), ('reducing', 'en'), ('win', 'en'), ('drill', 'en'), ('sufficient', 'en'), ('planetary', 'en'), ('lack', 'en'), ('sixties', 'en'), ('wheel', 'en'), ('petitioner', 'en'), ('wash', 'en'), ('even', 'en'), ('finish', 'en'), ('parade', 'en'), ('crease', 'en'), ('breath', 'en'), ('immediately', 'en'), ('acceptance', 'en'), ('fresh', 'en'), ('involve', 'en'), ('figured', 'en'), ('return', 'en'), ('gather', 'en'), ('viewed', 'en'), ('patrol', 'en'), ('serious', 'en'), ('handling', 'en'), ('drunk', 'en'), ('stone', 'en'), ('rose', 'en'), ('rocks', 'en'), ('precious', 'en'), ('keep', 'en'), ('better', 'en'), ('yours', 'en'), ('explicit', 'en'), ('stockholders', 'en'), ('supply', 'en'), ('em', 'en'), ('final', 'en'), ('and', 'en'), ('leaders', 'en'), ('panel', 'en'), ('surprise', 'en'), ('spend', 'en'), ('excess', 'en'), ('la', 'en'), ('already', 'en'), ('knowledge', 'en'), ('saying', 'en'), ('danger', 'en'), ('concepts', 'en'), ('grounds', 'en'), ('exchange', 'en'), ('eg', 'en'), ('square', 'en'), ('heavily', 'en'), ('veteran', 'en'), ('attractive', 'en'), ('triumph', 'en'), ('drugs', 'en'), ('latter', 'en'), ('approximately', 'en'), ('flash', 'en'), ('meanings', 'en'), ('pile', 'en'), ('champion', 'en'), ('evening', 'en'), ('devices', 'en'), ('building', 'en'), ('sees', 'en'), ('virtue', 'en'), ('position', 'en'), ('rent', 'en'), ('holding', 'en'), ('deal', 'en'), ('personal', 'en'), ('disposal', 'en'), ('owner', 'en'), ('primarily', 'en'), ('obligations', 'en'), ('continues', 'en'), ('happy', 'en'), ('financing', 'en'), ('table', 'en'), ('recognize', 'en'), ('milk', 'en'), ('green', 'en'), ('compromise', 'en'), ('nest', 'en'), ('ruling', 'en'), ('chairman', 'en'), ('furnish', 'en'), ('yellow', 'en'), ('historian', 'en'), ('serving', 'en'), ('exposure', 'en'), ('trading', 'en'), ('vast', 'en'), ('partner', 'en'), ('delay', 'en'), ('bearing', 'en'), ('substantial', 'en'), ('reflection', 'en'), ('web', 'en'), ('seconds', 'en'), ('burden', 'en'), ('partly', 'en'), ('votes', 'en'), ('set', 'en'), ('real', 'en'), ('country', 'en'), ('obvious', 'en'), ('architect', 'en'), ('wire', 'en'), ('explains', 'en'), ('job', 'en'), ('agent', 'en'), ('secrets', 'en'), ('met', 'en'), ('tooth', 'en'), ('white', 'en'), ('or', 'en'), ('missile', 'en'), ('wonder', 'en'), ('expression', 'en'), ('could', 'en'), ('organic', 'en'), ('atoms', 'en'), ('rates', 'en'), ('missed', 'en'), ('foods', 'en'), ('port', 'en'), ('logical', 'en'), ('referred', 'en'), ('learning', 'en'), ('research', 'en'), ('entertainment', 'en'), ('considering', 'en'), ('eliminated', 'en'), ('survey', 'en'), ('bring', 'en'), ('parties', 'en'), ('sight', 'en'), ('religion', 'en'), ('gentleman', 'en'), ('vote', 'en'), ('protected', 'en'), ('application', 'en'), ('arrived', 'en'), ('lid', 'en'), ('laws', 'en'), ('perfect', 'en'), ('driven', 'en'), ('cities', 'en'), ('sleep', 'en'), ('procedures', 'en'), ('birth', 'en'), ('you', 'en'), ('illusion', 'en'), ('appreciate', 'en'), ('both', 'en'), ('flew', 'en'), ('overcome', 'en'), ('eventually', 'en'), ('offered', 'en'), ('decisions', 'en'), ('shut', 'en'), ('male', 'en'), ('honored', 'en'), ('prior', 'en'), ('concrete', 'en'), ('mainly', 'en'), ('wines', 'en'), ('speed', 'en'), ('programs', 'en'), ('drinking', 'en'), ('uneasy', 'en'), ('regular', 'en'), ('located', 'en'), ('pass', 'en'), ('edt', 'en'), ('manager', 'en'), ('eating', 'en'), ('creation', 'en'), ('habits', 'en'), ('wet', 'en'), ('relief', 'en'), ('would', 'en'), ('signals', 'en'), ('no', 'en'), ('anxious', 'en'), ('probabilities', 'en'), ('valley', 'en'), ('excitement', 'en'), ('passengers', 'en'), ('object', 'en'), ('sticks', 'en'), ('invent', 'en'), ('existence', 'en'), ('doing', 'en'), ('get', 'en'), ('huge', 'en'), ('adopted', 'en'), ('deliberately', 'en'), ('go', 'en'), ('register', 'en'), ('brilliant', 'en'), ('recommendation', 'en'), ('boats', 'en'), ('universe', 'en'), ('ourselves', 'en'), ('justified', 'en'), ('tree', 'en'), ('third', 'en'), ('governments', 'en'), ('monday', 'en'), ('profound', 'en'), ('boy', 'en'), ('folks', 'en'), ('govern', 'en'), ('estate', 'en'), ('route', 'en'), ('disease', 'en'), ('mate', 'en'), ('accused', 'en'), ('behind', 'en'), ('shock', 'en'), ('planning', 'en'), ('bad', 'en'), ('occasion', 'en'), ('something', 'en'), ('streets', 'en'), ('edges', 'en'), ('afford', 'en'), ('academic', 'en'), ('bottom', 'en'), ('pulling', 'en'), ('approach', 'en'), ('hot', 'en'), ('self', 'en'), ('elsewhere', 'en'), ('eighteenth', 'en'), ('absence', 'en'), ('belong', 'en'), ('threat', 'en'), ('character', 'en'), ('restaurant', 'en'), ('prevention', 'en'), ('grateful', 'en'), ('enter', 'en'), ('bills', 'en'), ('foreign', 'en'), ('reporters', 'en'), ('rather', 'en'), ('stuff', 'en'), ('closely', 'en'), ('age', 'en'), ('plain', 'en'), ('sexual', 'en'), ('dancers', 'en'), ('cure', 'en'), ('expanded', 'en'), ('recommended', 'en'), ('volume', 'en'), ('difficulties', 'en'), ('luncheon', 'en'), ('quotient', 'en'), ('higher', 'en'), ('blog', 'en'), ('smell', 'en'), ('draft', 'en'), ('adding', 'en'), ('tests', 'en'), ('mental', 'en'), ('symbolic', 'en'), ('landscape', 'en'), ('results', 'en'), ('want', 'en'), ('cholesterol', 'en'), ('late', 'en'), ('valid', 'en'), ('key', 'en'), ('save', 'en'), ('printed', 'en'), ('matching', 'en'), ('foot', 'en'), ('experiences', 'en'), ('stored', 'en'), ('prime', 'en'), ('plan', 'en'), ('rector', 'en'), ('dynamic', 'en'), ('appear', 'en'), ('magnificent', 'en'), ('replaced', 'en'), ('worker', 'en'), ('charter', 'en'), ('usual', 'en'), ('gas', 'en'), ('area', 'en'), ('hope', 'en'), ('equation', 'en'), ('die', 'en'), ('bath', 'en'), ('knife', 'en'), ('becoming', 'en'), ('husband', 'en'), ('cook', 'en'), ('establishment', 'en'), ('sheet', 'en'), ('transformed', 'en'), ('exist', 'en'), ('preliminary', 'en'), ('gets', 'en'), ('novels', 'en'), ('right', 'en'), ('capable', 'en'), ('today', 'en'), ('construction', 'en'), ('necessary', 'en'), ('charming', 'en'), ('natural', 'en'), ('handled', 'en'), ('pot', 'en'), ('tail', 'en'), ('extent', 'en'), ('marriage', 'en'), ('ours', 'en'), ('released', 'en'), ('bond', 'en'), ('card', 'en'), ('thursday', 'en'), ('sick', 'en'), ('measures', 'en'), ('four', 'en'), ('bonds', 'en'), ('cotton', 'en'), ('opportunities', 'en'), ('publicly', 'en'), ('verse', 'en'), ('fire', 'en'), ('attend', 'en'), ('dawn', 'en'), ('recommendations', 'en'), ('meat', 'en'), ('hungry', 'en'), ('affected', 'en'), ('rolled', 'en'), ('experience', 'en'), ('legislative', 'en'), ('individual', 'en'), ('narrative', 'en'), ('who', 'en'), ('entire', 'en'), ('artists', 'en'), ('represented', 'en'), ('tools', 'en'), ('previously', 'en'), ('loans', 'en'), ('plastic', 'en'), ('wine', 'en'), ('simple', 'en'), ('purchased', 'en'), ('season', 'en'), ('officials', 'en'), ('luck', 'en'), ('witnesses', 'en'), ('furniture', 'en'), ('desired', 'en'), ('beach', 'en'), ('blame', 'en'), ('listened', 'en'), ('hair', 'en'), ('tie', 'en'), ('sea', 'en'), ('increases', 'en'), ('routine', 'en'), ('graduate', 'en'), ('worn', 'en'), ('continuously', 'en'), ('participation', 'en'), ('sharing', 'en'), ('hill', 'en'), ('jobs', 'en'), ('failure', 'en'), ('dream', 'en'), ('xml', 'en'), ('factors', 'en'), ('commonly', 'en'), ('chances', 'en'), ('inevitably', 'en'), ('institutions', 'en'), ('expected', 'en'), ('near', 'en'), ('shoot', 'en'), ('leading', 'en'), ('indication', 'en'), ('yesterday', 'en'), ('advance', 'en'), ('never', 'en'), ('silent', 'en'), ('player', 'en'), ('diameter', 'en'), ('expenses', 'en'), ('action', 'en'), ('assembled', 'en'), ('shine', 'en'), ('many', 'en'), ('fat', 'en'), ('atmosphere', 'en'), ('coffee', 'en'), ('talking', 'en'), ('swimming', 'en'), ('bare', 'en'), ('assessors', 'en'), ('surprised', 'en'), ('operated', 'en'), ('wearing', 'en'), ('church', 'en'), ('lips', 'en'), ('activities', 'en'), ('long-term', 'en'), ('author', 'en'), ('considerably', 'en'), ('productive', 'en'), ('fraction', 'en'), ('she', 'en'), ('widespread', 'en'), ('turning', 'en'), ('affairs', 'en'), ('utility', 'en'), ('act', 'en'), ('plastics', 'en'), ('planned', 'en'), ('inventory', 'en'), ('unconscious', 'en'), ('throw', 'en'), ('exact', 'en'), ('cheek', 'en'), ('fiber', 'en'), ('provide', 'en'), ('watching', 'en'), ('supporting', 'en'), ('way', 'en'), ('recovery', 'en'), ('neighbor', 'en'), ('lacking', 'en'), ('shelter', 'en'), ('wheels', 'en'), ('comments', 'en'), ('observers', 'en'), ('wrong', 'en'), ('in', 'en'), ('sitting', 'en'), ('electricity', 'en'), ('receives', 'en'), ('thrown', 'en'), ('alienation', 'en'), ('dark', 'en'), ('universities', 'en'), ('implications', 'en'), ('achievements', 'en'), ('colonel', 'en'), ('understanding', 'en'), ('remaining', 'en'), ('representative', 'en'), ('seized', 'en'), ('identified', 'en'), ('investment', 'en'), ('damn', 'en'), ('quick', 'en'), ('fast', 'en'), ('shows', 'en'), ('dare', 'en'), ('poet', 'en'), ('losses', 'en'), ('patient', 'en'), ('presented', 'en'), ('possibility', 'en'), ('concerning', 'en'), ('match', 'en'), ('description', 'en'), ('spiritual', 'en'), ('shopping', 'en'), ('riding', 'en'), ('series', 'en'), ('climb', 'en'), ('composition', 'en'), ('variety', 'en'), ('wedding', 'en'), ('syllable', 'en'), ('account', 'en'), ('purchase', 'en'), ('running', 'en'), ('nervous', 'en'), ('alone', 'en'), ('little', 'en'), ('switch', 'en'), ('pressures', 'en'), ('combination', 'en'), ('tended', 'en'), ('mst', 'en'), ('ie', 'en'), ('swept', 'en'), ('refused', 'en'), ('questions', 'en'), ('cents', 'en'), ('those', 'en'), ('relation', 'en'), ('away', 'en'), ('creating', 'en'), ('begun', 'en'), ('persuaded', 'en'), ('heights', 'en'), ('beyond', 'en'), ('rules', 'en'), ('bodies', 'en'), ('say', 'en'), ('qualities', 'en'), ('seldom', 'en'), ('original', 'en'), ('increase', 'en'), ('score', 'en'), ('picked', 'en'), ('outdoor', 'en'), ('absolute', 'en'), ('dirty', 'en'), ('brick', 'en'), ('relations', 'en'), ('stiff', 'en'), ('providing', 'en'), ('fortune', 'en'), ('suggested', 'en'), ('mysterious', 'en'), ('destroyed', 'en'), ('buying', 'en'), ('delightful', 'en'), ('published', 'en'), ('aids', 'en'), ('literary', 'en'), ('possible', 'en'), ('months', 'en'), ('earnings', 'en'), ('widely', 'en'), ('barn', 'en'), ('later', 'en'), ('kept', 'en'), ('prevented', 'en'), ('dear', 'en'), ('bold', 'en'), ('sampling', 'en'), ('proud', 'en'), ('members', 'en'), ('length', 'en'), ('metal', 'en'), ('were', 'en'), ('poured', 'en'), ('law', 'en'), ('containing', 'en'), ('collective', 'en'), ('understand', 'en'), ('turn', 'en'), ('specimen', 'en'), ('past', 'en'), ('iron', 'en'), ('cost', 'en'), ('chicken', 'en'), ('refund', 'en'), ('passed', 'en'), ('biological', 'en'), ('aug', 'en'), ('introduced', 'en'), ('gave', 'en'), ('anxiety', 'en'), ('engaged', 'en'), ('conversation', 'en'), ('letters', 'en'), ('under', 'en'), ('features', 'en'), ('contrast', 'en'), ('preferred', 'en'), ('harder', 'en'), ('lacked', 'en'), ('gathering', 'en'), ('feb', 'en'), ('inches', 'en'), ('sequence', 'en'), ('move', 'en'), ('officers', 'en'), ('our', 'en'), ('peas', 'en'), ('lo', 'en'), ('distinct', 'en'), ('its', 'en'), ('earlier', 'en'), ('over-all', 'en'), ('determined', 'en'), ('observations', 'en'), ('union', 'en'), ('enormous', 'en'), ('brother', 'en'), ('goes', 'en'), ('we', 'en'), ('hasn', 'en'), ('velocity', 'en'), ('notice', 'en'), ('group', 'en'), ('personally', 'en'), ('motion', 'en'), ('buildings', 'en'), ('payment', 'en'), ('blockquote', 'en'), ('angle', 'en'), ('radio', 'en'), ('desegregation', 'en'), ('managers', 'en'), ('accepted', 'en'), ('nineteenth', 'en'), ('ocean', 'en'), ('an', 'en'), ('bed', 'en'), ('neck', 'en'), ('medical', 'en'), ('movie', 'en'), ('soon', 'en'), ('paying', 'en'), ('permit', 'en'), ('records', 'en'), ('origin', 'en'), ('jump', 'en'), ('end', 'en'), ('roads', 'en'), ('splendid', 'en'), ('theory', 'en'), ('respective', 'en'), ('miss', 'en'), ('sending', 'en'), ('revenues', 'en'), ('diffusion', 'en'), ('pride', 'en'), ('urged', 'en'), ('reach', 'en'), ('purely', 'en'), ('dispute', 'en'), ('reliable', 'en'), ('patience', 'en'), ('pilot', 'en'), ('great', 'en'), ('scarcely', 'en'), ('particular', 'en'), ('expansion', 'en'), ('beautiful', 'en'), ('without', 'en'), ('than', 'en'), ('just', 'en'), ('pictures', 'en'), ('effective', 'en'), ('earliest', 'en'), ('department', 'en'), ('strike', 'en'), ('cafe', 'en'), ('congressional', 'en'), ('envelope', 'en'), ('supervision', 'en'), ('wounded', 'en'), ('symbol', 'en'), ('supplies', 'en'), ('smiled', 'en'), ('reduction', 'en'), ('living', 'en'), ('traditional', 'en'), ('technology', 'en'), ('fears', 'en'), ('follow', 'en'), ('appointment', 'en'), ('piece', 'en'), ('effort', 'en'), ('continued', 'en'), ('elected', 'en'), ('rapid', 'en'), ('endless', 'en'), ('suspected', 'en'), ('woman', 'en'), ('zero', 'en'), ('neat', 'en'), ('concentration', 'en'), ('cells', 'en'), ('demonstration', 'en'), ('image', 'en'), ('chiefly', 'en'), ('resistance', 'en'), ('saturday', 'en'), ('char', 'en'), ('enthusiastic', 'en'), ('attracted', 'en'), ('fractions', 'en'), ('chair', 'en'), ('narrow', 'en'), ('proceeded', 'en'), ('underlying', 'en'), ('copy', 'en'), ('practice', 'en'), ('order', 'en'), ('following', 'en'), ('restrictions', 'en'), ('prize', 'en'), ('parent', 'en'), ('sorry', 'en'), ('friday', 'en'), ('definitely', 'en'), ('data', 'en'), ('ignored', 'en'), ('fed', 'en'), ('year-old', 'en'), ('succession', 'en'), ('labour', 'en'), ('opposed', 'en'), ('after', 'en'), ('democracy', 'en'), ('breathing', 'en'), ('strongest', 'en'), ('enjoyment', 'en'), ('back', 'en'), ('as', 'en'), ('savings', 'en'), ('respond', 'en'), ('permitted', 'en'), ('opportunity', 'en'), ('discharge', 'en'), ('stumbled', 'en'), ('grants', 'en'), ('employment', 'en'), ('chest', 'en'), ('interested', 'en'), ('wondering', 'en'), ('growing', 'en'), ('eggs', 'en'), ('shadows', 'en'), ('difference', 'en'), ('ends', 'en'), ('theirs', 'en'), ('integration', 'en'), ('sheep', 'en'), ('came', 'en'), ('crossing', 'en'), ('span', 'en'), ('repeat', 'en'), ('customers', 'en'), ('wave', 'en'), ('theological', 'en'), ('recreation', 'en'), ('magnetic', 'en'), ('conversion', 'en'), ('sphere', 'en'), ('putting', 'en'), ('eternal', 'en'), ('variables', 'en'), ('protect', 'en'), ('eliminate', 'en'), ('someone', 'en'), ('plans', 'en'), ('kind', 'en'), ('herself', 'en'), ('takes', 'en'), ('stands', 'en'), ('agency', 'en'), ('smoke', 'en'), ('outlook', 'en'), ('assessment', 'en'), ('sure', 'en'), ('instrument', 'en'), ('strongly', 'en'), ('grown', 'en'), ('comfort', 'en'), ('pretty', 'en'), ('organizations', 'en'), ('telling', 'en'), ('onto', 'en'), ('attacks', 'en'), ('maid', 'en'), ('whole', 'en'), ('went', 'en'), ('new', 'en'), ('returning', 'en'), ('evaluation', 'en'), ('burst', 'en'), ('promote', 'en'), ('entirely', 'en'), ('stations', 'en'), ('skilled', 'en'), ('penny', 'en'), ('composed', 'en'), ('vital', 'en'), ('send', 'en'), ('limitations', 'en'), ('states', 'en'), ('respectively', 'en'), ('needed', 'en'), ('develop', 'en'), ('innocent', 'en'), ('sugar', 'en'), ('civilian', 'en'), ('prospective', 'en'), ('nose', 'en'), ('experienced', 'en'), ('prospects', 'en'), ('fathers', 'en'), ('communication', 'en'), ('norms', 'en'), ('expensive', 'en'), ('conclusion', 'en'), ('if', 'en'), ('characters', 'en'), ('him', 'en'), ('gentlemen', 'en'), ('wide', 'en'), ('examples', 'en'), ('finger', 'en'), ('cited', 'en'), ('drawing', 'en'), ('regions', 'en'), ('your', 'en'), ('describes', 'en'), ('seated', 'en'), ('start', 'en'), ('variable', 'en'), ('lying', 'en'), ('bone', 'en'), ('cutting', 'en'), ('consequences', 'en'), ('sacred', 'en'), ('offers', 'en'), ('ending', 'en'), ('actual', 'en'), ('calls', 'en'), ('demanded', 'en'), ('voice', 'en'), ('accordingly', 'en'), ('occurrence', 'en'), ('fields', 'en'), ('differ', 'en'), ('resulted', 'en'), ('holder', 'en'), ('electric', 'en'), ('begins', 'en'), ('discussions', 'en'), ('actually', 'en'), ('stores', 'en'), ('special', 'en'), ('occasional', 'en'), ('voices', 'en'), ('generally', 'en'), ('describe', 'en'), ('reflects', 'en'), ('father', 'en'), ('changed', 'en'), ('exceptions', 'en'), ('daughter', 'en'), ('audience', 'en'), ('money', 'en'), ('punishment', 'en'), ('anyone', 'en'), ('gin', 'en'), ('pleasant', 'en'), ('consistent', 'en'), ('plaster', 'en'), ('probability', 'en'), ('eight', 'en'), ('forward', 'en'), ('hoped', 'en'), ('enforced', 'en'), ('beat', 'en'), ('acquired', 'en'), ('compared', 'en'), ('characteristic', 'en'), ('cap', 'en'), ('nuts', 'en'), ('wednesday', 'en'), ('positions', 'en'), ('welcome', 'en'), ('trend', 'en'), ('colleagues', 'en'), ('substrate', 'en'), ('relationship', 'en'), ('did', 'en'), ('marks', 'en'), ('extensive', 'en'), ('ten', 'en'), ('disturbed', 'en'), ('meets', 'en'), ('eager', 'en'), ('sake', 'en'), ('from', 'en'), ('method', 'en'), ('sought', 'en'), ('her', 'en'), ('announced', 'en'), ('any', 'en'), ('define', 'en'), ('delicate', 'en'), ('more', 'en'), ('material', 'en'), ('subjects', 'en'), ('island', 'en'), ('cross', 'en'), ('prisoners', 'en'), ('patterns', 'en'), ('mathematics', 'en'), ('curious', 'en'), ('sweet', 'en'), ('amazing', 'en'), ('groups', 'en'), ('this', 'en'), ('carbon', 'en'), ('song', 'en'), ('ceiling', 'en'), ('fence', 'en'), ('cst', 'en'), ('hoping', 'en'), ('report', 'en'), ('authorities', 'en'), ('be', 'en'), ('class', 'en'), ('september', 'en'), ('utopian', 'en'), ('weekend', 'en'), ('detailed', 'en'), ('operational', 'en'), ('most', 'en'), ('writers', 'en'), ('thousands', 'en'), ('offering', 'en'), ('roof', 'en'), ('look', 'en'), ('passenger', 'en'), ('parked', 'en'), ('demonstrate', 'en'), ('hundreds', 'en'), ('had', 'en'), ('desert', 'en'), ('stop', 'en'), ('rang', 'en'), ('information', 'en'), ('venture', 'en'), ('slender', 'en'), ('equipped', 'en'), ('inspection', 'en'), ('leadership', 'en'), ('storm', 'en'), ('hat', 'en'), ('delivered', 'en'), ('stead', 'en'), ('reveals', 'en'), ('particle', 'en'), ('parking', 'en'), ('qualified', 'en'), ('measurement', 'en'), ('shapes', 'en'), ('excessive', 'en'), ('forming', 'en'), ('anticipated', 'en'), ('village', 'en'), ('captain', 'en'), ('sounds', 'en'), ('again', 'en'), ('wall', 'en'), ('quietly', 'en'), ('of', 'en'), ('employees', 'en'), ('relevant', 'en'), ('winter', 'en'), ('phase', 'en'), ('ugly', 'en'), ('platform', 'en'), ('much', 'en'), ('procedure', 'en'), ('neither', 'en'), ('anticipation', 'en'), ('explained', 'en'), ('part', 'en'), ('where', 'en'), ('flight', 'en'), ('settle', 'en'), ('doctor', 'en'), ('left', 'en'), ('experiment', 'en'), ('insisted', 'en'), ('hydrogen', 'en'), ('seen', 'en'), ('prestige', 'en'), ('constitute', 'en'), ('questioned', 'en'), ('aesthetic', 'en'), ('drug', 'en'), ('knows', 'en'), ('refer', 'en'), ('visual', 'en'), ('upon', 'en'), ('plates', 'en'), ('only', 'en'), ('stayed', 'en'), ('districts', 'en'), ('resumed', 'en'), ('disaster', 'en'), ('proposals', 'en'), ('means', 'en'), ('absolutely', 'en'), ('recorded', 'en'), ('structural', 'en'), ('blind', 'en'), ('requirements', 'en'), ('thoroughly', 'en'), ('socialism', 'en'), ('recall', 'en'), ('long-range', 'en'), ('sometimes', 'en'), ('confused', 'en'), ('remainder', 'en'), ('reasonable', 'en'), ('firing', 'en'), ('producing', 'en'), ('align', 'en'), ('strong', 'en'), ('voluntary', 'en'), ('calendar', 'en'), ('balanced', 'en'), ('forests', 'en'), ('empirical', 'en'), ('meet', 'en'), ('into', 'en'), ('seven', 'en'), ('advanced', 'en'), ('gorton', 'en'), ('weekly', 'en'), ('good', 'en'), ('making', 'en'), ('fabrics', 'en'), ('intention', 'en'), ('dependent', 'en'), ('myth', 'en'), ('master', 'en'), ('provision', 'en'), ('west', 'en'), ('solved', 'en'), ('concern', 'en'), ('thickness', 'en'), ('farmer', 'en'), ('reorganization', 'en'), ('screen', 'en'), ('fishing', 'en'), ('cleared', 'en'), ('shares', 'en'), ('pipe', 'en'), ('convinced', 'en'), ('roll', 'en'), ('axis', 'en'), ('state', 'en'), ('probably', 'en'), ('atom', 'en'), ('sand', 'en'), ('fallout', 'en'), ('ranks', 'en'), ('registered', 'en'), ('swim', 'en'), ('rub', 'en'), ('rode', 'en'), ('defense', 'en'), ('bell', 'en'), ('delayed', 'en'), ('heard', 'en'), ('numerous', 'en'), ('flat', 'en'), ('plenty', 'en'), ('ladder', 'en'), ('workshop', 'en'), ('joke', 'en'), ('senior', 'en'), ('mountain', 'en'), ('blues', 'en'), ('attitude', 'en'), ('behavior', 'en'), ('thyroid', 'en'), ('farmers', 'en'), ('arts', 'en'), ('conception', 'en'), ('increasing', 'en'), ('chairs', 'en'), ('come', 'en'), ('file', 'en'), ('fly', 'en'), ('bride', 'en'), ('dignity', 'en'), ('preparation', 'en'), ('degrees', 'en'), ('least', 'en'), ('less', 'en'), ('depending', 'en'), ('etc.', 'en'), ('tremendous', 'en'), ('percentage', 'en'), ('analysis', 'en'), ('insects', 'en'), ('dance', 'en'), ('decided', 'en'), ('household', 'en'), ('introduction', 'en'), ('virtually', 'en'), ('tall', 'en'), ('fixed', 'en'), ('declaration', 'en'), ('received', 'en'), ('paper', 'en'), ('stuck', 'en'), ('defend', 'en'), ('last', 'en'), ('uniform', 'en'), ('identity', 'en'), ('notes', 'en'), ('fascinating', 'en'), ('update', 'en'), ('next', 'en'), ('bid', 'en'), ('devil', 'en'), ('almost', 'en'), ('chapel', 'en'), ('instruments', 'en'), ('contacts', 'en'), ('toward', 'en'), ('oral', 'en'), ('circle', 'en'), ('finding', 'en'), ('duty', 'en'), ('contain', 'en'), ('displacement', 'en'), ('washed', 'en'), ('died', 'en'), ('fate', 'en'), ('frightened', 'en'), ('bus', 'en'), ('cast', 'en'), ('intended', 'en'), ('drivers', 'en'), ('hit', 'en'), ('attending', 'en'), ('merely', 'en'), ('company', 'en'), ('crowd', 'en'), ('make', 'en'), ('leather', 'en'), ('industry', 'en'), ('maintaining', 'en'), ('big', 'en'), ('churches', 'en'), ('themselves', 'en'), ('august', 'en'), ('burning', 'en'), ('flying', 'en'), ('worry', 'en'), ('mg', 'en'), ('passing', 'en'), ('input', 'en'), ('shoulders', 'en'), ('choose', 'en'), ('tip', 'en'), ('hanging', 'en'), ('slept', 'en'), ('movies', 'en'), ('leaving', 'en'), ('property', 'en'), ('middle', 'en'), ('grew', 'en'), ('ideological', 'en'), ('displayed', 'en'), ('younger', 'en'), ('individuals', 'en'), ('affects', 'en'), ('transformation', 'en'), ('schedule', 'en'), ('sophisticated', 'en'), ('launched', 'en'), ('adults', 'en'), ('secondary', 'en'), ('attitudes', 'en'), ('some', 'en'), ('applied', 'en'), ('with', 'en'), ('arrangement', 'en'), ('eat', 'en'), ('liked', 'en'), ('committee', 'en'), ('function', 'en'), ('advantages', 'en'), ('somehow', 'en'), ('propaganda', 'en'), ('shouting', 'en'), ('cat', 'en'), ('film', 'en'), ('handed', 'en'), ('concert', 'en'), ('impression', 'en'), ('liquor', 'en'), ('constant', 'en'), ('techniques', 'en'), ('improvement', 'en'), ('ahead', 'en'), ('crew', 'en'), ('ones', 'en'), ('mines', 'en'), ('financial', 'en'), ('wake', 'en'), ('respect', 'en'), ('glass', 'en'), ('grades', 'en'), ('against', 'en'), ('instructions', 'en'), ('regarded', 'en'), ('cultural', 'en'), ('certainly', 'en'), ('kitchen', 'en'), ('plays', 'en'), ('clothes', 'en'), ('music', 'en'), ('insure', 'en'), ('habit', 'en'), ('location', 'en'), ('carefully', 'en'), ('suffering', 'en'), ('playing', 'en'), ('occupation', 'en'), ('else', 'en'), ('ninth', 'en'), ('woods', 'en'), ('engine', 'en'), ('tomorrow', 'en'), ('society', 'en'), ('label', 'en'), ('busy', 'en'), ('souls', 'en'), ('dedication', 'en'), ('lists', 'en'), ('wives', 'en'), ('benefits', 'en'), ('needs', 'en'), ('forced', 'en'), ('missing', 'en'), ('brush', 'en'), ('focus', 'en'), ('basis', 'en'), ('begin', 'en'), ('inside', 'en'), ('ability', 'en'), ('improve', 'en'), ('suspect', 'en'), ('profit', 'en'), ('aspect', 'en'), ('likely', 'en'), ('kinds', 'en'), ('resources', 'en'), ('fourteen', 'en'), ('employed', 'en'), ('barely', 'en'), ('responded', 'en'), ('peculiar', 'en'), ('lengths', 'en'), ('discuss', 'en'), ('struck', 'en'), ('selling', 'en'), ('undoubtedly', 'en'), ('sighed', 'en'), ('political', 'en'), ('somebody', 'en'), ('positive', 'en'), ('steady', 'en'), ('necessarily', 'en'), ('settled', 'en'), ('garage', 'en'), ('adjustment', 'en'), ('laughing', 'en'), ('levels', 'en'), ('variation', 'en'), ('enterprise', 'en'), ('career', 'en'), ('ordered', 'en'), ('rapidly', 'en'), ('preparing', 'en'), ('accurate', 'en'), ('guests', 'en'), ('com', 'en'), ('behalf', 'en'), ('drinks', 'en'), ('remove', 'en'), ('spell', 'en'), ('display', 'en'), ('liberal', 'en'), ('vivid', 'en'), ('user', 'en'), ('disappeared', 'en'), ('pulmonary', 'en'), ('escape', 'en'), ('hopes', 'en'), ('salt', 'en'), ('ritual', 'en'), ('fans', 'en'), ('intense', 'en'), ('shift', 'en'), ('cent', 'en'), ('brave', 'en'), ('mind', 'en'), ('creative', 'en'), ('healthy', 'en'), ('colors', 'en'), ('older', 'en'), ('devoted', 'en'), ('bat', 'en'), ('saving', 'en'), ('exercises', 'en'), ('racing', 'en'), ('award', 'en'), ('hundred', 'en'), ('why', 'en'), ('product', 'en'), ('writes', 'en'), ('ideas', 'en'), ('destruction', 'en'), ('know', 'en'), ('consider', 'en'), ('legislators', 'en'), ('alliance', 'en'), ('lands', 'en'), ('posts', 'en'), ('break', 'en'), ('course', 'en'), ('commander', 'en'), ('upstairs', 'en'), ('lower', 'en'), ('foam', 'en'), ('freight', 'en'), ('sounded', 'en'), ('anti-trust', 'en'), ('specific', 'en'), ('follows', 'en'), ('seemed', 'en'), ('physics', 'en'), ('design', 'en'), ('agencies', 'en'), ('accounts', 'en'), ('internal', 'en'), ('chlorine', 'en'), ('harmony', 'en'), ('interests', 'en'), ('glanced', 'en'), ('worried', 'en'), ('civic', 'en'), ('package', 'en'), ('enemies', 'en'), ('pursue', 'en'), ('becomes', 'en'), ('males', 'en'), ('components', 'en'), ('full-time', 'en'), ('neighborhood', 'en'), ('concept', 'en'), ('happiness', 'en'), ('et', 'en'), ('paintings', 'en'), ('kids', 'en'), ('occurred', 'en'), ('psychological', 'en'), ('involved', 'en'), ('other', 'en'), ('parents', 'en'), ('intimate', 'en'), ('significance', 'en'), ('worthy', 'en'), ('gun', 'en'), ('term', 'en'), ('founded', 'en'), ('boys', 'en'), ('except', 'en'), ('loud', 'en'), ('friendly', 'en'), ('principle', 'en'), ('monument', 'en'), ('initiative', 'en'), ('visiting', 'en'), ('runs', 'en'), ('masses', 'en'), ('markets', 'en'), ('another', 'en'), ('nobody', 'en'), ('vein', 'en'), ('target', 'en'), ('view', 'en'), ('committed', 'en'), ('hero', 'en'), ('fiscal', 'en'), ('request', 'en'), ('formation', 'en'), ('ranging', 'en'), ('entered', 'en'), ('ship', 'en'), ('sit', 'en'), ('populate', 'en'), ('teams', 'en'), ('boating', 'en'), ('questioning', 'en'), ('policies', 'en'), ('forest', 'en'), ('hung', 'en'), ('powerful', 'en'), ('manner', 'en'), ('organ', 'en'), ('advertising', 'en'), ('honest', 'en'), ('besides', 'en'), ('salary', 'en'), ('wanted', 'en'), ('cottage', 'en'), ('ruled', 'en'), ('soft', 'en'), ('abbr', 'en'), ('obliged', 'en'), ('exclusively', 'en'), ('write', 'en'), ('culture', 'en'), ('movements', 'en'), ('curve', 'en'), ('liberty', 'en'), ('commercial', 'en'), ('guide', 'en'), ('carries', 'en'), ('worse', 'en'), ('lone', 'en'), ('wildly', 'en'), ('crazy', 'en'), ('dishes', 'en'), ('broken', 'en'), ('mad', 'en'), ('fill', 'en'), ('reports', 'en'), ('choice', 'en'), ('adjustments', 'en'), ('laid', 'en'), ('holes', 'en'), ('inadequate', 'en'), ('staining', 'en'), ('sex', 'en'), ('bedroom', 'en'), ('warmth', 'en'), ('increasingly', 'en'), ('well', 'en'), ('office', 'en'), ('anti-Semitism', 'en'), ('doctors', 'en'), ('critics', 'en'), ('recent', 'en'), ('news', 'en'), ('so-called', 'en'), ('worth', 'en'), ('potential', 'en'), ('newer', 'en'), ('concentrated', 'en'), ('possibilities', 'en'), ('row', 'en'), ('evidently', 'en'), ('headquarters', 'en'), ('sharp', 'en'), ('ball', 'en'), ('governing', 'en'), ('gang', 'en'), ('sources', 'en'), ('faith', 'en'), ('scenes', 'en'), ('none', 'en'), ('accomplished', 'en'), ('followed', 'en'), ('tangent', 'en'), ('sharply', 'en'), ('pool', 'en'), ('whip', 'en'), ('towards', 'en'), ('stood', 'en'), ('oct', 'en'), ('mere', 'en'), ('strange', 'en'), ('acts', 'en'), ('decide', 'en'), ('peered', 'en'), ('establishing', 'en'), ('divided', 'en'), ('cellar', 'en'), ('approaching', 'en'), ('craft', 'en'), ('directed', 'en'), ('suitable', 'en'), ('leaned', 'en'), ('so', 'en'), ('meant', 'en'), ('insurance', 'en'), ('killer', 'en'), ('tragedy', 'en'), ('weakness', 'en'), ('world', 'en'), ('flux', 'en'), ('emergency', 'en'), ('greatest', 'en'), ('straight', 'en'), ('prominent', 'en'), ('skirt', 'en'), ('abandoned', 'en'), ('early', 'en'), ('at', 'en'), ('fig', 'en'), ('murder', 'en'), ('trade', 'en'), ('builder', 'en'), ('hurry', 'en'), ('road', 'en'), ('air', 'en'), ('ideal', 'en'), ('rough', 'en'), ('form', 'en'), ('baby', 'en'), ('traveled', 'en'), ('respectable', 'en'), ('give', 'en'), ('unique', 'en'), ('supplement', 'en'), ('military', 'en'), ('smooth', 'en'), ('precision', 'en'), ('figure', 'en'), ('annual', 'en'), ('seed', 'en'), ('being', 'en'), ('executive', 'en'), ('constructed', 'en'), ('blonde', 'en'), ('december', 'en'), ('possessed', 'en'), ('lie', 'en'), ('examined', 'en'), ('various', 'en'), ('costs', 'en'), ('chief', 'en'), ('attempts', 'en'), ('derived', 'en'), ('man', 'en'), ('automatic', 'en'), ('overwhelming', 'en'), ('windows', 'en'), ('july', 'en'), ('possibly', 'en'), ('http', 'en'), ('walls', 'en'), ('friendship', 'en'), ('may', 'en'), ('leads', 'en'), ('permission', 'en'), ('operations', 'en'), ('planes', 'en'), ('different', 'en'), ('displays', 'en'), ('hall', 'en'), ('morality', 'en'), ('functional', 'en'), ('depression', 'en'), ('terms', 'en'), ('installed', 'en'), ('attached', 'en'), ('trained', 'en'), ('bridges', 'en'), ('immediate', 'en'), ('pleasure', 'en'), ('crucial', 'en'), ('novel', 'en'), ('sympathetic', 'en'), ('serves', 'en'), ('relieved', 'en'), ('hurried', 'en'), ('extraordinary', 'en'), ('allowances', 'en'), ('communities', 'en'), ('talents', 'en'), ('formulas', 'en'), ('dressing', 'en'), ('tactics', 'en'), ('blocks', 'en'), ('lighting', 'en'), ('conspiracy', 'en'), ('examination', 'en'), ('holds', 'en'), ('widow', 'en'), ('profession', 'en'), ('income', 'en'), ('hesitated', 'en'), ('regard', 'en'), ('impulse', 'en'), ('casual', 'en'), ('marginal', 'en'), ('walk', 'en'), ('manage', 'en'), ('hard', 'en'), ('width', 'en'), ('illustration', 'en'), ('number', 'en'), ('battle', 'en'), ('drawings', 'en'), ('nevertheless', 'en'), ('surely', 'en'), ('error', 'en'), ('attempted', 'en'), ('societies', 'en'), ('ownership', 'en'), ('organization', 'en'), ('remember', 'en'), ('starts', 'en'), ('managed', 'en'), ('now', 'en'), ('universal', 'en'), ('population', 'en'), ('waited', 'en'), ('test', 'en'), ('produce', 'en'), ('gross', 'en'), ('resist', 'en'), ('column', 'en'), ('proved', 'en'), ('how', 'en'), ('identical', 'en'), ('event', 'en'), ('their', 'en'), ('scale', 'en'), ('retained', 'en'), ('relating', 'en'), ('best', 'en'), ('automobiles', 'en'), ('therefore', 'en'), ('heels', 'en'), ('up', 'en'), ('lumber', 'en'), ('oxygen', 'en'), ('having', 'en'), ('slowly', 'en'), ('king', 'en'), ('tape', 'en'), ('included', 'en'), ('interpretation', 'en'), ('emission', 'en'), ('question', 'en'), ('breaking', 'en'), ('creatures', 'en'), ('pressure', 'en'), ('loan', 'en'), ('carried', 'en'), ('far', 'en'), ('winds', 'en'), ('required', 'en'), ('suffix', 'en'), ('admit', 'en'), ('makes', 'en'), ('animals', 'en'), ('sound', 'en'), ('also', 'en'), ('trim', 'en'), ('tissue', 'en'), ('distribution', 'en'), ('related', 'en'), ('stranger', 'en'), ('but', 'en'), ('too', 'en'), ('over', 'en'), ('reaching', 'en'), ('aboard', 'en'), ('federal', 'en'), ('largely', 'en'), ('heritage', 'en'), ('is', 'en'), ('fallen', 'en'), ('coat', 'en'), ('seventh', 'en'), ('mess', 'en'), ('prevent', 'en'), ('anger', 'en'), ('golden', 'en'), ('coach', 'en'), ('tendency', 'en'), ('attempting', 'en'), ('accordance', 'en'), ('lieutenant', 'en'), ('phases', 'en'), ('tubes', 'en'), ('assume', 'en'), ('pay', 'en'), ('historical', 'en'), ('like', 'en'), ('got', 'en'), ('collect', 'en'), ('shot', 'en'), ('camp', 'en'), ('believes', 'en'), ('receive', 'en'), ('tension', 'en'), ('toast', 'en'), ('fund', 'en'), ('subjected', 'en'), ('pink', 'en'), ('fear', 'en'), ('baseball', 'en'), ('especially', 'en'), ('include', 'en'), ('unlikely', 'en'), ('standing', 'en'), ('cut', 'en'), ('papers', 'en'), ('evidence', 'en'), ('his', 'en'), ('rush', 'en'), ('precisely', 'en'), ('profits', 'en'), ('easy', 'en'), ('realized', 'en'), ('coming', 'en'), ('basically', 'en'), ('stronger', 'en'), ('cash', 'en'), ('rise', 'en'), ('caused', 'en'), ('projects', 'en'), ('allows', 'en'), ('sacrifice', 'en'), ('bother', 'en'), ('shipping', 'en'), ('critic', 'en'), ('based', 'en'), ('conflict', 'en'), ('username', 'en'), ('join', 'en'), ('lake', 'en'), ('authentic', 'en'), ('assigned', 'en'), ('packed', 'en'), ('actions', 'en'), ('suite', 'en'), ('meaning', 'en'), ('libraries', 'en'), ('gentle', 'en'), ('rural', 'en'), ('likes', 'en'), ('depend', 'en'), ('device', 'en'), ('camera', 'en'), ('unit', 'en'), ('consonant', 'en'), ('fibers', 'en'), ('exciting', 'en'), ('pain', 'en'), ('made', 'en'), ('page', 'en'), ('browser', 'en'), ('representatives', 'en'), ('aren', 'en'), ('field', 'en'), ('completion', 'en'), ('uses', 'en'), ('such', 'en'), ('greater', 'en'), ('shape', 'en'), ('played', 'en'), ('degree', 'en'), ('politicians', 'en'), ('equivalent', 'en'), ('regularly', 'en'), ('dangerous', 'en'), ('apply', 'en'), ('badly', 'en'), ('current', 'en'), ('evil', 'en'), ('poetry', 'en'), ('edge', 'en'), ('guess', 'en'), ('swift', 'en'), ('example', 'en'), ('ancient', 'en'), ('yard', 'en'), ('discussed', 'en'), ('poems', 'en'), ('comfortable', 'en'), ('saddle', 'en'), ('june', 'en'), ('exception', 'en'), ('voted', 'en'), ('vigorous', 'en'), ('falling', 'en'), ('catch', 'en'), ('sponsor', 'en'), ('forth', 'en'), ('grain', 'en'), ('decades', 'en'), ('talent', 'en'), ('successfully', 'en'), ('quarrel', 'en'), ('fun', 'en'), ('north', 'en'), ('existed', 'en'), ('member', 'en'), ('clerk', 'en'), ('waves', 'en'), ('candidates', 'en'), ('dealt', 'en'), ('exhibition', 'en'), ('ward', 'en'), ('time', 'en'), ('round', 'en'), ('maintain', 'en'), ('engagement', 'en'), ('convenience', 'en'), ('orderly', 'en'), ('sponsored', 'en'), ('afraid', 'en'), ('dining', 'en'), ('mustard', 'en'), ('shortly', 'en'), ('son', 'en'), ('cooling', 'en'), ('human', 'en'), ('classical', 'en'), ('personality', 'en'), ('twice', 'en'), ('vacuum', 'en'), ('participate', 'en'), ('locate', 'en'), ('voters', 'en'), ('procurement', 'en'), ('teeth', 'en'), ('complex', 'en'), ('angry', 'en'), ('midnight', 'en'), ('dirt', 'en'), ('tossed', 'en'), ('nine', 'en'), ('cell', 'en'), ('closed', 'en'), ('cry', 'en'), ('phone', 'en'), ('bridge', 'en'), ('apparent', 'en'), ('journey', 'en'), ('showed', 'en'), ('ranch', 'en'), ('filled', 'en'), ('international', 'en'), ('largest', 'en'), ('apparatus', 'en'), ('represent', 'en'), ('dying', 'en'), ('has', 'en'), ('housing', 'en'), ('string', 'en'), ('reaction', 'en'), ('turned', 'en'), ('secretary', 'en'), ('dominant', 'en'), ('imitation', 'en'), ('reason', 'en'), ('conditioned', 'en'), ('element', 'en'), ('thereafter', 'en'), ('suddenly', 'en'), ('apart', 'en'), ('pages', 'en'), ('achieve', 'en'), ('bottle', 'en'), ('entitled', 'en'), ('cuts', 'en'), ('stretched', 'en'), ('poets', 'en'), ('production', 'en'), ('demanding', 'en'), ('intentions', 'en'), ('alert', 'en'), ('interviews', 'en'), ('days', 'en'), ('subtract', 'en'), ('assist', 'en'), ('useful', 'en'), ('mud', 'en'), ('keeps', 'en'), ('flower', 'en'), ('sentence', 'en'), ('rid', 'en'), ('desperate', 'en'), ('throat', 'en'), ('clarity', 'en'), ('scholars', 'en'), ('contributed', 'en'), ('continue', 'en'), ('guys', 'en'), ('grant', 'en'), ('sufficiently', 'en'), ('consequence', 'en'), ('clothe', 'en'), ('shooting', 'en'), ('concerts', 'en'), ('band', 'en'), ('library', 'en'), ('words', 'en'), ('trends', 'en'), ('use', 'en'), ('theme', 'en'), ('filling', 'en'), ('week', 'en'), ('apple', 'en'), ('noun', 'en'), ('despite', 'en'), ('commerce', 'en'), ('until', 'en'), ('charm', 'en'), ('goals', 'en'), ('citizen', 'en'), ('halign', 'en'), ('clothing', 'en'), ('junior', 'en'), ('achievement', 'en'), ('confronted', 'en'), ('approaches', 'en'), ('temperature', 'en'), ('overseas', 'en'), ('minutes', 'en'), ('est', 'en'), ('doesn', 'en'), ('arise', 'en'), ('panels', 'en'), ('porch', 'en'), ('meals', 'en'), ('confidence', 'en'), ('box', 'en'), ('hired', 'en'), ('appeal', 'en'), ('dried', 'en'), ('powers', 'en'), ('views', 'en'), ('forces', 'en'), ('conference', 'en'), ('mentioned', 'en'), ('distant', 'en'), ('sympathy', 'en'), ('aim', 'en'), ('banks', 'en'), ('charge', 'en'), ('suspended', 'en'), ('events', 'en'), ('bill', 'en'), ('thrust', 'en'), ('rare', 'en'), ('heading', 'en'), ('switches', 'en'), ('grabbed', 'en'), ('threatened', 'en'), ('rises', 'en'), ('essential', 'en'), ('march', 'en'), ('href', 'en'), ('typical', 'en'), ('anniversary', 'en'), ('safe', 'en'), ('star', 'en'), ('fellow', 'en'), ('replace', 'en'), ('merchants', 'en'), ('personnel', 'en'), ('establish', 'en'), ('columns', 'en'), ('clouds', 'en'), ('ice', 'en'), ('lot', 'en'), ('pointed', 'en'), ('observed', 'en'), ('stages', 'en'), ('fool', 'en'), ('growth', 'en'), ('languages', 'en'), ('notion', 'en'), ('established', 'en'), ('searching', 'en'), ('mean', 'en'), ('message', 'en'), ('belly', 'en'), ('via', 'en'), ('began', 'en'), ('textile', 'en'), ('spent', 'en'), ('stick', 'en'), ('responsibility', 'en'), ('suggestion', 'en'), ('enthusiasm', 'en'), ('allied', 'en'), ('mutual', 'en'), ('scope', 'en'), ('magic', 'en'), ('harm', 'en'), ('negotiations', 'en'), ('perform', 'en'), ('styles', 'en'), ('suburban', 'en'), ('binomial', 'en'), ('rail', 'en'), ('nodded', 'en'), ('gift', 'en'), ('issued', 'en'), ('softly', 'en'), ('involves', 'en'), ('curiosity', 'en'), ('onset', 'en'), ('efficiency', 'en'), ('planets', 'en'), ('inc', 'en'), ('taste', 'en'), ('nearby', 'en'), ('conscience', 'en'), ('bet', 'en'), ('range', 'en'), ('retirement', 'en'), ('returned', 'en'), ('planet', 'en'), ('medium', 'en'), ('root', 'en'), ('grinned', 'en'), ('particularly', 'en'), ('moved', 'en'), ('effect', 'en'), ('stay', 'en'), ('perhaps', 'en'), ('burns', 'en'), ('decimal', 'en'), ('cdt', 'en'), ('piano', 'en'), ('search', 'en'), ('sudden', 'en'), ('apartment', 'en'), ('perspective', 'en'), ('jul', 'en'), ('trying', 'en'), ('lay', 'en'), ('gain', 'en'), ('gives', 'en'), ('entrance', 'en'), ('placing', 'en'), ('abstract', 'en'), ('major', 'en'), ('issues', 'en'), ('wholly', 'en'), ('primary', 'en'), ('place', 'en'), ('instruction', 'en'), ('forget', 'en'), ('absent', 'en'), ('above', 'en'), ('help', 'en'), ('competition', 'en'), ('combined', 'en'), ('thin', 'en'), ('consciousness', 'en'), ('play', 'en'), ('snow', 'en'), ('winning', 'en'), ('particles', 'en'), ('belongs', 'en'), ('noise', 'en'), ('startled', 'en'), ('manufacturers', 'en'), ('cope', 'en'), ('performances', 'en'), ('thinks', 'en'), ('recommend', 'en'), ('learned', 'en'), ('email', 'en'), ('aid', 'en'), ('million', 'en'), ('modest', 'en'), ('serve', 'en'), ('obviously', 'en'), ('mighty', 'en'), ('materials', 'en'), ('molecule', 'en'), ('tied', 'en'), ('practical', 'en'), ('matters', 'en'), ('charges', 'en'), ('previous', 'en'), ('consistently', 'en'), ('regional', 'en'), ('remarks', 'en'), ('reduce', 'en'), ('dancing', 'en'), ('imagine', 'en'), ('insight', 'en'), ('access', 'en'), ('incredible', 'en'), ('board', 'en'), ('figures', 'en'), ('land', 'en'), ('regiment', 'en'), ('elaborate', 'en'), ('lights', 'en'), ('bit', 'en'), ('senator', 'en'), ('activity', 'en'), ('lots', 'en'), ('birds', 'en'), ('placed', 'en'), ('presence', 'en'), ('urban', 'en'), ('knew', 'en'), ('setting', 'en'), ('shouted', 'en'), ('goal', 'en'), ('thank', 'en'), ('publicity', 'en'), ('type', 'en'), ('authority', 'en'), ('outstanding', 'en'), ('worked', 'en'), ('ladies', 'en'), ('lonely', 'en'), ('flesh', 'en'), ('ended', 'en'), ('check', 'en'), ('cow', 'en'), ('philosophical', 'en'), ('beauty', 'en'), ('context', 'en'), ('definite', 'en'), ('communism', 'en'), ('situation', 'en'), ('naked', 'en'), ('snakes', 'en'), ('nature', 'en'), ('note', 'en'), ('supported', 'en'), ('there', 'en'), ('eye', 'en'), ('keeping', 'en'), ('patent', 'en'), ('jungle', 'en'), ('market', 'en'), ('fight', 'en'), ('despair', 'en'), ('stretch', 'en'), ('faced', 'en'), ('dictionary', 'en'), ('race', 'en'), ('south', 'en'), ('video', 'en'), ('formerly', 'en'), ('distinctive', 'en'), ('quart', 'en'), ('astronomy', 'en'), ('partially', 'en'), ('provides', 'en'), ('afternoon', 'en'), ('top', 'en'), ('beard', 'en'), ('front', 'en'), ('measuring', 'en'), ('certain', 'en'), ('barrel', 'en'), ('asking', 'en'), ('occur', 'en'), ('readily', 'en'), ('professional', 'en'), ('pursuant', 'en'), ('del', 'en'), ('meeting', 'en'), ('destiny', 'en'), ('varying', 'en'), ('directly', 'en'), ('amount', 'en'), ('things', 'en'), ('excuse', 'en'), ('wildlife', 'en'), ('marked', 'en'), ('police', 'en'), ('occurs', 'en'), ('seeds', 'en'), ('confirmed', 'en'), ('few', 'en'), ('authors', 'en'), ('government', 'en'), ('told', 'en'), ('rejected', 'en'), ('companion', 'en'), ('york', 'en'), ('assistant', 'en'), ('occasionally', 'en'), ('colored', 'en'), ('price', 'en'), ('responses', 'en'), ('killing', 'en'), ('reactionary', 'en'), ('contact', 'en'), ('bombs', 'en'), ('famous', 'en'), ('should', 'en'), ('promised', 'en'), ('plane', 'en'), ('he', 'en'), ('sail', 'en'), ('artist', 'en'), ('calculated', 'en'), ('intervals', 'en'), ('selected', 'en'), ('gardens', 'en'), ('seek', 'en'), ('necessity', 'en'), ('applying', 'en'), ('mention', 'en'), ('bar', 'en'), ('liquid', 'en'), ('river', 'en'), ('fine', 'en'), ('heads', 'en'), ('composer', 'en'), ('products', 'en'), ('fighting', 'en'), ('dancer', 'en'), ('firms', 'en'), ('valign', 'en'), ('pace', 'en'), ('jet', 'en'), ('taught', 'en'), ('interest', 'en'), ('chick', 'en'), ('reaches', 'en'), ('frequently', 'en'), ('poor', 'en'), ('hide', 'en'), ('injured', 'en'), ('answered', 'en'), ('utterly', 'en'), ('men', 'en'), ('resulting', 'en'), ('watch', 'en'), ('waiting', 'en'), ('clinical', 'en'), ('doors', 'en'), ('lucky', 'en'), ('think', 'en'), ('properly', 'en'), ('painter', 'en'), ('build', 'en'), ('settlement', 'en'), ('index', 'en'), ('dealer', 'en'), ('awareness', 'en'), ('yourself', 'en'), ('warfare', 'en'), ('proposed', 'en'), ('off', 'en'), ('horse', 'en'), ('passion', 'en'), ('known', 'en'), ('era', 'en'), ('differences', 'en'), ('word', 'en'), ('additional', 'en'), ('pm', 'en'), ('hell', 'en'), ('ham', 'en'), ('assumption', 'en'), ('director', 'en'), ('city', 'en'), ('musicians', 'en'), ('maintenance', 'en'), ('person', 'en'), ('aimed', 'en'), ('correspondence', 'en'), ('shell', 'en'), ('towns', 'en'), ('jazz', 'en'), ('kingdom', 'en'), ('nothing', 'en'), ('ambiguous', 'en'), ('hated', 'en'), ('objectives', 'en'), ('slide', 'en'), ('that', 'en'), ('fiction', 'en'), ('perception', 'en'), ('knee', 'en'), ('line', 'en'), ('dollar', 'en'), ('shout', 'en'), ('fled', 'en'), ('neighboring', 'en'), ('part-time', 'en'), ('helpful', 'en'), ('result', 'en'), ('wonderful', 'en'), ('impressed', 'en'), ('constitutional', 'en'), ('dec', 'en'), ('registration', 'en'), ('helpless', 'en'), ('shorts', 'en'), ('quarters', 'en'), ('text', 'en'), ('findings', 'en'), ('stated', 'en'), ('house', 'en'), ('pointing', 'en'), ('detail', 'en'), ('plus', 'en'), ('automatically', 'en'), ('giant', 'en'), ('investigations', 'en'), ('houses', 'en'), ('need', 'en'), ('earth', 'en'), ('does', 'en'), ('musician', 'en'), ('outside', 'en'), ('per', 'en'), ('performed', 'en'), ('fewer', 'en'), ('process', 'en'), ('whereas', 'en'), ('everything', 'en'), ('hr', 'en'), ('computed', 'en'), ('belief', 'en'), ('young', 'en'), ('center', 'en'), ('lunch', 'en'), ('slipped', 'en'), ('trials', 'en'), ('laugh', 'en'), ('physical', 'en'), ('arrange', 'en'), ('working', 'en'), ('knowing', 'en'), ('cleaning', 'en'), ('sensitive', 'en'), ('tale', 'en'), ('difficult', 'en'), ('indicated', 'en'), ('wiped', 'en'), ('melody', 'en'), ('mm', 'en'), ('policeman', 'en'), ('documents', 'en'), ('statistics', 'en'), ('print', 'en'), ('program', 'en'), ('threw', 'en'), ('reform', 'en'), ('remarked', 'en'), ('business', 'en'), ('pound', 'en'), ('power', 'en'), ('lighted', 'en'), ('code', 'en'), ('filing', 'en'), ('reception', 'en'), ('telephone', 'en'), ('although', 'en'), ('period', 'en'), ('substance', 'en'), ('claim', 'en'), ('pertinent', 'en'), ('important', 'en'), ('tears', 'en'), ('old', 'en'), ('presents', 'en'), ('rendered', 'en'), ('spectacular', 'en'), ('comparison', 'en'), ('players', 'en'), ('wondered', 'en'), ('bore', 'en'), ('girl', 'en'), ('all', 'en'), ('periods', 'en'), ('plants', 'en'), ('atomic', 'en'), ('traditions', 'en'), ('attain', 'en'), ('humanity', 'en'), ('units', 'en'), ('aunt', 'en'), ('speaking', 'en'), ('systems', 'en'), ('notable', 'en'), ('red', 'en'), ('admission', 'en'), ('practically', 'en'), ('total', 'en'), ('twenty-five', 'en'), ('contains', 'en'), ('grip', 'en'), ('development', 'en'), ('mine', 'en'), ('support', 'en'), ('commissioner', 'en'), ('add', 'en'), ('factor', 'en'), ('forgive', 'en'), ('trucks', 'en'), ('control', 'en'), ('including', 'en'), ('don', 'en'), ('provisions', 'en'), ('car', 'en'), ('unless', 'en'), ('created', 'en'), ('bureau', 'en'), ('february', 'en'), ('complete', 'en'), ('itself', 'en'), ('portion', 'en'), ('content', 'en'), ('happens', 'en'), ('preceding', 'en'), ('flowers', 'en'), ('orchestra', 'en'), ('mostly', 'en'), ('authorized', 'en'), ('gains', 'en'), ('hours', 'en'), ('fingers', 'en'), ('pathology', 'en'), ('applications', 'en'), ('wrote', 'en'), ('black', 'en'), ('agree', 'en'), ('negative', 'en'), ('religious', 'en'), ('historic', 'en'), ('wood', 'en'), ('hate', 'en'), ('solely', 'en'), ('budget', 'en'), ('anyway', 'en'), ('html', 'en'), ('colorful', 'en'), ('lightly', 'en'), ('below', 'en'), ('create', 'en'), ('during', 'en'), ('sentiment', 'en'), ('continuity', 'en'), ('bread', 'en'), ('altered', 'en'), ('frozen', 'en'), ('centers', 'en'), ('encountered', 'en'), ('schools', 'en'), ('probable', 'en'), ('greatly', 'en'), ('multiple', 'en'), ('deeper', 'en'), ('cases', 'en'), ('speak', 'en'), ('grade', 'en'), ('pounds', 'en'), ('temporary', 'en'), ('chosen', 'en'), ('mystery', 'en'), ('equally', 'en'), ('methods', 'en'), ('contracts', 'en'), ('drink', 'en'), ('darkness', 'en'), ('recently', 'en'), ('moves', 'en'), ('sad', 'en'), ('depth', 'en'), ('warned', 'en'), ('resolution', 'en'), ('lady', 'en'), ('painted', 'en'), ('whispered', 'en'), ('writing', 'en'), ('faint', 'en'), ('call', 'en'), ('classes', 'en'), ('excite', 'en'), ('environment', 'en'), ('comment', 'en'), ('filed', 'en'), ('rising', 'en'), ('symbols', 'en'), ('distributed', 'en'), ('capital', 'en'), ('items', 'en'), ('plural', 'en'), ('waters', 'en'), ('entries', 'en'), ('steel', 'en'), ('suits', 'en'), ('distinction', 'en'), ('relative', 'en'), ('contract', 'en'), ('pst', 'en'), ('courts', 'en'), ('details', 'en'), ('open', 'en'), ('significant', 'en'), ('expressing', 'en'), ('extended', 'en'), ('magnitude', 'en'), ('forms', 'en'), ('mobile', 'en'), ('neutral', 'en'), ('argument', 'en'), ('have', 'en'), ('contained', 'en'), ('skywave', 'en'), ('height', 'en'), ('pressed', 'en'), ('encounter', 'en'), ('singing', 'en'), ('spirits', 'en'), ('talk', 'en'), ('dog', 'en'), ('bod', 'en'), ('designed', 'en'), ('bundle', 'en'), ('pupil', 'en'), ('desperately', 'en'), ('post', 'en'), ('easily', 'en'), ('shared', 'en'), ('biggest', 'en'), ('subtle', 'en'), ('comes', 'en'), ('legislation', 'en'), ('ears', 'en'), ('paused', 'en'), ('arrive', 'en'), ('polynomial', 'en'), ('opposite', 'en'), ('pack', 'en'), ('connection', 'en'), ('they', 'en'), ('listening', 'en'), ('definition', 'en'), ('factories', 'en'), ('fairly', 'en'), ('wished', 'en'), ('arrival', 'en'), ('climbed', 'en'), ('honey', 'en'), ('remarkable', 'en'), ('sales', 'en'), ('troops', 'en'), ('meaningful', 'en'), ('rational', 'en'), ('whatever', 'en'), ('removal', 'en'), ('horses', 'en'), ('successes', 'en'), ('weak', 'en'), ('caught', 'en'), ('according', 'en'), ('grass', 'en'), ('region', 'en'), ('arranged', 'en'), ('monthly', 'en'), ('cycle', 'en'), ('ships', 'en'), ('allotment', 'en'), ('oh', 'en'), ('joint', 'en'), ('replied', 'en'), ('helping', 'en'), ('showing', 'en'), ('feet', 'en'), ('language', 'en'), ('suggest', 'en'), ('bag', 'en'), ('staff', 'en'), ('colonial', 'en'), ('release', 'en'), ('satisfied', 'en'), ('cooking', 'en'), ('promotion', 'en'), ('directions', 'en'), ('carrying', 'en'), ('manufacturer', 'en'), ('amounts', 'en'), ('snake', 'en'), ('recording', 'en'), ('assured', 'en'), ('anybody', 'en'), ('radiation', 'en'), ('for', 'en'), ('submitted', 'en'), ('pick', 'en'), ('clean', 'en'), ('consisting', 'en'), ('main', 'en'), ('questionnaire', 'en'), ('morning', 'en'), ('nerves', 'en'), ('lesson', 'en'), ('upward', 'en'), ('classification', 'en'), ('reminded', 'en'), ('conferences', 'en'), ('basic', 'en'), ('blue', 'en'), ('drive', 'en'), ('interior', 'en'), ('alternative', 'en'), ('constantly', 'en'), ('mail', 'en'), ('advice', 'en'), ('substantially', 'en'), ('testimony', 'en'), ('saved', 'en'), ('effects', 'en'), ('suggestions', 'en'), ('receiving', 'en'), ('excellent', 'en'), ('protein', 'en'), ('roots', 'en'), ('newly', 'en'), ('somewhere', 'en'), ('date', 'en'), ('meal', 'en'), ('concerned', 'en'), ('opening', 'en'), ('buy', 'en'), ('autumn', 'en'), ('reasonably', 'en'), ('conventional', 'en'), ('marble', 'en'), ('life', 'en'), ('extend', 'en'), ('tested', 'en'), ('subject', 'en'), ('wit', 'en'), ('basement', 'en'), ('marketing', 'en'), ('happening', 'en'), ('moment', 'en'), ('odd', 'en'), ('permanent', 'en'), ('energy', 'en'), ('system', 'en'), ('mirror', 'en'), ('month', 'en'), ('brings', 'en'), ('lb.', 'en'), ('maturity', 'en'), ('dir', 'en'), ('uncertain', 'en'), ('closing', 'en'), ('nowhere', 'en'), ('desirable', 'en'), ('every', 'en'), ('isolated', 'en'), ('a', 'en'), ('model', 'en'), ('appears', 'en'), ('national', 'en'), ('ma', 'en'), ('reserve', 'en'), ('policy', 'en'), ('terrible', 'en'), ('stand', 'en'), ('pushed', 'en'), ('addition', 'en'), ('lifted', 'en'), ('neighbors', 'en'), ('fist', 'en'), ('eyes', 'en'), ('risk', 'en'), ('aroused', 'en'), ('care', 'en'), ('addressed', 'en'), ('isn', 'en'), ('cattle', 'en'), ('hypothalamic', 'en'), ('charoff', 'en'), ('explain', 'en'), ('sky', 'en'), ('beginning', 'en'), ('pressing', 'en'), ('desires', 'en'), ('spoke', 'en'), ('drama', 'en'), ('heat', 'en'), ('permits', 'en'), ('numeral', 'en'), ('rank', 'en'), ('highly', 'en'), ('rifles', 'en'), ('slow', 'en'), ('similarly', 'en'), ('soil', 'en'), ('map', 'en'), ('on', 'en'), ('inner', 'en'), ('measured', 'en'), ('strictly', 'en'), ('benefit', 'en'), ('dreamed', 'en'), ('beside', 'en'), ('affect', 'en'), ('survive', 'en'), ('standards', 'en'), ('injury', 'en'), ('dull', 'en'), ('passages', 'en'), ('errors', 'en'), ('visit', 'en'), ('case', 'en'), ('ages', 'en'), ('regime', 'en'), ('proper', 'en'), ('mouth', 'en'), ('places', 'en'), ('slight', 'en'), ('frequencies', 'en'), ('ago', 'en'), ('observation', 'en'), ('organized', 'en'), ('silver', 'en'), ('mistake', 'en'), ('shoe', 'en'), ('effectively', 'en'), ('attract', 'en'), ('surface', 'en'), ('facing', 'en'), ('slip', 'en'), ('lean', 'en'), ('blow', 'en'), ('mass', 'en'), ('about', 'en'), ('down', 'en'), ('crack', 'en'), ('lively', 'en'), ('roles', 'en'), ('articles', 'en'), ('defined', 'en'), ('stressed', 'en'), ('prove', 'en'), ('chart', 'en'), ('rhythm', 'en'), ('reputation', 'en'), ('dead', 'en'), ('log', 'en'), ('together', 'en'), ('girls', 'en'), ('maybe', 'en'), ('addresses', 'en'), ('continuous', 'en'), ('animal', 'en'), ('formed', 'en'), ('unusual', 'en'), ('changing', 'en'), ('primitive', 'en'), ('cold', 'en'), ('normal', 'en'), ('circles', 'en'), ('technical', 'en'), ('trace', 'en'), ('extra', 'en'), ('own', 'en'), ('speeches', 'en'), ('touched', 'en'), ('contemporary', 'en'), ('equate', 'en'), ('rope', 'en'), ('goods', 'en'), ('east', 'en'), ('yards', 'en'), ('continent', 'en'), ('improved', 'en'), ('security', 'en'), ('statements', 'en'), ('games', 'en'), ('cheap', 'en'), ('wages', 'en'), ('advised', 'en'), ('dilemma', 'en'), ('stared', 'en'), ('representing', 'en'), ('operation', 'en'), ('naval', 'en'), ('transportation', 'en'), ('crawled', 'en'), ('station', 'en'), ('art', 'en'), ('wound', 'en'), ('departments', 'en'), ('truly', 'en'), ('feelings', 'en'), ('varied', 'en'), ('lived', 'en'), ('yield', 'en'), ('innocence', 'en'), ('includes', 'en'), ('citizens', 'en'), ('understood', 'en'), ('returns', 'en'), ('active', 'en'), ('resolved', 'en'), ('seat', 'en'), ('happened', 'en'), ('shelters', 'en'), ('gear', 'en'), ('thick', 'en'), ('impact', 'en'), ('shame', 'en'), ('thoughts', 'en'), ('evident', 'en'), ('examine', 'en'), ('diplomatic', 'en'), ('achieved', 'en'), ('manufacturing', 'en'), ('run', 'en'), ('smart', 'en'), ('raising', 'en'), ('answer', 'en'), ('letting', 'en'), ('getting', 'en'), ('oil', 'en'), ('please', 'en'), ('everybody', 'en'), ('distance', 'en'), ('slim', 'en'), ('pistol', 'en'), ('deck', 'en'), ('triangle', 'en'), ('civilization', 'en'), ('destructive', 'en'), ('currently', 'en'), ('satisfactory', 'en'), ('farm', 'en'), ('opened', 'en'), ('conceived', 'en'), ('signs', 'en'), ('aside', 'en'), ('speaker', 'en'), ('defeat', 'en'), ('added', 'en'), ('collection', 'en'), ('frame', 'en'), ('face', 'en'), ('remains', 'en'), ('to', 'en'), ('decent', 'en'), ('home', 'en'), ('revealed', 'en'), ('hotels', 'en'), ('ease', 'en'), ('movement', 'en'), ('tsunami', 'en'), ('provided', 'en'), ('couple', 'en'), ('large', 'en'), ('plug', 'en'), ('variations', 'en'), ('removed', 'en'), ('relatively', 'en'), ('sense', 'en'), ('counties', 'en'), ('lawyers', 'en'), ('literally', 'en'), ('hands', 'en'), ('sidewalk', 'en'), ('attempt', 'en'), ('physically', 'en'), ('laughter', 'en'), ('lawyer', 'en'), ('rock', 'en'), ('built', 'en'), ('grand', 'en'), ('legend', 'en'), ('luxury', 'en'), ('chemical', 'en'), ('puts', 'en'), ('acquire', 'en'), ('worries', 'en'), ('memory', 'en'), ('treatment', 'en'), ('strain', 'en'), ('gathered', 'en'), ('thorough', 'en'), ('accuracy', 'en'), ('nation', 'en'), ('separated', 'en'), ('damage', 'en'), ('types', 'en'), ('disk', 'en'), ('bones', 'en'), ('accepting', 'en'), ('opposition', 'en'), ('judgments', 'en'), ('are', 'en'), ('abroad', 'en'), ('long', 'en'), ('hunt', 'en'), ('credit', 'en'), ('determine', 'en'), ('specialists', 'en'), ('force', 'en'), ('trembling', 'en'), ('hen', 'en'), ('experiments', 'en'), ('vs', 'en'), ('measure', 'en'), ('become', 'en'), ('initial', 'en'), ('phenomenon', 'en'), ('snapped', 'en'), ('my', 'en'), ('took', 'en'), ('diet', 'en'), ('category', 'en'), ('central', 'en'), ('expect', 'en'), ('burned', 'en'), ('saline', 'en'), ('presentation', 'en'), ('history', 'en'), ('elements', 'en'), ('fit', 'en'), ('entering', 'en'), ('letter', 'en'), ('seems', 'en'), ('consideration', 'en'), ('awake', 'en'), ('inclined', 'en'), ('careful', 'en'), ('peoples', 'en'), ('several', 'en'), ('rest', 'en'), ('tournament', 'en'), ('temperatures', 'en'), ('operator', 'en'), ('generations', 'en'), ('pleased', 'en'), ('arrangements', 'en'), ('assistance', 'en'), ('said', 'en'), ('sovereign', 'en'), ('swing', 'en'), ('generation', 'en'), ('aware', 'en'), ('chose', 'en'), ('driver', 'en'), ('lead', 'en'), ('leaves', 'en'), ('comparable', 'en'), ('discipline', 'en'), ('plate', 'en'), ('wishes', 'en'), ('feature', 'en'), ('fitted', 'en'), ('might', 'en'), ('record', 'en'), ('sing', 'en'), ('sixty', 'en'), ('emphasize', 'en'), ('tend', 'en'), ('feel', 'en'), ('stable', 'en'), ('unhappy', 'en'), ('favorable', 'en'), ('mechanism', 'en'), ('cousin', 'en'), ('treat', 'en'), ('whose', 'en'), ('corresponding', 'en'), ('livestock', 'en'), ('council', 'en'), ('thinking', 'en'), ('discussion', 'en'), ('guards', 'en'), ('mile', 'en'), ('channels', 'en'), ('heart', 'en'), ('romantic', 'en'), ('legs', 'en'), ('water', 'en'), ('border', 'en'), ('openly', 'en'), ('started', 'en'), ('talked', 'en'), ('moderate', 'en'), ('beings', 'en'), ('shear', 'en'), ('whisky', 'en'), ('encourage', 'en'), ('engineering', 'en'), ('second', 'en'), ('mechanical', 'en'), ('scheme', 'en'), ('bound', 'en'), ('skill', 'en'), ('uncle', 'en'), ('nov', 'en'), ('minds', 'en'), ('hearts', 'en'), ('heaven', 'en'), ('leg', 'en'), ('october', 'en'), ('suitcase', 'en'), ('impressions', 'en'), ('furthermore', 'en'), ('reported', 'en'), ('fall', 'en'), ('tagged', 'en'), ('lobby', 'en'), ('mature', 'en'), ('foams', 'en'), ('work', 'en'), ('generous', 'en'), ('stories', 'en'), ('downtown', 'en'), ('naturally', 'en'), ('lock', 'en'), ('landing', 'en'), ('sell', 'en'), ('readers', 'en'), ('miles', 'en'), ('conducted', 'en'), ('images', 'en'), ('service', 'en'), ('ways', 'en'), ('emerged', 'en'), ('whether', 'en'), ('forever', 'en'), ('tones', 'en'), ('guidance', 'en'), ('three', 'en'), ('coating', 'en'), ('april', 'en'), ('reference', 'en'), ('electron', 'en'), ('feeling', 'en'), ('specifically', 'en'), ('loyalty', 'en'), ('hearing', 'en'), ('mix', 'en'), ('was', 'en'), ('mathematical', 'en'), ('wisdom', 'en'), ('thanks', 'en'), ('enjoyed', 'en'), ('two', 'en'), ('silence', 'en'), ('arbitrary', 'en'), ('grave', 'en'), ('count', 'en'), ('each', 'en'), ('deny', 'en'), ('parks', 'en'), ('succeeded', 'en'), ('wish', 'en'), ('controlling', 'en'), ('engineers', 'en'), ('promise', 'en'), ('role', 'en'), ('court', 'en'), ('technique', 'en'), ('combat', 'en'), ('backed', 'en'), ('pencil', 'en'), ('unfortunate', 'en'), ('attention', 'en'), ('it', 'en'), ('progress', 'en'), ('minor', 'en'), ('structures', 'en'), ('plot', 'en'), ('mixed', 'en'), ('motives', 'en'), ('greeted', 'en'), ('wouldn', 'en'), ('weight', 'en'), ('countries', 'en'), ('suggests', 'en'), ('full', 'en'), ('identification', 'en'), ('stock', 'en'), ('tough', 'en'), ('reality', 'en'), ('whereby', 'en'), ('fees', 'en'), ('pieces', 'en'), ('finished', 'en'), ('smile', 'en'), ('half', 'en'), ('bars', 'en'), ('excited', 'en'), ('fifth', 'en'), ('efficient', 'en'), ('realistic', 'en'), ('keys', 'en'), ('close', 'en'), ('recognized', 'en'), ('limit', 'en'), ('available', 'en'), ('boat', 'en'), ('doubt', 'en'), ('drawn', 'en'), ('panic', 'en'), ('approved', 'en'), ('customer', 'en'), ('block', 'en'), ('hidden', 'en'), ('however', 'en'), ('servants', 'en'), ('testing', 'en'), ('accompanied', 'en'), ('municipal', 'en'), ('grow', 'en'), ('bird', 'en'), ('double', 'en'), ('session', 'en'), ('patients', 'en'), ('realism', 'en'), ('sum', 'en'), ('passes', 'en'), ('shots', 'en'), ('curt', 'en'), ('output', 'en'), ('muscle', 'en'), ('one', 'en'), ('valuable', 'en'), ('hardly', 'en'), ('minimum', 'en'), ('mission', 'en'), ('brief', 'en'), ('find', 'en'), ('rooms', 'en'), ('employee', 'en'), ('estimated', 'en'), ('prepare', 'en'), ('kid', 'en'), ('decision', 'en'), ('allowed', 'en'), ('site', 'en'), ('unions', 'en'), ('year', 'en'), ('institution', 'en'), ('speech', 'en'), ('net', 'en'), ('let', 'en'), ('races', 'en'), ('laboratory', 'en'), ('assignment', 'en'), ('among', 'en'), ('considered', 'en'), ('muscles', 'en'), ('comedy', 'en'), ('threatening', 'en'), ('soap', 'en'), ('worst', 'en'), ('witness', 'en'), ('approval', 'en'), ('golf', 'en'), ('imagined', 'en'), ('sep', 'en'), ('electronics', 'en'), ('shouldn', 'en'), ('conscious', 'en'), ('proportion', 'en'), ('january', 'en'), ('trees', 'en'), ('jumped', 'en'), ('thousand', 'en'), ('breakfast', 'en'), ('competitive', 'en'), ('nights', 'en'), ('possession', 'en'), ('reasons', 'en'), ('private', 'en'), ('eighth', 'en'), ('light', 'en'), ('tonight', 'en'), ('listed', 'en'), ('happen', 'en'), ('title', 'en'), ('poverty', 'en'), ('rule', 'en'), ('objective', 'en'), ('still', 'en'), ('wind', 'en'), ('direction', 'en'), ('cover', 'en'), ('knocked', 'en'), ('will', 'en'), ('developments', 'en'), ('command', 'en'), ('covering', 'en'), ('skin', 'en'), ('firm', 'en'), ('magnet', 'en'), ('average', 'en'), ('warning', 'en'), ('convictions', 'en'), ('arrest', 'en'), ('practices', 'en'), ('rain', 'en'), ('network', 'en'), ('surplus', 'en'), ('theories', 'en'), ('future', 'en'), ('cause', 'en'), ('quite', 'en'), ('sample', 'en'), ('ecumenical', 'en'), ('very', 'en'), ('emphasis', 'en'), ('que', 'en'), ('inch', 'en'), ('usually', 'en'), ('sewage', 'en'), ('status', 'en'), ('nice', 'en'), ('problem', 'en'), ('sitter', 'en'), ('warrant', 'en'), ('avoid', 'en'), ('produced', 'en'), ('tight', 'en'), ('unexpected', 'en'), ('signed', 'en'), ('component', 'en'), ('dry', 'en'), ('became', 'en'), ('stairs', 'en'), ('payments', 'en'), ('sweat', 'en'), ('popular', 'en'), ('numbers', 'en'), ('percent', 'en'), ('warm', 'en'), ('exactly', 'en'), ('judges', 'en'), ('ft.', 'en'), ('theater', 'en'), ('strip', 'en'), ('thereby', 'en'), ('feels', 'en'), ('traffic', 'en'), ('stars', 'en'), ('moral', 'en'), ('handle', 'en'), ('equipment', 'en'), ('dress', 'en'), ('alike', 'en'), ('doubtful', 'en'), ('indeed', 'en'), ('dedicated', 'en'), ('bringing', 'en'), ('demand', 'en'), ('ratio', 'en'), ('blanket', 'en'), ('hunting', 'en'), ('deeply', 'en'), ('stopped', 'en'), ('retired', 'en'), ('then', 'en'), ('dominated', 'en'), ('storage', 'en'), ('coverage', 'en'), ('fifteen', 'en'), ('segment', 'en'), ('fantastic', 'en'), ('bear', 'en'), ('governmental', 'en'), ('pitch', 'en'), ('converted', 'en'), ('ring', 'en'), ('steam', 'en'), ('satisfaction', 'en'), ('milligrams', 'en'), ('reserved', 'en'), ('suit', 'en'), ('direct', 'en'), ('interference', 'en'), ('bomb', 'en'), ('tired', 'en'), ('connected', 'en'), ('billion', 'en'), ('quantity', 'en'), ('blood', 'en'), ('involving', 'en'), ('sat', 'en'), ('weeks', 'en'), ('facilities', 'en'), ('bases', 'en'), ('civil', 'en'), ('dimensions', 'en'), ('cars', 'en'), ('formula', 'en'), ('thermal', 'en'), ('losing', 'en'), ('deep', 'en'), ('horizon', 'en'), ('due', 'en'), ('argued', 'en'), ('guided', 'en'), ('collected', 'en'), ('sun', 'en'), ('increased', 'en'), ('toes', 'en'), ('frequent', 'en'), ('offices', 'en'), ('six', 'en'), ('frames', 'en'), ('approached', 'en'), ('moreover', 'en'), ('quality', 'en'), ('extreme', 'en'), ('suited', 'en'), ('food', 'en'), ('often', 'en'), ('remembered', 'en'), ('vacation', 'en'), ('intensity', 'en'), ('reflect', 'en'), ('raw', 'en'), ('replacement', 'en'), ('vision', 'en'), ('decline', 'en'), ('thus', 'en'), ('daily', 'en'), ('similar', 'en'), ('cloth', 'en'), ('guns', 'en'), ('bright', 'en'), ('difficulty', 'en'), ('summer', 'en'), ('waste', 'en'), ('push', 'en'), ('changes', 'en'), ('hold', 'en'), ('asleep', 'en'), ('areas', 'en'), ('precise', 'en'), ('described', 'en'), ('felt', 'en'), ('relationships', 'en'), ('indicate', 'en'), ('same', 'en'), ('reporter', 'en'), ('believed', 'en'), ('named', 'en'), ('criticism', 'en'), ('rich', 'en'), ('expects', 'en'), ('branches', 'en'), ('situations', 'en'), ('factory', 'en'), ('operate', 'en'), ('prison', 'en'), ('license', 'en'), ('explanation', 'en'), ('glory', 'en'), ('allow', 'en'), ('merit', 'en'), ('temporarily', 'en'), ('hour', 'en'), ('larger', 'en'), ('loose', 'en'), ('others', 'en'), ('scholarship', 'en'), ('tool', 'en'), ('sent', 'en'), ('president', 'en'), ('politics', 'en'), ('transition', 'en'), ('occupied', 'en'), ('nude', 'en'), ('mount', 'en'), ('impressive', 'en'), ('grows', 'en'), ('fault', 'en'), ('here', 'en'), ('orders', 'en'), ('born', 'en'), ('motor', 'en'), ('shade', 'en'), ('guilty', 'en'), ('complained', 'en'), ('aspects', 'en'), ('mothers', 'en'), ('boss', 'en'), ('raised', 'en'), ('spot', 'en'), ('spread', 'en'), ('expressed', 'en'), ('partisan', 'en'), ('given', 'en'), ('yet', 'en'), ('friends', 'en'), ('juniors', 'en'), ('denied', 'en'), ('duties', 'en'), ('broke', 'en'), ('protest', 'en'), ('encouraged', 'en'), ('chain', 'en'), ('shining', 'en'), ('separate', 'en'), ('official', 'en'), ('solve', 'en'), ('principal', 'en'), ('ad', 'en'), ('insect', 'en'), ('article', 'en'), ('connect', 'en'), ('link', 'en'), ('feed', 'en'), ('between', 'en'), ('convenient', 'en'), ('complement', 'en'), ('sailing', 'en'), ('lives', 'en'), ('reply', 'en'), ('residential', 'en'), ('sessions', 'en'), ('book', 'en'), ('parts', 'en'), ('determining', 'en'), ('hollywood', 'en'), ('truth', 'en'), ('parallel', 'en'), ('judgment', 'en'), ('passage', 'en'), ('put', 'en'), ('dealers', 'en'), ('pdt', 'en'), ('verb', 'en'), ('listeners', 'en'), ('controls', 'en'), ('won', 'en'), ('arrested', 'en'), ('slid', 'en'), ('fail', 'en'), ('fort', 'en'), ('acceptable', 'en'), ('bronchial', 'en'), ('willing', 'en'), ('female', 'en'), ('expert', 'en'), ('dinner', 'en'), ('though', 'en'), ('heroic', 'en'), ('incident', 'en'), ('attack', 'en'), ('gesture', 'en'), ('operating', 'en'), ('enable', 'en'), ('secret', 'en'), ('branch', 'en'), ('urge', 'en'), ('visited', 'en'), ('motive', 'en'), ('thought', 'en'), ('trial', 'en'), ('mixture', 'en'), ('offer', 'en'), ('kill', 'en'), ('continuing', 'en'), ('dramatic', 'en'), ('transferred', 'en'), ('urgent', 'en'), ('tuesday', 'en'), ('assure', 'en'), ('jail', 'en'), ('presumably', 'en'), ('publication', 'en'), ('www', 'en'), ('women', 'en'), ('adequate', 'en'), ('looked', 'en'), ('beer', 'en'), ('controlled', 'en'), ('gay', 'en'), ('am', 'en'), ('mood', 'en'), ('public', 'en'), ('youngsters', 'en'), ('pattern', 'en'), ('fully', 'en'), ('essentially', 'en'), ('entry', 'en'), ('taking', 'en'), ('beliefs', 'en'), ('racial', 'en'), ('day', 'en'), ('tractor', 'en'), ('determination', 'en'), ('problems', 'en'), ('cooperative', 'en'), ('agents', 'en'), ('lowered', 'en'), ('engineer', 'en'), ('suppose', 'en'), ('magazines', 'en'), ('reflected', 'en'), ('must', 'en'), ('eleven', 'en'), ('body', 'en'), ('anyhow', 'en'), ('characteristics', 'en'), ('imagination', 'en'), ('illness', 'en'), ('sections', 'en'), ('washing', 'en'), ('able', 'en'), ('faces', 'en'), ('style', 'en'), ('companies', 'en'), ('path', 'en'), ('using', 'en'), ('justify', 'en'), ('hole', 'en'), ('single', 'en'), ('observe', 'en'), ('signal', 'en'), ('drew', 'en'), ('pocket', 'en'), ('advantage', 'en'), ('since', 'en'), ('multiply', 'en'), ('reactions', 'en'), ('critical', 'en'), ('residents', 'en'), ('television', 'en'), ('contrary', 'en'), ('dive', 'en'), ('bent', 'en'), ('stresses', 'en'), ('ethics', 'en'), ('pulled', 'en'), ('shoes', 'en'), ('swung', 'en'), ('ever', 'en'), ('sale', 'en'), ('normally', 'en'), ('admitted', 'en'), ('fourth', 'en'), ('namely', 'en'), ('consumer', 'en'), ('radar', 'en'), ('appeared', 'en'), ('sets', 'en'), ('inspired', 'en'), ('upper', 'en'), ('backward', 'en'), ('wooden', 'en'), ('assumptions', 'en'), ('encouraging', 'en'), ('dogs', 'en'), ('compare', 'en'), ('literature', 'en'), ('clearly', 'en'), ('scheduled', 'en'), ('himself', 'en'), ('always', 'en'), ('lost', 'en'), ('glad', 'en'), ('tone', 'en'), ('watched', 'en'), ('boards', 'en'), ('properties', 'en'), ('machines', 'en'), ('county', 'en'), ('noon', 'en'), ('touch', 'en'), ('tire', 'en'), ('written', 'en'), ('limited', 'en'), ('unable', 'en'), ('bullets', 'en'), ('alive', 'en'), ('completed', 'en'), ('easier', 'en'), ('directors', 'en'), ('species', 'en'), ('suffered', 'en'), ('actor', 'en'), ('within', 'en'), ('obtained', 'en'), ('crash', 'en'), ('led', 'en'), ('points', 'en'), ('wants', 'en'), ('tongue', 'en'), ('concluded', 'en'), ('covers', 'en'), ('jun', 'en'), ('exclusive', 'en'), ('obtain', 'en'), ('apparently', 'en'), ('somewhat', 'en'), ('dressed', 'en'), ('stream', 'en'), ('painful', 'en'), ('justice', 'en'), ('myself', 'en'), ('ready', 'en'), ('spare', 'en'), ('grains', 'en'), ('painting', 'en'), ('processing', 'en'), ('select', 'en'), ('store', 'en'), ('dreams', 'en'), ('response', 'en'), ('track', 'en'), ('trip', 'en'), ('domestic', 'en'), ('wear', 'en'), ('trips', 'en'), ('because', 'en'), ('depends', 'en'), ('condition', 'en'), ('rights', 'en'), ('meetings', 'en'), ('skills', 'en'), ('pale', 'en'), ('dollars', 'en'), ('tiny', 'en'), ('machinery', 'en'), ('travel', 'en'), ('legal', 'en'), ('prefer', 'en'), ('walking', 'en'), ('altogether', 'en'), ('attended', 'en'), ('bank', 'en'), ('decade', 'en'), ('level', 'en'), ('covered', 'en'), ('tour', 'en'), ('vehicle', 'en'), ('train', 'en'), ('automobile', 'en'), ('friend', 'en'), ('do', 'en'), ('invariably', 'en'), ('garden', 'en'), ('shoulder', 'en'), ('perfectly', 'en'), ('smaller', 'en'), ('pioneer', 'en'), ('says', 'en'), ('asked', 'en'), ('spite', 'en'), ('occurring', 'en'), ('faculty', 'en'), ('recalled', 'en'), ('knees', 'en'), ('the', 'en'), ('done', 'en'), ('newspapers', 'en'), ('troubled', 'en'), ('frequency', 'en'), ('spring', 'en'), ('struggle', 'en'), ('remark', 'en'), ('wife', 'en'), ('mar', 'en'), ('locking', 'en'), ('makers', 'en'), ('duck', 'en'), ('idea', 'en'), ('corner', 'en'), ('management', 'en'), ('cocktail', 'en'), ('vehicles', 'en'), ('complicated', 'en'), ('slightly', 'en'), ('experimental', 'en'), ('empty', 'en'), ('joy', 'en'), ('equal', 'en'), ('mother', 'en'), ('honor', 'en'), ('absorbed', 'en'), ('broad', 'en'), ('shook', 'en'), ('delivery', 'en'), ('vary', 'en'), ('guy', 'en'), ('dust', 'en'), ('shore', 'en'), ('longer', 'en'), ('sixteen', 'en'), ('surprising', 'en'), ('survival', 'en'), ('mankind', 'en'), ('indicates', 'en'), ('originally', 'en'), ('them', 'en'), ('tribute', 'en'), ('optimal', 'en'), ('rifle', 'en'), ('used', 'en'), ('expanding', 'en'), ('steps', 'en'), ('tends', 'en'), ('story', 'en'), ('datetime', 'en'), ('hits', 'en'), ('considerable', 'en'), ('nor', 'en'), ('press', 'en'), ('when', 'en'), ('scared', 'en'), ('responsible', 'en'), ('background', 'en'), ('looks', 'en'), ('ground', 'en'), ('beneath', 'en'), ('reached', 'en'), ('exists', 'en'), ('vocational', 'en'), ('historians', 'en'), ('argue', 'en'), ('coast', 'en'), ('beef', 'en'), ('wage', 'en'), ('remote', 'en'), ('challenge', 'en'), ('proposal', 'en'), ('pond', 'en'), ('finance', 'en'), ('furnished', 'en'), ('efforts', 'en'), ('drying', 'en'), ('enforcement', 'en'), ('fashion', 'en'), ('believe', 'en'), ('quickly', 'en'), ('melting', 'en'), ('otherwise', 'en'), ('desire', 'en'), ('reveal', 'en'), ('festival', 'en'), ('noble', 'en'), ('moving', 'en'), ('hence', 'en'), ('conduct', 'en'), ('promises', 'en'), ('accept', 'en'), ('fruit', 'en'), ('rushed', 'en'), ('host', 'en'), ('served', 'en'), ('anywhere', 'en'), ('yes', 'en'), ('solid', 'en'), ('debate', 'en'), ('mode', 'en'), ('secure', 'en'), ('continually', 'en'), ('shall', 'en'), ('further', 'en'), ('happily', 'en'), ('mounted', 'en'), ('sign', 'en'), ('finds', 'en'), ('base', 'en'), ('colony', 'en'), ('relatives', 'en'), ('lang', 'en'), ('change', 'en'), ('revolution', 'en'), ('laughed', 'en'), ('ought', 'en'), ('november', 'en'), ('poem', 'en'), ('developing', 'en'), ('newspaper', 'en'), ('summary', 'en'), ('thirty', 'en'), ('influence', 'en'), ('called', 'en'), ('realize', 'en'), ('responsibilities', 'en'), ('causes', 'en'), ('associate', 'en'), ('consisted', 'en'), ('sold', 'en'), ('insist', 'en'), ('step', 'en'), ('stress', 'en'), ('brushed', 'en'), ('charged', 'en'), ('communications', 'en'), ('considerations', 'en'), ('wing', 'en'), ('administration', 'en'), ('tradition', 'en'), ('objects', 'en'), ('promising', 'en'), ('unknown', 'en'), ('familiar', 'en'), ('machine', 'en'), ('musical', 'en'), ('names', 'en'), ('roughly', 'en'), ('adjusted', 'en'), ('centuries', 'en'), ('crowded', 'en'), ('capacity', 'en'), ('point', 'en'), ('hand', 'en'), ('assumed', 'en'), ('minority', 'en'), ('performance', 'en'), ('gained', 'en'), ('strategic', 'en'), ('which', 'en'), ('loop', 'en'), ('realization', 'en'), ('flow', 'en'), ('trap', 'en'), ('egg', 'en'), ('wagon', 'en'), ('functions', 'en'), ('specified', 'en'), ('pull', 'en'), ('shaking', 'en'), ('small', 'en'), ('divide', 'en'), ('found', 'en'), ('briefly', 'en'), ('conviction', 'en'), ('limits', 'en'), ('color', 'en'), ('struggling', 'en'), ('association', 'en'), ('lungs', 'en'), ('locked', 'en'), ('visible', 'en'), ('instead', 'en'), ('click', 'en'), ('sort', 'en'), ('sauce', 'en'), ('game', 'en'), ('drank', 'en'), ('exposed', 'en'), ('reading', 'en'), ('staring', 'en'), ('accomplish', 'en'), ('bigger', 'en'), ('former', 'en'), ('vice', 'en'), ('repeated', 'en'), ('studio', 'en'), ('expectations', 'en'), ('ask', 'en'), ('dozen', 'en'), ('training', 'en'), ('plant', 'en'), ('low', 'en'), ('nearly', 'en'), ('classic', 'en'), ('pure', 'en'), ('extension', 'en'), ('fifty', 'en'), ('contest', 'en'), ('across', 'en'), ('through', 'en'), ('while', 'en'), ('poetic', 'en'), ('tons', 'en'), ('theoretical', 'en'), ('inevitable', 'en'), ('herd', 'en'), ('refrigerator', 'en'), ('railroad', 'en'), ('funds', 'en'), ('held', 'en'), ('tangible', 'en'), ('mold', 'en'), ('gyro', 'en'), ('divorce', 'en'), ('clear', 'en'), ('oxidation', 'en'), ('torn', 'en'), ('take', 'en'), ('regulations', 'en'), ('developed', 'en'), ('seeking', 'en'), ('cigarette', 'en'), ('minute', 'en'), ('purposes', 'en'), ('impossible', 'en'), ('occasions', 'en'), ('claimed', 'en'), ('bullet', 'en'), ('linear', 'en'), ('books', 'en'), ('forum', 'en'), ('volunteers', 'en'), ('bgcolor', 'en'), ('formal', 'en'), ('transfer', 'en'), ('school', 'en'), ('maintained', 'en'), ('finally', 'en'), ('suffer', 'en'), ('try', 'en'), ('officer', 'en'), ('stems', 'en'), ('lose', 'en'), ('spoken', 'en'), ('circumstances', 'en'), ('amendment', 'en'), ('wings', 'en'), ('fell', 'en'), ('pause', 'en'), ('turns', 'en'), ('surrounded', 'en'), ('capabilities', 'en'), ('read', 'en'), ('stake', 'en'), ('stepped', 'en'), ('once', 'en'), ('unity', 'en'), ('killed', 'en'), ('respects', 'en'), ('enemy', 'en'), ('courage', 'en'), ('allies', 'en'), ('paint', 'en'), ('wore', 'en'), ('sunday', 'en'), ('window', 'en'), ('strategy', 'en'), ('films', 'en'), ('purpose', 'en'), ('owners', 'en'), ('scattered', 'en'), ('inherent', 'en'), ('prospect', 'en'), ('instances', 'en'), ('stroke', 'en'), ('middle-class', 'en'), ('successful', 'en'), ('present', 'en'), ('apr', 'en'), ('republic', 'en'), ('minimal', 'en'), ('fog', 'en'), ('these', 'en'), ('floor', 'en'), ('intervention', 'en'), ('heavy', 'en'), ('see', 'en'), ('headed', 'en'), ('statement', 'en'), ('didn', 'en'), ('peace', 'en'), ('preserve', 'en'), ('pose', 'en'), ('night', 'en'), ('wherever', 'en'), ('drove', 'en'), ('joined', 'en'), ('hatred', 'en'), ('voting', 'en'), ('stem', 'en'), ('weather', 'en'), ('exhibit', 'en'), ('draw', 'en'), ('version', 'en'), ('either', 'en'), ('common', 'en'), ('health', 'en'), ('throughout', 'en'), ('paragraph', 'en'), ('driving', 'en'), ('review', 'en'), ('chord', 'en'), ('supposed', 'en'), ('tried', 'en'), ('tag', 'en'), ('tells', 'en'), ('governor', 'en'), ('walked', 'en'), ('correct', 'en'), ('enjoy', 'en'), ('gradually', 'en'), ('conductor', 'en'), ('rehabilitation', 'en'), ('substances', 'en'), ('interpreted', 'en'), ('sang', 'en'), ('artery', 'en'), ('general', 'en'), ('maximum', 'en'), ('noticed', 'en'), ('century', 'en'), ('tent', 'en'), ('merger', 'en'), ('agreed', 'en'), ('phrase', 'en'), ('flexible', 'en'), ('quiet', 'en'), ('five', 'en'), ('task', 'en'), ('bitter', 'en'), ('confusion', 'en'), ('auto', 'en'), ('traders', 'en'), ('clock', 'en'), ('ultimate', 'en'), ('closer', 'en'), ('concerns', 'en'), ('quoted', 'en'), ('club', 'en'), ('gently', 'en'), ('prepared', 'en'), ('adult', 'en'), ('list', 'en'), ('out', 'en'), ('accident', 'en'), ('promptly', 'en'), ('compete', 'en'), ('tell', 'en'), ('glance', 'en'), ('marshall', 'en'), ('seem', 'en'), ('leave', 'en'), ('core', 'en'), ('massive', 'en'), ('stage', 'en'), ('supper', 'en'), ('section', 'en'), ('desk', 'en'), ('size', 'en'), ('appointed', 'en'), ('estimates', 'en'), ('extremely', 'en'), ('editor', 'en'), ('dances', 'en'), ('categories', 'en'), ('spending', 'en'), ('side', 'en'), ('fired', 'en'), ('street', 'en'), ('shirt', 'en'), ('town', 'en'), ('font', 'en'), ('cloud', 'en'), ('membership', 'en'), ('expense', 'en'), ('tasks', 'en'), ('enough', 'en'), ('acting', 'en'), ('raise', 'en'), ('social', 'en'), ('affair', 'en'), ('helped', 'en'), ('designs', 'en'), ('balance', 'en'), ('modern', 'en'), ('completely', 'en'), ('electrical', 'en'), ('songs', 'en'), ('homes', 'en'), ('lines', 'en'), ('safety', 'en'), ('short', 'en'), ('gold', 'en'), ('been', 'en'), ('requires', 'en'), ('unlike', 'en'), ('troubles', 'en'), ('looking', 'en'), ('controversy', 'en'), ('genuine', 'en'), ('doctrine', 'en'), ('firmly', 'en'), ('paid', 'en'), ('chance', 'en'), ('eve', 'en'), ('announcement', 'en'), ('instant', 'en'), ('millions', 'en'), ('majority', 'en'), ('carry', 'en'), ('name', 'en'), ('spirit', 'en'), ('instance', 'en'), ('simply', 'en'), ('door', 'en'), ('tube', 'en'), ('learn', 'en'), ('prices', 'en'), ('guilt', 'en'), ('detergent', 'en'), ('extending', 'en'), ('attacked', 'en'), ('edition', 'en'), ('investigation', 'en'), ('mdt', 'en'), ('certainty', 'en'), ('dealing', 'en'), ('existing', 'en'), ('revolutionary', 'en'), ('appreciation', 'en'), ('require', 'en'), ('philosophy', 'en'), ('sides', 'en'), ('attorney', 'en'), ('strikes', 'en'), ('substitute', 'en'), ('glasses', 'en'), ('ethical', 'en'), ('requirement', 'en'), ('visitors', 'en'), ('ride', 'en'), ('spots', 'en'), ('head', 'en'), ('claims', 'en'), ('sister', 'en'), ('anything', 'en'), ('sin', 'en'), ('burn', 'en'), ('delight', 'en'), ('starting', 'en'), ('consists', 'en'), ('latest', 'en'), ('works', 'en'), ('noted', 'en'), ('stomach', 'en'), ('volumes', 'en'), ('cream', 'en'), ('us', 'en'), ('genius', 'en'), ('hang', 'en'), ('highest', 'en'), ('anode', 'en'), ('residence', 'en'), ('crossed', 'en'), ('granted', 'en'), ('brought', 'en'), ('peaceful', 'en'), ('shop', 'en'), ('thing', 'en'), ('rear', 'en'), ('around', 'en'), ('avoided', 'en'), ('phenomena', 'en'), ('cannot', 'en'), ('binding', 'en'), ('along', 'en'), ('not', 'en'), ('butter', 'en'), ('leaped', 'en'), ('mark', 'en'), ('years', 'en'), ('listen', 'en'), ('brown', 'en'), ('bought', 'en'), ('loss', 'en'), ('folk', 'en'), ('improvements', 'en'), ('principles', 'en'), ('effectiveness', 'en'), ('editorial', 'en'), ('marriages', 'en'), ('remain', 'en'), ('circular', 'en'), ('jacket', 'en'), ('division', 'en'), ('drop', 'en'), ('quarter', 'en'), ('football', 'en'), ('invention', 'en'), ('underground', 'en'), ('informed', 'en'), ('stupid', 'en'), ('selection', 'en'), ('severe', 'en'), ('services', 'en'), ('reduced', 'en'), ('party', 'en'), ('repair', 'en'), ('tags', 'en'), ('demands', 'en'), ('live', 'en'), ('remained', 'en'), ('counter', 'en'), ('times', 'en'), ('sergeant', 'en'), ('standard', 'en'), ('workers', 'en'), ('courses', 'en'), ('soul', 'en'), ('community', 'en'), ('pair', 'en'), ('before', 'en'), ('answers', 'en'), ('pupils', 'en'), ('success', 'en'), ('represents', 'en'), ('fought', 'en'), ('persons', 'en'), ('checked', 'en'), ('team', 'en'), ('room', 'en'), ('appearance', 'en'), ('high', 'en'), ('availability', 'en'), ('saw', 'en'), ('demonstrated', 'en'), ('destroy', 'en'), ('picture', 'en'), ('exploration', 'en'), ('first', 'en'), ('scene', 'en'), ('seriously', 'en'), ('candidate', 'en'), ('structure', 'en'), ('totally', 'en');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('cope', 'en'), ('recommend', 'en'), ('email', 'en'), ('aid', 'en'), ('modest', 'en'), ('million', 'en'), ('calm', 'en'), ('taken', 'en'), ('nearest', 'en'), ('sole', 'en'), ('vigor', 'en'), ('load', 'en'), ('what', 'en'), ('religi', 'en'), ('prompt', 'en'), ('rigid', 'en'), ('compos', 'en'), ('gray', 'en'), ('saddl', 'en'), ('wherev', 'en'), ('atmospher', 'en'), ('shown', 'en'), ('custom', 'en'), ('trust', 'en'), ('serv', 'en'), ('corn', 'en'), ('arteri', 'en'), ('previous', 'en'), ('jan', 'en'), ('occupi', 'en'), ('interview', 'en'), ('bay', 'en'), ('insight', 'en'), ('access', 'en'), ('confer', 'en'), ('board', 'en'), ('facil', 'en'), ('can', 'en'), ('wild', 'en'), ('land', 'en'), ('regiment', 'en'), ('wait', 'en'), ('favor', 'en'), ('bit', 'en'), ('merchant', 'en'), ('fluid', 'en'), ('bench', 'en'), ('arc', 'en'), ('address', 'en'), ('urban', 'en'), ('knew', 'en'), ('crop', 'en'), ('sympathet', 'en'), ('me', 'en'), ('encount', 'en'), ('by', 'en'), ('goal', 'en'), ('thank', 'en'), ('type', 'en'), ('whatev', 'en'), ('fair', 'en'), ('tragic', 'en'), ('injur', 'en'), ('project', 'en'), ('assessor', 'en'), ('cool', 'en'), ('flesh', 'en'), ('secur', 'en'), ('marbl', 'en'), ('cow', 'en'), ('check', 'en'), ('share', 'en'), ('context', 'en'), ('death', 'en'), ('farther', 'en'), ('communism', 'en'), ('complet', 'en'), ('dine', 'en'), ('note', 'en'), ('proof', 'en'), ('there', 'en'), ('indirect', 'en'), ('reliabl', 'en'), ('amend', 'en'), ('verbal', 'en'), ('eye', 'en'), ('patent', 'en'), ('crazi', 'en'), ('market', 'en'), ('temporari', 'en'), ('fight', 'en'), ('despair', 'en'), ('stretch', 'en'), ('guard', 'en'), ('rate', 'en'), ('gone', 'en'), ('matter', 'en'), ('race', 'en'), ('ran', 'en'), ('south', 'en'), ('recreat', 'en'), ('video', 'en'), ('quart', 'en'), ('crown', 'en'), ('remaind', 'en'), ('reprint', 'en'), ('lift', 'en'), ('top', 'en'), ('afternoon', 'en'), ('precis', 'en'), ('novemb', 'en'), ('sharpli', 'en'), ('guest', 'en'), ('beard', 'en'), ('front', 'en'), ('stair', 'en'), ('local', 'en'), ('certain', 'en'), ('sink', 'en'), ('revolutionari', 'en'), ('barrel', 'en'), ('occur', 'en'), ('pursuant', 'en'), ('toss', 'en'), ('del', 'en'), ('vowel', 'en'), ('employ', 'en'), ('fish', 'en'), ('powder', 'en'), ('amount', 'en'), ('ear', 'en'), ('entir', 'en'), ('reader', 'en'), ('vari', 'en'), ('accompani', 'en'), ('aesthet', 'en'), ('item', 'en'), ('hear', 'en'), ('octob', 'en'), ('truck', 'en'), ('express', 'en'), ('regist', 'en'), ('satisfi', 'en'), ('few', 'en'), ('suspicion', 'en'), ('whom', 'en'), ('government', 'en'), ('told', 'en'), ('show', 'en'), ('win', 'en'), ('drill', 'en'), ('companion', 'en'), ('york', 'en'), ('lack', 'en'), ('veloc', 'en'), ('price', 'en'), ('divorc', 'en'), ('wheel', 'en'), ('wash', 'en'), ('even', 'en'), ('finish', 'en'), ('contact', 'en'), ('part-tim', 'en'), ('famous', 'en'), ('posit', 'en'), ('breath', 'en'), ('penni', 'en'), ('medicin', 'en'), ('should', 'en'), ('experienc', 'en'), ('ourselv', 'en'), ('pretti', 'en'), ('divis', 'en'), ('plane', 'en'), ('he', 'en'), ('sail', 'en'), ('fresh', 'en'), ('condemn', 'en'), ('return', 'en'), ('tendenc', 'en'), ('gather', 'en'), ('artist', 'en'), ('patrol', 'en'), ('necessarili', 'en'), ('attitud', 'en'), ('purchas', 'en'), ('gestur', 'en'), ('seek', 'en'), ('serious', 'en'), ('urg', 'en'), ('drunk', 'en'), ('mention', 'en'), ('stone', 'en'), ('automat', 'en'), ('rose', 'en'), ('river', 'en'), ('liquid', 'en'), ('bar', 'en'), ('fine', 'en'), ('precious', 'en'), ('keep', 'en'), ('better', 'en'), ('worri', 'en'), ('explicit', 'en'), ('dancer', 'en'), ('pace', 'en'), ('valign', 'en'), ('taught', 'en'), ('jet', 'en'), ('interest', 'en'), ('chick', 'en'), ('em', 'en'), ('poor', 'en'), ('final', 'en'), ('hide', 'en'), ('and', 'en'), ('chanc', 'en'), ('dri', 'en'), ('panel', 'en'), ('spend', 'en'), ('excess', 'en'), ('men', 'en'), ('la', 'en'), ('watch', 'en'), ('think', 'en'), ('histor', 'en'), ('landscap', 'en'), ('cycl', 'en'), ('pour', 'en'), ('marshal', 'en'), ('painter', 'en'), ('danger', 'en'), ('settlement', 'en'), ('inform', 'en'), ('build', 'en'), ('commerc', 'en'), ('index', 'en'), ('emerg', 'en'), ('dealer', 'en'), ('academ', 'en'), ('sixti', 'en'), ('yourself', 'en'), ('eg', 'en'), ('off', 'en'), ('updat', 'en'), ('tradit', 'en'), ('veteran', 'en'), ('mainten', 'en'), ('triumph', 'en'), ('secondari', 'en'), ('persuad', 'en'), ('passion', 'en'), ('latter', 'en'), ('known', 'en'), ('garag', 'en'), ('era', 'en'), ('flash', 'en'), ('introduc', 'en'), ('newspap', 'en'), ('pleasur', 'en'), ('pile', 'en'), ('champion', 'en'), ('word', 'en'), ('displac', 'en'), ('achiev', 'en'), ('pm', 'en'), ('hell', 'en'), ('inevit', 'en'), ('stare', 'en'), ('reorgan', 'en'), ('disput', 'en'), ('struggl', 'en'), ('ham', 'en'), ('rent', 'en'), ('director', 'en'), ('financi', 'en'), ('norm', 'en'), ('deal', 'en'), ('nearbi', 'en'), ('person', 'en'), ('elimin', 'en'), ('shell', 'en'), ('sympathi', 'en'), ('punish', 'en'), ('silenc', 'en'), ('clariti', 'en'), ('owner', 'en'), ('kingdom', 'en'), ('jazz', 'en'), ('cri', 'en'), ('vacat', 'en'), ('magnitud', 'en'), ('onli', 'en'), ('that', 'en'), ('slide', 'en'), ('fiction', 'en'), ('milk', 'en'), ('green', 'en'), ('nest', 'en'), ('activ', 'en'), ('graduat', 'en'), ('chairman', 'en'), ('furnish', 'en'), ('line', 'en'), ('knee', 'en'), ('someth', 'en'), ('yellow', 'en'), ('emphasi', 'en'), ('historian', 'en'), ('enorm', 'en'), ('dollar', 'en'), ('theori', 'en'), ('shout', 'en'), ('fled', 'en'), ('probabl', 'en'), ('somebodi', 'en'), ('veri', 'en'), ('result', 'en'), ('vast', 'en'), ('partner', 'en'), ('logic', 'en'), ('healthi', 'en'), ('delay', 'en'), ('dec', 'en'), ('helpless', 'en'), ('web', 'en'), ('factori', 'en'), ('burden', 'en'), ('text', 'en'), ('cooper', 'en'), ('set', 'en'), ('real', 'en'), ('typic', 'en'), ('obvious', 'en'), ('candid', 'en'), ('architect', 'en'), ('wire', 'en'), ('rehabilit', 'en'), ('simpl', 'en'), ('job', 'en'), ('plus', 'en'), ('necess', 'en'), ('detail', 'en'), ('agent', 'en'), ('giant', 'en'), ('circl', 'en'), ('leader', 'en'), ('met', 'en'), ('stumbl', 'en'), ('need', 'en'), ('tooth', 'en'), ('earth', 'en'), ('white', 'en'), ('oper', 'en'), ('or', 'en'), ('potenti', 'en'), ('wonder', 'en'), ('could', 'en'), ('musician', 'en'), ('compar', 'en'), ('januari', 'en'), ('per', 'en'), ('port', 'en'), ('fewer', 'en'), ('process', 'en'), ('hr', 'en'), ('belief', 'en'), ('relat', 'en'), ('ceil', 'en'), ('ambigu', 'en'), ('research', 'en'), ('young', 'en'), ('distanc', 'en'), ('lunch', 'en'), ('center', 'en'), ('visitor', 'en'), ('voic', 'en'), ('survey', 'en'), ('bring', 'en'), ('laugh', 'en'), ('impuls', 'en'), ('sight', 'en'), ('religion', 'en'), ('gentleman', 'en'), ('vote', 'en'), ('lid', 'en'), ('perfect', 'en'), ('tale', 'en'), ('margin', 'en'), ('driven', 'en'), ('difficult', 'en'), ('sleep', 'en'), ('birth', 'en'), ('policeman', 'en'), ('mm', 'en'), ('you', 'en'), ('intens', 'en'), ('both', 'en'), ('print', 'en'), ('threw', 'en'), ('program', 'en'), ('flew', 'en'), ('reform', 'en'), ('pound', 'en'), ('power', 'en'), ('becam', 'en'), ('code', 'en'), ('diseas', 'en'), ('period', 'en'), ('although', 'en'), ('shut', 'en'), ('male', 'en'), ('claim', 'en'), ('old', 'en'), ('entri', 'en'), ('inspect', 'en'), ('earn', 'en'), ('nerv', 'en'), ('transit', 'en'), ('spectacular', 'en'), ('prior', 'en'), ('comparison', 'en'), ('becom', 'en'), ('individu', 'en'), ('girl', 'en'), ('bore', 'en'), ('all', 'en'), ('speed', 'en'), ('categori', 'en'), ('attach', 'en'), ('regular', 'en'), ('pass', 'en'), ('attain', 'en'), ('edt', 'en'), ('compet', 'en'), ('announc', 'en'), ('locat', 'en'), ('alik', 'en'), ('creation', 'en'), ('averag', 'en'), ('signific', 'en'), ('bodi', 'en'), ('youngster', 'en'), ('wet', 'en'), ('relief', 'en'), ('satisfactori', 'en'), ('would', 'en'), ('aunt', 'en'), ('no', 'en'), ('storag', 'en'), ('earli', 'en'), ('anxious', 'en'), ('red', 'en'), ('steadi', 'en'), ('sentenc', 'en'), ('valley', 'en'), ('total', 'en'), ('grip', 'en'), ('voluntari', 'en'), ('mine', 'en'), ('support', 'en'), ('object', 'en'), ('add', 'en'), ('submit', 'en'), ('invent', 'en'), ('despit', 'en'), ('factor', 'en'), ('get', 'en'), ('control', 'en'), ('huge', 'en'), ('uncl', 'en'), ('definit', 'en'), ('don', 'en'), ('plenti', 'en'), ('car', 'en'), ('lobbi', 'en'), ('unless', 'en'), ('go', 'en'), ('brilliant', 'en'), ('bureau', 'en'), ('believ', 'en'), ('terribl', 'en'), ('portion', 'en'), ('itself', 'en'), ('surfac', 'en'), ('dish', 'en'), ('industri', 'en'), ('tree', 'en'), ('content', 'en'), ('third', 'en'), ('monday', 'en'), ('profound', 'en'), ('orchestra', 'en'), ('boy', 'en'), ('senat', 'en'), ('govern', 'en'), ('stain', 'en'), ('wrote', 'en'), ('black', 'en'), ('mate', 'en'), ('behind', 'en'), ('transport', 'en'), ('wood', 'en'), ('fee', 'en'), ('hate', 'en'), ('shock', 'en'), ('budget', 'en'), ('anyway', 'en'), ('bad', 'en'), ('html', 'en'), ('genuin', 'en'), ('occasion', 'en'), ('increas', 'en'), ('initi', 'en'), ('below', 'en'), ('distribut', 'en'), ('recogn', 'en'), ('sentiment', 'en'), ('purpos', 'en'), ('afford', 'en'), ('bread', 'en'), ('bottom', 'en'), ('frozen', 'en'), ('expand', 'en'), ('approach', 'en'), ('deeper', 'en'), ('hot', 'en'), ('deni', 'en'), ('speak', 'en'), ('grade', 'en'), ('self', 'en'), ('domin', 'en'), ('produc', 'en'), ('eighteenth', 'en'), ('chosen', 'en'), ('belong', 'en'), ('drink', 'en'), ('empir', 'en'), ('decis', 'en'), ('regul', 'en'), ('consequ', 'en'), ('threat', 'en'), ('character', 'en'), ('sad', 'en'), ('depth', 'en'), ('whi', 'en'), ('mobil', 'en'), ('enter', 'en'), ('foreign', 'en'), ('rather', 'en'), ('tast', 'en'), ('stuff', 'en'), ('exclus', 'en'), ('popul', 'en'), ('district', 'en'), ('basi', 'en'), ('age', 'en'), ('faint', 'en'), ('sewag', 'en'), ('resid', 'en'), ('colleagu', 'en'), ('plain', 'en'), ('sexual', 'en'), ('cure', 'en'), ('call', 'en'), ('hous', 'en'), ('luncheon', 'en'), ('nois', 'en'), ('quotient', 'en'), ('higher', 'en'), ('blog', 'en'), ('datetim', 'en'), ('smell', 'en'), ('draft', 'en'), ('comment', 'en'), ('isol', 'en'), ('petition', 'en'), ('mental', 'en'), ('substanc', 'en'), ('capabl', 'en'), ('want', 'en'), ('plural', 'en'), ('busi', 'en'), ('cholesterol', 'en'), ('relev', 'en'), ('late', 'en'), ('steel', 'en'), ('valid', 'en'), ('key', 'en'), ('save', 'en'), ('foot', 'en'), ('prime', 'en'), ('plan', 'en'), ('rector', 'en'), ('particip', 'en'), ('contract', 'en'), ('pst', 'en'), ('simpli', 'en'), ('assumpt', 'en'), ('full-tim', 'en'), ('nobl', 'en'), ('avail', 'en'), ('open', 'en'), ('appear', 'en'), ('flexibl', 'en'), ('frequenc', 'en'), ('worker', 'en'), ('charter', 'en'), ('usual', 'en'), ('intervent', 'en'), ('howev', 'en'), ('compromis', 'en'), ('passag', 'en'), ('sourc', 'en'), ('gas', 'en'), ('area', 'en'), ('hope', 'en'), ('neutral', 'en'), ('die', 'en'), ('bath', 'en'), ('argument', 'en'), ('knife', 'en'), ('loos', 'en'), ('have', 'en'), ('littl', 'en'), ('husband', 'en'), ('cook', 'en'), ('ecumen', 'en'), ('handl', 'en'), ('sheet', 'en'), ('exist', 'en'), ('height', 'en'), ('scholar', 'en'), ('right', 'en'), ('appli', 'en'), ('today', 'en'), ('talk', 'en'), ('dog', 'en'), ('bod', 'en'), ('intim', 'en'), ('pupil', 'en'), ('post', 'en'), ('disappear', 'en'), ('approv', 'en'), ('besid', 'en'), ('biggest', 'en'), ('pot', 'en'), ('tail', 'en'), ('extent', 'en'), ('fix', 'en'), ('doubl', 'en'), ('bond', 'en'), ('card', 'en'), ('thursday', 'en'), ('sick', 'en'), ('legisl', 'en'), ('pack', 'en'), ('four', 'en'), ('tremend', 'en'), ('they', 'en'), ('counti', 'en'), ('provid', 'en'), ('adopt', 'en'), ('cotton', 'en'), ('fire', 'en'), ('resolut', 'en'), ('attend', 'en'), ('fli', 'en'), ('dawn', 'en'), ('welcom', 'en'), ('theoret', 'en'), ('preliminari', 'en'), ('meat', 'en'), ('machineri', 'en'), ('lip', 'en'), ('agenc', 'en'), ('honey', 'en'), ('absorb', 'en'), ('speci', 'en'), ('occas', 'en'), ('wast', 'en'), ('who', 'en'), ('clinic', 'en'), ('caught', 'en'), ('weak', 'en'), ('grass', 'en'), ('insur', 'en'), ('region', 'en'), ('plastic', 'en'), ('wine', 'en'), ('preced', 'en'), ('season', 'en'), ('therebi', 'en'), ('diplomat', 'en'), ('luck', 'en'), ('beach', 'en'), ('blame', 'en'), ('oh', 'en'), ('joint', 'en'), ('massiv', 'en'), ('hair', 'en'), ('feet', 'en'), ('suggest', 'en'), ('staff', 'en'), ('bag', 'en'), ('tie', 'en'), ('sea', 'en'), ('enemi', 'en'), ('worthi', 'en'), ('worn', 'en'), ('permiss', 'en'), ('diffus', 'en'), ('hill', 'en'), ('hatr', 'en'), ('snake', 'en'), ('hypothalam', 'en'), ('convinc', 'en'), ('revenu', 'en'), ('duti', 'en'), ('theolog', 'en'), ('awar', 'en'), ('dream', 'en'), ('xml', 'en'), ('capit', 'en'), ('themselv', 'en'), ('for', 'en'), ('empti', 'en'), ('pick', 'en'), ('clean', 'en'), ('main', 'en'), ('near', 'en'), ('bridg', 'en'), ('shoot', 'en'), ('twenty-f', 'en'), ('upward', 'en'), ('lesson', 'en'), ('yesterday', 'en'), ('awak', 'en'), ('basic', 'en'), ('vehicl', 'en'), ('never', 'en'), ('matur', 'en'), ('interior', 'en'), ('drive', 'en'), ('blue', 'en'), ('tear', 'en'), ('mail', 'en'), ('distinguish', 'en'), ('piec', 'en'), ('silent', 'en'), ('player', 'en'), ('lung', 'en'), ('protein', 'en'), ('action', 'en'), ('multipli', 'en'), ('memori', 'en'), ('shine', 'en'), ('meal', 'en'), ('date', 'en'), ('deepli', 'en'), ('moreov', 'en'), ('buy', 'en'), ('autumn', 'en'), ('fat', 'en'), ('approxim', 'en'), ('technic', 'en'), ('battl', 'en'), ('life', 'en'), ('extend', 'en'), ('bare', 'en'), ('telephon', 'en'), ('subject', 'en'), ('absolut', 'en'), ('wit', 'en'), ('basement', 'en'), ('moment', 'en'), ('odd', 'en'), ('arriv', 'en'), ('explan', 'en'), ('tabl', 'en'), ('system', 'en'), ('church', 'en'), ('mirror', 'en'), ('month', 'en'), ('lb.', 'en'), ('long-term', 'en'), ('author', 'en'), ('fraction', 'en'), ('uncertain', 'en'), ('dir', 'en'), ('magnific', 'en'), ('she', 'en'), ('widespread', 'en'), ('juli', 'en'), ('act', 'en'), ('throw', 'en'), ('valuabl', 'en'), ('exact', 'en'), ('virtual', 'en'), ('cheek', 'en'), ('fiber', 'en'), ('advantag', 'en'), ('angl', 'en'), ('defin', 'en'), ('way', 'en'), ('a', 'en'), ('neighbor', 'en'), ('model', 'en'), ('shelter', 'en'), ('realli', 'en'), ('environ', 'en'), ('salari', 'en'), ('ma', 'en'), ('wrong', 'en'), ('in', 'en'), ('everyth', 'en'), ('dispos', 'en'), ('stand', 'en'), ('occurr', 'en'), ('blockquot', 'en'), ('thrown', 'en'), ('fist', 'en'), ('dark', 'en'), ('risk', 'en'), ('care', 'en'), ('isn', 'en'), ('colonel', 'en'), ('greet', 'en'), ('safeti', 'en'), ('charoff', 'en'), ('explain', 'en'), ('sky', 'en'), ('spoke', 'en'), ('drama', 'en'), ('heat', 'en'), ('librari', 'en'), ('harmoni', 'en'), ('singl', 'en'), ('titl', 'en'), ('damn', 'en'), ('rank', 'en'), ('quick', 'en'), ('fast', 'en'), ('slow', 'en'), ('vocat', 'en'), ('soil', 'en'), ('dare', 'en'), ('succeed', 'en'), ('languag', 'en'), ('poet', 'en'), ('on', 'en'), ('map', 'en'), ('patient', 'en'), ('inner', 'en'), ('gentl', 'en'), ('pursu', 'en'), ('benefit', 'en'), ('expenditur', 'en'), ('match', 'en'), ('affect', 'en'), ('escap', 'en'), ('quantiti', 'en'), ('dull', 'en'), ('temperatur', 'en'), ('questionnair', 'en'), ('visit', 'en'), ('oversea', 'en'), ('case', 'en'), ('wildlif', 'en'), ('climb', 'en'), ('proper', 'en'), ('mouth', 'en'), ('slight', 'en'), ('variat', 'en'), ('account', 'en'), ('unhappi', 'en'), ('philosoph', 'en'), ('festiv', 'en'), ('damag', 'en'), ('ago', 'en'), ('nervous', 'en'), ('silver', 'en'), ('incid', 'en'), ('switch', 'en'), ('shoe', 'en'), ('attract', 'en'), ('slip', 'en'), ('lean', 'en'), ('blow', 'en'), ('jungl', 'en'), ('anyon', 'en'), ('futur', 'en'), ('outstand', 'en'), ('mass', 'en'), ('down', 'en'), ('about', 'en'), ('mst', 'en'), ('crack', 'en'), ('ie', 'en'), ('swept', 'en'), ('concentr', 'en'), ('incom', 'en'), ('prove', 'en'), ('those', 'en'), ('volunt', 'en'), ('hire', 'en'), ('entertain', 'en'), ('restaur', 'en'), ('away', 'en'), ('begun', 'en'), ('chart', 'en'), ('rhythm', 'en'), ('beyond', 'en'), ('threaten', 'en'), ('dead', 'en'), ('hesit', 'en'), ('say', 'en'), ('log', 'en'), ('enterpris', 'en'), ('narrat', 'en'), ('releas', 'en'), ('seldom', 'en'), ('sacr', 'en'), ('scatter', 'en'), ('score', 'en'), ('energi', 'en'), ('laboratori', 'en'), ('format', 'en'), ('certainti', 'en'), ('outdoor', 'en'), ('normal', 'en'), ('cold', 'en'), ('liberti', 'en'), ('mistak', 'en'), ('brick', 'en'), ('resum', 'en'), ('stiff', 'en'), ('trace', 'en'), ('extra', 'en'), ('own', 'en'), ('concret', 'en'), ('discharg', 'en'), ('substrat', 'en'), ('rope', 'en'), ('east', 'en'), ('scarc', 'en'), ('peac', 'en'), ('somewher', 'en'), ('util', 'en'), ('upstair', 'en'), ('electr', 'en'), ('cheap', 'en'), ('barn', 'en'), ('later', 'en'), ('poetri', 'en'), ('creativ', 'en'), ('kept', 'en'), ('biolog', 'en'), ('equival', 'en'), ('emiss', 'en'), ('dear', 'en'), ('dilemma', 'en'), ('naval', 'en'), ('station', 'en'), ('art', 'en'), ('bold', 'en'), ('wound', 'en'), ('proud', 'en'), ('length', 'en'), ('ignor', 'en'), ('mathemat', 'en'), ('suitabl', 'en'), ('committe', 'en'), ('grin', 'en'), ('metal', 'en'), ('were', 'en'), ('procedur', 'en'), ('excus', 'en'), ('law', 'en'), ('understand', 'en'), ('yield', 'en'), ('turn', 'en'), ('nake', 'en'), ('trembl', 'en'), ('specimen', 'en'), ('past', 'en'), ('iron', 'en'), ('ultim', 'en'), ('understood', 'en'), ('cost', 'en'), ('chicken', 'en'), ('refund', 'en'), ('seat', 'en'), ('uniqu', 'en'), ('aug', 'en'), ('over-al', 'en'), ('gave', 'en'), ('gear', 'en'), ('thick', 'en'), ('impact', 'en'), ('shame', 'en'), ('elsewher', 'en'), ('invit', 'en'), ('run', 'en'), ('applic', 'en'), ('smart', 'en'), ('agre', 'en'), ('triangl', 'en'), ('radiat', 'en'), ('notabl', 'en'), ('under', 'en'), ('answer', 'en'), ('contrast', 'en'), ('oil', 'en'), ('harder', 'en'), ('cattl', 'en'), ('feb', 'en'), ('presum', 'en'), ('move', 'en'), ('appoint', 'en'), ('slim', 'en'), ('pistol', 'en'), ('deck', 'en'), ('our', 'en'), ('contrari', 'en'), ('absenc', 'en'), ('lo', 'en'), ('distinct', 'en'), ('danc', 'en'), ('earlier', 'en'), ('textil', 'en'), ('farm', 'en'), ('surround', 'en'), ('femal', 'en'), ('pertin', 'en'), ('union', 'en'), ('liber', 'en'), ('brother', 'en'), ('otherwis', 'en'), ('speaker', 'en'), ('administr', 'en'), ('defeat', 'en'), ('pulmonari', 'en'), ('channel', 'en'), ('frame', 'en'), ('appreci', 'en'), ('we', 'en'), ('hasn', 'en'), ('face', 'en'), ('ideolog', 'en'), ('briefli', 'en'), ('to', 'en'), ('group', 'en'), ('unexpect', 'en'), ('motion', 'en'), ('home', 'en'), ('decent', 'en'), ('grab', 'en'), ('payment', 'en'), ('tsunami', 'en'), ('movement', 'en'), ('radio', 'en'), ('plug', 'en'), ('proceed', 'en'), ('nineteenth', 'en'), ('ocean', 'en'), ('an', 'en'), ('binomi', 'en'), ('bed', 'en'), ('bottl', 'en'), ('neck', 'en'), ('sidewalk', 'en'), ('attempt', 'en'), ('laughter', 'en'), ('lawyer', 'en'), ('coloni', 'en'), ('soon', 'en'), ('rock', 'en'), ('built', 'en'), ('permit', 'en'), ('wipe', 'en'), ('legend', 'en'), ('grand', 'en'), ('origin', 'en'), ('romant', 'en'), ('jump', 'en'), ('end', 'en'), ('treatment', 'en'), ('splendid', 'en'), ('strain', 'en'), ('excel', 'en'), ('miss', 'en'), ('pride', 'en'), ('thorough', 'en'), ('nation', 'en'), ('reach', 'en'), ('disk', 'en'), ('are', 'en'), ('abroad', 'en'), ('unlik', 'en'), ('beauti', 'en'), ('long', 'en'), ('pilot', 'en'), ('great', 'en'), ('hunt', 'en'), ('particular', 'en'), ('credit', 'en'), ('without', 'en'), ('than', 'en'), ('mayb', 'en'), ('hen', 'en'), ('just', 'en'), ('long-rang', 'en'), ('vs', 'en'), ('phenomenon', 'en'), ('automobil', 'en'), ('usernam', 'en'), ('earliest', 'en'), ('strike', 'en'), ('composit', 'en'), ('cafe', 'en'), ('my', 'en'), ('took', 'en'), ('improv', 'en'), ('diet', 'en'), ('central', 'en'), ('expect', 'en'), ('consum', 'en'), ('symbol', 'en'), ('fit', 'en'), ('technolog', 'en'), ('provis', 'en'), ('letter', 'en'), ('settl', 'en'), ('happi', 'en'), ('follow', 'en'), ('effort', 'en'), ('rest', 'en'), ('tournament', 'en'), ('rapid', 'en'), ('endless', 'en'), ('woman', 'en'), ('impress', 'en'), ('zero', 'en'), ('commission', 'en'), ('altogeth', 'en'), ('said', 'en'), ('neat', 'en'), ('sovereign', 'en'), ('swing', 'en'), ('conspiraci', 'en'), ('saturday', 'en'), ('char', 'en'), ('chose', 'en'), ('driver', 'en'), ('lead', 'en'), ('chair', 'en'), ('narrow', 'en'), ('depart', 'en'), ('yell', 'en'), ('plate', 'en'), ('might', 'en'), ('record', 'en'), ('fifti', 'en'), ('order', 'en'), ('sing', 'en'), ('prize', 'en'), ('qualifi', 'en'), ('parent', 'en'), ('destini', 'en'), ('figur', 'en'), ('friday', 'en'), ('tend', 'en'), ('data', 'en'), ('fed', 'en'), ('year-old', 'en'), ('feel', 'en'), ('labour', 'en'), ('fan', 'en'), ('cousin', 'en'), ('treat', 'en'), ('livestock', 'en'), ('whose', 'en'), ('peopl', 'en'), ('council', 'en'), ('intend', 'en'), ('abil', 'en'), ('after', 'en'), ('injuri', 'en'), ('marri', 'en'), ('depress', 'en'), ('summari', 'en'), ('mile', 'en'), ('heart', 'en'), ('strongest', 'en'), ('observ', 'en'), ('back', 'en'), ('water', 'en'), ('border', 'en'), ('estim', 'en'), ('as', 'en'), ('qualiti', 'en'), ('declar', 'en'), ('respond', 'en'), ('readi', 'en'), ('rout', 'en'), ('shear', 'en'), ('mechan', 'en'), ('democraci', 'en'), ('dictionari', 'en'), ('chest', 'en'), ('procur', 'en'), ('registr', 'en'), ('anniversari', 'en'), ('second', 'en'), ('basebal', 'en'), ('scheme', 'en'), ('remot', 'en'), ('justifi', 'en'), ('bound', 'en'), ('reduc', 'en'), ('skill', 'en'), ('sheep', 'en'), ('came', 'en'), ('overcom', 'en'), ('span', 'en'), ('repeat', 'en'), ('nov', 'en'), ('wave', 'en'), ('heaven', 'en'), ('leg', 'en'), ('sphere', 'en'), ('protect', 'en'), ('effici', 'en'), ('amaz', 'en'), ('deliveri', 'en'), ('presenc', 'en'), ('kind', 'en'), ('herself', 'en'), ('fall', 'en'), ('courag', 'en'), ('audienc', 'en'), ('citi', 'en'), ('smoke', 'en'), ('outlook', 'en'), ('inadequ', 'en'), ('sure', 'en'), ('work', 'en'), ('instrument', 'en'), ('grown', 'en'), ('comfort', 'en'), ('generous', 'en'), ('entitl', 'en'), ('divid', 'en'), ('identifi', 'en'), ('downtown', 'en'), ('onto', 'en'), ('lieuten', 'en'), ('lock', 'en'), ('capac', 'en'), ('document', 'en'), ('sell', 'en'), ('maid', 'en'), ('whole', 'en'), ('construct', 'en'), ('suffici', 'en'), ('went', 'en'), ('new', 'en'), ('balanc', 'en'), ('realiz', 'en'), ('burst', 'en'), ('whether', 'en'), ('everyon', 'en'), ('vital', 'en'), ('deriv', 'en'), ('intern', 'en'), ('three', 'en'), ('februari', 'en'), ('possess', 'en'), ('send', 'en'), ('develop', 'en'), ('april', 'en'), ('electron', 'en'), ('entranc', 'en'), ('cite', 'en'), ('sugar', 'en'), ('civilian', 'en'), ('exchang', 'en'), ('properti', 'en'), ('sever', 'en'), ('mix', 'en'), ('was', 'en'), ('nose', 'en'), ('identif', 'en'), ('exposur', 'en'), ('primari', 'en'), ('wisdom', 'en'), ('two', 'en'), ('refus', 'en'), ('if', 'en'), ('compani', 'en'), ('him', 'en'), ('grave', 'en'), ('gentlemen', 'en'), ('wide', 'en'), ('each', 'en'), ('count', 'en'), ('finger', 'en'), ('densiti', 'en'), ('preserv', 'en'), ('wish', 'en'), ('your', 'en'), ('quot', 'en'), ('articl', 'en'), ('wherea', 'en'), ('packag', 'en'), ('role', 'en'), ('start', 'en'), ('manufactur', 'en'), ('court', 'en'), ('assess', 'en'), ('bone', 'en'), ('engag', 'en'), ('larg', 'en'), ('squar', 'en'), ('adequ', 'en'), ('combat', 'en'), ('pencil', 'en'), ('actual', 'en'), ('experi', 'en'), ('leav', 'en'), ('it', 'en'), ('progress', 'en'), ('minor', 'en'), ('differ', 'en'), ('philosophi', 'en'), ('plot', 'en'), ('digniti', 'en'), ('footbal', 'en'), ('belli', 'en'), ('easi', 'en'), ('slowli', 'en'), ('holder', 'en'), ('arous', 'en'), ('hors', 'en'), ('primit', 'en'), ('special', 'en'), ('wouldn', 'en'), ('weight', 'en'), ('nod', 'en'), ('rais', 'en'), ('stock', 'en'), ('full', 'en'), ('manag', 'en'), ('tough', 'en'), ('appar', 'en'), ('father', 'en'), ('perspect', 'en'), ('convict', 'en'), ('smile', 'en'), ('half', 'en'), ('daughter', 'en'), ('fifth', 'en'), ('money', 'en'), ('descript', 'en'), ('close', 'en'), ('limit', 'en'), ('boat', 'en'), ('gin', 'en'), ('pleasant', 'en'), ('plaster', 'en'), ('doubt', 'en'), ('drawn', 'en'), ('physic', 'en'), ('anim', 'en'), ('panic', 'en'), ('happili', 'en'), ('hidden', 'en'), ('block', 'en'), ('reject', 'en'), ('eight', 'en'), ('negoti', 'en'), ('interpret', 'en'), ('forward', 'en'), ('grow', 'en'), ('bird', 'en'), ('innoc', 'en'), ('appl', 'en'), ('financ', 'en'), ('session', 'en'), ('beat', 'en'), ('characterist', 'en'), ('sum', 'en'), ('realism', 'en'), ('cap', 'en'), ('curt', 'en'), ('output', 'en'), ('repli', 'en'), ('one', 'en'), ('wednesday', 'en'), ('wive', 'en'), ('prestig', 'en'), ('trend', 'en'), ('minimum', 'en'), ('intent', 'en'), ('centuri', 'en'), ('mission', 'en'), ('relationship', 'en'), ('did', 'en'), ('edit', 'en'), ('brief', 'en'), ('find', 'en'), ('ten', 'en'), ('wholli', 'en'), ('involv', 'en'), ('eager', 'en'), ('sake', 'en'), ('from', 'en'), ('method', 'en'), ('determin', 'en'), ('sought', 'en'), ('her', 'en'), ('movi', 'en'), ('kid', 'en'), ('vers', 'en'), ('aliv', 'en'), ('site', 'en'), ('more', 'en'), ('year', 'en'), ('congression', 'en'), ('cross', 'en'), ('island', 'en'), ('speech', 'en'), ('net', 'en'), ('ugli', 'en'), ('curious', 'en'), ('let', 'en'), ('sweet', 'en'), ('this', 'en'), ('carbon', 'en'), ('challeng', 'en'), ('shadow', 'en'), ('song', 'en'), ('molecul', 'en'), ('togeth', 'en'), ('among', 'en'), ('cst', 'en'), ('report', 'en'), ('be', 'en'), ('reserv', 'en'), ('illustr', 'en'), ('class', 'en'), ('utopian', 'en'), ('commod', 'en'), ('weekend', 'en'), ('most', 'en'), ('soap', 'en'), ('promin', 'en'), ('sensit', 'en'), ('fantast', 'en'), ('roof', 'en'), ('remind', 'en'), ('look', 'en'), ('worst', 'en'), ('integr', 'en'), ('had', 'en'), ('desert', 'en'), ('golf', 'en'), ('stop', 'en'), ('daili', 'en'), ('rang', 'en'), ('advic', 'en'), ('sep', 'en'), ('slender', 'en'), ('shouldn', 'en'), ('perman', 'en'), ('leadership', 'en'), ('aris', 'en'), ('conscious', 'en'), ('storm', 'en'), ('hat', 'en'), ('stead', 'en'), ('thousand', 'en'), ('breakfast', 'en'), ('henc', 'en'), ('eighth', 'en'), ('desegreg', 'en'), ('light', 'en'), ('tonight', 'en'), ('perhap', 'en'), ('captain', 'en'), ('encourag', 'en'), ('again', 'en'), ('happen', 'en'), ('seri', 'en'), ('wall', 'en'), ('of', 'en'), ('heel', 'en'), ('rule', 'en'), ('expans', 'en'), ('still', 'en'), ('nowher', 'en'), ('wind', 'en'), ('cover', 'en'), ('reliev', 'en'), ('winter', 'en'), ('will', 'en'), ('phase', 'en'), ('platform', 'en'), ('much', 'en'), ('servant', 'en'), ('neither', 'en'), ('command', 'en'), ('skin', 'en'), ('institut', 'en'), ('part', 'en'), ('envelop', 'en'), ('firm', 'en'), ('magnet', 'en'), ('where', 'en'), ('flight', 'en'), ('glanc', 'en'), ('doctor', 'en'), ('arrest', 'en'), ('rain', 'en'), ('left', 'en'), ('experiment', 'en'), ('network', 'en'), ('surplus', 'en'), ('hydrogen', 'en'), ('seen', 'en'), ('furthermor', 'en'), ('drug', 'en'), ('refer', 'en'), ('visual', 'en'), ('upon', 'en'), ('controversi', 'en'), ('militari', 'en'), ('furnitur', 'en'), ('ill', 'en'), ('que', 'en'), ('sovereignti', 'en'), ('inch', 'en'), ('status', 'en'), ('pleas', 'en'), ('alway', 'en'), ('nice', 'en'), ('problem', 'en'), ('blind', 'en'), ('sitter', 'en'), ('expos', 'en'), ('warrant', 'en'), ('avoid', 'en'), ('tight', 'en'), ('evid', 'en'), ('sweat', 'en'), ('popular', 'en'), ('percent', 'en'), ('heavili', 'en'), ('warm', 'en'), ('blond', 'en'), ('strang', 'en'), ('park', 'en'), ('theater', 'en'), ('ft.', 'en'), ('strip', 'en'), ('satisfact', 'en'), ('traffic', 'en'), ('align', 'en'), ('moral', 'en'), ('strong', 'en'), ('knowledg', 'en'), ('disturb', 'en'), ('calendar', 'en'), ('desper', 'en'), ('dress', 'en'), ('meet', 'en'), ('into', 'en'), ('seven', 'en'), ('gorton', 'en'), ('fascin', 'en'), ('good', 'en'), ('charg', 'en'), ('demand', 'en'), ('ratio', 'en'), ('myth', 'en'), ('master', 'en'), ('blanket', 'en'), ('west', 'en'), ('emphas', 'en'), ('cultur', 'en'), ('concern', 'en'), ('farmer', 'en'), ('suspend', 'en'), ('then', 'en'), ('screen', 'en'), ('fifteen', 'en'), ('dramat', 'en'), ('spiritu', 'en'), ('pipe', 'en'), ('segment', 'en'), ('roll', 'en'), ('state', 'en'), ('bear', 'en'), ('atom', 'en'), ('pitch', 'en'), ('sand', 'en'), ('fallout', 'en'), ('creas', 'en'), ('swim', 'en'), ('rub', 'en'), ('rode', 'en'), ('ring', 'en'), ('steam', 'en'), ('offici', 'en'), ('bell', 'en'), ('comput', 'en'), ('chemic', 'en'), ('heard', 'en'), ('suit', 'en'), ('flat', 'en'), ('direct', 'en'), ('ladder', 'en'), ('workshop', 'en'), ('conscienc', 'en'), ('bomb', 'en'), ('joke', 'en'), ('schedul', 'en'), ('senior', 'en'), ('mountain', 'en'), ('billion', 'en'), ('behavior', 'en'), ('thyroid', 'en'), ('blood', 'en'), ('conclud', 'en'), ('sat', 'en'), ('civil', 'en'), ('come', 'en'), ('file', 'en'), ('bride', 'en'), ('necessari', 'en'), ('least', 'en'), ('thermal', 'en'), ('less', 'en'), ('formula', 'en'), ('commerci', 'en'), ('generat', 'en'), ('anod', 'en'), ('etc.', 'en'), ('deep', 'en'), ('due', 'en'), ('horizon', 'en'), ('parti', 'en'), ('substanti', 'en'), ('sun', 'en'), ('fenc', 'en'), ('household', 'en'), ('stockhold', 'en'), ('frequent', 'en'), ('six', 'en'), ('accus', 'en'), ('requir', 'en'), ('tall', 'en'), ('marriag', 'en'), ('anybodi', 'en'), ('paper', 'en'), ('stuck', 'en'), ('defend', 'en'), ('last', 'en'), ('notic', 'en'), ('food', 'en'), ('supervis', 'en'), ('uniform', 'en'), ('often', 'en'), ('reflect', 'en'), ('mysteri', 'en'), ('raw', 'en'), ('vision', 'en'), ('villag', 'en'), ('next', 'en'), ('bid', 'en'), ('devil', 'en'), ('almost', 'en'), ('instal', 'en'), ('investig', 'en'), ('thus', 'en'), ('grate', 'en'), ('becaus', 'en'), ('similar', 'en'), ('chapel', 'en'), ('argu', 'en'), ('cloth', 'en'), ('bright', 'en'), ('toward', 'en'), ('oral', 'en'), ('summer', 'en'), ('muscl', 'en'), ('contain', 'en'), ('push', 'en'), ('hold', 'en'), ('asleep', 'en'), ('fate', 'en'), ('percept', 'en'), ('refriger', 'en'), ('bus', 'en'), ('cast', 'en'), ('arrang', 'en'), ('felt', 'en'), ('surviv', 'en'), ('same', 'en'), ('hit', 'en'), ('natur', 'en'), ('rich', 'en'), ('substitut', 'en'), ('convert', 'en'), ('crowd', 'en'), ('make', 'en'), ('imit', 'en'), ('extrem', 'en'), ('prison', 'en'), ('leather', 'en'), ('poverti', 'en'), ('big', 'en'), ('august', 'en'), ('allow', 'en'), ('merit', 'en'), ('mg', 'en'), ('degre', 'en'), ('hour', 'en'), ('input', 'en'), ('larger', 'en'), ('tip', 'en'), ('slept', 'en'), ('scholarship', 'en'), ('tool', 'en'), ('copi', 'en'), ('excit', 'en'), ('sent', 'en'), ('messag', 'en'), ('grew', 'en'), ('conveni', 'en'), ('nude', 'en'), ('mount', 'en'), ('younger', 'en'), ('materi', 'en'), ('ident', 'en'), ('judg', 'en'), ('dedic', 'en'), ('fabric', 'en'), ('some', 'en'), ('with', 'en'), ('fault', 'en'), ('crawl', 'en'), ('estat', 'en'), ('here', 'en'), ('eat', 'en'), ('replac', 'en'), ('acquir', 'en'), ('salin', 'en'), ('function', 'en'), ('shade', 'en'), ('motor', 'en'), ('born', 'en'), ('wed', 'en'), ('somehow', 'en'), ('resolv', 'en'), ('propaganda', 'en'), ('cat', 'en'), ('passeng', 'en'), ('film', 'en'), ('concert', 'en'), ('liquor', 'en'), ('boss', 'en'), ('seiz', 'en'), ('constant', 'en'), ('spot', 'en'), ('spread', 'en'), ('ahead', 'en'), ('crew', 'en'), ('reactionari', 'en'), ('respect', 'en'), ('wake', 'en'), ('advertis', 'en'), ('partisan', 'en'), ('glass', 'en'), ('given', 'en'), ('against', 'en'), ('yet', 'en'), ('kitchen', 'en'), ('chlorin', 'en'), ('music', 'en'), ('conceiv', 'en'), ('habit', 'en'), ('missil', 'en'), ('analysi', 'en'), ('broke', 'en'), ('variabl', 'en'), ('bind', 'en'), ('ninth', 'en'), ('protest', 'en'), ('someon', 'en'), ('chain', 'en'), ('politician', 'en'), ('subtl', 'en'), ('tomorrow', 'en'), ('label', 'en'), ('surpris', 'en'), ('rifl', 'en'), ('brush', 'en'), ('focus', 'en'), ('ad', 'en'), ('republ', 'en'), ('insect', 'en'), ('begin', 'en'), ('doctrin', 'en'), ('connect', 'en'), ('link', 'en'), ('feed', 'en'), ('oxid', 'en'), ('between', 'en'), ('suspect', 'en'), ('defens', 'en'), ('profit', 'en'), ('aspect', 'en'), ('complement', 'en'), ('minim', 'en'), ('book', 'en'), ('peer', 'en'), ('truth', 'en'), ('hollywood', 'en'), ('examin', 'en'), ('stori', 'en'), ('fourteen', 'en'), ('parallel', 'en'), ('judgment', 'en'), ('put', 'en'), ('structur', 'en'), ('verb', 'en'), ('pdt', 'en'), ('peculiar', 'en'), ('slid', 'en'), ('won', 'en'), ('discuss', 'en'), ('struck', 'en'), ('fail', 'en'), ('fort', 'en'), ('bronchial', 'en'), ('expert', 'en'), ('though', 'en'), ('dinner', 'en'), ('circumst', 'en'), ('heroic', 'en'), ('mixtur', 'en'), ('compon', 'en'), ('attack', 'en'), ('specif', 'en'), ('situat', 'en'), ('career', 'en'), ('secret', 'en'), ('branch', 'en'), ('enthusiast', 'en'), ('principl', 'en'), ('axi', 'en'), ('heavi', 'en'), ('editori', 'en'), ('com', 'en'), ('behalf', 'en'), ('thought', 'en'), ('trial', 'en'), ('offer', 'en'), ('spell', 'en'), ('proport', 'en'), ('competit', 'en'), ('kill', 'en'), ('display', 'en'), ('septemb', 'en'), ('vivid', 'en'), ('user', 'en'), ('inde', 'en'), ('salt', 'en'), ('expens', 'en'), ('inher', 'en'), ('urgent', 'en'), ('ritual', 'en'), ('tuesday', 'en'), ('outcom', 'en'), ('jail', 'en'), ('cent', 'en'), ('shift', 'en'), ('brave', 'en'), ('mind', 'en'), ('women', 'en'), ('www', 'en'), ('temporarili', 'en'), ('minut', 'en'), ('doe', 'en'), ('municip', 'en'), ('beer', 'en'), ('numer', 'en'), ('am', 'en'), ('gay', 'en'), ('older', 'en'), ('mood', 'en'), ('public', 'en'), ('hungri', 'en'), ('bat', 'en'), ('award', 'en'), ('import', 'en'), ('pattern', 'en'), ('product', 'en'), ('know', 'en'), ('sigh', 'en'), ('consider', 'en'), ('racial', 'en'), ('tractor', 'en'), ('evalu', 'en'), ('day', 'en'), ('break', 'en'), ('frighten', 'en'), ('curios', 'en'), ('accord', 'en'), ('especi', 'en'), ('lower', 'en'), ('foam', 'en'), ('freight', 'en'), ('medic', 'en'), ('anti-trust', 'en'), ('must', 'en'), ('tri', 'en'), ('advis', 'en'), ('eleven', 'en'), ('design', 'en'), ('anyhow', 'en'), ('mighti', 'en'), ('instruct', 'en'), ('cours', 'en'), ('astronomi', 'en'), ('charact', 'en'), ('style', 'en'), ('path', 'en'), ('civic', 'en'), ('caus', 'en'), ('hole', 'en'), ('signal', 'en'), ('drew', 'en'), ('pocket', 'en'), ('everi', 'en'), ('routin', 'en'), ('planetari', 'en'), ('concept', 'en'), ('neighborhood', 'en'), ('et', 'en'), ('dive', 'en'), ('bent', 'en'), ('other', 'en'), ('fortun', 'en'), ('sophist', 'en'), ('suppli', 'en'), ('swung', 'en'), ('ever', 'en'), ('sale', 'en'), ('domest', 'en'), ('gun', 'en'), ('term', 'en'), ('fourth', 'en'), ('surrend', 'en'), ('except', 'en'), ('hurri', 'en'), ('radar', 'en'), ('loud', 'en'), ('quit', 'en'), ('immedi', 'en'), ('upper', 'en'), ('backward', 'en'), ('wooden', 'en'), ('uneasi', 'en'), ('monument', 'en'), ('ani', 'en'), ('equip', 'en'), ('suppos', 'en'), ('particl', 'en'), ('psycholog', 'en'), ('vein', 'en'), ('target', 'en'), ('himself', 'en'), ('view', 'en'), ('repres', 'en'), ('middl', 'en'), ('hero', 'en'), ('abov', 'en'), ('fiscal', 'en'), ('lost', 'en'), ('request', 'en'), ('tangibl', 'en'), ('glad', 'en'), ('tone', 'en'), ('ship', 'en'), ('sit', 'en'), ('calcul', 'en'), ('tribut', 'en'), ('noon', 'en'), ('touch', 'en'), ('tire', 'en'), ('written', 'en'), ('anywher', 'en'), ('everybodi', 'en'), ('forest', 'en'), ('hung', 'en'), ('complic', 'en'), ('warn', 'en'), ('manner', 'en'), ('transform', 'en'), ('retir', 'en'), ('easier', 'en'), ('asid', 'en'), ('launch', 'en'), ('organ', 'en'), ('honest', 'en'), ('actor', 'en'), ('instanc', 'en'), ('dirti', 'en'), ('within', 'en'), ('toe', 'en'), ('crash', 'en'), ('led', 'en'), ('soft', 'en'), ('deliv', 'en'), ('disast', 'en'), ('abbr', 'en'), ('jun', 'en'), ('forev', 'en'), ('obtain', 'en'), ('write', 'en'), ('somewhat', 'en'), ('stream', 'en'), ('elect', 'en'), ('startl', 'en'), ('consid', 'en'), ('myself', 'en'), ('assum', 'en'), ('spare', 'en'), ('failur', 'en'), ('lone', 'en'), ('broken', 'en'), ('partial', 'en'), ('mad', 'en'), ('sauc', 'en'), ('fill', 'en'), ('justic', 'en'), ('recal', 'en'), ('select', 'en'), ('store', 'en'), ('volum', 'en'), ('laid', 'en'), ('trip', 'en'), ('track', 'en'), ('sex', 'en'), ('bedroom', 'en'), ('warmth', 'en'), ('well', 'en'), ('wear', 'en'), ('recent', 'en'), ('news', 'en'), ('implic', 'en'), ('worth', 'en'), ('newer', 'en'), ('hundr', 'en'), ('pale', 'en'), ('occup', 'en'), ('travel', 'en'), ('lucki', 'en'), ('row', 'en'), ('legal', 'en'), ('sharp', 'en'), ('ball', 'en'), ('allianc', 'en'), ('convent', 'en'), ('prefer', 'en'), ('stabl', 'en'), ('tongu', 'en'), ('patholog', 'en'), ('gang', 'en'), ('bank', 'en'), ('level', 'en'), ('tour', 'en'), ('faith', 'en'), ('train', 'en'), ('abl', 'en'), ('friend', 'en'), ('none', 'en'), ('receiv', 'en'), ('do', 'en'), ('undoubt', 'en'), ('contin', 'en'), ('shoulder', 'en'), ('garden', 'en'), ('tangent', 'en'), ('altern', 'en'), ('smaller', 'en'), ('pool', 'en'), ('whip', 'en'), ('stood', 'en'), ('pioneer', 'en'), ('spite', 'en'), ('oct', 'en'), ('mere', 'en'), ('strategi', 'en'), ('babi', 'en'), ('the', 'en'), ('done', 'en'), ('ton', 'en'), ('deterg', 'en'), ('cellar', 'en'), ('spring', 'en'), ('craft', 'en'), ('so', 'en'), ('newli', 'en'), ('remark', 'en'), ('wife', 'en'), ('mar', 'en'), ('meant', 'en'), ('duck', 'en'), ('killer', 'en'), ('noth', 'en'), ('so-cal', 'en'), ('idea', 'en'), ('corner', 'en'), ('cocktail', 'en'), ('testimoni', 'en'), ('world', 'en'), ('flux', 'en'), ('impos', 'en'), ('greatest', 'en'), ('confirm', 'en'), ('straight', 'en'), ('skirt', 'en'), ('at', 'en'), ('fig', 'en'), ('joy', 'en'), ('licens', 'en'), ('equal', 'en'), ('enforc', 'en'), ('murder', 'en'), ('outsid', 'en'), ('mother', 'en'), ('trade', 'en'), ('honor', 'en'), ('builder', 'en'), ('broad', 'en'), ('road', 'en'), ('air', 'en'), ('ideal', 'en'), ('shook', 'en'), ('tini', 'en'), ('rough', 'en'), ('form', 'en'), ('shore', 'en'), ('dust', 'en'), ('guy', 'en'), ('longer', 'en'), ('sixteen', 'en'), ('polici', 'en'), ('give', 'en'), ('render', 'en'), ('sens', 'en'), ('propos', 'en'), ('mankind', 'en'), ('supplement', 'en'), ('alli', 'en'), ('them', 'en'), ('abandon', 'en'), ('voter', 'en'), ('smooth', 'en'), ('annual', 'en'), ('scare', 'en'), ('seed', 'en'), ('nor', 'en'), ('press', 'en'), ('associ', 'en'), ('lie', 'en'), ('extraordinari', 'en'), ('when', 'en'), ('various', 'en'), ('chief', 'en'), ('background', 'en'), ('ground', 'en'), ('man', 'en'), ('beneath', 'en'), ('eas', 'en'), ('moder', 'en'), ('coast', 'en'), ('beef', 'en'), ('wage', 'en'), ('http', 'en'), ('friendship', 'en'), ('elabor', 'en'), ('pond', 'en'), ('dynam', 'en'), ('may', 'en'), ('hall', 'en'), ('fashion', 'en'), ('opportun', 'en'), ('heritag', 'en'), ('alon', 'en'), ('coupl', 'en'), ('realist', 'en'), ('wors', 'en'), ('reveal', 'en'), ('creat', 'en'), ('forc', 'en'), ('describ', 'en'), ('crucial', 'en'), ('novel', 'en'), ('conduct', 'en'), ('accept', 'en'), ('twenti', 'en'), ('fruit', 'en'), ('edg', 'en'), ('host', 'en'), ('yes', 'en'), ('solid', 'en'), ('mode', 'en'), ('separ', 'en'), ('pressur', 'en'), ('exercis', 'en'), ('shall', 'en'), ('widow', 'en'), ('profession', 'en'), ('further', 'en'), ('sign', 'en'), ('base', 'en'), ('lang', 'en'), ('regard', 'en'), ('carri', 'en'), ('steadili', 'en'), ('casual', 'en'), ('chang', 'en'), ('ought', 'en'), ('walk', 'en'), ('comedi', 'en'), ('poem', 'en'), ('hard', 'en'), ('width', 'en'), ('knock', 'en'), ('number', 'en'), ('debat', 'en'), ('exampl', 'en'), ('nevertheless', 'en'), ('error', 'en'), ('ownership', 'en'), ('ration', 'en'), ('trader', 'en'), ('now', 'en'), ('thereaft', 'en'), ('sold', 'en'), ('insist', 'en'), ('test', 'en'), ('optim', 'en'), ('gross', 'en'), ('visibl', 'en'), ('morn', 'en'), ('skywav', 'en'), ('step', 'en'), ('mani', 'en'), ('princip', 'en'), ('resist', 'en'), ('column', 'en'), ('stress', 'en'), ('how', 'en'), ('event', 'en'), ('continu', 'en'), ('their', 'en'), ('oblig', 'en'), ('scale', 'en'), ('best', 'en'), ('confid', 'en'), ('wing', 'en'), ('up', 'en'), ('lumber', 'en'), ('oxygen', 'en'), ('onc', 'en'), ('multipl', 'en'), ('writer', 'en'), ('forgiv', 'en'), ('unknown', 'en'), ('king', 'en'), ('inventori', 'en'), ('tape', 'en'), ('decim', 'en'), ('familiar', 'en'), ('whiski', 'en'), ('tissu', 'en'), ('question', 'en'), ('condit', 'en'), ('point', 'en'), ('hand', 'en'), ('anoth', 'en'), ('goe', 'en'), ('loan', 'en'), ('which', 'en'), ('far', 'en'), ('loop', 'en'), ('recoveri', 'en'), ('pea', 'en'), ('measur', 'en'), ('suffix', 'en'), ('admit', 'en'), ('trap', 'en'), ('flow', 'en'), ('constitut', 'en'), ('addit', 'en'), ('sound', 'en'), ('egg', 'en'), ('also', 'en'), ('wagon', 'en'), ('polic', 'en'), ('trim', 'en'), ('attent', 'en'), ('pull', 'en'), ('adjust', 'en'), ('stranger', 'en'), ('but', 'en'), ('too', 'en'), ('anyth', 'en'), ('small', 'en'), ('over', 'en'), ('aboard', 'en'), ('found', 'en'), ('is', 'en'), ('fallen', 'en'), ('coat', 'en'), ('color', 'en'), ('seventh', 'en'), ('mess', 'en'), ('prevent', 'en'), ('anger', 'en'), ('golden', 'en'), ('coach', 'en'), ('instead', 'en'), ('sort', 'en'), ('click', 'en'), ('sinc', 'en'), ('drank', 'en'), ('game', 'en'), ('tactic', 'en'), ('accomplish', 'en'), ('pay', 'en'), ('bigger', 'en'), ('correspond', 'en'), ('cigarett', 'en'), ('like', 'en'), ('unconsci', 'en'), ('got', 'en'), ('coffe', 'en'), ('former', 'en'), ('collect', 'en'), ('polynomi', 'en'), ('vice', 'en'), ('shot', 'en'), ('camp', 'en'), ('assur', 'en'), ('ladi', 'en'), ('studio', 'en'), ('strateg', 'en'), ('ask', 'en'), ('negat', 'en'), ('choic', 'en'), ('tension', 'en'), ('dozen', 'en'), ('declin', 'en'), ('toast', 'en'), ('guidanc', 'en'), ('fund', 'en'), ('befor', 'en'), ('pink', 'en'), ('fear', 'en'), ('polit', 'en'), ('cut', 'en'), ('essenti', 'en'), ('plant', 'en'), ('low', 'en'), ('his', 'en'), ('admiss', 'en'), ('classic', 'en'), ('dimens', 'en'), ('pure', 'en'), ('contest', 'en'), ('rush', 'en'), ('divin', 'en'), ('across', 'en'), ('interv', 'en'), ('through', 'en'), ('deliber', 'en'), ('poetic', 'en'), ('while', 'en'), ('resourc', 'en'), ('herd', 'en'), ('stronger', 'en'), ('railroad', 'en'), ('influenc', 'en'), ('cash', 'en'), ('held', 'en'), ('syllabl', 'en'), ('rise', 'en'), ('mold', 'en'), ('torn', 'en'), ('gyro', 'en'), ('clear', 'en'), ('take', 'en'), ('communiti', 'en'), ('unfortun', 'en'), ('bother', 'en'), ('ventur', 'en'), ('sacrific', 'en'), ('critic', 'en'), ('conflict', 'en'), ('privat', 'en'), ('join', 'en'), ('lake', 'en'), ('bullet', 'en'), ('linear', 'en'), ('meaning', 'en'), ('forum', 'en'), ('truli', 'en'), ('bgcolor', 'en'), ('possibl', 'en'), ('formal', 'en'), ('transfer', 'en'), ('rural', 'en'), ('school', 'en'), ('inclin', 'en'), ('depend', 'en'), ('camera', 'en'), ('contemporari', 'en'), ('unusu', 'en'), ('suffer', 'en'), ('unit', 'en'), ('nut', 'en'), ('lose', 'en'), ('loyalti', 'en'), ('solv', 'en'), ('spoken', 'en'), ('patienc', 'en'), ('diamet', 'en'), ('accur', 'en'), ('pain', 'en'), ('made', 'en'), ('engin', 'en'), ('page', 'en'), ('browser', 'en'), ('fell', 'en'), ('aren', 'en'), ('stake', 'en'), ('read', 'en'), ('literari', 'en'), ('field', 'en'), ('delic', 'en'), ('residenti', 'en'), ('decemb', 'en'), ('paint', 'en'), ('such', 'en'), ('greater', 'en'), ('suitcas', 'en'), ('snap', 'en'), ('shape', 'en'), ('wore', 'en'), ('contribut', 'en'), ('sunday', 'en'), ('reput', 'en'), ('secretari', 'en'), ('window', 'en'), ('faculti', 'en'), ('accid', 'en'), ('tragedi', 'en'), ('countri', 'en'), ('imposs', 'en'), ('includ', 'en'), ('current', 'en'), ('evil', 'en'), ('featur', 'en'), ('prospect', 'en'), ('guess', 'en'), ('assembl', 'en'), ('swift', 'en'), ('stroke', 'en'), ('ancient', 'en'), ('middle-class', 'en'), ('headquart', 'en'), ('yard', 'en'), ('uniti', 'en'), ('invari', 'en'), ('oppos', 'en'), ('june', 'en'), ('convers', 'en'), ('present', 'en'), ('techniqu', 'en'), ('apr', 'en'), ('catch', 'en'), ('sponsor', 'en'), ('these', 'en'), ('fog', 'en'), ('forth', 'en'), ('grain', 'en'), ('floor', 'en'), ('talent', 'en'), ('quarrel', 'en'), ('fun', 'en'), ('see', 'en'), ('promis', 'en'), ('offic', 'en'), ('north', 'en'), ('statist', 'en'), ('statement', 'en'), ('didn', 'en'), ('liter', 'en'), ('pose', 'en'), ('night', 'en'), ('member', 'en'), ('clerk', 'en'), ('drove', 'en'), ('troop', 'en'), ('incred', 'en'), ('dure', 'en'), ('respons', 'en'), ('dealt', 'en'), ('stem', 'en'), ('ward', 'en'), ('time', 'en'), ('maintain', 'en'), ('round', 'en'), ('presid', 'en'), ('paus', 'en'), ('weather', 'en'), ('magazin', 'en'), ('exhibit', 'en'), ('draw', 'en'), ('afraid', 'en'), ('version', 'en'), ('either', 'en'), ('common', 'en'), ('mustard', 'en'), ('health', 'en'), ('pictur', 'en'), ('throughout', 'en'), ('paragraph', 'en'), ('coverag', 'en'), ('classif', 'en'), ('son', 'en'), ('human', 'en'), ('review', 'en'), ('promot', 'en'), ('chord', 'en'), ('twice', 'en'), ('revolut', 'en'), ('primarili', 'en'), ('wherebi', 'en'), ('vacuum', 'en'), ('tag', 'en'), ('governor', 'en'), ('correct', 'en'), ('easili', 'en'), ('enjoy', 'en'), ('varieti', 'en'), ('favorit', 'en'), ('conductor', 'en'), ('teeth', 'en'), ('sang', 'en'), ('general', 'en'), ('maximum', 'en'), ('complex', 'en'), ('tent', 'en'), ('restrict', 'en'), ('midnight', 'en'), ('merger', 'en'), ('imag', 'en'), ('dirt', 'en'), ('guid', 'en'), ('phrase', 'en'), ('cell', 'en'), ('nine', 'en'), ('interfer', 'en'), ('quiet', 'en'), ('devic', 'en'), ('five', 'en'), ('task', 'en'), ('employe', 'en'), ('utter', 'en'), ('phone', 'en'), ('bitter', 'en'), ('servic', 'en'), ('conson', 'en'), ('auto', 'en'), ('journey', 'en'), ('clock', 'en'), ('closer', 'en'), ('ranch', 'en'), ('televis', 'en'), ('remov', 'en'), ('largest', 'en'), ('apparatus', 'en'), ('has', 'en'), ('club', 'en'), ('string', 'en'), ('reaction', 'en'), ('chiefli', 'en'), ('creatur', 'en'), ('adult', 'en'), ('list', 'en'), ('out', 'en'), ('unabl', 'en'), ('reason', 'en'), ('arbitrari', 'en'), ('element', 'en'), ('practic', 'en'), ('tell', 'en'), ('seem', 'en'), ('decad', 'en'), ('melodi', 'en'), ('apart', 'en'), ('core', 'en'), ('devot', 'en'), ('stage', 'en'), ('supper', 'en'), ('warfar', 'en'), ('section', 'en'), ('specialist', 'en'), ('consist', 'en'), ('size', 'en'), ('desk', 'en'), ('curv', 'en'), ('insid', 'en'), ('editor', 'en'), ('choos', 'en'), ('invest', 'en'), ('alert', 'en'), ('complain', 'en'), ('parad', 'en'), ('communic', 'en'), ('hotel', 'en'), ('gradual', 'en'), ('demonstr', 'en'), ('publish', 'en'), ('side', 'en'), ('subtract', 'en'), ('confront', 'en'), ('assist', 'en'), ('street', 'en'), ('prepar', 'en'), ('shirt', 'en'), ('town', 'en'), ('font', 'en'), ('cottag', 'en'), ('mud', 'en'), ('cloud', 'en'), ('alter', 'en'), ('membership', 'en'), ('flower', 'en'), ('milligram', 'en'), ('enough', 'en'), ('social', 'en'), ('execut', 'en'), ('rid', 'en'), ('affair', 'en'), ('conclus', 'en'), ('throat', 'en'), ('ethic', 'en'), ('thirti', 'en'), ('modern', 'en'), ('luxuri', 'en'), ('grant', 'en'), ('glori', 'en'), ('band', 'en'), ('recept', 'en'), ('short', 'en'), ('gold', 'en'), ('use', 'en'), ('been', 'en'), ('theme', 'en'), ('retain', 'en'), ('bundl', 'en'), ('guilti', 'en'), ('week', 'en'), ('noun', 'en'), ('etern', 'en'), ('readili', 'en'), ('paid', 'en'), ('melt', 'en'), ('charm', 'en'), ('until', 'en'), ('eve', 'en'), ('univers', 'en'), ('citizen', 'en'), ('halign', 'en'), ('equat', 'en'), ('introduct', 'en'), ('instant', 'en'), ('junior', 'en'), ('maker', 'en'), ('disciplin', 'en'), ('therefor', 'en'), ('name', 'en'), ('spirit', 'en'), ('commit', 'en'), ('sampl', 'en'), ('est', 'en'), ('doesn', 'en'), ('door', 'en'), ('tube', 'en'), ('assign', 'en'), ('porch', 'en'), ('learn', 'en'), ('box', 'en'), ('troubl', 'en'), ('guilt', 'en'), ('fulli', 'en'), ('appeal', 'en'), ('enabl', 'en'), ('decid', 'en'), ('authent', 'en'), ('alien', 'en'), ('mdt', 'en'), ('profess', 'en'), ('distant', 'en'), ('aim', 'en'), ('explor', 'en'), ('bill', 'en'), ('eventu', 'en'), ('els', 'en'), ('destruct', 'en'), ('illus', 'en'), ('attorney', 'en'), ('thrust', 'en'), ('rare', 'en'), ('ride', 'en'), ('march', 'en'), ('href', 'en'), ('combin', 'en'), ('safe', 'en'), ('machin', 'en'), ('head', 'en'), ('sister', 'en'), ('star', 'en'), ('fellow', 'en'), ('sin', 'en'), ('overwhelm', 'en'), ('burn', 'en'), ('delight', 'en'), ('establish', 'en'), ('personnel', 'en'), ('sequenc', 'en'), ('ice', 'en'), ('opposit', 'en'), ('indic', 'en'), ('desir', 'en'), ('issu', 'en'), ('lot', 'en'), ('latest', 'en'), ('stomach', 'en'), ('inspir', 'en'), ('virtu', 'en'), ('accuraci', 'en'), ('fool', 'en'), ('nobodi', 'en'), ('cream', 'en'), ('growth', 'en'), ('us', 'en'), ('genius', 'en'), ('notion', 'en'), ('hang', 'en'), ('highest', 'en'), ('mean', 'en'), ('brought', 'en'), ('feder', 'en'), ('shop', 'en'), ('via', 'en'), ('began', 'en'), ('thing', 'en'), ('rear', 'en'), ('whisper', 'en'), ('confus', 'en'), ('spent', 'en'), ('around', 'en'), ('phenomena', 'en'), ('stick', 'en'), ('along', 'en'), ('cannot', 'en'), ('not', 'en'), ('angri', 'en'), ('butter', 'en'), ('mark', 'en'), ('literatur', 'en'), ('enthusiasm', 'en'), ('listen', 'en'), ('rememb', 'en'), ('bought', 'en'), ('brown', 'en'), ('loss', 'en'), ('mutual', 'en'), ('scope', 'en'), ('magic', 'en'), ('harm', 'en'), ('perform', 'en'), ('sometim', 'en'), ('suburban', 'en'), ('folk', 'en'), ('strict', 'en'), ('rail', 'en'), ('regim', 'en'), ('sorri', 'en'), ('gift', 'en'), ('shake', 'en'), ('remain', 'en'), ('onset', 'en'), ('circular', 'en'), ('extens', 'en'), ('societi', 'en'), ('jacket', 'en'), ('inc', 'en'), ('bet', 'en'), ('anticip', 'en'), ('drop', 'en'), ('quarter', 'en'), ('underground', 'en'), ('allot', 'en'), ('stupid', 'en'), ('motiv', 'en'), ('planet', 'en'), ('medium', 'en'), ('root', 'en'), ('effect', 'en'), ('repair', 'en'), ('specifi', 'en'), ('stay', 'en'), ('live', 'en'), ('counter', 'en'), ('sergeant', 'en'), ('cdt', 'en'), ('piano', 'en'), ('standard', 'en'), ('search', 'en'), ('sudden', 'en'), ('soul', 'en'), ('pair', 'en'), ('advanc', 'en'), ('jul', 'en'), ('success', 'en'), ('leap', 'en'), ('lay', 'en'), ('fought', 'en'), ('gain', 'en'), ('anti-semit', 'en'), ('anxieti', 'en'), ('abstract', 'en'), ('team', 'en'), ('major', 'en'), ('realiti', 'en'), ('alreadi', 'en'), ('imagin', 'en'), ('place', 'en'), ('room', 'en'), ('forget', 'en'), ('absent', 'en'), ('high', 'en'), ('saw', 'en'), ('reduct', 'en'), ('help', 'en'), ('destroy', 'en'), ('first', 'en'), ('thin', 'en'), ('scene', 'en'), ('play', 'en'), ('snow', 'en'), ('histori', 'en'), ('percentag', 'en'), ('difficulti', 'en');INSERT INTO stopwords_long (stopword, language) VALUES ('estuvierais', 'es'), ('fueras', 'es'), ('fuiste', 'es'), ('para', 'es'), ('otro', 'es'), ('siendo', 'es'), ('estuviesen', 'es'), ('e', 'es'), ('tuvieron', 'es'), ('tendrás', 'es'), ('hubisteis', 'es'), ('tuvo', 'es'), ('eran', 'es'), ('eres', 'es'), ('está', 'es'), ('estaba', 'es'), ('ti', 'es'), ('seáis', 'es'), ('estáis', 'es'), ('tuvieran', 'es'), ('estuviste', 'es'), ('mi', 'es'), ('tiene', 'es'), ('por', 'es'), ('tuya', 'es'), ('cual', 'es'), ('habrá', 'es'), ('otros', 'es'), ('algunas', 'es'), ('contra', 'es'), ('tenías', 'es'), ('sido', 'es'), ('se', 'es'), ('mío', 'es'), ('estábamos', 'es'), ('les', 'es'), ('son', 'es'), ('tuvieseis', 'es'), ('a', 'es'), ('no', 'es'), ('ese', 'es'), ('hubiste', 'es'), ('tuyos', 'es'), ('tengo', 'es'), ('una', 'es'), ('más', 'es'), ('mías', 'es'), ('estaréis', 'es'), ('estuviese', 'es'), ('me', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('hayan', 'es'), ('estaríais', 'es'), ('que', 'es'), ('estabas', 'es'), ('hubimos', 'es'), ('nuestra', 'es'), ('seríamos', 'es'), ('eso', 'es'), ('tuve', 'es'), ('estas', 'es'), ('sobre', 'es'), ('habido', 'es'), ('fuimos', 'es'), ('vuestra', 'es'), ('también', 'es'), ('unos', 'es'), ('hemos', 'es'), ('esos', 'es'), ('vuestros', 'es'), ('estuviéramos', 'es'), ('sería', 'es'), ('o', 'es'), ('tenían', 'es'), ('has', 'es'), ('tenemos', 'es'), ('estuvieron', 'es'), ('estemos', 'es'), ('hay', 'es'), ('sí', 'es'), ('fuera', 'es'), ('tuvisteis', 'es'), ('estuvieseis', 'es'), ('habrían', 'es'), ('al', 'es'), ('fuéramos', 'es'), ('habidas', 'es'), ('teníais', 'es'), ('fuese', 'es'), ('habíamos', 'es'), ('estarán', 'es'), ('yo', 'es'), ('qué', 'es'), ('vosotras', 'es'), ('habrás', 'es'), ('hayamos', 'es'), ('estuviera', 'es'), ('suyas', 'es'), ('del', 'es'), ('habías', 'es'), ('estarás', 'es'), ('hayáis', 'es'), ('estará', 'es'), ('somos', 'es'), ('otra', 'es'), ('tendrías', 'es'), ('habían', 'es'), ('estuvieran', 'es'), ('es', 'es'), ('fuésemos', 'es'), ('habríamos', 'es'), ('otras', 'es'), ('estuvisteis', 'es'), ('vuestro', 'es'), ('esas', 'es'), ('tuviéramos', 'es'), ('hasta', 'es'), ('estad', 'es'), ('estaban', 'es'), ('mí', 'es'), ('míos', 'es'), ('tuviste', 'es'), ('estado', 'es'), ('en', 'es'), ('de', 'es'), ('un', 'es'), ('serían', 'es'), ('fuerais', 'es'), ('estadas', 'es'), ('teniendo', 'es'), ('estando', 'es'), ('como', 'es'), ('habrán', 'es'), ('estuviésemos', 'es'), ('fui', 'es'), ('seréis', 'es'), ('he', 'es'), ('estaríamos', 'es'), ('quienes', 'es'), ('esto', 'es'), ('los', 'es'), ('habida', 'es'), ('tuviésemos', 'es'), ('estar', 'es'), ('estuve', 'es'), ('sus', 'es'), ('tendrían', 'es'), ('tienen', 'es'), ('nuestros', 'es'), ('le', 'es'), ('estoy', 'es'), ('y', 'es'), ('nos', 'es'), ('estás', 'es'), ('hubieses', 'es'), ('sean', 'es'), ('tengáis', 'es'), ('estuvimos', 'es'), ('hayas', 'es'), ('nuestras', 'es'), ('pero', 'es'), ('porque', 'es'), ('fuesen', 'es'), ('tened', 'es'), ('esté', 'es'), ('seremos', 'es'), ('tuvieses', 'es'), ('mis', 'es'), ('sois', 'es'), ('habidos', 'es'), ('seré', 'es'), ('suyos', 'es'), ('hubieseis', 'es'), ('tendrán', 'es'), ('uno', 'es'), ('la', 'es'), ('serán', 'es'), ('estaremos', 'es'), ('con', 'es'), ('nosotras', 'es'), ('tenidas', 'es'), ('tuvierais', 'es'), ('todo', 'es'), ('estaría', 'es'), ('nosotros', 'es'), ('ellos', 'es'), ('ellas', 'es'), ('tuviesen', 'es'), ('han', 'es'), ('fue', 'es'), ('tendríais', 'es'), ('habréis', 'es'), ('tuviese', 'es'), ('cuando', 'es'), ('hubieran', 'es'), ('tenidos', 'es'), ('ya', 'es'), ('fueran', 'es'), ('teníamos', 'es'), ('os', 'es'), ('tenía', 'es'), ('tenéis', 'es'), ('durante', 'es'), ('tu', 'es'), ('habríais', 'es'), ('tenida', 'es'), ('era', 'es'), ('hubierais', 'es'), ('suyo', 'es'), ('fueron', 'es'), ('tuvieras', 'es'), ('tendremos', 'es'), ('estés', 'es'), ('estuvo', 'es'), ('sin', 'es'), ('tenido', 'es'), ('tuvimos', 'es'), ('tengas', 'es'), ('algunos', 'es'), ('este', 'es'), ('habrías', 'es'), ('habría', 'es'), ('ha', 'es'), ('habéis', 'es'), ('las', 'es'), ('había', 'es'), ('tienes', 'es'), ('estén', 'es'), ('nada', 'es'), ('habíais', 'es'), ('tú', 'es'), ('tuviera', 'es'), ('éramos', 'es'), ('tuyas', 'es'), ('hubieron', 'es'), ('tendría', 'es'), ('estuvieras', 'es'), ('antes', 'es'), ('su', 'es'), ('estabais', 'es'), ('suya', 'es'), ('mucho', 'es'), ('hubiésemos', 'es'), ('quien', 'es'), ('habré', 'es'), ('lo', 'es'), ('hubo', 'es'), ('soy', 'es'), ('todos', 'es'), ('tus', 'es'), ('estos', 'es'), ('vuestras', 'es'), ('hubiesen', 'es'), ('será', 'es'), ('tengamos', 'es'), ('poco', 'es'), ('tengan', 'es'), ('tenga', 'es'), ('estaré', 'es'), ('ni', 'es'), ('desde', 'es'), ('ella', 'es'), ('estada', 'es'), ('estarían', 'es'), ('haya', 'es'), ('sea', 'es'), ('tuyo', 'es'), ('entre', 'es'), ('tendréis', 'es'), ('vosotros', 'es'), ('muchos', 'es'), ('estéis', 'es'), ('eras', 'es'), ('donde', 'es'), ('tendríamos', 'es'), ('fueses', 'es'), ('estados', 'es'), ('algo', 'es'), ('hubiese', 'es'), ('seas', 'es'), ('estarías', 'es'), ('serías', 'es'), ('muy', 'es'), ('hubiera', 'es'), ('hubiéramos', 'es'), ('esta', 'es'), ('estuvieses', 'es'), ('tendrá', 'es'), ('erais', 'es'), ('mía', 'es'), ('estamos', 'es'), ('hube', 'es'), ('hubieras', 'es'), ('habiendo', 'es'), ('habremos', 'es'), ('seamos', 'es'), ('seríais', 'es'), ('él', 'es'), ('están', 'es'), ('ante', 'es'), ('esa', 'es'), ('te', 'es'), ('tendré', 'es'), ('serás', 'es'), ('nuestro', 'es'), ('el', 'es'), ('tanto', 'es');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('sus', 'es'), ('sid', 'es'), ('som', 'es'), ('esteis', 'es'), ('le', 'es'), ('tendras', 'es'), ('estoy', 'es'), ('otro', 'es'), ('nos', 'es'), ('y', 'es'), ('sean', 'es'), ('e', 'es'), ('qui', 'es'), ('porqu', 'es'), ('nad', 'es'), ('mis', 'es'), ('eran', 'es'), ('eres', 'es'), ('sois', 'es'), ('habran', 'es'), ('hem', 'es'), ('tien', 'es'), ('uno', 'es'), ('ti', 'es'), ('la', 'es'), ('con', 'es'), ('much', 'es'), ('estem', 'es'), ('mi', 'es'), ('fues', 'es'), ('ellas', 'es'), ('ellos', 'es'), ('por', 'es'), ('cual', 'es'), ('otros', 'es'), ('fue', 'es'), ('han', 'es'), ('vosotr', 'es'), ('esten', 'es'), ('se', 'es'), ('sobr', 'es'), ('com', 'es'), ('les', 'es'), ('son', 'es'), ('a', 'es'), ('ya', 'es'), ('dond', 'es'), ('no', 'es'), ('ese', 'es'), ('estam', 'es'), ('tuy', 'es'), ('os', 'es'), ('tu', 'es'), ('una', 'es'), ('seras', 'es'), ('estes', 'es'), ('era', 'es'), ('mas', 'es'), ('si', 'es'), ('fueron', 'es'), ('me', 'es'), ('sin', 'es'), ('mios', 'es'), ('fueseis', 'es'), ('fuisteis', 'es'), ('que', 'es'), ('desd', 'es'), ('este', 'es'), ('estaran', 'es'), ('ha', 'es'), ('eso', 'es'), ('par', 'es'), ('estas', 'es'), ('las', 'es'), ('estand', 'es'), ('unos', 'es'), ('mia', 'es'), ('poc', 'es'), ('esos', 'es'), ('estaras', 'es'), ('estuv', 'es'), ('tambien', 'es'), ('o', 'es'), ('su', 'es'), ('antes', 'es'), ('estabais', 'es'), ('tuv', 'es'), ('has', 'es'), ('durant', 'es'), ('fuer', 'es'), ('quien', 'es'), ('lo', 'es'), ('hay', 'es'), ('tendran', 'es'), ('soy', 'es'), ('nosotr', 'es'), ('eram', 'es'), ('estos', 'es'), ('tus', 'es'), ('hub', 'es'), ('siend', 'es'), ('seais', 'es'), ('al', 'es'), ('suy', 'es'), ('ten', 'es'), ('mio', 'es'), ('yo', 'es'), ('seam', 'es'), ('hab', 'es'), ('ni', 'es'), ('ser', 'es'), ('ella', 'es'), ('hast', 'es'), ('del', 'es'), ('sea', 'es'), ('algun', 'es'), ('entre', 'es'), ('tendr', 'es'), ('tant', 'es'), ('estab', 'es'), ('eras', 'es'), ('habr', 'es'), ('habras', 'es'), ('otra', 'es'), ('tod', 'es'), ('es', 'es'), ('otras', 'es'), ('algo', 'es'), ('cuand', 'es'), ('seas', 'es'), ('esas', 'es'), ('muy', 'es'), ('fuim', 'es'), ('estad', 'es'), ('fuist', 'es'), ('per', 'es'), ('esta', 'es'), ('en', 'es'), ('de', 'es'), ('erais', 'es'), ('un', 'es'), ('vuestr', 'es'), ('estan', 'es'), ('fuerais', 'es'), ('teng', 'es'), ('mias', 'es'), ('contr', 'es'), ('seran', 'es'), ('ante', 'es'), ('esa', 'es'), ('estais', 'es'), ('fui', 'es'), ('te', 'es'), ('he', 'es'), ('nuestr', 'es'), ('esto', 'es'), ('los', 'es'), ('el', 'es'), ('estar', 'es');INSERT INTO stopwords_long (stopword, language) VALUES ('ne', 'fi'), ('teillä', 'fi'), ('vaikka', 'fi'), ('jota', 'fi'), ('hänestä', 'fi'), ('yli', 'fi'), ('heidät', 'fi'), ('noille', 'fi'), ('tallä', 'fi'), ('joksi', 'fi'), ('joilta', 'fi'), ('tästä', 'fi'), ('teitä', 'fi'), ('siihen', 'fi'), ('tuoksi', 'fi'), ('niille', 'fi'), ('tässä', 'fi'), ('niissä', 'fi'), ('keissä', 'fi'), ('tuossa', 'fi'), ('hänellä', 'fi'), ('vaan', 'fi'), ('tuon', 'fi'), ('meihin', 'fi'), ('näinä', 'fi'), ('meiltä', 'fi'), ('ollut', 'fi'), ('näitä', 'fi'), ('jolta', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('heissä', 'fi'), ('teidän', 'fi'), ('tähän', 'fi'), ('tuona', 'fi'), ('meille', 'fi'), ('näille', 'fi'), ('sinä', 'fi'), ('näiksi', 'fi'), ('täksi', 'fi'), ('niillä', 'fi'), ('tänä', 'fi'), ('keneen', 'fi'), ('nämä', 'fi'), ('kanssa', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sille', 'fi'), ('mistä', 'fi'), ('noiksi', 'fi'), ('tälle', 'fi'), ('jotka', 'fi'), ('mutta', 'fi'), ('jonka', 'fi'), ('sinua', 'fi'), ('minulla', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('joiden', 'fi'), ('noita', 'fi'), ('olin', 'fi'), ('meidät', 'fi'), ('joka', 'fi'), ('niiden', 'fi'), ('tätä', 'fi'), ('näissä', 'fi'), ('mihin', 'fi'), ('meidän', 'fi'), ('kenessä', 'fi'), ('kenen', 'fi'), ('noissa', 'fi'), ('me', 'fi'), ('kenellä', 'fi'), ('keille', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('miltä', 'fi'), ('siinä', 'fi'), ('tuolta', 'fi'), ('tältä', 'fi'), ('kenenä', 'fi'), ('ei', 'fi'), ('keillä', 'fi'), ('minut', 'fi'), ('häntä', 'fi'), ('minä', 'fi'), ('keiltä', 'fi'), ('teidät', 'fi'), ('sinulta', 'fi'), ('koska', 'fi'), ('sitä', 'fi'), ('tuolle', 'fi'), ('nuo', 'fi'), ('joista', 'fi'), ('noilta', 'fi'), ('näiltä', 'fi'), ('olivat', 'fi'), ('olisivat', 'fi'), ('mukaan', 'fi'), ('näillä', 'fi'), ('heille', 'fi'), ('niitä', 'fi'), ('niistä', 'fi'), ('heillä', 'fi'), ('olette', 'fi'), ('olimme', 'fi'), ('kenet', 'fi'), ('että', 'fi'), ('joita', 'fi'), ('mille', 'fi'), ('näistä', 'fi'), ('olleet', 'fi'), ('sinulla', 'fi'), ('heistä', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('keneltä', 'fi'), ('olen', 'fi'), ('jossa', 'fi'), ('näiden', 'fi'), ('joissa', 'fi'), ('häneen', 'fi'), ('minkä', 'fi'), ('olisi', 'fi'), ('itse', 'fi'), ('olet', 'fi'), ('on', 'fi'), ('josta', 'fi'), ('sinuun', 'fi'), ('jolla', 'fi'), ('ovat', 'fi'), ('noin', 'fi'), ('joihin', 'fi'), ('mikä', 'fi'), ('heiltä', 'fi'), ('oli', 'fi'), ('minussa', 'fi'), ('miksi', 'fi'), ('niiksi', 'fi'), ('tämä', 'fi'), ('keihin', 'fi'), ('jos', 'fi'), ('olisitte', 'fi'), ('minulta', 'fi'), ('meistä', 'fi'), ('siitä', 'fi'), ('millä', 'fi'), ('sillä', 'fi'), ('ketkä', 'fi'), ('ketä', 'fi'), ('poikki', 'fi'), ('noiden', 'fi'), ('joilla', 'fi'), ('sinun', 'fi'), ('tuo', 'fi'), ('niiltä', 'fi'), ('hänessä', 'fi'), ('ole', 'fi'), ('tuotä', 'fi'), ('sinusta', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('kenestä', 'fi'), ('sinussa', 'fi'), ('eivät', 'fi'), ('teille', 'fi'), ('tuohon', 'fi'), ('hänen', 'fi'), ('näihin', 'fi'), ('keistä', 'fi'), ('teihin', 'fi'), ('olisin', 'fi'), ('olisimme', 'fi'), ('teistä', 'fi'), ('keneksi', 'fi'), ('ette', 'fi'), ('teissä', 'fi'), ('siltä', 'fi'), ('jolle', 'fi'), ('kuka', 'fi'), ('noista', 'fi'), ('jona', 'fi'), ('keiksi', 'fi'), ('teiltä', 'fi'), ('johon', 'fi'), ('olla', 'fi'), ('sinulle', 'fi'), ('keitä', 'fi'), ('häneltä', 'fi'), ('tuolla', 'fi'), ('nyt', 'fi'), ('noihin', 'fi'), ('hänet', 'fi'), ('heidän', 'fi'), ('noina', 'fi'), ('minun', 'fi'), ('et', 'fi'), ('heihin', 'fi'), ('siksi', 'fi'), ('noilla', 'fi'), ('tuosta', 'fi'), ('kenelle', 'fi'), ('meissä', 'fi'), ('minulle', 'fi'), ('en', 'fi'), ('niihin', 'fi'), ('joiksi', 'fi'), ('hänelle', 'fi'), ('mitkä', 'fi'), ('minua', 'fi'), ('olitte', 'fi'), ('kun', 'fi'), ('ja', 'fi'), ('sinut', 'fi'), ('mitä', 'fi'), ('niinä', 'fi'), ('minuun', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('joille', 'fi'), ('olemme', 'fi'), ('meillä', 'fi'), ('minusta', 'fi'), ('te', 'fi'), ('he', 'fi'), ('missä', 'fi'), ('emme', 'fi'), ('tämän', 'fi'), ('keiden', 'fi');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ne', 'fi'), ('jota', 'fi'), ('yli', 'fi'), ('ketk', 'fi'), ('teitä', 'fi'), ('vaik', 'fi'), ('täs', 'fi'), ('niide', 'fi'), ('näide', 'fi'), ('em', 'fi'), ('täl', 'fi'), ('joih', 'fi'), ('jol', 'fi'), ('tuon', 'fi'), ('vaan', 'fi'), ('mis', 'fi'), ('näinä', 'fi'), ('näitä', 'fi'), ('tai', 'fi'), ('niin', 'fi'), ('tuona', 'fi'), ('meil', 'fi'), ('sinä', 'fi'), ('tänä', 'fi'), ('nämä', 'fi'), ('vai', 'fi'), ('se', 'fi'), ('sinu', 'fi'), ('sil', 'fi'), ('joho', 'fi'), ('olit', 'fi'), ('kuin', 'fi'), ('noita', 'fi'), ('joka', 'fi'), ('tätä', 'fi'), ('teidä', 'fi'), ('olisiv', 'fi'), ('me', 'fi'), ('joina', 'fi'), ('keinä', 'fi'), ('siinä', 'fi'), ('tuos', 'fi'), ('joks', 'fi'), ('mitk', 'fi'), ('poik', 'fi'), ('ei', 'fi'), ('minä', 'fi'), ('heis', 'fi'), ('mut', 'fi'), ('keiks', 'fi'), ('sitä', 'fi'), ('noil', 'fi'), ('nuo', 'fi'), ('jotk', 'fi'), ('tuol', 'fi'), ('olu', 'fi'), ('niitä', 'fi'), ('heil', 'fi'), ('joita', 'fi'), ('noih', 'fi'), ('oliv', 'fi'), ('heitä', 'fi'), ('sen', 'fi'), ('mink', 'fi'), ('näil', 'fi'), ('näih', 'fi'), ('eivä', 'fi'), ('olet', 'fi'), ('häne', 'fi'), ('on', 'fi'), ('joil', 'fi'), ('its', 'fi'), ('olis', 'fi'), ('noin', 'fi'), ('mikä', 'fi'), ('oli', 'fi'), ('tämä', 'fi'), ('tuoks', 'fi'), ('kan', 'fi'), ('ova', 'fi'), ('kene', 'fi'), ('jos', 'fi'), ('näiks', 'fi'), ('meis', 'fi'), ('siitä', 'fi'), ('tähä', 'fi'), ('keih', 'fi'), ('ketä', 'fi'), ('muka', 'fi'), ('tuo', 'fi'), ('ole', 'fi'), ('mih', 'fi'), ('tuotä', 'fi'), ('ol', 'fi'), ('hän', 'fi'), ('meitä', 'fi'), ('meih', 'fi'), ('teis', 'fi'), ('teil', 'fi'), ('jonk', 'fi'), ('kuka', 'fi'), ('niih', 'fi'), ('jona', 'fi'), ('joiks', 'fi'), ('näis', 'fi'), ('tal', 'fi'), ('meidä', 'fi'), ('heih', 'fi'), ('keitä', 'fi'), ('miks', 'fi'), ('nyt', 'fi'), ('et', 'fi'), ('noina', 'fi'), ('joide', 'fi'), ('keil', 'fi'), ('täks', 'fi'), ('heidä', 'fi'), ('en', 'fi'), ('niil', 'fi'), ('mil', 'fi'), ('siks', 'fi'), ('noide', 'fi'), ('minu', 'fi'), ('keide', 'fi'), ('kun', 'fi'), ('teih', 'fi'), ('ja', 'fi'), ('niinä', 'fi'), ('mitä', 'fi'), ('tuoho', 'fi'), ('olisit', 'fi'), ('sekä', 'fi'), ('jois', 'fi'), ('he', 'fi'), ('te', 'fi'), ('siihe', 'fi'), ('nois', 'fi'), ('niiks', 'fi'), ('keis', 'fi'), ('kosk', 'fi'), ('noiks', 'fi'), ('niis', 'fi');INSERT INTO stopwords_long (stopword, language) VALUES ('ne', 'fr'), ('le', 'fr'), ('serais', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('ayons', 'fr'), ('eus', 'fr'), ('ta', 'fr'), ('mais', 'fr'), ('ayante', 'fr'), ('êtes', 'fr'), ('serions', 'fr'), ('qui', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('étantes', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('aurons', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('soyons', 'fr'), ('serez', 'fr'), ('pour', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('eussiez', 'fr'), ('ayez', 'fr'), ('aies', 'fr'), ('aurions', 'fr'), ('ayantes', 'fr'), ('se', 'fr'), ('eûtes', 'fr'), ('ai', 'fr'), ('fussiez', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('ayants', 'fr'), ('aux', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('soyez', 'fr'), ('seras', 'fr'), ('étants', 'fr'), ('ces', 'fr'), ('me', 'fr'), ('aurez', 'fr'), ('s', 'fr'), ('eussions', 'fr'), ('étiez', 'fr'), ('c', 'fr'), ('que', 'fr'), ('vos', 'fr'), ('eusse', 'fr'), ('des', 'fr'), ('étions', 'fr'), ('aurai', 'fr'), ('par', 'fr'), ('mes', 'fr'), ('soit', 'fr'), ('même', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('mon', 'fr'), ('votre', 'fr'), ('aviez', 'fr'), ('m', 'fr'), ('serai', 'fr'), ('as', 'fr'), ('aurait', 'fr'), ('étante', 'fr'), ('nous', 'fr'), ('une', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('étée', 'fr'), ('d', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('fusse', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('eussent', 'fr'), ('eusses', 'fr'), ('étais', 'fr'), ('sont', 'fr'), ('fussions', 'fr'), ('étés', 'fr'), ('dans', 'fr'), ('fûtes', 'fr'), ('eu', 'fr'), ('auront', 'fr'), ('fusses', 'fr'), ('auraient', 'fr'), ('avions', 'fr'), ('serons', 'fr'), ('eûmes', 'fr'), ('avais', 'fr'), ('fus', 'fr'), ('aura', 'fr'), ('lui', 'fr'), ('notre', 'fr'), ('eux', 'fr'), ('auras', 'fr'), ('sommes', 'fr'), ('ait', 'fr'), ('ton', 'fr'), ('sera', 'fr'), ('es', 'fr'), ('avez', 'fr'), ('je', 'fr'), ('j', 'fr'), ('étées', 'fr'), ('avons', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('sa', 'fr'), ('seriez', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('en', 'fr'), ('de', 'fr'), ('un', 'fr'), ('elle', 'fr'), ('aie', 'fr'), ('furent', 'fr'), ('pas', 'fr'), ('serait', 'fr'), ('ce', 'fr'), ('seraient', 'fr'), ('aurais', 'fr'), ('moi', 'fr'), ('n', 'fr'), ('ou', 'fr'), ('te', 'fr'), ('ses', 'fr'), ('tes', 'fr'), ('fûmes', 'fr'), ('auriez', 'fr'), ('l', 'fr'), ('eues', 'fr'), ('à', 'fr'), ('avec', 'fr');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ayon', 'fr'), ('ne', 'fr'), ('fuss', 'fr'), ('soyon', 'fr'), ('auron', 'fr'), ('ayant', 'fr'), ('du', 'fr'), ('le', 'fr'), ('d', 'fr'), ('suis', 'fr'), ('leur', 'fr'), ('eue', 'fr'), ('on', 'fr'), ('y', 'fr'), ('nos', 'fr'), ('été', 'fr'), ('fussent', 'fr'), ('ta', 'fr'), ('eus', 'fr'), ('mais', 'fr'), ('e', 'fr'), ('qui', 'fr'), ('seron', 'fr'), ('eussent', 'fr'), ('étaient', 'fr'), ('est', 'fr'), ('étais', 'fr'), ('ête', 'fr'), ('sont', 'fr'), ('sois', 'fr'), ('avaient', 'fr'), ('seront', 'fr'), ('fûm', 'fr'), ('euss', 'fr'), ('toi', 'fr'), ('fût', 'fr'), ('qu', 'fr'), ('fut', 'fr'), ('la', 'fr'), ('avait', 'fr'), ('eu', 'fr'), ('fussion', 'fr'), ('auront', 'fr'), ('ser', 'fr'), ('pour', 'fr'), ('avon', 'fr'), ('soi', 'fr'), ('notr', 'fr'), ('eurent', 'fr'), ('vous', 'fr'), ('avais', 'fr'), ('ayez', 'fr'), ('fus', 'fr'), ('se', 'fr'), ('lui', 'fr'), ('ai', 'fr'), ('eux', 'fr'), ('eut', 'fr'), ('il', 'fr'), ('son', 'fr'), ('aurion', 'fr'), ('ton', 'fr'), ('ait', 'fr'), ('eûm', 'fr'), ('avez', 'fr'), ('dan', 'fr'), ('je', 'fr'), ('aux', 'fr'), ('j', 'fr'), ('ont', 'fr'), ('était', 'fr'), ('eût', 'fr'), ('ma', 'fr'), ('tu', 'fr'), ('sa', 'fr'), ('aur', 'fr'), ('étion', 'fr'), ('t', 'fr'), ('et', 'fr'), ('étant', 'fr'), ('eussion', 'fr'), ('soient', 'fr'), ('sur', 'fr'), ('me', 'fr'), ('s', 'fr'), ('en', 'fr'), ('mêm', 'fr'), ('c', 'fr'), ('de', 'fr'), ('un', 'fr'), ('que', 'fr'), ('elle', 'fr'), ('vos', 'fr'), ('furent', 'fr'), ('aie', 'fr'), ('avi', 'fr'), ('pas', 'fr'), ('par', 'fr'), ('soit', 'fr'), ('ce', 'fr'), ('aient', 'fr'), ('au', 'fr'), ('serion', 'fr'), ('mon', 'fr'), ('moi', 'fr'), ('ou', 'fr'), ('n', 'fr'), ('te', 'fr'), ('m', 'fr'), ('éti', 'fr'), ('avion', 'fr'), ('l', 'fr'), ('somm', 'fr'), ('as', 'fr'), ('à', 'fr'), ('avec', 'fr'), ('votr', 'fr'), ('une', 'fr'), ('nous', 'fr');INSERT INTO stopwords_long (stopword, language) VALUES ('szinte', 'hu'), ('ne', 'hu'), ('továbbá', 'hu'), ('lenni', 'hu'), ('ki', 'hu'), ('igen', 'hu'), ('való', 'hu'), ('voltak', 'hu'), ('ellen', 'hu'), ('mindent', 'hu'), ('illetve', 'hu'), ('cikkek', 'hu'), ('azzal', 'hu'), ('mindig', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('annak', 'hu'), ('úgy', 'hu'), ('számára', 'hu'), ('szerint', 'hu'), ('ismét', 'hu'), ('nem', 'hu'), ('azután', 'hu'), ('sem', 'hu'), ('mert', 'hu'), ('valamint', 'hu'), ('jobban', 'hu'), ('teljes', 'hu'), ('amelyek', 'hu'), ('vagyok', 'hu'), ('arról', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('mindenki', 'hu'), ('olyan', 'hu'), ('mi', 'hu'), ('között', 'hu'), ('egyes', 'hu'), ('maga', 'hu'), ('legalább', 'hu'), ('általában', 'hu'), ('vissza', 'hu'), ('õk', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('lehetett', 'hu'), ('azok', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('amelynek', 'hu'), ('ehhez', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('saját', 'hu'), ('hiszen', 'hu'), ('aki', 'hu'), ('mikor', 'hu'), ('újabb', 'hu'), ('itt', 'hu'), ('ami', 'hu'), ('ugyanis', 'hu'), ('egyre', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('kívül', 'hu'), ('valami', 'hu'), ('s', 'hu'), ('meg', 'hu'), ('ahhoz', 'hu'), ('azon', 'hu'), ('eddig', 'hu'), ('egyik', 'hu'), ('valaki', 'hu'), ('ill', 'hu'), ('ison', 'hu'), ('vagyis', 'hu'), ('pedig', 'hu'), ('keressünk', 'hu'), ('át', 'hu'), ('amelyekben', 'hu'), ('alatt', 'hu'), ('mellett', 'hu'), ('új', 'hu'), ('elõször', 'hu'), ('õket', 'hu'), ('én', 'hu'), ('elõtt', 'hu'), ('egyetlen', 'hu'), ('azonban', 'hu'), ('éppen', 'hu'), ('tovább', 'hu'), ('ott', 'hu'), ('közül', 'hu'), ('ebben', 'hu'), ('melyek', 'hu'), ('lenne', 'hu'), ('mit', 'hu'), ('magát', 'hu'), ('szemben', 'hu'), ('õ', 'hu'), ('ezzel', 'hu'), ('belül', 'hu'), ('lett', 'hu'), ('ahol', 'hu'), ('amikor', 'hu'), ('viszont', 'hu'), ('azért', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('semmi', 'hu'), ('kellett', 'hu'), ('ezt', 'hu'), ('több', 'hu'), ('nagyon', 'hu'), ('abban', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('ilyenkor', 'hu'), ('nagyobb', 'hu'), ('még', 'hu'), ('amely', 'hu'), ('ekkor', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('újra', 'hu'), ('neki', 'hu'), ('mivel', 'hu'), ('voltam', 'hu'), ('bár', 'hu'), ('benne', 'hu'), ('felé', 'hu'), ('nélkül', 'hu'), ('által', 'hu'), ('ezen', 'hu'), ('amelyet', 'hu'), ('elég', 'hu'), ('miért', 'hu'), ('talán', 'hu'), ('minden', 'hu'), ('amolyan', 'hu'), ('hogy', 'hu'), ('persze', 'hu'), ('egyéb', 'hu'), ('milyen', 'hu'), ('utána', 'hu'), ('sokkal', 'hu'), ('akkor', 'hu'), ('emilyen', 'hu'), ('mint', 'hu'), ('mely', 'hu'), ('ilyen', 'hu'), ('voltunk', 'hu'), ('nekem', 'hu'), ('amit', 'hu'), ('vele', 'hu'), ('míg', 'hu'), ('sok', 'hu'), ('keresztül', 'hu'), ('elsõ', 'hu'), ('cikkeket', 'hu'), ('mintha', 'hu'), ('hogyan', 'hu'), ('vannak', 'hu'), ('csak', 'hu'), ('után', 'hu'), ('nagy', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('be', 'hu'), ('lehet', 'hu'), ('most', 'hu'), ('fel', 'hu'), ('hanem', 'hu'), ('arra', 'hu'), ('másik', 'hu'), ('és', 'hu'), ('majd', 'hu'), ('aztán', 'hu'), ('amelyeket', 'hu'), ('volna', 'hu'), ('össze', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('ennek', 'hu'), ('tehát', 'hu'), ('rá', 'hu'), ('erre', 'hu'), ('sokat', 'hu'), ('néha', 'hu'), ('ezek', 'hu'), ('ezért', 'hu'), ('legyen', 'hu'), ('akik', 'hu'), ('el', 'hu'), ('van', 'hu'), ('egy', 'hu');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ne', 'hu'), ('eb', 'hu'), ('ki', 'hu'), ('való', 'hu'), ('lehetet', 'hu'), ('mos', 'hu'), ('egész', 'hu'), ('e', 'hu'), ('úgy', 'hu'), ('nem', 'hu'), ('hisz', 'hu'), ('ak', 'hu'), ('sem', 'hu'), ('teljes', 'hu'), ('lenn', 'hu'), ('közöt', 'hu'), ('cik', 'hu'), ('is', 'hu'), ('ut', 'hu'), ('így', 'hu'), ('volt', 'hu'), ('ek', 'hu'), ('viszon', 'hu'), ('maj', 'hu'), ('mi', 'hu'), ('vel', 'hu'), ('it', 'hu'), ('egyes', 'hu'), ('leh', 'hu'), ('val', 'hu'), ('legalább', 'hu'), ('õk', 'hu'), ('han', 'hu'), ('jó', 'hu'), ('lesz', 'hu'), ('nincs', 'hu'), ('jól', 'hu'), ('valamin', 'hu'), ('a', 'hu'), ('amíg', 'hu'), ('újabb', 'hu'), ('ugyanis', 'hu'), ('más', 'hu'), ('elõ', 'hu'), ('valam', 'hu'), ('azut', 'hu'), ('s', 'hu'), ('keress', 'hu'), ('meg', 'hu'), ('ab', 'hu'), ('azon', 'hu'), ('ill', 'hu'), ('mily', 'hu'), ('ped', 'hu'), ('vagyis', 'hu'), ('egyetl', 'hu'), ('át', 'hu'), ('elõször', 'hu'), ('új', 'hu'), ('am', 'hu'), ('én', 'hu'), ('kereszt', 'hu'), ('oly', 'hu'), ('tovább', 'hu'), ('ot', 'hu'), ('ah', 'hu'), ('köz', 'hu'), ('õ', 'hu'), ('szám', 'hu'), ('ahol', 'hu'), ('elõt', 'hu'), ('az', 'hu'), ('ez', 'hu'), ('épp', 'hu'), ('szint', 'hu'), ('mellet', 'hu'), ('több', 'hu'), ('utolsó', 'hu'), ('ahogy', 'hu'), ('emily', 'hu'), ('nagyobb', 'hu'), ('bel', 'hu'), ('még', 'hu'), ('eh', 'hu'), ('amely', 'hu'), ('ill.', 'hu'), ('kell', 'hu'), ('ell', 'hu'), ('néhány', 'hu'), ('vagy', 'hu'), ('mer', 'hu'), ('szem', 'hu'), ('bár', 'hu'), ('által', 'hu'), ('alat', 'hu'), ('elég', 'hu'), ('vol', 'hu'), ('persz', 'hu'), ('amoly', 'hu'), ('minden', 'hu'), ('job', 'hu'), ('hogy', 'hu'), ('egyéb', 'hu'), ('legy', 'hu'), ('vissz', 'hu'), ('mely', 'hu'), ('an', 'hu'), ('ilyen', 'hu'), ('ism', 'hu'), ('tal', 'hu'), ('míg', 'hu'), ('let', 'hu'), ('teh', 'hu'), ('semm', 'hu'), ('néh', 'hu'), ('minth', 'hu'), ('sa', 'hu'), ('sok', 'hu'), ('er', 'hu'), ('elsõ', 'hu'), ('csak', 'hu'), ('en', 'hu'), ('nagy', 'hu'), ('után', 'hu'), ('de', 'hu'), ('cikk', 'hu'), ('ed', 'hu'), ('be', 'hu'), ('továb', 'hu'), ('voln', 'hu'), ('fel', 'hu'), ('mag', 'hu'), ('és', 'hu'), ('benn', 'hu'), ('kív', 'hu'), ('illetv', 'hu'), ('már', 'hu'), ('azt', 'hu'), ('kellet', 'hu'), ('min', 'hu'), ('rá', 'hu'), ('ig', 'hu'), ('ar', 'hu'), ('nek', 'hu'), ('ily', 'hu'), ('nél', 'hu'), ('el', 'hu'), ('össz', 'hu'), ('van', 'hu'), ('egy', 'hu'), ('szerin', 'hu');INSERT INTO stopwords_long (stopword, language) VALUES ('ne', 'it'), ('sugli', 'it'), ('delle', 'it'), ('avevo', 'it'), ('avete', 'it'), ('avessimo', 'it'), ('stemmo', 'it'), ('e', 'it'), ('queste', 'it'), ('ebbero', 'it'), ('facevano', 'it'), ('noi', 'it'), ('alle', 'it'), ('avesti', 'it'), ('tutto', 'it'), ('aveva', 'it'), ('sarete', 'it'), ('sua', 'it'), ('all', 'it'), ('nello', 'it'), ('chi', 'it'), ('ti', 'it'), ('faceste', 'it'), ('mi', 'it'), ('tua', 'it'), ('vostro', 'it'), ('fosti', 'it'), ('se', 'it'), ('gli', 'it'), ('sulla', 'it'), ('quelli', 'it'), ('miei', 'it'), ('ai', 'it'), ('avevano', 'it'), ('a', 'it'), ('nostra', 'it'), ('eravamo', 'it'), ('ma', 'it'), ('in', 'it'), ('una', 'it'), ('facessero', 'it'), ('si', 'it'), ('stavamo', 'it'), ('staremmo', 'it'), ('saremo', 'it'), ('quanti', 'it'), ('avessi', 'it'), ('anche', 'it'), ('siamo', 'it'), ('li', 'it'), ('avreste', 'it'), ('staresti', 'it'), ('sia', 'it'), ('questi', 'it'), ('facemmo', 'it'), ('stavo', 'it'), ('coi', 'it'), ('avrete', 'it'), ('dei', 'it'), ('sulle', 'it'), ('stettero', 'it'), ('faremmo', 'it'), ('sue', 'it'), ('ebbe', 'it'), ('o', 'it'), ('fecero', 'it'), ('stetti', 'it'), ('farai', 'it'), ('quante', 'it'), ('stesse', 'it'), ('della', 'it'), ('saremmo', 'it'), ('vi', 'it'), ('avrai', 'it'), ('stessimo', 'it'), ('farebbero', 'it'), ('dov', 'it'), ('starei', 'it'), ('sullo', 'it'), ('starà', 'it'), ('avessero', 'it'), ('al', 'it'), ('dello', 'it'), ('starai', 'it'), ('avrò', 'it'), ('avevate', 'it'), ('eravate', 'it'), ('quale', 'it'), ('stai', 'it'), ('avuto', 'it'), ('sarò', 'it'), ('perché', 'it'), ('mio', 'it'), ('aveste', 'it'), ('dagli', 'it'), ('avuti', 'it'), ('del', 'it'), ('stanno', 'it'), ('farà', 'it'), ('eri', 'it'), ('facesse', 'it'), ('facevo', 'it'), ('nei', 'it'), ('fu', 'it'), ('farebbe', 'it'), ('farete', 'it'), ('facciate', 'it'), ('stia', 'it'), ('suoi', 'it'), ('quanta', 'it'), ('essendo', 'it'), ('abbia', 'it'), ('quella', 'it'), ('stiamo', 'it'), ('avevi', 'it'), ('sarai', 'it'), ('avrei', 'it'), ('ed', 'it'), ('un', 'it'), ('stessero', 'it'), ('avrebbero', 'it'), ('sei', 'it'), ('sul', 'it'), ('sarà', 'it'), ('sto', 'it'), ('questo', 'it'), ('fareste', 'it'), ('dall', 'it'), ('stavi', 'it'), ('più', 'it'), ('faranno', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('stava', 'it'), ('fossero', 'it'), ('dal', 'it'), ('sarei', 'it'), ('voi', 'it'), ('abbiate', 'it'), ('facciamo', 'it'), ('avremo', 'it'), ('sarebbe', 'it'), ('da', 'it'), ('degli', 'it'), ('le', 'it'), ('come', 'it'), ('mie', 'it'), ('vostre', 'it'), ('ci', 'it'), ('negl', 'it'), ('saresti', 'it'), ('foste', 'it'), ('ebbi', 'it'), ('faccia', 'it'), ('steste', 'it'), ('sta', 'it'), ('stesti', 'it'), ('dalla', 'it'), ('furono', 'it'), ('fanno', 'it'), ('saranno', 'it'), ('fece', 'it'), ('dallo', 'it'), ('quelle', 'it'), ('siate', 'it'), ('stette', 'it'), ('dell', 'it'), ('avevamo', 'it'), ('sull', 'it'), ('uno', 'it'), ('la', 'it'), ('con', 'it'), ('vostri', 'it'), ('dagl', 'it'), ('starebbero', 'it'), ('stessi', 'it'), ('hai', 'it'), ('faccio', 'it'), ('farò', 'it'), ('il', 'it'), ('nostro', 'it'), ('ho', 'it'), ('facciano', 'it'), ('avrà', 'it'), ('loro', 'it'), ('tu', 'it'), ('cui', 'it'), ('era', 'it'), ('starebbe', 'it'), ('fossi', 'it'), ('facendo', 'it'), ('hanno', 'it'), ('c', 'it'), ('facessi', 'it'), ('alla', 'it'), ('vostra', 'it'), ('sarebbero', 'it'), ('ha', 'it'), ('dalle', 'it'), ('feci', 'it'), ('tue', 'it'), ('tra', 'it'), ('nell', 'it'), ('i', 'it'), ('faresti', 'it'), ('mia', 'it'), ('stiate', 'it'), ('nel', 'it'), ('avesse', 'it'), ('faceva', 'it'), ('degl', 'it'), ('sono', 'it'), ('su', 'it'), ('sareste', 'it'), ('quanto', 'it'), ('contro', 'it'), ('stareste', 'it'), ('lei', 'it'), ('faremo', 'it'), ('lo', 'it'), ('nostri', 'it'), ('avuta', 'it'), ('facevamo', 'it'), ('siete', 'it'), ('questa', 'it'), ('avemmo', 'it'), ('facevate', 'it'), ('sui', 'it'), ('facesti', 'it'), ('fai', 'it'), ('avranno', 'it'), ('starete', 'it'), ('dove', 'it'), ('di', 'it'), ('tuo', 'it'), ('sugl', 'it'), ('abbiano', 'it'), ('avresti', 'it'), ('stiano', 'it'), ('starò', 'it'), ('siano', 'it'), ('nelle', 'it'), ('quello', 'it'), ('fummo', 'it'), ('farei', 'it'), ('lui', 'it'), ('fossimo', 'it'), ('avute', 'it'), ('non', 'it'), ('fosse', 'it'), ('ero', 'it'), ('io', 'it'), ('avrebbe', 'it'), ('per', 'it'), ('nella', 'it'), ('nostre', 'it'), ('staremo', 'it'), ('tuoi', 'it'), ('facevi', 'it'), ('facessimo', 'it'), ('stavate', 'it'), ('dai', 'it'), ('stavano', 'it'), ('è', 'it'), ('erano', 'it'), ('ad', 'it'), ('agl', 'it'), ('col', 'it'), ('stando', 'it'), ('negli', 'it'), ('avremmo', 'it'), ('l', 'it'), ('abbiamo', 'it'), ('tutti', 'it'), ('allo', 'it'), ('avendo', 'it'), ('staranno', 'it');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ne', 'it'), ('facess', 'it'), ('le', 'it'), ('da', 'it'), ('stessim', 'it'), ('stav', 'it'), ('avrebb', 'it'), ('far', 'it'), ('mie', 'it'), ('hann', 'it'), ('stiam', 'it'), ('tutt', 'it'), ('ci', 'it'), ('negl', 'it'), ('avessim', 'it'), ('e', 'it'), ('ebbi', 'it'), ('stett', 'it'), ('avre', 'it'), ('sta', 'it'), ('fann', 'it'), ('noi', 'it'), ('alle', 'it'), ('abbi', 'it'), ('avest', 'it'), ('essend', 'it'), ('avra', 'it'), ('eran', 'it'), ('quest', 'it'), ('sua', 'it'), ('stest', 'it'), ('dell', 'it'), ('all', 'it'), ('farann', 'it'), ('uno', 'it'), ('fecer', 'it'), ('nostr', 'it'), ('chi', 'it'), ('ti', 'it'), ('sull', 'it'), ('la', 'it'), ('avesser', 'it'), ('con', 'it'), ('farem', 'it'), ('mi', 'it'), ('tua', 'it'), ('dagl', 'it'), ('perc', 'it'), ('fare', 'it'), ('siat', 'it'), ('hai', 'it'), ('starann', 'it'), ('starem', 'it'), ('sara', 'it'), ('se', 'it'), ('sar', 'it'), ('gli', 'it'), ('ai', 'it'), ('sarest', 'it'), ('sarem', 'it'), ('com', 'it'), ('il', 'it'), ('son', 'it'), ('a', 'it'), ('farebb', 'it'), ('avrann', 'it'), ('ho', 'it'), ('avrà', 'it'), ('avrem', 'it'), ('tu', 'it'), ('ma', 'it'), ('in', 'it'), ('cui', 'it'), ('una', 'it'), ('stess', 'it'), ('era', 'it'), ('stand', 'it'), ('si', 'it'), ('star', 'it'), ('c', 'it'), ('anche', 'it'), ('stavam', 'it'), ('li', 'it'), ('alla', 'it'), ('sia', 'it'), ('siam', 'it'), ('stare', 'it'), ('ha', 'it'), ('avevam', 'it'), ('starest', 'it'), ('tue', 'it'), ('coi', 'it'), ('dei', 'it'), ('lor', 'it'), ('fac', 'it'), ('i', 'it'), ('tra', 'it'), ('nell', 'it'), ('avrest', 'it'), ('sue', 'it'), ('mia', 'it'), ('nel', 'it'), ('ebbe', 'it'), ('degl', 'it'), ('siet', 'it'), ('avet', 'it'), ('o', 'it'), ('su', 'it'), ('facessim', 'it'), ('stemm', 'it'), ('sare', 'it'), ('lei', 'it'), ('stetter', 'it'), ('facc', 'it'), ('lo', 'it'), ('vi', 'it'), ('erav', 'it'), ('sian', 'it'), ('fur', 'it'), ('dov', 'it'), ('avret', 'it'), ('al', 'it'), ('avrò', 'it'), ('sui', 'it'), ('fai', 'it'), ('fossim', 'it'), ('sarann', 'it'), ('stesser', 'it'), ('di', 'it'), ('vostr', 'it'), ('tuo', 'it'), ('mio', 'it'), ('avev', 'it'), ('stara', 'it'), ('avend', 'it'), ('sugl', 'it'), ('sarebber', 'it'), ('quant', 'it'), ('avemm', 'it'), ('del', 'it'), ('eri', 'it'), ('lui', 'it'), ('non', 'it'), ('fost', 'it'), ('nei', 'it'), ('avremm', 'it'), ('fu', 'it'), ('starebb', 'it'), ('eravam', 'it'), ('avrebber', 'it'), ('abbiam', 'it'), ('fara', 'it'), ('sarebb', 'it'), ('ero', 'it'), ('facest', 'it'), ('ebber', 'it'), ('quell', 'it'), ('io', 'it'), ('qual', 'it'), ('per', 'it'), ('ed', 'it'), ('un', 'it'), ('stann', 'it'), ('sti', 'it'), ('sei', 'it'), ('starebber', 'it'), ('sul', 'it'), ('sto', 'it'), ('dai', 'it'), ('farebber', 'it'), ('è', 'it'), ('fosser', 'it'), ('fec', 'it'), ('contr', 'it'), ('dall', 'it'), ('agl', 'it'), ('ad', 'it'), ('col', 'it'), ('avess', 'it'), ('più', 'it'), ('foss', 'it'), ('fumm', 'it'), ('farest', 'it'), ('fui', 'it'), ('agli', 'it'), ('che', 'it'), ('suo', 'it'), ('l', 'it'), ('dal', 'it'), ('voi', 'it'), ('allo', 'it'), ('avut', 'it');INSERT INTO stopwords_long (stopword, language) VALUES ('su', 'lt'), ('ne', 'lt'), ('į', 'lt'), ('pasakė', 'lt'), ('kurių', 'lt'), ('d', 'lt'), ('lietuva', 'lt'), ('kai', 'lt'), ('arba', 'lt'), ('po', 'lt'), ('kurios', 'lt'), ('kam', 'lt'), ('darbo', 'lt'), ('daugiau', 'lt'), ('niekada', 'lt'), ('tas', 'lt'), ('metus', 'lt'), ('kovo', 'lt'), ('negali', 'lt'), ('kaip', 'lt'), ('kada', 'lt'), ('už', 'lt'), ('jos', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('savo', 'lt'), ('labai', 'lt'), ('tik', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('pat', 'lt'), ('negu', 'lt'), ('gali', 'lt'), ('čia', 'lt'), ('turi', 'lt'), ('bet', 'lt'), ('tuo', 'lt'), ('mažas', 'lt'), ('bei', 'lt'), ('tai', 'lt'), ('prašau', 'lt'), ('to', 'lt'), ('reikia', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jau', 'lt'), ('metu', 'lt'), ('lietuvos', 'lt'), ('jog', 'lt'), ('mažiau', 'lt'), ('val', 'lt'), ('lietuvoje', 'lt'), ('jei', 'lt'), ('jį', 'lt'), ('man', 'lt'), ('turėjo', 'lt'), ('nes', 'lt'), ('vienas', 'lt'), ('nei', 'lt'), ('daugiausia', 'lt'), ('mūsų', 'lt'), ('tarp', 'lt'), ('a', 'lt'), ('tačiau', 'lt'), ('net', 'lt'), ('šalia', 'lt'), ('niekas', 'lt'), ('mažai', 'lt'), ('todėl', 'lt'), ('yra', 'lt'), ('jie', 'lt'), ('pasak', 'lt'), ('per', 'lt'), ('dar', 'lt'), ('apie', 'lt'), ('kuris', 'lt'), ('žmonių', 'lt'), ('šalies', 'lt'), ('metais', 'lt'), ('ir', 'lt'), ('be', 'lt'), ('gauna', 'lt'), ('dėl', 'lt'), ('taip', 'lt'), ('kurie', 'lt'), ('dieną', 'lt'), ('jam', 'lt'), ('jų', 'lt'), ('kad', 'lt'), ('visi', 'lt'), ('r', 'lt'), ('bus', 'lt'), ('klausti', 'lt'), ('mano', 'lt'), ('mes', 'lt'), ('sakė', 'lt'), ('būti', 'lt'), ('dažnai', 'lt'), ('šios', 'lt'), ('jis', 'lt'), ('nuo', 'lt'), ('nors', 'lt'), ('iki', 'lt'), ('buvo', 'lt'), ('kitas', 'lt'), ('metų', 'lt'), ('kiekvienas', 'lt'), ('tikrai', 'lt'), ('iš', 'lt'), ('būtų', 'lt'), ('jo', 'lt'), ('tavo', 'lt'), ('nėra', 'lt'), ('prie', 'lt'), ('sakyti', 'lt'), ('m', 'lt'), ('daryti', 'lt'), ('kartą', 'lt'), ('ar', 'lt'), ('kas', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('klausimas', 'lt'), ('o', 'lt'), ('gauti', 'lt');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('', 'lt'), ('d', 'lt'), ('vis', 'lt'), ('neg', 'lt'), ('kam', 'lt'), ('tikr', 'lt'), ('klaus', 'lt'), ('tur', 'lt'), ('kaip', 'lt'), ('už', 'lt'), ('gau', 'lt'), ('pagal', 'lt'), ('prieš', 'lt'), ('b', 'lt'), ('ji', 'lt'), ('aš', 'lt'), ('tik', 'lt'), ('pat', 'lt'), ('bet', 'lt'), ('nor', 'lt'), ('dažn', 'lt'), ('lie', 'lt'), ('daug', 'lt'), ('taip pat', 'lt'), ('jog', 'lt'), ('val', 'lt'), ('būt', 'lt'), ('buv', 'lt'), ('gal', 'lt'), ('man', 'lt'), ('negal', 'lt'), ('lab', 'lt'), ('bū', 'lt'), ('mūs', 'lt'), ('darb', 'lt'), ('tarp', 'lt'), ('met', 'lt'), ('š', 'lt'), ('maž', 'lt'), ('net', 'lt'), ('niek', 'lt'), ('j', 'lt'), ('praša', 'lt'), ('kuri', 'lt'), ('k', 'lt'), ('todėl', 'lt'), ('pri', 'lt'), ('nėr', 'lt'), ('pasak', 'lt'), ('t', 'lt'), ('re', 'lt'), ('v', 'lt'), ('per', 'lt'), ('s', 'lt'), ('dar', 'lt'), ('šal', 'lt'), ('ir', 'lt'), ('dėl', 'lt'), ('arb', 'lt'), ('taip', 'lt'), ('kart', 'lt'), ('kiekv', 'lt'), ('jam', 'lt'), ('kad', 'lt'), ('gaun', 'lt'), ('yr', 'lt'), ('r', 'lt'), ('kov', 'lt'), ('api', 'lt'), ('ja', 'lt'), ('niekad', 'lt'), ('iš', 'lt'), ('n', 'lt'), ('klausim', 'lt'), ('m', 'lt'), ('ar', 'lt'), ('p', 'lt'), ('dabar', 'lt'), ('kur', 'lt'), ('kit', 'lt'), ('sak', 'lt'), ('žmon', 'lt'), ('tat', 'lt');INSERT INTO stopwords_long (stopword, language) VALUES ('want', 'nl'), ('deze', 'nl'), ('wil', 'nl'), ('kon', 'nl'), ('nu', 'nl'), ('heeft', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('wie', 'nl'), ('zijn', 'nl'), ('wat', 'nl'), ('niets', 'nl'), ('kunnen', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('iemand', 'nl'), ('doen', 'nl'), ('worden', 'nl'), ('door', 'nl'), ('haar', 'nl'), ('ge', 'nl'), ('tegen', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('veel', 'nl'), ('of', 'nl'), ('zij', 'nl'), ('uit', 'nl'), ('alles', 'nl'), ('toen', 'nl'), ('voor', 'nl'), ('hem', 'nl'), ('die', 'nl'), ('is', 'nl'), ('iets', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('daar', 'nl'), ('zelf', 'nl'), ('ze', 'nl'), ('hebben', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('andere', 'nl'), ('in', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('waren', 'nl'), ('geen', 'nl'), ('wezen', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('naar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('reeds', 'nl'), ('meer', 'nl'), ('en', 'nl'), ('heb', 'nl'), ('de', 'nl'), ('hier', 'nl'), ('bij', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('maar', 'nl'), ('op', 'nl'), ('eens', 'nl'), ('hun', 'nl'), ('toch', 'nl'), ('mij', 'nl'), ('geweest', 'nl'), ('tot', 'nl'), ('werd', 'nl'), ('ja', 'nl'), ('dat', 'nl'), ('moet', 'nl'), ('hij', 'nl'), ('ook', 'nl'), ('ben', 'nl'), ('had', 'nl'), ('te', 'nl'), ('hoe', 'nl'), ('nog', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('wordt', 'nl'), ('ik', 'nl'), ('was', 'nl'), ('van', 'nl');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('want', 'nl'), ('wil', 'nl'), ('kunn', 'nl'), ('kon', 'nl'), ('dor', 'nl'), ('heeft', 'nl'), ('nu', 'nl'), ('ons', 'nl'), ('onder', 'nl'), ('zijn', 'nl'), ('wie', 'nl'), ('wat', 'nl'), ('iet', 'nl'), ('zonder', 'nl'), ('zal', 'nl'), ('een', 'nl'), ('zo', 'nl'), ('al', 'nl'), ('kan', 'nl'), ('doen', 'nl'), ('iemand', 'nl'), ('ge', 'nl'), ('over', 'nl'), ('uw', 'nl'), ('doch', 'nl'), ('of', 'nl'), ('mer', 'nl'), ('uit', 'nl'), ('zij', 'nl'), ('hem', 'nl'), ('toen', 'nl'), ('alles', 'nl'), ('is', 'nl'), ('die', 'nl'), ('hebb', 'nl'), ('men', 'nl'), ('niet', 'nl'), ('zelf', 'nl'), ('vel', 'nl'), ('ze', 'nl'), ('nar', 'nl'), ('har', 'nl'), ('ander', 'nl'), ('dez', 'nl'), ('dus', 'nl'), ('na', 'nl'), ('teg', 'nl'), ('der', 'nl'), ('met', 'nl'), ('dan', 'nl'), ('je', 'nl'), ('u', 'nl'), ('in', 'nl'), ('red', 'nl'), ('het', 'nl'), ('zich', 'nl'), ('altijd', 'nl'), ('aan', 'nl'), ('er', 'nl'), ('mijn', 'nl'), ('mar', 'nl'), ('me', 'nl'), ('om', 'nl'), ('dar', 'nl'), ('heb', 'nl'), ('en', 'nl'), ('vor', 'nl'), ('de', 'nl'), ('word', 'nl'), ('bij', 'nl'), ('hier', 'nl'), ('dit', 'nl'), ('als', 'nl'), ('op', 'nl'), ('hun', 'nl'), ('mij', 'nl'), ('toch', 'nl'), ('war', 'nl'), ('tot', 'nl'), ('geweest', 'nl'), ('dat', 'nl'), ('ja', 'nl'), ('werd', 'nl'), ('ook', 'nl'), ('hij', 'nl'), ('moet', 'nl'), ('gen', 'nl'), ('had', 'nl'), ('ben', 'nl'), ('te', 'nl'), ('nog', 'nl'), ('hoe', 'nl'), ('zou', 'nl'), ('omdat', 'nl'), ('ik', 'nl'), ('wordt', 'nl'), ('was', 'nl'), ('van', 'nl'), ('wez', 'nl');INSERT INTO stopwords_long (stopword, language) VALUES ('fra', 'no'), ('som', 'no'), ('da', 'no'), ('ingen', 'no'), ('kvi', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('bare', 'no'), ('ville', 'no'), ('det', 'no'), ('alle', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('honom', 'no'), ('hoss', 'no'), ('ut', 'no'), ('somme', 'no'), ('men', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('vors', 'no'), ('han', 'no'), ('noka', 'no'), ('man', 'no'), ('hjå', 'no'), ('har', 'no'), ('deim', 'no'), ('eg', 'no'), ('samme', 'no'), ('elles', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('hennar', 'no'), ('seg', 'no'), ('vort', 'no'), ('inkje', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('mine', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('deira', 'no'), ('sånn', 'no'), ('å', 'no'), ('skal', 'no'), ('dei', 'no'), ('nå', 'no'), ('nokor', 'no'), ('hvordan', 'no'), ('i', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('dykkar', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('dette', 'no'), ('verte', 'no'), ('du', 'no'), ('vere', 'no'), ('vart', 'no'), ('deires', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('begge', 'no'), ('kunne', 'no'), ('med', 'no'), ('kan', 'no'), ('varte', 'no'), ('somt', 'no'), ('før', 'no'), ('di', 'no'), ('inni', 'no'), ('siden', 'no'), ('noko', 'no'), ('ikkje', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('også', 'no'), ('dykk', 'no'), ('ble', 'no'), ('etter', 'no'), ('den', 'no'), ('vore', 'no'), ('hvilke', 'no'), ('nokon', 'no'), ('deg', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('hans', 'no'), ('denne', 'no'), ('ingi', 'no'), ('være', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('hadde', 'no'), ('vår', 'no'), ('så', 'no'), ('nokre', 'no'), ('hvor', 'no'), ('både', 'no'), ('so', 'no'), ('et', 'no'), ('hvilken', 'no'), ('henne', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('dere', 'no'), ('disse', 'no'), ('jeg', 'no'), ('en', 'no'), ('mange', 'no'), ('de', 'no'), ('meget', 'no'), ('deres', 'no'), ('din', 'no'), ('hun', 'no'), ('medan', 'no'), ('uten', 'no'), ('mykje', 'no'), ('hennes', 'no'), ('noen', 'no'), ('ikke', 'no'), ('sine', 'no'), ('kun', 'no'), ('skulle', 'no'), ('ja', 'no'), ('at', 'no'), ('hossen', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('inn', 'no'), ('eller', 'no'), ('hvorfor', 'no'), ('mot', 'no');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('sid', 'no'), ('fra', 'no'), ('som', 'no'), ('kunn', 'no'), ('da', 'no'), ('dett', 'no'), ('bar', 'no'), ('kvi', 'no'), ('mykj', 'no'), ('hver', 'no'), ('til', 'no'), ('vært', 'no'), ('selv', 'no'), ('bli', 'no'), ('det', 'no'), ('når', 'no'), ('ditt', 'no'), ('over', 'no'), ('av', 'no'), ('henn', 'no'), ('honom', 'no'), ('nok', 'no'), ('all', 'no'), ('hoss', 'no'), ('ut', 'no'), ('men', 'no'), ('skull', 'no'), ('ned', 'no'), ('på', 'no'), ('mi', 'no'), ('fordi', 'no'), ('hadd', 'no'), ('han', 'no'), ('man', 'no'), ('har', 'no'), ('hjå', 'no'), ('deir', 'no'), ('deim', 'no'), ('nokr', 'no'), ('eg', 'no'), ('um', 'no'), ('no', 'no'), ('ein', 'no'), ('og', 'no'), ('ho', 'no'), ('hvis', 'no'), ('blir', 'no'), ('seg', 'no'), ('vort', 'no'), ('si', 'no'), ('eitt', 'no'), ('eit', 'no'), ('me', 'no'), ('om', 'no'), ('meg', 'no'), ('dem', 'no'), ('sin', 'no'), ('sjøl', 'no'), ('hvem', 'no'), ('hva', 'no'), ('sia', 'no'), ('kvar', 'no'), ('ha', 'no'), ('sånn', 'no'), ('skal', 'no'), ('å', 'no'), ('dei', 'no'), ('nå', 'no'), ('i', 'no'), ('hvordan', 'no'), ('nokor', 'no'), ('mitt', 'no'), ('båe', 'no'), ('blei', 'no'), ('oss', 'no'), ('kven', 'no'), ('sidan', 'no'), ('ikkj', 'no'), ('du', 'no'), ('vart', 'no'), ('begg', 'no'), ('mellom', 'no'), ('vi', 'no'), ('ved', 'no'), ('hvilk', 'no'), ('med', 'no'), ('kan', 'no'), ('somt', 'no'), ('vær', 'no'), ('båd', 'no'), ('før', 'no'), ('di', 'no'), ('ell', 'no'), ('inni', 'no'), ('noko', 'no'), ('vil', 'no'), ('kva', 'no'), ('noe', 'no'), ('var', 'no'), ('korleis', 'no'), ('her', 'no'), ('korso', 'no'), ('kom', 'no'), ('ett', 'no'), ('opp', 'no'), ('dykk', 'no'), ('ver', 'no'), ('også', 'no'), ('ble', 'no'), ('denn', 'no'), ('vill', 'no'), ('den', 'no'), ('ikk', 'no'), ('nokon', 'no'), ('deg', 'no'), ('samm', 'no'), ('slik', 'no'), ('der', 'no'), ('kvifor', 'no'), ('inkj', 'no'), ('ingi', 'no'), ('då', 'no'), ('upp', 'no'), ('enn', 'no'), ('vår', 'no'), ('så', 'no'), ('hvor', 'no'), ('mang', 'no'), ('et', 'no'), ('so', 'no'), ('er', 'no'), ('for', 'no'), ('blitt', 'no'), ('ing', 'no'), ('jeg', 'no'), ('en', 'no'), ('vor', 'no'), ('de', 'no'), ('vert', 'no'), ('din', 'no'), ('hun', 'no'), ('uten', 'no'), ('medan', 'no'), ('noen', 'no'), ('diss', 'no'), ('kun', 'no'), ('ja', 'no'), ('at', 'no'), ('kvarhelst', 'no'), ('min', 'no'), ('sitt', 'no'), ('hoe', 'no'), ('hvorfor', 'no'), ('inn', 'no'), ('mot', 'no'), ('somm', 'no');INSERT INTO stopwords_long (stopword, language) VALUES ('terá', 'pt'), ('havemos', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('da', 'pt'), ('houvéramos', 'pt'), ('para', 'pt'), ('nosso', 'pt'), ('pelos', 'pt'), ('nos', 'pt'), ('são', 'pt'), ('há', 'pt'), ('tenham', 'pt'), ('delas', 'pt'), ('mais', 'pt'), ('e', 'pt'), ('fôramos', 'pt'), ('estávamos', 'pt'), ('isso', 'pt'), ('nem', 'pt'), ('formos', 'pt'), ('hão', 'pt'), ('pelas', 'pt'), ('em', 'pt'), ('não', 'pt'), ('estejam', 'pt'), ('tive', 'pt'), ('seremos', 'pt'), ('sem', 'pt'), ('houvera', 'pt'), ('está', 'pt'), ('houver', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('aquilo', 'pt'), ('houveremos', 'pt'), ('fossem', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('terão', 'pt'), ('por', 'pt'), ('das', 'pt'), ('vocês', 'pt'), ('essa', 'pt'), ('estivera', 'pt'), ('essas', 'pt'), ('houvermos', 'pt'), ('nossa', 'pt'), ('muito', 'pt'), ('tivéramos', 'pt'), ('seria', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('se', 'pt'), ('dos', 'pt'), ('hajamos', 'pt'), ('com', 'pt'), ('na', 'pt'), ('tivéssemos', 'pt'), ('a', 'pt'), ('foram', 'pt'), ('estão', 'pt'), ('um', 'pt'), ('no', 'pt'), ('tenhamos', 'pt'), ('os', 'pt'), ('hajam', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('aquelas', 'pt'), ('aqueles', 'pt'), ('tivemos', 'pt'), ('estes', 'pt'), ('teriam', 'pt'), ('era', 'pt'), ('esteve', 'pt'), ('nossas', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('serei', 'pt'), ('terei', 'pt'), ('tem', 'pt'), ('teve', 'pt'), ('houverá', 'pt'), ('tivesse', 'pt'), ('tenha', 'pt'), ('fôssemos', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('serão', 'pt'), ('este', 'pt'), ('seríamos', 'pt'), ('estou', 'pt'), ('estas', 'pt'), ('só', 'pt'), ('lhe', 'pt'), ('houveria', 'pt'), ('houveram', 'pt'), ('pela', 'pt'), ('tivessem', 'pt'), ('nossos', 'pt'), ('teu', 'pt'), ('tiver', 'pt'), ('lhes', 'pt'), ('tivermos', 'pt'), ('éramos', 'pt'), ('estivéramos', 'pt'), ('meu', 'pt'), ('sejamos', 'pt'), ('estiver', 'pt'), ('estiveram', 'pt'), ('as', 'pt'), ('houveriam', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('houvesse', 'pt'), ('houverei', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quando', 'pt'), ('sejam', 'pt'), ('hei', 'pt'), ('houvessem', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('pelo', 'pt'), ('minhas', 'pt'), ('fora', 'pt'), ('houveríamos', 'pt'), ('será', 'pt'), ('esse', 'pt'), ('houvéssemos', 'pt'), ('temos', 'pt'), ('num', 'pt'), ('estivessem', 'pt'), ('do', 'pt'), ('tinha', 'pt'), ('aquela', 'pt'), ('já', 'pt'), ('fomos', 'pt'), ('teria', 'pt'), ('eu', 'pt'), ('tenho', 'pt'), ('numa', 'pt'), ('elas', 'pt'), ('deles', 'pt'), ('haja', 'pt'), ('esteja', 'pt'), ('estejamos', 'pt'), ('entre', 'pt'), ('estivemos', 'pt'), ('somos', 'pt'), ('houverão', 'pt'), ('teríamos', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('tivera', 'pt'), ('fosse', 'pt'), ('tiverem', 'pt'), ('dela', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('houvemos', 'pt'), ('teus', 'pt'), ('estive', 'pt'), ('tínhamos', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('mesmo', 'pt'), ('ela', 'pt'), ('teremos', 'pt'), ('seja', 'pt'), ('de', 'pt'), ('estivesse', 'pt'), ('dele', 'pt'), ('estamos', 'pt'), ('houve', 'pt'), ('nas', 'pt'), ('estavam', 'pt'), ('tinham', 'pt'), ('uma', 'pt'), ('aquele', 'pt'), ('como', 'pt'), ('estava', 'pt'), ('forem', 'pt'), ('esses', 'pt'), ('depois', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('estivéssemos', 'pt'), ('estiverem', 'pt'), ('houverem', 'pt'), ('minha', 'pt'), ('à', 'pt'), ('estivermos', 'pt'), ('tiveram', 'pt'), ('seriam', 'pt'), ('você', 'pt');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('tev', 'pt'), ('som', 'pt'), ('às', 'pt'), ('quem', 'pt'), ('form', 'pt'), ('da', 'pt'), ('nos', 'pt'), ('estivéss', 'pt'), ('há', 'pt'), ('mesm', 'pt'), ('terã', 'pt'), ('mais', 'pt'), ('tenh', 'pt'), ('e', 'pt'), ('haj', 'pt'), ('isso', 'pt'), ('hav', 'pt'), ('nem', 'pt'), ('em', 'pt'), ('aquel', 'pt'), ('sem', 'pt'), ('está', 'pt'), ('ao', 'pt'), ('sua', 'pt'), ('minh', 'pt'), ('seus', 'pt'), ('tua', 'pt'), ('por', 'pt'), ('das', 'pt'), ('tiv', 'pt'), ('essa', 'pt'), ('vocês', 'pt'), ('serã', 'pt'), ('essas', 'pt'), ('seu', 'pt'), ('meus', 'pt'), ('muit', 'pt'), ('dos', 'pt'), ('se', 'pt'), ('na', 'pt'), ('com', 'pt'), ('a', 'pt'), ('um', 'pt'), ('no', 'pt'), ('estam', 'pt'), ('estej', 'pt'), ('os', 'pt'), ('tuas', 'pt'), ('tu', 'pt'), ('tính', 'pt'), ('estes', 'pt'), ('era', 'pt'), ('mas', 'pt'), ('me', 'pt'), ('tem', 'pt'), ('que', 'pt'), ('vos', 'pt'), ('este', 'pt'), ('só', 'pt'), ('estas', 'pt'), ('estou', 'pt'), ('par', 'pt'), ('lhe', 'pt'), ('estáv', 'pt'), ('teu', 'pt'), ('estav', 'pt'), ('lhes', 'pt'), ('estã', 'pt'), ('meu', 'pt'), ('as', 'pt'), ('aos', 'pt'), ('eles', 'pt'), ('o', 'pt'), ('suas', 'pt'), ('sej', 'pt'), ('também', 'pt'), ('isto', 'pt'), ('nós', 'pt'), ('quand', 'pt'), ('tinh', 'pt'), ('hei', 'pt'), ('tém', 'pt'), ('eram', 'pt'), ('esse', 'pt'), ('num', 'pt'), ('do', 'pt'), ('noss', 'pt'), ('já', 'pt'), ('eu', 'pt'), ('elas', 'pt'), ('ter', 'pt'), ('ser', 'pt'), ('del', 'pt'), ('sã', 'pt'), ('hã', 'pt'), ('éram', 'pt'), ('entre', 'pt'), ('tivéss', 'pt'), ('depo', 'pt'), ('houv', 'pt'), ('fom', 'pt'), ('até', 'pt'), ('foi', 'pt'), ('pel', 'pt'), ('ele', 'pt'), ('for', 'pt'), ('qual', 'pt'), ('teus', 'pt'), ('estev', 'pt'), ('sou', 'pt'), ('esta', 'pt'), ('ela', 'pt'), ('de', 'pt'), ('nas', 'pt'), ('uma', 'pt'), ('foss', 'pt'), ('fôss', 'pt'), ('nã', 'pt'), ('esses', 'pt'), ('fôr', 'pt'), ('aquil', 'pt'), ('ou', 'pt'), ('fui', 'pt'), ('te', 'pt'), ('voc', 'pt'), ('estiv', 'pt'), ('à', 'pt'), ('houvéss', 'pt');INSERT INTO stopwords_long (stopword, language) VALUES ('încă', 'ro'), ('toţi', 'ro'), ('ne', 'ro'), ('multă', 'ro'), ('înseşi', 'ro'), ('oricât', 'ro'), ('totuşi', 'ro'), ('puţini', 'ro'), ('înşine', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('veţi', 'ro'), ('unii', 'ro'), ('altcineva', 'ro'), ('aceiaşi', 'ro'), ('atât', 'ro'), ('parte', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('primii', 'ro'), ('doime', 'ro'), ('sine ', 'ro'), ('oi', 'ro'), ('tău', 'ro'), ('câtă', 'ro'), ('unei', 'ro'), ('mare', 'ro'), ('însăşi', 'ro'), ('celorlalte', 'ro'), ('oricăruia', 'ro'), ('celei', 'ro'), ('mi', 'ro'), ('acelaşi', 'ro'), ('ălora', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('cine', 'ro'), ('puţină', 'ro'), ('primelor', 'ro'), ('alea', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('suntem', 'ro'), ('acelui', 'ro'), ('însele', 'ro'), ('una', 'ro'), ('însămi', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('foarte', 'ro'), ('acea', 'ro'), ('mine', 'ro'), ('puţine', 'ro'), ('şapte', 'ro'), ('multe', 'ro'), ('atâţi', 'ro'), ('care', 'ro'), ('cei', 'ro'), ('oricâtă', 'ro'), ('am', 'ro'), ('acelei', 'ro'), ('lor', 'ro'), ('dintre', 'ro'), ('celorlalţi', 'ro'), ('alţii', 'ro'), ('alt', 'ro'), ('orice', 'ro'), ('vreunii', 'ro'), ('aceasta', 'ro'), ('alta', 'ro'), ('aceşti', 'ro'), ('eşti', 'ro'), ('amândoi', 'ro'), ('noastră', 'ro'), ('meu', 'ro'), ('unul', 'ro'), ('fiţi', 'ro'), ('fiind', 'ro'), ('ambele', 'ro'), ('altora', 'ro'), ('celuilalt', 'ro'), ('aceea', 'ro'), ('spre', 'ro'), ('o', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('fiecărei', 'ro'), ('vreuna', 'ro'), ('bine', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('vreunele', 'ro'), ('alte', 'ro'), ('celelalte', 'ro'), ('eram', 'ro'), ('celălalt', 'ro'), ('există', 'ro'), ('amândurora', 'ro'), ('al', 'ro'), ('numai', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('aceştia', 'ro'), ('spune', 'ro'), ('cutare', 'ro'), ('fiu', 'ro'), ('câţi', 'ro'), ('mult', 'ro'), ('altcuiva ', 'ro'), ('oricui', 'ro'), ('ea', 'ro'), ('cinci', 'ro'), ('aceloraşi', 'ro'), ('vreunuia', 'ro'), ('mei', 'ro'), ('deşi', 'ro'), ('mele', 'ro'), ('primul', 'ro'), ('însevă', 'ro'), ('vreuneia', 'ro'), ('acelor', 'ro'), ('câte', 'ro'), ('oricăreia', 'ro'), ('ca', 'ro'), ('oricâtor', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('primilor', 'ro'), ('oricărora', 'ro'), ('aceia', 'ro'), ('câteva', 'ro'), ('fost', 'ro'), ('vreunul', 'ro'), ('vreo', 'ro'), ('ăstora', 'ro'), ('despre', 'ro'), ('după', 'ro'), ('însăţi', 'ro'), ('fii', 'ro'), ('destui', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('nimeni', 'ro'), ('noştri', 'ro'), ('oricine', 'ro'), ('aceleaşi', 'ro'), ('oţi', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('ăia', 'ro'), ('vor', 'ro'), ('fără', 'ro'), ('de', 'ro'), ('atâtora', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('întâiul', 'ro'), ('ăştia', 'ro'), ('faţă', 'ro'), ('ambii', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('sale', 'ro'), ('nişte', 'ro'), ('trei', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('atunci', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('că', 'ro'), ('eraţi', 'ro'), ('vreunora', 'ro'), ('cele', 'ro'), ('ambelor', 'ro'), ('ar', 'ro'), ('atâtor', 'ro'), ('cineva', 'ro'), ('iar', 'ro'), ('deci', 'ro'), ('îndoit', 'ro'), ('voi', 'ro'), ('avut', 'ro'), ('însuşi', 'ro'), ('cărei', 'ro'), ('până', 'ro'), ('aceluiaşi', 'ro'), ('căror ', 'ro'), ('cât', 'ro'), ('noua', 'ro'), ('altceva', 'ro'), ('le', 'ro'), ('astea', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('oricâte', 'ro'), ('ci', 'ro'), ('cel', 'ro'), ('acele', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('sunteţi', 'ro'), ('ăla', 'ro'), ('alteia', 'ro'), ('câtorva', 'ro'), ('ambilor', 'ro'), ('acesteia', 'ro'), ('asupra', 'ro'), ('putea', 'ro'), ('pentru', 'ro'), ('va', 'ro'), ('aveaţi', 'ro'), ('când', 'ro'), ('aveai', 'ro'), ('aproape', 'ro'), ('la', 'ro'), ('celui', 'ro'), ('aceleia', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('oricărui', 'ro'), ('puţin', 'ro'), ('ăsta', 'ro'), ('acei', 'ro'), ('întâia', 'ro'), ('oricâţi', 'ro'), ('acestea', 'ro'), ('dacă', 'ro'), ('acestei', 'ro'), ('însumi', 'ro'), ('treime', 'ro'), ('oricare', 'ro'), ('altuia', 'ro'), ('unde', 'ro'), ('altei', 'ro'), ('înşivă', 'ro'), ('altele', 'ro'), ('fiecare', 'ro'), ('ăluia', 'ro'), ('erai', 'ro'), ('îşi', 'ro'), ('aceeaşi', 'ro'), ('fiecăreia', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('acesta', 'ro'), ('primului', 'ro'), ('s', 'ro'), ('această', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('primei', 'ro'), ('trebuie', 'ro'), ('primele', 'ro'), ('este', 'ro'), ('înşişi', 'ro'), ('însutit', 'ro'), ('patru', 'ro'), ('nimănui', 'ro'), ('ei', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('au', 'ro'), ('voastre', 'ro'), ('şase', 'ro'), ('noastre', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('doar', 'ro'), ('ăsteia', 'ro'), ('oarecare', 'ro'), ('atâta', 'ro'), ('astfel', 'ro'), ('oricâtora', 'ro'), ('întreit', 'ro'), ('acelea', 'ro'), ('poate', 'ro'), ('chiar', 'ro'), ('unuia', 'ro'), ('nimic', 'ro'), ('acela', 'ro'), ('atâţia', 'ro'), ('dumneavoastră', 'ro'), ('către', 'ro'), ('fiecăruia', 'ro'), ('ţie', 'ro'), ('destule', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('ceva', 'ro'), ('sutime', 'ro'), ('voastră ', 'ro'), ('cealaltă', 'ro'), ('aceste', 'ro'), ('aceleiaşi', 'ro'), ('face', 'ro'), ('aş', 'ro'), ('toată', 'ro'), ('cu', 'ro'), ('ăstuia', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('mulţi', 'ro'), ('cum', 'ro'), ('peste', 'ro'), ('acelora', 'ro'), ('cărui', 'ro'), ('vreunui', 'ro'), ('nouă', 'ro'), ('lui', 'ro'), ('voştri', 'ro'), ('unele', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('uneia', 'ro'), ('unora ', 'ro'), ('prima', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('atâtea', 'ro'), ('zece', 'ro'), ('aşadar', 'ro'), ('ieri', 'ro'), ('tăi', 'ro'), ('destulă', 'ro'), ('or', 'ro'), ('acestor', 'ro'), ('acel', 'ro'), ('avea', 'ro'), ('câtor', 'ro'), ('câtva', 'ro'), ('v', 'ro'), ('dar', 'ro'), ('totul', 'ro'), ('însuţi', 'ro'), ('câţiva', 'ro'), ('spus', 'ro'), ('doua', 'ro'), ('câtora', 'ro'), ('din', 'ro'), ('săi', 'ro'), ('însene', 'ro'), ('acestuia', 'ro'), ('altor', 'ro'), ('acestora', 'ro'), ('ţi', 'ro'), ('are', 'ro'), ('ceilalţi', 'ro'), ('ce', 'ro'), ('amândouă', 'ro'), ('unu', 'ro'), ('aia', 'ro'), ('vouă', 'ro'), ('oricăror', 'ro'), ('sieşi', 'ro'), ('nici', 'ro'), ('printre', 'ro'), ('decât', 'ro'), ('avem', 'ro'), ('toate', 'ro'), ('ale ', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('aveţi', 'ro'), ('celeilalte', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('aveam', 'ro'), ('tuturor', 'ro'), ('aţi', 'ro'), ('tale', 'ro'), ('tine', 'ro'), ('lângă', 'ro'), ('el', 'ro'), ('altul', 'ro'), ('ăleia', 'ro');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ne', 'ro'), ('încă', 'ro'), ('oricât', 'ro'), ('put', 'ro'), ('vom', 'ro'), ('e', 'ro'), ('nou', 'ro'), ('atât', 'ro'), ('noi', 'ro'), ('înspre', 'ro'), ('şas', 'ro'), ('sine ', 'ro'), ('tău', 'ro'), ('oi', 'ro'), ('oricin', 'ro'), ('dou', 'ro'), ('unei', 'ro'), ('aproap', 'ro'), ('amândou', 'ro'), ('celeilalt', 'ro'), ('mi', 'ro'), ('sal', 'ro'), ('mel', 'ro'), ('part', 'ro'), ('poat', 'ro'), ('oricăr', 'ro'), ('aceleaş', 'ro'), ('noastr', 'ro'), ('asupr', 'ro'), ('dup', 'ro'), ('se', 'ro'), ('ai', 'ro'), ('bin', 'ro'), ('a', 'ro'), ('sunt', 'ro'), ('prim', 'ro'), ('cev', 'ro'), ('acelui', 'ro'), ('celu', 'ro'), ('înşin', 'ro'), ('una', 'ro'), ('toţ', 'ro'), ('încât', 'ro'), ('om', 'ro'), ('aveau', 'ro'), ('altu', 'ro'), ('fi', 'ro'), ('mai', 'ro'), ('mea', 'ro'), ('însă', 'ro'), ('li', 'ro'), ('însut', 'ro'), ('ambe', 'ro'), ('ălui', 'ro'), ('dumneavoastr', 'ro'), ('cei', 'ro'), ('acelei', 'ro'), ('am', 'ro'), ('lor', 'ro'), ('celelalt', 'ro'), ('ace', 'ro'), ('alt', 'ro'), ('car', 'ro'), ('alta', 'ro'), ('eşti', 'ro'), ('spun', 'ro'), ('meu', 'ro'), ('foart', 'ro'), ('căru', 'ro'), ('fiind', 'ro'), ('o', 'ro'), ('spre', 'ro'), ('celuilalt', 'ro'), ('căre', 'ro'), ('amânduror', 'ro'), ('vă', 'ro'), ('sau', 'ro'), ('altcinev', 'ro'), ('fiecărei', 'ro'), ('noştr', 'ro'), ('însăş', 'ro'), ('vi', 'ro'), ('făcut', 'ro'), ('alte', 'ro'), ('însăm', 'ro'), ('eram', 'ro'), ('aceşt', 'ro'), ('celălalt', 'ro'), ('toat', 'ro'), ('al', 'ro'), ('înşiş', 'ro'), ('vreun', 'ro'), ('prin', 'ro'), ('celorlalţ', 'ro'), ('fiu', 'ro'), ('mult', 'ro'), ('câţiv', 'ro'), ('altcuiva ', 'ro'), ('câtev', 'ro'), ('vou', 'ro'), ('ea', 'ro'), ('numa', 'ro'), ('tin', 'ro'), ('mei', 'ro'), ('celorlalt', 'ro'), ('acelor', 'ro'), ('ca', 'ro'), ('ăşti', 'ro'), ('oricâtor', 'ro'), ('însum', 'ro'), ('oricărei', 'ro'), ('mă', 'ro'), ('altă', 'ro'), ('şapt', 'ro'), ('deş', 'ro'), ('fost', 'ro'), ('cin', 'ro'), ('aceast', 'ro'), ('vreo', 'ro'), ('fii', 'ro'), ('tal', 'ro'), ('acest', 'ro'), ('sa', 'ro'), ('înşiv', 'ro'), ('ăstui', 'ro'), ('fiecărui', 'ro'), ('ele', 'ro'), ('oţi', 'ro'), ('ăia', 'ro'), ('însuţ', 'ro'), ('vor', 'ro'), ('de', 'ro'), ('un', 'ro'), ('cea', 'ro'), ('vei', 'ro'), ('îţi', 'ro'), ('aşa', 'ro'), ('fim', 'ro'), ('însăţ', 'ro'), ('oric', 'ro'), ('tot', 'ro'), ('acestui', 'ro'), ('unui', 'ro'), ('aceeaş', 'ro'), ('unor', 'ro'), ('dat', 'ro'), ('ăstor', 'ro'), ('că', 'ro'), ('eraţ', 'ro'), ('cele', 'ro'), ('min', 'ro'), ('înseş', 'ro'), ('ar', 'ro'), ('cutar', 'ro'), ('atâtor', 'ro'), ('iar', 'ro'), ('voi', 'ro'), ('îndoit', 'ro'), ('avut', 'ro'), ('cât', 'ro'), ('căror ', 'ro'), ('le', 'ro'), ('mie', 'ro'), ('nostru', 'ro'), ('nu', 'ro'), ('cel', 'ro'), ('ci', 'ro'), ('acele', 'ro'), ('prime', 'ro'), ('ta', 'ro'), ('doi', 'ro'), ('îi', 'ro'), ('aveţ', 'ro'), ('ăla', 'ro'), ('oricăre', 'ro'), ('tre', 'ro'), ('câţ', 'ro'), ('pentru', 'ro'), ('nic', 'ro'), ('va', 'ro'), ('când', 'ro'), ('la', 'ro'), ('aste', 'ro'), ('oricărui', 'ro'), ('veţ', 'ro'), ('erau', 'ro'), ('în', 'ro'), ('puţin', 'ro'), ('amândo', 'ro'), ('acei', 'ro'), ('ăsta', 'ro'), ('totuş', 'ro'), ('exist', 'ro'), ('zec', 'ro'), ('ale', 'ro'), ('acestei', 'ro'), ('înse', 'ro'), ('atunc', 'ro'), ('ier', 'ro'), ('altei', 'ro'), ('unde', 'ro'), ('pân', 'ro'), ('ambi', 'ro'), ('îşi', 'ro'), ('uni', 'ro'), ('tu', 'ro'), ('pot', 'ro'), ('cui', 'ro'), ('era', 'ro'), ('s', 'ro'), ('cuiva ', 'ro'), ('ori', 'ro'), ('îl', 'ro'), ('ave', 'ro'), ('însen', 'ro'), ('cinc', 'ro'), ('este', 'ro'), ('oarec', 'ro'), ('câtorv', 'ro'), ('patru', 'ro'), ('ei', 'ro'), ('vreune', 'ro'), ('îmi', 'ro'), ('său', 'ro'), ('alţi', 'ro'), ('despr', 'ro'), ('sutim', 'ro'), ('au', 'ro'), ('fac', 'ro'), ('şi', 'ro'), ('i', 'ro'), ('asta', 'ro'), ('cealalt', 'ro'), ('celor', 'ro'), ('m', 'ro'), ('altcev', 'ro'), ('ăstei', 'ro'), ('doar', 'ro'), ('nişt', 'ro'), ('întâi', 'ro'), ('astfel', 'ro'), ('cinev', 'ro'), ('întreit', 'ro'), ('ălor', 'ro'), ('une', 'ro'), ('chiar', 'ro'), ('fiec', 'ro'), ('nimic', 'ro'), ('făr', 'ro'), ('treim', 'ro'), ('pest', 'ro'), ('dintr', 'ro'), ('ţie', 'ro'), ('ceilalţ', 'ro'), ('destul', 'ro'), ('fie', 'ro'), ('vostru', 'ro'), ('dest', 'ro'), ('voastră ', 'ro'), ('cătr', 'ro'), ('dec', 'ro'), ('aceste', 'ro'), ('aş', 'ro'), ('cu', 'ro'), ('eu', 'ro'), ('ni', 'ro'), ('câtv', 'ro'), ('cum', 'ro'), ('lâng', 'ro'), ('mulţ', 'ro'), ('vreunui', 'ro'), ('lui', 'ro'), ('fiţ', 'ro'), ('pe', 'ro'), ('altui', 'ro'), ('unora ', 'ro'), ('printr', 'ro'), ('acum', 'ro'), ('opt', 'ro'), ('aşadar', 'ro'), ('tăi', 'ro'), ('ălei', 'ro'), ('acestor', 'ro'), ('or', 'ro'), ('acel', 'ro'), ('nimăn', 'ro'), ('sieş', 'ro'), ('câtor', 'ro'), ('avea', 'ro'), ('mar', 'ro'), ('atâţ', 'ro'), ('v', 'ro'), ('nimen', 'ro'), ('faţ', 'ro'), ('dar', 'ro'), ('însev', 'ro'), ('spus', 'ro'), ('voştr', 'ro'), ('din', 'ro'), ('voastr', 'ro'), ('săi', 'ro'), ('altor', 'ro'), ('ţi', 'ro'), ('fiecăr', 'ro'), ('trebui', 'ro'), ('are', 'ro'), ('doim', 'ro'), ('ce', 'ro'), ('aia', 'ro'), ('fiecăre', 'ro'), ('unu', 'ro'), ('oricăror', 'ro'), ('decât', 'ro'), ('ale ', 'ro'), ('avem', 'ro'), ('sie', 'ro'), ('te', 'ro'), ('vreunei', 'ro'), ('l', 'ro'), ('dac', 'ro'), ('vreunor', 'ro'), ('sub', 'ro'), ('tuturor', 'ro'), ('aveam', 'ro'), ('aţi', 'ro'), ('el', 'ro'), ('însuş', 'ro');INSERT INTO stopwords_long (stopword, language) VALUES ('sid', 'ru'), ('группа', 'ru'), ('каким', 'ru'), ('при', 'ru'), ('кстате', 'ru'), ('знаю', 'ru'), ('мог', 'ru'), ('как', 'ru'), ('помощи', 'ru'), ('для', 'ru'), ('над', 'ru'), ('дело', 'ru'), ('завода', 'ru'), ('нельзя', 'ru'), ('место', 'ru'), ('последний', 'ru'), ('последнее', 'ru'), ('проект', 'ru'), ('вообще', 'ru'), ('еще', 'ru'), ('дать', 'ru'), ('хорошо', 'ru'), ('века', 'ru'), ('лишь', 'ru'), ('Чтобы', 'ru'), ('жизни', 'ru'), ('недавно', 'ru'), ('него', 'ru'), ('будет', 'ru'), ('каждая', 'ru'), ('Пресс', 'ru'), ('года', 'ru'), ('срок', 'ru'), ('НЕ', 'ru'), ('новой', 'ru'), ('количество', 'ru'), ('февраля', 'ru'), ('руководитель', 'ru'), ('главы', 'ru'), ('закон', 'ru'), ('похоже', 'ru'), ('одну', 'ru'), ('жителей', 'ru'), ('семьи', 'ru'), ('коп', 'ru'), ('Если', 'ru'), ('Я', 'ru'), ('мая', 'ru'), ('ранее', 'ru'), ('естественно', 'ru'), ('быстро', 'ru'), ('меня', 'ru'), ('представителей', 'ru'), ('первой', 'ru'), ('http', 'ru'), ('дом', 'ru'), ('они', 'ru'), ('новостей', 'ru'), ('вице', 'ru'), ('Когда', 'ru'), ('понять', 'ru'), ('становится', 'ru'), ('кто', 'ru'), ('корреспонденту', 'ru'), ('имеют', 'ru'), ('В', 'ru'), ('начальник', 'ru'), ('которой', 'ru'), ('неделю', 'ru'), ('это', 'ru'), ('мне', 'ru'), ('части', 'ru'), ('если', 'ru'), ('дня', 'ru'), ('этой', 'ru'), ('livejournal', 'ru'), ('не', 'ru'), ('часть', 'ru'), ('заявление', 'ru'), ('того', 'ru'), ('председателя', 'ru'), ('период', 'ru'), ('друга', 'ru'), ('нему', 'ru'), ('рук', 'ru'), ('А', 'ru'), ('каждого', 'ru'), ('премьера', 'ru'), ('доме', 'ru'), ('мере', 'ru'), ('свой', 'ru'), ('история', 'ru'), ('который', 'ru'), ('раньше', 'ru'), ('деле', 'ru'), ('Мы', 'ru'), ('главный', 'ru'), ('получили', 'ru'), ('работе', 'ru'), ('решили', 'ru'), ('нужно', 'ru'), ('ведь', 'ru'), ('лучше', 'ru'), ('делаем', 'ru'), ('лично', 'ru'), ('перед', 'ru'), ('прямо', 'ru'), ('около', 'ru'), ('места', 'ru'), ('первых', 'ru'), ('игры', 'ru'), ('об', 'ru'), ('уровне', 'ru'), ('главным', 'ru'), ('Причем', 'ru'), ('Тем', 'ru'), ('свет', 'ru'), ('их', 'ru'), ('хочет', 'ru'), ('стал', 'ru'), ('целом', 'ru'), ('И', 'ru'), ('чего', 'ru'), ('необходимо', 'ru'), ('одной', 'ru'), ('представители', 'ru'), ('П', 'ru'), ('т', 'ru'), ('более', 'ru'), ('про', 'ru'), ('вы', 'ru'), ('главного', 'ru'), ('октября', 'ru'), ('массовой', 'ru'), ('обычно', 'ru'), ('апрель', 'ru'), ('сообщил', 'ru'), ('чтобы', 'ru'), ('второй', 'ru'), ('одна', 'ru'), ('и', 'ru'), ('интересы', 'ru'), ('он', 'ru'), ('могли', 'ru'), ('никаких', 'ru'), ('лица', 'ru'), ('Ну', 'ru'), ('ее', 'ru'), ('совершенно', 'ru'), ('руководителей', 'ru'), ('процесс', 'ru'), ('Борис', 'ru'), ('самый', 'ru'), ('руках', 'ru'), ('Кроме', 'ru'), ('едва', 'ru'), ('жить', 'ru'), ('человека', 'ru'), ('районе', 'ru'), ('которым', 'ru'), ('возможность', 'ru'), ('уж', 'ru'), ('возможности', 'ru'), ('выборах', 'ru'), ('От', 'ru'), ('сообщили', 'ru'), ('точнее', 'ru'), ('отдела', 'ru'), ('начале', 'ru'), ('столько', 'ru'), ('комментариев', 'ru'), ('его', 'ru'), ('свои', 'ru'), ('партия', 'ru'), ('новые', 'ru'), ('суббота', 'ru'), ('Об', 'ru'), ('прежде', 'ru'), ('трех', 'ru'), ('без', 'ru'), ('Тогда', 'ru'), ('О', 'ru'), ('трудно', 'ru'), ('найти', 'ru'), ('руки', 'ru'), ('кстати', 'ru'), ('хотя', 'ru'), ('ним', 'ru'), ('adriver', 'ru'), ('вопрос', 'ru'), ('Среди', 'ru'), ('декабря', 'ru'), ('amp', 'ru'), ('где', 'ru'), ('была', 'ru'), ('правда', 'ru'), ('вместе', 'ru'), ('Иллюстрация', 'ru'), ('власть', 'ru'), ('я', 'ru'), ('утверждают', 'ru'), ('Г', 'ru'), ('нибудь', 'ru'), ('против', 'ru'), ('месяца', 'ru'), ('Во', 'ru'), ('значит', 'ru'), ('встречи', 'ru'), ('несколько', 'ru'), ('никогда', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('Уже', 'ru'), ('остается', 'ru'), ('своими', 'ru'), ('Вот', 'ru'), ('оказались', 'ru'), ('Ю', 'ru'), ('скорее', 'ru'), ('практически', 'ru'), ('конечно', 'ru'), ('команды', 'ru'), ('почта', 'ru'), ('взять', 'ru'), ('многие', 'ru'), ('иметь', 'ru'), ('самой', 'ru'), ('городе', 'ru'), ('любая', 'ru'), ('из', 'ru'), ('Эти', 'ru'), ('Ассошиэйтед', 'ru'), ('пока', 'ru'), ('Нет', 'ru'), ('получил', 'ru'), ('Теперь', 'ru'), ('есть', 'ru'), ('также', 'ru'), ('от', 'ru'), ('всего', 'ru'), ('только', 'ru'), ('разных', 'ru'), ('одновременно', 'ru'), ('самое', 'ru'), ('одном', 'ru'), ('эта', 'ru'), ('этому', 'ru'), ('у', 'ru'), ('путь', 'ru'), ('Дело', 'ru'), ('должна', 'ru'), ('фонда', 'ru'), ('мировой', 'ru'), ('cgi', 'ru'), ('уже', 'ru'), ('быть', 'ru'), ('факт', 'ru'), ('жизнь', 'ru'), ('своим', 'ru'), ('Л', 'ru'), ('сборной', 'ru'), ('июля', 'ru'), ('утверждает', 'ru'), ('пять', 'ru'), ('год', 'ru'), ('теперь', 'ru'), ('потому', 'ru'), ('о', 'ru'), ('Д', 'ru'), ('сотрудников', 'ru'), ('говоря', 'ru'), ('той', 'ru'), ('якобы', 'ru'), ('условия', 'ru'), ('сами', 'ru'), ('миллион', 'ru'), ('сути', 'ru'), ('больше', 'ru'), ('все', 'ru'), ('удалось', 'ru'), ('Есть', 'ru'), ('политики', 'ru'), ('событий', 'ru'), ('им', 'ru'), ('своего', 'ru'), ('Через', 'ru'), ('именно', 'ru'), ('одного', 'ru'), ('стало', 'ru'), ('женщины', 'ru'), ('местных', 'ru'), ('главе', 'ru'), ('Она', 'ru'), ('г', 'ru'), ('которую', 'ru'), ('особенно', 'ru'), ('многих', 'ru'), ('Однако', 'ru'), ('Сергей', 'ru'), ('х', 'ru'), ('являетесь', 'ru'), ('наша', 'ru'), ('этот', 'ru'), ('равно', 'ru'), ('Еще', 'ru'), ('работает', 'ru'), ('число', 'ru'), ('каких', 'ru'), ('После', 'ru'), ('производства', 'ru'), ('под', 'ru'), ('связи', 'ru'), ('вас', 'ru'), ('сегодня', 'ru'), ('собой', 'ru'), ('Ведь', 'ru'), ('году', 'ru'), ('было', 'ru'), ('в', 'ru'), ('сих', 'ru'), ('всегда', 'ru'), ('говорится', 'ru'), ('С', 'ru'), ('наши', 'ru'), ('своему', 'ru'), ('другой', 'ru'), ('бы', 'ru'), ('первый', 'ru'), ('даже', 'ru'), ('качестве', 'ru'), ('подобная', 'ru'), ('нем', 'ru'), ('такое', 'ru'), ('лидер', 'ru'), ('оказался', 'ru'), ('миллиарда', 'ru'), ('несмотря', 'ru'), ('слишком', 'ru'), ('придется', 'ru'), ('Кстати', 'ru'), ('У', 'ru'), ('пост', 'ru'), ('мире', 'ru'), ('первого', 'ru'), ('ссылка', 'ru'), ('сделаем', 'ru'), ('времена', 'ru'), ('течение', 'ru'), ('слова', 'ru'), ('Е', 'ru'), ('Правда', 'ru'), ('другое', 'ru'), ('которому', 'ru'), ('участие', 'ru'), ('сейчас', 'ru'), ('руб', 'ru'), ('просто', 'ru'), ('да', 'ru'), ('руководство', 'ru'), ('развития', 'ru'), ('видимо', 'ru'), ('нынешнего', 'ru'), ('после', 'ru'), ('сентября', 'ru'), ('весьма', 'ru'), ('наших', 'ru'), ('двух', 'ru'), ('резко', 'ru'), ('например', 'ru'), ('января', 'ru'), ('шесть', 'ru'), ('вопросы', 'ru'), ('должен', 'ru'), ('Таким', 'ru'), ('долго', 'ru'), ('были', 'ru'), ('тоже', 'ru'), ('вся', 'ru'), ('предприятия', 'ru'), ('рядом', 'ru'), ('же', 'ru'), ('среди', 'ru'), ('дел', 'ru'), ('четыре', 'ru'), ('й', 'ru'), ('себя', 'ru'), ('rnd', 'ru'), ('пришлось', 'ru'), ('комитета', 'ru'), ('те', 'ru'), ('находившегося', 'ru'), ('тебе', 'ru'), ('состоянии', 'ru'), ('но', 'ru'), ('любой', 'ru'), ('говорят', 'ru'), ('на', 'ru'), ('что', 'ru'), ('говорил', 'ru'), ('силы', 'ru'), ('права', 'ru'), ('будто', 'ru'), ('включая', 'ru'), ('тех', 'ru'), ('считать', 'ru'), ('документы', 'ru'), ('кого', 'ru'), ('ты', 'ru'), ('себе', 'ru'), ('Кто', 'ru'), ('На', 'ru'), ('тысячи', 'ru'), ('кроме', 'ru'), ('Александр', 'ru'), ('Рейтер', 'ru'), ('земли', 'ru'), ('конца', 'ru'), ('то', 'ru'), ('числе', 'ru'), ('работать', 'ru'), ('говорить', 'ru'), ('Это', 'ru'), ('людям', 'ru'), ('центре', 'ru'), ('два', 'ru'), ('тем', 'ru'), ('Можно', 'ru'), ('дома', 'ru'), ('системы', 'ru'), ('каждый', 'ru'), ('какой', 'ru'), ('случае', 'ru'), ('средства', 'ru'), ('всё', 'ru'), ('ваш', 'ru'), ('дал', 'ru'), ('производство', 'ru'), ('стать', 'ru'), ('а', 'ru'), ('почти', 'ru'), ('первую', 'ru'), ('глава', 'ru'), ('роль', 'ru'), ('такая', 'ru'), ('столь', 'ru'), ('Александра', 'ru'), ('оказалась', 'ru'), ('НА', 'ru'), ('ответ', 'ru'), ('процентов', 'ru'), ('деятельности', 'ru'), ('часто', 'ru'), ('делам', 'ru'), ('никак', 'ru'), ('сторону', 'ru'), ('ныне', 'ru'), ('ходе', 'ru'), ('оно', 'ru'), ('деятельность', 'ru'), ('день', 'ru'), ('link', 'ru'), ('касается', 'ru'), ('градусов', 'ru'), ('прокуратуры', 'ru'), ('много', 'ru'), ('новых', 'ru'), ('отличие', 'ru'), ('Сергея', 'ru'), ('либо', 'ru'), ('таких', 'ru'), ('Сейчас', 'ru'), ('такие', 'ru'), ('три', 'ru'), ('времени', 'ru'), ('нее', 'ru'), ('Андрей', 'ru'), ('знает', 'ru'), ('Владимира', 'ru'), ('решения', 'ru'), ('выяснилось', 'ru'), ('пресс', 'ru'), ('марта', 'ru'), ('позиции', 'ru'), ('работу', 'ru'), ('е', 'ru'), ('снова', 'ru'), ('внимание', 'ru'), ('Как', 'ru'), ('весь', 'ru'), ('примеру', 'ru'), ('эти', 'ru'), ('сказать', 'ru'), ('люди', 'ru'), ('движения', 'ru'), ('выше', 'ru'), ('нам', 'ru'), ('к', 'ru'), ('интервью', 'ru'), ('территории', 'ru'), ('таки', 'ru'), ('ей', 'ru'), ('метра', 'ru'), ('деньги', 'ru'), ('Фото', 'ru'), ('отношении', 'ru'), ('последние', 'ru'), ('когда', 'ru'), ('вдруг', 'ru'), ('всю', 'ru'), ('Вы', 'ru'), ('Конечно', 'ru'), ('этим', 'ru'), ('Их', 'ru'), ('Нью', 'ru'), ('степени', 'ru'), ('апреля', 'ru'), ('большинство', 'ru'), ('дела', 'ru'), ('за', 'ru'), ('десять', 'ru'), ('должно', 'ru'), ('километров', 'ru'), ('крупных', 'ru'), ('там', 'ru'), ('назад', 'ru'), ('наш', 'ru'), ('площади', 'ru'), ('bin', 'ru'), ('Там', 'ru'), ('д', 'ru'), ('по', 'ru'), ('наиболее', 'ru'), ('могла', 'ru'), ('квартиры', 'ru'), ('мы', 'ru'), ('нас', 'ru'), ('явно', 'ru'), ('работа', 'ru'), ('женщин', 'ru'), ('ни', 'ru'), ('вряд', 'ru'), ('лет', 'ru'), ('столице', 'ru'), ('зрения', 'ru'), ('час', 'ru'), ('своих', 'ru'), ('операции', 'ru'), ('тот', 'ru'), ('сказал', 'ru'), ('невозможно', 'ru'), ('За', 'ru'), ('Известия', 'ru'), ('иначе', 'ru'), ('Поэтому', 'ru'), ('процента', 'ru'), ('www', 'ru'), ('ряд', 'ru'), ('Ни', 'ru'), ('минут', 'ru'), ('недели', 'ru'), ('опять', 'ru'), ('вовсе', 'ru'), ('специалистов', 'ru'), ('полностью', 'ru'), ('То', 'ru'), ('свою', 'ru'), ('этих', 'ru'), ('считает', 'ru'), ('своем', 'ru'), ('мене', 'ru'), ('фирмы', 'ru'), ('результате', 'ru'), ('кому', 'ru'), ('очередь', 'ru'), ('довольно', 'ru'), ('новый', 'ru'), ('Он', 'ru'), ('делу', 'ru'), ('Известиям', 'ru'), ('примерно', 'ru'), ('между', 'ru'), ('заместитель', 'ru'), ('знают', 'ru'), ('миллиона', 'ru'), ('время', 'ru'), ('которая', 'ru'), ('годы', 'ru'), ('всей', 'ru'), ('премьер', 'ru'), ('Хотя', 'ru'), ('края', 'ru'), ('До', 'ru'), ('тому', 'ru'), ('Грозном', 'ru'), ('комиссии', 'ru'), ('центр', 'ru'), ('ему', 'ru'), ('людей', 'ru'), ('дней', 'ru'), ('Но', 'ru'), ('куда', 'ru'), ('силу', 'ru'), ('вполне', 'ru'), ('ничего', 'ru'), ('которые', 'ru'), ('ситуация', 'ru'), ('Из', 'ru'), ('может', 'ru'), ('так', 'ru'), ('сотрудники', 'ru'), ('делать', 'ru'), ('html', 'ru'), ('далеко', 'ru'), ('легко', 'ru'), ('проблем', 'ru'), ('группы', 'ru'), ('вокруг', 'ru'), ('которыми', 'ru'), ('виде', 'ru'), ('августа', 'ru'), ('конце', 'ru'), ('Более', 'ru'), ('К', 'ru'), ('работавшую', 'ru'), ('style', 'ru'), ('впервые', 'ru'), ('самым', 'ru'), ('Виктор', 'ru'), ('района', 'ru'), ('действий', 'ru'), ('поскольку', 'ru'), ('можно', 'ru'), ('месяцев', 'ru'), ('события', 'ru'), ('действительно', 'ru'), ('стала', 'ru'), ('Алексей', 'ru'), ('Один', 'ru'), ('метров', 'ru'), ('Все', 'ru'), ('Впрочем', 'ru'), ('сама', 'ru'), ('некоторого', 'ru'), ('тогда', 'ru'), ('письмо', 'ru'), ('информацию', 'ru'), ('счет', 'ru'), ('сделать', 'ru'), ('дает', 'ru'), ('Его', 'ru'), ('войск', 'ru'), ('Пока', 'ru'), ('которого', 'ru'), ('пор', 'ru'), ('существует', 'ru'), ('вроде', 'ru'), ('немало', 'ru'), ('бывшего', 'ru'), ('эту', 'ru'), ('первая', 'ru'), ('Эта', 'ru'), ('смерти', 'ru'), ('ТАСС', 'ru'), ('членов', 'ru'), ('частности', 'ru'), ('специалисты', 'ru'), ('Этот', 'ru'), ('которых', 'ru'), ('При', 'ru'), ('стате', 'ru'), ('народа', 'ru'), ('могут', 'ru'), ('некоторых', 'ru'), ('данным', 'ru'), ('говорит', 'ru'), ('лиц', 'ru'), ('сто', 'ru'), ('Может', 'ru'), ('сколько', 'ru'), ('считают', 'ru'), ('вот', 'ru'), ('предприятий', 'ru'), ('которое', 'ru'), ('средств', 'ru'), ('Многие', 'ru'), ('итоге', 'ru'), ('дальше', 'ru'), ('тонн', 'ru'), ('является', 'ru'), ('пяти', 'ru'), ('проблемы', 'ru'), ('Потом', 'ru'), ('следует', 'ru'), ('раза', 'ru'), ('Почему', 'ru'), ('сообща', 'ru'), ('quot', 'ru'), ('друг', 'ru'), ('ИТАР', 'ru'), ('Они', 'ru'), ('Сегодня', 'ru'), ('основном', 'ru'), ('известно', 'ru'), ('такой', 'ru'), ('Между', 'ru'), ('ли', 'ru'), ('чаще', 'ru'), ('должны', 'ru'), ('большой', 'ru'), ('вторая', 'ru'), ('Да', 'ru'), ('через', 'ru'), ('точки', 'ru'), ('прежнему', 'ru'), ('самые', 'ru'), ('го', 'ru'), ('Даже', 'ru'), ('часов', 'ru'), ('чем', 'ru'), ('решил', 'ru'), ('директора', 'ru'), ('взгляд', 'ru'), ('часа', 'ru'), ('со', 'ru'), ('ситуацию', 'ru'), ('принять', 'ru'), ('правило', 'ru'), ('Б', 'ru'), ('уровень', 'ru'), ('помощь', 'ru'), ('поводу', 'ru'), ('собственности', 'ru'), ('четырех', 'ru'), ('первые', 'ru'), ('самом', 'ru'), ('одно', 'ru'), ('тысяч', 'ru'), ('сразу', 'ru'), ('Николай', 'ru'), ('органы', 'ru'), ('фирм', 'ru'), ('порядке', 'ru'), ('ясно', 'ru'), ('таким', 'ru'), ('М', 'ru'), ('опыт', 'ru'), ('однако', 'ru'), ('href', 'ru'), ('произошло', 'ru'), ('блог', 'ru'), ('почему', 'ru'), ('По', 'ru'), ('суда', 'ru'), ('том', 'ru'), ('всему', 'ru'), ('постоянно', 'ru'), ('человек', 'ru'), ('посколько', 'ru'), ('свое', 'ru'), ('тут', 'ru'), ('наконец', 'ru'), ('ними', 'ru'), ('м', 'ru'), ('прошлого', 'ru'), ('мало', 'ru'), ('центра', 'ru'), ('пятая', 'ru'), ('помощью', 'ru'), ('воды', 'ru'), ('приходится', 'ru'), ('две', 'ru'), ('прошла', 'ru'), ('кажется', 'ru'), ('чуть', 'ru'), ('последних', 'ru'), ('образом', 'ru'), ('раз', 'ru'), ('имени', 'ru'), ('компания', 'ru'), ('условиях', 'ru'), ('поэтому', 'ru'), ('Анатолий', 'ru'), ('прошлом', 'ru'), ('нет', 'ru'), ('туда', 'ru'), ('АО', 'ru'), ('стали', 'ru'), ('другом', 'ru'), ('этом', 'ru'), ('труда', 'ru'), ('меньше', 'ru'), ('нашей', 'ru'), ('отношения', 'ru'), ('самого', 'ru'), ('воскресение', 'ru'), ('некоторые', 'ru'), ('возможно', 'ru'), ('этого', 'ru'), ('какие', 'ru'), ('Михаил', 'ru'), ('котором', 'ru'), ('цены', 'ru'), ('нового', 'ru'), ('вновь', 'ru'), ('нашего', 'ru'), ('участников', 'ru'), ('сумму', 'ru'), ('проблема', 'ru'), ('Что', 'ru'), ('провести', 'ru'), ('июня', 'ru'), ('имеет', 'ru'), ('нескольких', 'ru'), ('главное', 'ru'), ('словам', 'ru'), ('дни', 'ru'), ('первым', 'ru'), ('вести', 'ru'), ('никто', 'ru'), ('Н', 'ru'), ('имя', 'ru'), ('ней', 'ru'), ('создать', 'ru'), ('города', 'ru'), ('продукции', 'ru'), ('месяц', 'ru'), ('потом', 'ru'), ('Так', 'ru'), ('бывший', 'ru'), ('зам', 'ru'), ('она', 'ru'), ('действия', 'ru'), ('днях', 'ru'), ('вместо', 'ru'), ('положение', 'ru'), ('сам', 'ru'), ('идет', 'ru'), ('находится', 'ru'), ('всем', 'ru'), ('области', 'ru'), ('один', 'ru'), ('Только', 'ru'), ('самых', 'ru'), ('театра', 'ru'), ('своей', 'ru'), ('давно', 'ru'), ('меры', 'ru'), ('Для', 'ru'), ('месте', 'ru'), ('Юрий', 'ru'), ('совсем', 'ru'), ('речь', 'ru'), ('одним', 'ru'), ('до', 'ru'), ('момент', 'ru'), ('начальника', 'ru'), ('вам', 'ru'), ('другие', 'ru'), ('затем', 'ru'), ('случай', 'ru'), ('пути', 'ru'), ('менее', 'ru'), ('заявил', 'ru'), ('право', 'ru'), ('стоит', 'ru'), ('очередной', 'ru'), ('Не', 'ru'), ('получить', 'ru'), ('конференции', 'ru'), ('станет', 'ru'), ('система', 'ru'), ('был', 'ru'), ('мой', 'ru'), ('оказалось', 'ru'), ('очень', 'ru'), ('скажем', 'ru'), ('во', 'ru'), ('такого', 'ru'), ('акций', 'ru'), ('фирма', 'ru'), ('ноября', 'ru'), ('с', 'ru'), ('хоть', 'ru'), ('Здесь', 'ru'), ('другим', 'ru'), ('полтора', 'ru'), ('других', 'ru'), ('здесь', 'ru'), ('команда', 'ru'), ('сделал', 'ru'), ('или', 'ru'), ('будут', 'ru'), ('достаточно', 'ru'), ('Без', 'ru'), ('Известий', 'ru'), ('надо', 'ru'), ('рода', 'ru'), ('стороны', 'ru'), ('город', 'ru'), ('происходит', 'ru'), ('Мне', 'ru'), ('Именно', 'ru'), ('начала', 'ru'), ('летний', 'ru'), ('ситуации', 'ru');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('sid', 'ru'), ('при', 'ru'), ('мог', 'ru'), ('как', 'ru'), ('деятельн', 'ru'), ('пришл', 'ru'), ('для', 'ru'), ('над', 'ru'), ('ма', 'ru'), ('проект', 'ru'), ('вперв', 'ru'), ('легк', 'ru'), ('систем', 'ru'), ('реш', 'ru'), ('будет', 'ru'), ('взят', 'ru'), ('поч', 'ru'), ('воскресен', 'ru'), ('равн', 'ru'), ('срок', 'ru'), ('теб', 'ru'), ('юр', 'ru'), ('закон', 'ru'), ('отношен', 'ru'), ('коп', 'ru'), ('туд', 'ru'), ('резк', 'ru'), ('недел', 'ru'), ('письм', 'ru'), ('площад', 'ru'), ('http', 'ru'), ('дом', 'ru'), ('вест', 'ru'), ('впроч', 'ru'), ('уда', 'ru'), ('кто', 'ru'), ('сил', 'ru'), ('имеют', 'ru'), ('себ', 'ru'), ('оп', 'ru'), ('начальник', 'ru'), ('прот', 'ru'), ('могл', 'ru'), ('мне', 'ru'), ('слишк', 'ru'), ('фот', 'ru'), ('дня', 'ru'), ('livejournal', 'ru'), ('не', 'ru'), ('народ', 'ru'), ('полтор', 'ru'), ('имет', 'ru'), ('период', 'ru'), ('известн', 'ru'), ('рук', 'ru'), ('отлич', 'ru'), ('одновремен', 'ru'), ('акц', 'ru'), ('врод', 'ru'), ('должн', 'ru'), ('перед', 'ru'), ('част', 'ru'), ('август', 'ru'), ('наход', 'ru'), ('основн', 'ru'), ('об', 'ru'), ('межд', 'ru'), ('внов', 'ru'), ('большинств', 'ru'), ('свет', 'ru'), ('их', 'ru'), ('хочет', 'ru'), ('стал', 'ru'), ('род', 'ru'), ('след', 'ru'), ('т', 'ru'), ('про', 'ru'), ('вы', 'ru'), ('тогд', 'ru'), ('кром', 'ru'), ('сут', 'ru'), ('и', 'ru'), ('реч', 'ru'), ('скаж', 'ru'), ('он', 'ru'), ('смерт', 'ru'), ('стольк', 'ru'), ('процесс', 'ru'), ('истор', 'ru'), ('земл', 'ru'), ('нег', 'ru'), ('конференц', 'ru'), ('уж', 'ru'), ('ве', 'ru'), ('завод', 'ru'), ('тасс', 'ru'), ('представител', 'ru'), ('очеред', 'ru'), ('получ', 'ru'), ('пут', 'ru'), ('произошл', 'ru'), ('территор', 'ru'), ('без', 'ru'), ('трех', 'ru'), ('найт', 'ru'), ('информац', 'ru'), ('ним', 'ru'), ('вопрос', 'ru'), ('adriver', 'ru'), ('сегодн', 'ru'), ('снов', 'ru'), ('amp', 'ru'), ('где', 'ru'), ('я', 'ru'), ('слов', 'ru'), ('такж', 'ru'), ('позиц', 'ru'), ('рол', 'ru'), ('ещ', 'ru'), ('всех', 'ru'), ('них', 'ru'), ('качеств', 'ru'), ('рейтер', 'ru'), ('сдела', 'ru'), ('каса', 'ru'), ('постоя', 'ru'), ('корреспондент', 'ru'), ('окол', 'ru'), ('из', 'ru'), ('январ', 'ru'), ('некотор', 'ru'), ('наибол', 'ru'), ('от', 'ru'), ('анатол', 'ru'), ('б', 'ru'), ('зрен', 'ru'), ('у', 'ru'), ('cgi', 'ru'), ('факт', 'ru'), ('жизн', 'ru'), ('вовс', 'ru'), ('сторон', 'ru'), ('год', 'ru'), ('о', 'ru'), ('личн', 'ru'), ('оста', 'ru'), ('люб', 'ru'), ('помощ', 'ru'), ('район', 'ru'), ('сво', 'ru'), ('практическ', 'ru'), ('миллион', 'ru'), ('все', 'ru'), ('им', 'ru'), ('хорош', 'ru'), ('октябр', 'ru'), ('говор', 'ru'), ('г', 'ru'), ('кра', 'ru'), ('х', 'ru'), ('несмотр', 'ru'), ('этот', 'ru'), ('под', 'ru'), ('поскольк', 'ru'), ('нужн', 'ru'), ('вас', 'ru'), ('созда', 'ru'), ('театр', 'ru'), ('ком', 'ru'), ('движен', 'ru'), ('в', 'ru'), ('мер', 'ru'), ('поня', 'ru'), ('когд', 'ru'), ('сих', 'ru'), ('дне', 'ru'), ('фонд', 'ru'), ('дальш', 'ru'), ('работ', 'ru'), ('собствен', 'ru'), ('бы', 'ru'), ('куд', 'ru'), ('чащ', 'ru'), ('прав', 'ru'), ('нем', 'ru'), ('лидер', 'ru'), ('нич', 'ru'), ('решен', 'ru'), ('главн', 'ru'), ('прежн', 'ru'), ('руководств', 'ru'), ('трудн', 'ru'), ('март', 'ru'), ('пост', 'ru'), ('област', 'ru'), ('деся', 'ru'), ('соб', 'ru'), ('видим', 'ru'), ('нача', 'ru'), ('почт', 'ru'), ('вниман', 'ru'), ('руб', 'ru'), ('сейчас', 'ru'), ('да', 'ru'), ('п', 'ru'), ('существ', 'ru'), ('двух', 'ru'), ('оказа', 'ru'), ('например', 'ru'), ('новост', 'ru'), ('зна', 'ru'), ('комисс', 'ru'), ('документ', 'ru'), ('частност', 'ru'), ('вся', 'ru'), ('утвержда', 'ru'), ('же', 'ru'), ('дел', 'ru'), ('й', 'ru'), ('rnd', 'ru'), ('те', 'ru'), ('но', 'ru'), ('на', 'ru'), ('что', 'ru'), ('выш', 'ru'), ('производств', 'ru'), ('борис', 'ru'), ('тех', 'ru'), ('довольн', 'ru'), ('комментар', 'ru'), ('быстр', 'ru'), ('нын', 'ru'), ('сумм', 'ru'), ('ты', 'ru'), ('приня', 'ru'), ('инач', 'ru'), ('декабр', 'ru'), ('дан', 'ru'), ('жит', 'ru'), ('всег', 'ru'), ('то', 'ru'), ('мал', 'ru'), ('грозн', 'ru'), ('выясн', 'ru'), ('орга', 'ru'), ('два', 'ru'), ('тем', 'ru'), ('сред', 'ru'), ('ваш', 'ru'), ('всё', 'ru'), ('количеств', 'ru'), ('результат', 'ru'), ('дал', 'ru'), ('а', 'ru'), ('кажет', 'ru'), ('специалист', 'ru'), ('ответ', 'ru'), ('ден', 'ru'), ('никак', 'ru'), ('сентябр', 'ru'), ('есл', 'ru'), ('придет', 'ru'), ('чтоб', 'ru'), ('link', 'ru'), ('начал', 'ru'), ('посл', 'ru'), ('раньш', 'ru'), ('век', 'ru'), ('можн', 'ru'), ('глав', 'ru'), ('три', 'ru'), ('вед', 'ru'), ('якоб', 'ru'), ('знает', 'ru'), ('тепер', 'ru'), ('пресс', 'ru'), ('е', 'ru'), ('суд', 'ru'), ('сообщ', 'ru'), ('команд', 'ru'), ('четыр', 'ru'), ('нема', 'ru'), ('всегд', 'ru'), ('никогд', 'ru'), ('тольк', 'ru'), ('нам', 'ru'), ('к', 'ru'), ('втор', 'ru'), ('вдруг', 'ru'), ('всю', 'ru'), ('участник', 'ru'), ('хот', 'ru'), ('либ', 'ru'), ('дела', 'ru'), ('за', 'ru'), ('игр', 'ru'), ('точн', 'ru'), ('андр', 'ru'), ('заявлен', 'ru'), ('там', 'ru'), ('назад', 'ru'), ('июл', 'ru'), ('наш', 'ru'), ('очередн', 'ru'), ('политик', 'ru'), ('ситуац', 'ru'), ('bin', 'ru'), ('ег', 'ru'), ('д', 'ru'), ('по', 'ru'), ('мы', 'ru'), ('прост', 'ru'), ('нас', 'ru'), ('нескольк', 'ru'), ('женщин', 'ru'), ('работа', 'ru'), ('ни', 'ru'), ('вряд', 'ru'), ('лет', 'ru'), ('пят', 'ru'), ('час', 'ru'), ('крупн', 'ru'), ('тот', 'ru'), ('вид', 'ru'), ('однак', 'ru'), ('шест', 'ru'), ('мо', 'ru'), ('вод', 'ru'), ('квартир', 'ru'), ('www', 'ru'), ('заяв', 'ru'), ('ряд', 'ru'), ('стат', 'ru'), ('минут', 'ru'), ('недавн', 'ru'), ('достаточн', 'ru'), ('тог', 'ru'), ('степен', 'ru'), ('труд', 'ru'), ('услов', 'ru'), ('образ', 'ru'), ('миров', 'ru'), ('счита', 'ru'), ('пот', 'ru'), ('долж', 'ru'), ('метр', 'ru'), ('знают', 'ru'), ('уровен', 'ru'), ('приход', 'ru'), ('премьер', 'ru'), ('встреч', 'ru'), ('перв', 'ru'), ('возможн', 'ru'), ('центр', 'ru'), ('вмест', 'ru'), ('тон', 'ru'), ('так', 'ru'), ('может', 'ru'), ('миллиард', 'ru'), ('виц', 'ru'), ('html', 'ru'), ('времен', 'ru'), ('проблем', 'ru'), ('конечн', 'ru'), ('вокруг', 'ru'), ('явля', 'ru'), ('опя', 'ru'), ('эт', 'ru'), ('style', 'ru'), ('зде', 'ru'), ('особен', 'ru'), ('чут', 'ru'), ('подобн', 'ru'), ('заместител', 'ru'), ('власт', 'ru'), ('цел', 'ru'), ('пок', 'ru'), ('примерн', 'ru'), ('естествен', 'ru'), ('нельз', 'ru'), ('быт', 'ru'), ('мест', 'ru'), ('сем', 'ru'), ('сраз', 'ru'), ('лиш', 'ru'), ('счет', 'ru'), ('иллюстрац', 'ru'), ('развит', 'ru'), ('дает', 'ru'), ('войск', 'ru'), ('повод', 'ru'), ('необходим', 'ru'), ('пор', 'ru'), ('связ', 'ru'), ('никола', 'ru'), ('полност', 'ru'), ('уровн', 'ru'), ('ког', 'ru'), ('пример', 'ru'), ('могут', 'ru'), ('ссылк', 'ru'), ('алекс', 'ru'), ('лиц', 'ru'), ('котор', 'ru'), ('прокуратур', 'ru'), ('сто', 'ru'), ('лучш', 'ru'), ('давн', 'ru'), ('вот', 'ru'), ('включ', 'ru'), ('скор', 'ru'), ('председател', 'ru'), ('ноябр', 'ru'), ('средств', 'ru'), ('прошл', 'ru'), ('выбор', 'ru'), ('скольк', 'ru'), ('директор', 'ru'), ('едв', 'ru'), ('нынешн', 'ru'), ('разн', 'ru'), ('друг', 'ru'), ('quot', 'ru'), ('точк', 'ru'), ('деньг', 'ru'), ('ли', 'ru'), ('через', 'ru'), ('александр', 'ru'), ('последн', 'ru'), ('километр', 'ru'), ('член', 'ru'), ('го', 'ru'), ('знач', 'ru'), ('чем', 'ru'), ('весьм', 'ru'), ('станов', 'ru'), ('ест', 'ru'), ('даж', 'ru'), ('взгляд', 'ru'), ('со', 'ru'), ('нов', 'ru'), ('никт', 'ru'), ('компан', 'ru'), ('кажд', 'ru'), ('бывш', 'ru'), ('серге', 'ru'), ('сказа', 'ru'), ('числ', 'ru'), ('четырех', 'ru'), ('тысяч', 'ru'), ('происход', 'ru'), ('врем', 'ru'), ('фирм', 'ru'), ('прям', 'ru'), ('предприят', 'ru'), ('совершен', 'ru'), ('href', 'ru'), ('очен', 'ru'), ('блог', 'ru'), ('имен', 'ru'), ('долг', 'ru'), ('вполн', 'ru'), ('том', 'ru'), ('мен', 'ru'), ('человек', 'ru'), ('обычн', 'ru'), ('мног', 'ru'), ('н', 'ru'), ('столиц', 'ru'), ('тут', 'ru'), ('наконец', 'ru'), ('м', 'ru'), ('жител', 'ru'), ('конц', 'ru'), ('итог', 'ru'), ('ну', 'ru'), ('интерв', 'ru'), ('две', 'ru'), ('действ', 'ru'), ('сборн', 'ru'), ('градус', 'ru'), ('нью', 'ru'), ('чег', 'ru'), ('раз', 'ru'), ('положен', 'ru'), ('ем', 'ru'), ('бол', 'ru'), ('стол', 'ru'), ('июн', 'ru'), ('будт', 'ru'), ('прич', 'ru'), ('ю', 'ru'), ('нибуд', 'ru'), ('нет', 'ru'), ('массов', 'ru'), ('пробл', 'ru'), ('итар', 'ru'), ('ассошиэйтед', 'ru'), ('поэт', 'ru'), ('провест', 'ru'), ('порядк', 'ru'), ('совс', 'ru'), ('прежд', 'ru'), ('случа', 'ru'), ('ил', 'ru'), ('местн', 'ru'), ('имеет', 'ru'), ('дни', 'ru'), ('состоян', 'ru'), ('ран', 'ru'), ('л', 'ru'), ('течен', 'ru'), ('месяц', 'ru'), ('феврал', 'ru'), ('люд', 'ru'), ('продукц', 'ru'), ('зам', 'ru'), ('днях', 'ru'), ('мир', 'ru'), ('извест', 'ru'), ('сам', 'ru'), ('интерес', 'ru'), ('идет', 'ru'), ('один', 'ru'), ('всем', 'ru'), ('меньш', 'ru'), ('ход', 'ru'), ('апрел', 'ru'), ('явн', 'ru'), ('момент', 'ru'), ('одн', 'ru'), ('вообщ', 'ru'), ('до', 'ru'), ('вам', 'ru'), ('отдел', 'ru'), ('серг', 'ru'), ('тож', 'ru'), ('далек', 'ru'), ('комитет', 'ru'), ('сотрудник', 'ru'), ('летн', 'ru'), ('руководител', 'ru'), ('операц', 'ru'), ('похож', 'ru'), ('виктор', 'ru'), ('станет', 'ru'), ('кстат', 'ru'), ('цен', 'ru'), ('правд', 'ru'), ('был', 'ru'), ('во', 'ru'), ('суббот', 'ru'), ('с', 'ru'), ('дат', 'ru'), ('действительн', 'ru'), ('миха', 'ru'), ('событ', 'ru'), ('участ', 'ru'), ('будут', 'ru'), ('невозможн', 'ru'), ('ясн', 'ru'), ('владимир', 'ru'), ('зат', 'ru'), ('парт', 'ru'), ('город', 'ru'), ('групп', 'ru'), ('больш', 'ru'), ('процент', 'ru');INSERT INTO stopwords_long (stopword, language) VALUES ('som', 'sv'), ('du', 'sv'), ('att', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('sådan', 'sv'), ('utan', 'sv'), ('nu', 'sv'), ('sådana', 'sv'), ('ingen', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('samma', 'sv'), ('bli', 'sv'), ('vars', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('med', 'sv'), ('ert', 'sv'), ('vilken', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('detta', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('ut', 'sv'), ('inom', 'sv'), ('någon', 'sv'), ('men', 'sv'), ('inte', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('vara', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('mina', 'sv'), ('till', 'sv'), ('ett', 'sv'), ('sitta', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('när', 'sv'), ('man', 'sv'), ('den', 'sv'), ('dess', 'sv'), ('har', 'sv'), ('sådant', 'sv'), ('hade', 'sv'), ('hans', 'sv'), ('vid', 'sv'), ('dessa', 'sv'), ('dina', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('vilka', 'sv'), ('deras', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('henne', 'sv'), ('blivit', 'sv'), ('varje', 'sv'), ('vilkas', 'sv'), ('om', 'sv'), ('dem', 'sv'), ('en', 'sv'), ('sin', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('alla', 'sv'), ('kunde', 'sv'), ('våra', 'sv'), ('hennes', 'sv'), ('åt', 'sv'), ('något', 'sv'), ('ha', 'sv'), ('vem', 'sv'), ('efter', 'sv'), ('skulle', 'sv'), ('icke', 'sv'), ('i', 'sv'), ('över', 'sv'), ('än', 'sv'), ('sina', 'sv'), ('denna', 'sv'), ('mitt', 'sv'), ('under', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('några', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('eller', 'sv'), ('mot', 'sv'), ('blev', 'sv'), ('är', 'sv');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('som', 'sv'), ('att', 'sv'), ('du', 'sv'), ('dett', 'sv'), ('ju', 'sv'), ('vart', 'sv'), ('nu', 'sv'), ('utan', 'sv'), ('sådan', 'sv'), ('vi', 'sv'), ('vilket', 'sv'), ('bli', 'sv'), ('med', 'sv'), ('kan', 'sv'), ('det', 'sv'), ('ert', 'sv'), ('från', 'sv'), ('dig', 'sv'), ('här', 'sv'), ('ditt', 'sv'), ('och', 'sv'), ('för', 'sv'), ('av', 'sv'), ('ell', 'sv'), ('henn', 'sv'), ('int', 'sv'), ('där', 'sv'), ('honom', 'sv'), ('vad', 'sv'), ('all', 'sv'), ('ut', 'sv'), ('mellan', 'sv'), ('allt', 'sv'), ('inom', 'sv'), ('men', 'sv'), ('någon', 'sv'), ('skull', 'sv'), ('sedan', 'sv'), ('var', 'sv'), ('på', 'sv'), ('ni', 'sv'), ('vilk', 'sv'), ('till', 'sv'), ('und', 'sv'), ('ett', 'sv'), ('denn', 'sv'), ('ej', 'sv'), ('han', 'sv'), ('den', 'sv'), ('man', 'sv'), ('när', 'sv'), ('har', 'sv'), ('dess', 'sv'), ('sådant', 'sv'), ('varj', 'sv'), ('samm', 'sv'), ('der', 'sv'), ('vid', 'sv'), ('då', 'sv'), ('upp', 'sv'), ('kund', 'sv'), ('vår', 'sv'), ('blir', 'sv'), ('jag', 'sv'), ('så', 'sv'), ('själv', 'sv'), ('vårt', 'sv'), ('era', 'sv'), ('varit', 'sv'), ('er', 'sv'), ('blivit', 'sv'), ('om', 'sv'), ('ing', 'sv'), ('någr', 'sv'), ('dem', 'sv'), ('sin', 'sv'), ('en', 'sv'), ('de', 'sv'), ('hur', 'sv'), ('ick', 'sv'), ('hon', 'sv'), ('din', 'sv'), ('ha', 'sv'), ('något', 'sv'), ('åt', 'sv'), ('vem', 'sv'), ('eft', 'sv'), ('än', 'sv'), ('över', 'sv'), ('i', 'sv'), ('had', 'sv'), ('mitt', 'sv'), ('mig', 'sv'), ('min', 'sv'), ('sitt', 'sv'), ('mycket', 'sv'), ('oss', 'sv'), ('varför', 'sv'), ('sig', 'sv'), ('mot', 'sv'), ('är', 'sv'), ('blev', 'sv');INSERT INTO stopwords_long (stopword, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('filan', 'tr'), ('böylece', 'tr'), ('biz', 'tr'), ('hepsi', 'tr'), ('senin', 'tr'), ('hepsine', 'tr'), ('hiçbirini', 'tr'), ('hangisi', 'tr'), ('e', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('bazısı', 'tr'), ('þey', 'tr'), ('kendi', 'tr'), ('dokuz', 'tr'), ('siz', 'tr'), ('kendine', 'tr'), ('diğeri', 'tr'), ('bu', 'tr'), ('seni', 'tr'), ('hem', 'tr'), ('onu', 'tr'), ('kimi', 'tr'), ('bizim', 'tr'), ('defa', 'tr'), ('burada', 'tr'), ('mi', 'tr'), ('hiç kimse', 'tr'), ('biri', 'tr'), ('bütün', 'tr'), ('ve', 'tr'), ('altmýþ', 'tr'), ('son', 'tr'), ('onun', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('dört', 'tr'), ('milyon', 'tr'), ('u', 'tr'), ('yerine', 'tr'), ('k', 'tr'), ('orada', 'tr'), ('beş', 'tr'), ('mý', 'tr'), ('buna', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('þuna', 'tr'), ('dahi', 'tr'), ('içinde', 'tr'), ('çoğuna', 'tr'), ('kaç', 'tr'), ('en gibi', 'tr'), ('böyle', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('beni', 'tr'), ('kadar', 'tr'), ('şunda', 'tr'), ('zaten', 'tr'), ('dolayı', 'tr'), ('o', 'tr'), ('bunun', 'tr'), ('hangi', 'tr'), ('şeye', 'tr'), ('d', 'tr'), ('on', 'tr'), ('herkese', 'tr'), ('yüz', 'tr'), ('ya da', 'tr'), ('bazıları', 'tr'), ('herkes', 'tr'), ('nereden', 'tr'), ('bile', 'tr'), ('doksan', 'tr'), ('hatta', 'tr'), ('bana', 'tr'), ('onlari', 'tr'), ('z', 'tr'), ('artık', 'tr'), ('şimdi', 'tr'), ('bazen', 'tr'), ('onlara', 'tr'), ('fakat', 'tr'), ('size', 'tr'), ('birşey', 'tr'), ('var', 'tr'), ('neden', 'tr'), ('hep', 'tr'), ('nereye', 'tr'), ('bunu', 'tr'), ('bunda', 'tr'), ('ğ', 'tr'), ('birkez', 'tr'), ('oysa', 'tr'), ('birkaç', 'tr'), ('yine', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('t', 'tr'), ('bizden', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('onların', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('birçoğu', 'tr'), ('madem', 'tr'), ('yedi', 'tr'), ('onlar', 'tr'), ('ötürü', 'tr'), ('þeyi', 'tr'), ('herkesi', 'tr'), ('bir', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('yoksa', 'tr'), ('yetmiþ', 'tr'), ('þeyler', 'tr'), ('birþeyi', 'tr'), ('birisi', 'tr'), ('n', 'tr'), ('şeyi', 'tr'), ('p', 'tr'), ('INSERmi', 'tr'), ('hiç', 'tr'), ('diye', 'tr'), ('kimin', 'tr'), ('hepsini', 'tr'), ('da', 'tr'), ('y', 'tr'), ('önce', 'tr'), ('g', 'tr'), ('her biri', 'tr'), ('nerede', 'tr'), ('tabi', 'tr'), ('ne zaman', 'tr'), ('bazı', 'tr'), ('daha', 'tr'), ('sizden', 'tr'), ('benim', 'tr'), ('değil', 'tr'), ('benden', 'tr'), ('katrilyon', 'tr'), ('acaba', 'tr'), ('sizi', 'tr'), ('ç', 'tr'), ('kendini', 'tr'), ('belki', 'tr'), ('elli', 'tr'), ('trilyon', 'tr'), ('şundan', 'tr'), ('ya', 'tr'), ('şeyden', 'tr'), ('asla', 'tr'), ('ü', 'tr'), ('tüm', 'tr'), ('mı', 'tr'), ('tümü', 'tr'), ('çünkü', 'tr'), ('ama', 'tr'), ('işte', 'tr'), ('birkaçı', 'tr'), ('s', 'tr'), ('rağmen', 'tr'), ('c', 'tr'), ('üzere', 'tr'), ('altı', 'tr'), ('şunu', 'tr'), ('diğerleri', 'tr'), ('bize', 'tr'), ('veyahut', 'tr'), ('şunun', 'tr'), ('senden', 'tr'), ('nasýl', 'tr'), ('üç', 'tr'), ('i', 'tr'), ('m', 'tr'), ('gibi', 'tr'), ('hiçbiri', 'tr'), ('ile', 'tr'), ('sen', 'tr'), ('niçin', 'tr'), ('sizin', 'tr'), ('yirmi', 'tr'), ('gene', 'tr'), ('kim', 'tr'), ('az', 'tr'), ('felan', 'tr'), ('altý', 'tr'), ('b', 'tr'), ('nesi', 'tr'), ('nerde', 'tr'), ('kimisi', 'tr'), ('þu', 'tr'), ('birşeyi', 'tr'), ('elbette', 'tr'), ('şöyle', 'tr'), ('milyar', 'tr'), ('oysaki', 'tr'), ('şunlar', 'tr'), ('kime', 'tr'), ('bazý', 'tr'), ('mu', 'tr'), ('her', 'tr'), ('öbürü', 'tr'), ('bundan', 'tr'), ('þundan', 'tr'), ('onlardan', 'tr'), ('seksen', 'tr'), ('kimden', 'tr'), ('bizi', 'tr'), ('birçokları', 'tr'), ('henüz', 'tr'), ('diğer', 'tr'), ('þunu', 'tr'), ('birçok', 'tr'), ('ş', 'tr'), ('ı', 'tr'), ('şeyler', 'tr'), ('neyse', 'tr'), ('tamam', 'tr'), ('sekiz', 'tr'), ('öyle', 'tr'), ('hani', 'tr'), ('onlarýn', 'tr'), ('ise', 'tr'), ('birþey', 'tr'), ('niye', 'tr'), ('v', 'tr'), ('ondan', 'tr'), ('zira', 'tr'), ('hiçbirine', 'tr'), ('ne kadar', 'tr'), ('çoğu', 'tr'), ('r', 'tr'), ('þunda', 'tr'), ('onu otuz', 'tr'), ('h', 'tr'), ('falan', 'tr'), ('sonra', 'tr'), ('iki', 'tr'), ('nedir', 'tr'), ('ona', 'tr'), ('f', 'tr'), ('demek', 'tr'), ('sana', 'tr'), ('þeyden', 'tr'), ('çoğunu', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('aslında', 'tr'), ('yani', 'tr'), ('şuna', 'tr'), ('için', 'tr');INSERT INTO stopword_stems_long (stopword_stem, language) VALUES ('ne', 'tr'), ('ki', 'tr'), ('biz', 'tr'), ('da', 'tr'), ('hepsi', 'tr'), ('y', 'tr'), ('hat', 'tr'), ('g', 'tr'), ('tama', 'tr'), ('ner', 'tr'), ('tabi', 'tr'), ('iş', 'tr'), ('e', 'tr'), ('bazı', 'tr'), ('ön', 'tr'), ('şey', 'tr'), ('kendi', 'tr'), ('þey', 'tr'), ('en gip', 'tr'), ('siz', 'tr'), ('ke', 'tr'), ('þe', 'tr'), ('değil', 'tr'), ('üzer', 'tr'), ('ge', 'tr'), ('bu', 'tr'), ('şun', 'tr'), ('hem', 'tr'), ('is', 'tr'), ('dok', 'tr'), ('defa', 'tr'), ('hiçbir', 'tr'), ('mi', 'tr'), ('ç', 'tr'), ('heps', 'tr'), ('belki', 'tr'), ('se', 'tr'), ('öbür', 'tr'), ('elli', 'tr'), ('ve', 'tr'), ('il', 'tr'), ('altmýþ', 'tr'), ('fala', 'tr'), ('sek', 'tr'), ('son', 'tr'), ('niç', 'tr'), ('baze', 'tr'), ('ba', 'tr'), ('a', 'tr'), ('beþ', 'tr'), ('bin', 'tr'), ('zate', 'tr'), ('ya', 'tr'), ('dört', 'tr'), ('yer', 'tr'), ('ü', 'tr'), ('u', 'tr'), ('tüm', 'tr'), ('k', 'tr'), ('mı', 'tr'), ('böyleç', 'tr'), ('beş', 'tr'), ('katrilyo', 'tr'), ('mý', 'tr'), ('çünkü', 'tr'), ('biris', 'tr'), ('çok', 'tr'), ('sanki', 'tr'), ('s', 'tr'), ('ya ', 'tr'), ('dahi', 'tr'), ('dah', 'tr'), ('c', 'tr'), ('bazıs', 'tr'), ('bura', 'tr'), ('kaç', 'tr'), ('kimis', 'tr'), ('iç', 'tr'), ('veyahut', 'tr'), ('birþe', 'tr'), ('am', 'tr'), ('üç', 'tr'), ('nasýl', 'tr'), ('milyo', 'tr'), ('i', 'tr'), ('ö', 'tr'), ('ben', 'tr'), ('kýrk', 'tr'), ('þ', 'tr'), ('kadar', 'tr'), ('m', 'tr'), ('ik', 'tr'), ('as', 'tr'), ('onu ot', 'tr'), ('sen', 'tr'), ('acap', 'tr'), ('ne zama', 'tr'), ('o', 'tr'), ('hangi', 'tr'), ('yirmi', 'tr'), ('d', 'tr'), ('kim', 'tr'), ('on', 'tr'), ('yüz', 'tr'), ('az', 'tr'), ('herkes', 'tr'), ('gip', 'tr'), ('ötür', 'tr'), ('altý', 'tr'), ('onlarý', 'tr'), ('al', 'tr'), ('onlari', 'tr'), ('hangis', 'tr'), ('b', 'tr'), ('büt', 'tr'), ('z', 'tr'), ('bil', 'tr'), ('trilyo', 'tr'), ('di', 'tr'), ('þu', 'tr'), ('fakat', 'tr'), ('milyar', 'tr'), ('birşey', 'tr'), ('oysaki', 'tr'), ('var', 'tr'), ('mu', 'tr'), ('bazý', 'tr'), ('ni', 'tr'), ('her', 'tr'), ('hep', 'tr'), ('fila', 'tr'), ('nes', 'tr'), ('asl', 'tr'), ('ğ', 'tr'), ('rağme', 'tr'), ('birkez', 'tr'), ('made', 'tr'), ('bö', 'tr'), ('diğer', 'tr'), ('ora', 'tr'), ('birkaç', 'tr'), ('birçok', 'tr'), ('şö', 'tr'), ('şayet', 'tr'), ('j', 'tr'), ('ş', 'tr'), ('insermi', 'tr'), ('ı', 'tr'), ('sa', 'tr'), ('hiç k', 'tr'), ('her bir', 'tr'), ('t', 'tr'), ('hani', 'tr'), ('birþey', 'tr'), ('v', 'tr'), ('elbet', 'tr'), ('zira', 'tr'), ('yok', 'tr'), ('en', 'tr'), ('şu', 'tr'), ('kez', 'tr'), ('nasıl', 'tr'), ('de', 'tr'), ('be', 'tr'), ('ne kadar', 'tr'), ('yedi', 'tr'), ('dola', 'tr'), ('r', 'tr'), ('şe', 'tr'), ('bir', 'tr'), ('sonra', 'tr'), ('h', 'tr'), ('birşe', 'tr'), ('mü', 'tr'), ('ancak', 'tr'), ('hâlâ', 'tr'), ('nere', 'tr'), ('demek', 'tr'), ('f', 'tr'), ('yetmiþ', 'tr'), ('n', 'tr'), ('baz', 'tr'), ('he', 'tr'), ('ar', 'tr'), ('l', 'tr'), ('veya', 'tr'), ('p', 'tr'), ('hiç', 'tr'), ('yani', 'tr'), ('fela', 'tr'); - -CREATE OR REPLACE FUNCTION is_stop_stem(p_size TEXT, p_stem TEXT, p_language TEXT) - RETURNS BOOLEAN AS $$ -DECLARE - result BOOLEAN; -BEGIN - - -- Tiny - IF p_size = 'tiny' THEN - SELECT 't' INTO result FROM stopword_stems_tiny - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - - -- Short - ELSIF p_size = 'short' THEN - SELECT 't' INTO result FROM stopword_stems_short - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - - -- Long - ELSIF p_size = 'long' THEN - SELECT 't' INTO result FROM stopword_stems_long - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - - -- unknown size - ELSE - RAISE EXCEPTION 'Unknown stopword stem size: "%" (expected "tiny", "short" or "long")', p_size; - result := 'f'; - END IF; - - RETURN result; -END; -$$ LANGUAGE plpgsql; - - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4408-4409.sql b/apps/postgresql-server/schema/migrations/mediawords-4408-4409.sql deleted file mode 100644 index fc40168cb9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4408-4409.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4408 and 4409. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4408, and you would like to upgrade both the Media Cloud and the --- database to be at version 4409, import this SQL file: --- --- psql mediacloud < mediawords-4408-4409.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE story_subsets DROP CONSTRAINT story_subsets_media_id_fkey; -ALTER TABLE story_subsets ADD FOREIGN KEY (media_id) REFERENCES media; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4409; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4409-4410.sql b/apps/postgresql-server/schema/migrations/mediawords-4409-4410.sql deleted file mode 100644 index c34ec318e2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4409-4410.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4409 and 4410. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4409, and you would like to upgrade both the Media Cloud and the --- database to be at version 4410, import this SQL file: --- --- psql mediacloud < mediawords-4409-4410.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4410; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4410-4411.sql b/apps/postgresql-server/schema/migrations/mediawords-4410-4411.sql deleted file mode 100644 index 33fc75d4be..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4410-4411.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4410 and 4411. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4409, and you would like to upgrade both the Media Cloud and the --- database to be at version 4411, import this SQL file: --- --- psql mediacloud < mediawords-4410-4411.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4411; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE feeds - RENAME COLUMN last_download_time TO last_attempted_download_time; -ALTER TABLE feeds - ADD COLUMN last_successful_download_time TIMESTAMP WITH TIME ZONE; -UPDATE feeds - SET last_new_story_time = GREATEST( last_attempted_download_time, last_new_story_time ); -ALTER INDEX feeds_last_download_time - RENAME TO feeds_last_attempted_download_time; -CREATE INDEX feeds_last_successful_download_time - ON feeds(last_successful_download_time); - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4411-4412.sql b/apps/postgresql-server/schema/migrations/mediawords-4411-4412.sql deleted file mode 100644 index b623d5ca1e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4411-4412.sql +++ /dev/null @@ -1,344 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4410 and 4411. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4409, and you would like to upgrade both the Media Cloud and the --- database to be at version 4411, import this SQL file: --- --- psql mediacloud < mediawords-4410-4411.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4412; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - -alter table dashboard_topics alter language drop not null; -alter table story_sentence_words alter language drop not null; -alter table daily_words drop language; -alter table weekly_words drop language; -alter table top_500_weekly_words drop language; -alter table daily_country_counts drop language; -alter table daily_author_words drop language; -alter table weekly_author_words drop language; -alter table top_500_weekly_author_words drop language; - -create view controversies_with_search_info as - select c.*, q.start_date::date, q.end_date::date, qss.pattern, qss.queries_id - from controversies c - left join query_story_searches qss on ( c.query_story_searches_id = qss.query_story_searches_id ) - left join queries q on ( qss.queries_id = q.queries_id ); - -create or replace view controversy_links_cross_media as - select s.stories_id, sm.name as media_name, r.stories_id as ref_stories_id, rm.name as ref_media_name, cl.url as url, cs.controversies_id from media sm, media rm, controversy_links cl, stories s, stories r, controversy_stories cs where cl.ref_stories_id <> cl.stories_id and s.stories_id = cl.stories_id and cl.ref_stories_id = r.stories_id and s.media_id <> r.media_id and sm.media_id = s.media_id and rm.media_id = r.media_id and cs.stories_id = cl.ref_stories_id and cs.controversies_id = cl.controversies_id; - -create table controversy_dumps ( - controversy_dumps_id serial primary key, - controversies_id int not null references controversies on delete cascade, - dump_date timestamp not null, - start_date timestamp not null, - end_date timestamp not null, - note text, - daily_counts_csv text, - weekly_counts_csv text -); - -create index controversy_dumps_controversy on controversy_dumps ( controversies_id ); - -create type cd_period_type AS ENUM ( 'overall', 'weekly', 'monthly', 'custom' ); - --- individual time slices within a controversy dump -create table controversy_dump_time_slices ( - controversy_dump_time_slices_id serial primary key, - controversy_dumps_id int not null references controversy_dumps on delete cascade, - start_date timestamp not null, - end_date timestamp not null, - period cd_period_type not null, - gexf text, - stories_csv text, - story_links_csv text, - media_csv text, - medium_links_csv text, - model_r2_mean float, - model_r2_stddev float, - model_num_media int -); - -alter table controversy_dump_time_slices add constraint cdts_confidence - check ( confidence is null or ( confidence >= 0 and confidence <= 100 ) ); - -create index controversy_dump_time_slices_dump on controversy_dump_time_slices ( controversy_dumps_id ); - --- schema to hold all of the controversy dump snapshot tables -create schema cd; - -create table cd.stories ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - stories_id int, - media_id int not null, - url varchar(1024) not null, - guid varchar(1024) not null, - title text not null, - description text null, - publish_date timestamp not null, - collect_date timestamp not null, - full_text_rss boolean not null default 'f', - language varchar(3) null -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset -); -create index stories_id on cd.stories ( controversy_dumps_id, stories_id ); - -create table cd.controversy_stories ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - controversy_stories_id int, - controversies_id int not null, - stories_id int not null, - link_mined boolean, - iteration int, - link_weight real, - redirect_url text, - valid_foreign_rss_story boolean -); -create index controversy_stories_id on cd.controversy_stories ( controversy_dumps_id, stories_id ); - -create table cd.controversy_links_cross_media ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - controversy_links_id int, - controversies_id int not null, - stories_id int not null, - url text not null, - ref_stories_id int, - media_name text, - ref_media_name text -); -create index controversy_links_story on cd.controversy_links_cross_media ( controversy_dumps_id, stories_id ); -create index controversy_links_ref on cd.controversy_links_cross_media ( controversy_dumps_id, ref_stories_id ); - -create table cd.controversy_media_codes ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - controversies_id int not null, - media_id int not null, - code_type text, - code text -); -create index controversy_media_codes_medium on cd.controversy_media_codes ( controversy_dumps_id, media_id ); - -create table cd.media ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - media_id int, - url varchar(1024) not null, - name varchar(128) not null, - moderated boolean not null, - feeds_added boolean not null, - moderation_notes text null, - full_text_rss boolean, - extract_author boolean default(false), - sw_data_start_date date default(null), - sw_data_end_date date default(null), - foreign_rss_links boolean not null default( false ), - dup_media_id int null, - is_not_dup boolean null, - use_pager boolean null, - unpaged_stories int not null default 0 -); -create index media_id on cd.media ( controversy_dumps_id, media_id ); - -create table cd.media_tags_map ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - media_tags_map_id int, - media_id int not null, - tags_id int not null -); -create index media_tags_map_medium on cd.media_tags_map ( controversy_dumps_id, media_id ); -create index media_tags_map_tag on cd.media_tags_map ( controversy_dumps_id, tags_id ); - -create table cd.stories_tags_map -( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - stories_tags_map_id int, - stories_id int, - tags_id int -); -create index stories_tags_map_story on cd.stories_tags_map ( controversy_dumps_id, stories_id ); -create index stories_tags_map_tag on cd.stories_tags_map ( controversy_dumps_id, tags_id ); - -create table cd.tags ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - tags_id int, - tag_sets_id int, - tag varchar(512) -); -create index tags_id on cd.tags ( controversy_dumps_id, tags_id ); - -create table cd.tag_sets ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - tag_sets_id int, - name varchar(512) -); -create index tag_sets_id on cd.tag_sets ( controversy_dumps_id, tag_sets_id ); - --- story -> story links within a cdts -create table cd.story_links ( - controversy_dump_time_slices_id int not null - references controversy_dump_time_slices on delete cascade, - source_stories_id int not null, - ref_stories_id int not null -); - --- TODO: add complex foreign key to check that *_stories_id exist for the controversy_dump stories snapshot -create index story_links_source on cd.story_links( controversy_dump_time_slices_id, source_stories_id ); -create index story_links_ref on cd.story_links( controversy_dump_time_slices_id, ref_stories_id ); - --- link counts for stories within a cdts -create table cd.story_link_counts ( - controversy_dump_time_slices_id int not null - references controversy_dump_time_slices on delete cascade, - stories_id int not null, - inlink_count int not null, - outlink_count int not null -); - --- TODO: add complex foreign key to check that stories_id exists for the controversy_dump stories snapshot -create index story_link_counts_story on cd.story_link_counts ( controversy_dump_time_slices_id, stories_id ); - --- links counts for media within a cdts -create table cd.medium_link_counts ( - controversy_dump_time_slices_id int not null - references controversy_dump_time_slices on delete cascade, - media_id int not null, - inlink_count int not null, - outlink_count int not null, - story_count int not null -); - --- TODO: add complex foreign key to check that media_id exists for the controversy_dump media snapshot -create index medium_link_counts_medium on cd.medium_link_counts ( controversy_dump_time_slices_id, media_id ); - -create table cd.medium_links ( - controversy_dump_time_slices_id int not null - references controversy_dump_time_slices on delete cascade, - source_media_id int not null, - ref_media_id int not null, - link_count int not null -); - --- TODO: add complex foreign key to check that *_media_id exist for the controversy_dump media snapshot -create index medium_links_source on cd.medium_links( controversy_dump_time_slices_id, source_media_id ); -create index medium_links_ref on cd.medium_links( controversy_dump_time_slices_id, ref_media_id ); - -create table cd.daily_date_counts ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - publish_date date not null, - story_count int not null, - tags_id int -); - -create index daily_date_counts_date on cd.daily_date_counts( controversy_dumps_id, publish_date ); -create index daily_date_counts_tag on cd.daily_date_counts( controversy_dumps_id, tags_id ); - -create table cd.weekly_date_counts ( - controversy_dumps_id int not null references controversy_dumps on delete cascade, - publish_date date not null, - story_count int not null, - tags_id int -); - -create index weekly_date_counts_date on cd.weekly_date_counts( controversy_dumps_id, publish_date ); -create index weekly_date_counts_tag on cd.weekly_date_counts( controversy_dumps_id, tags_id ); - - -alter table media_edits alter reason drop not null; -alter table media_edits drop constraint reason_not_empty; - -alter table story_edits alter reason drop not null; -alter table story_edits drop constraint reason_not_empty; - -CREATE OR REPLACE FUNCTION is_stop_stem(p_size TEXT, p_stem TEXT, p_language TEXT) - RETURNS BOOLEAN AS \$\$ -DECLARE - result BOOLEAN; -BEGIN - - -- Tiny - IF p_size = 'tiny' THEN - IF p_language IS NULL THEN - SELECT 't' INTO result FROM stopword_stems_tiny - WHERE stopword_stem = p_stem; - IF NOT FOUND THEN - result := 'f'; - END IF; - ELSE - SELECT 't' INTO result FROM stopword_stems_tiny - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - END IF; - - -- Short - ELSIF p_size = 'short' THEN - IF p_language IS NULL THEN - SELECT 't' INTO result FROM stopword_stems_short - WHERE stopword_stem = p_stem; - IF NOT FOUND THEN - result := 'f'; - END IF; - ELSE - SELECT 't' INTO result FROM stopword_stems_short - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - END IF; - - -- Long - ELSIF p_size = 'long' THEN - IF p_language IS NULL THEN - SELECT 't' INTO result FROM stopword_stems_long - WHERE stopword_stem = p_stem; - IF NOT FOUND THEN - result := 'f'; - END IF; - ELSE - SELECT 't' INTO result FROM stopword_stems_long - WHERE stopword_stem = p_stem AND language = p_language; - IF NOT FOUND THEN - result := 'f'; - END IF; - END IF; - - -- unknown size - ELSE - RAISE EXCEPTION 'Unknown stopword stem size: "%" (expected "tiny", "short" or "long")', p_size; - result := 'f'; - END IF; - - RETURN result; -END; -\$\$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4412-4413.sql b/apps/postgresql-server/schema/migrations/mediawords-4412-4413.sql deleted file mode 100644 index f86bcf20c4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4412-4413.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4412 and 4413. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4409, and you would like to upgrade both the Media Cloud and the --- database to be at version 4412, import this SQL file: --- --- psql mediacloud < mediawords-4412-4413.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4413; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -INSERT INTO auth_roles (role, description) VALUES - ('cm', 'Controversy mapper; includes media and story editing'), - ('stories-api', 'Access to the stories api'); - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4413-4414.sql b/apps/postgresql-server/schema/migrations/mediawords-4413-4414.sql deleted file mode 100644 index d3a22b7649..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4413-4414.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4413 and 4414. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4413, and you would like to upgrade both the Media Cloud and the --- database to be at version 4414, import this SQL file: --- --- psql mediacloud < mediawords-4413-4414.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4414; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -create view controversy_links_cross_media as - select s.stories_id, sm.name as media_name, r.stories_id as ref_stories_id, rm.name as ref_media_name, cl.url as url, cs.controversies_id, cl.controversy_links_id from media sm, media rm, controversy_links cl, stories s, stories r, controversy_stories cs where cl.ref_stories_id <> cl.stories_id and s.stories_id = cl.stories_id and cl.ref_stories_id = r.stories_id and s.media_id <> r.media_id and sm.media_id = s.media_id and rm.media_id = r.media_id and cs.stories_id = cl.ref_stories_id and cs.controversies_id = cl.controversies_id; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4414-4415.sql b/apps/postgresql-server/schema/migrations/mediawords-4414-4415.sql deleted file mode 100644 index 83a90165d2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4414-4415.sql +++ /dev/null @@ -1,99 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4414 and 4415. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4414, and you would like to upgrade both the Media Cloud and the --- database to be at version 4415, import this SQL file: --- --- psql mediacloud < mediawords-4414-4415.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4415; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - -begin; - -alter table controversy_dump_time_slices add story_count int; -alter table controversy_dump_time_slices add story_link_count int; -alter table controversy_dump_time_slices add medium_count int; -alter table controversy_dump_time_slices add medium_link_count int; - -update controversy_dump_time_slices cdts set story_count = q.count - from ( - select count(*) count, c.controversy_dump_time_slices_id - from cd.story_link_counts c - group by c.controversy_dump_time_slices_id - ) q - where - q.controversy_dump_time_slices_id = cdts.controversy_dump_time_slices_id; - -update controversy_dump_time_slices set story_count = 0 where story_count is null; - -update controversy_dump_time_slices cdts set story_link_count = q.count - from ( - select count(*) count, c.controversy_dump_time_slices_id - from cd.story_links c - group by c.controversy_dump_time_slices_id - ) q - where - q.controversy_dump_time_slices_id = cdts.controversy_dump_time_slices_id; - -update controversy_dump_time_slices set story_link_count = 0 where story_link_count is null; - -update controversy_dump_time_slices cdts set medium_count = q.count - from ( - select count(*) count, c.controversy_dump_time_slices_id - from cd.medium_link_counts c - group by c.controversy_dump_time_slices_id - ) q - where - q.controversy_dump_time_slices_id = cdts.controversy_dump_time_slices_id; - -update controversy_dump_time_slices set medium_count = 0 where medium_count is null; - -update controversy_dump_time_slices cdts set medium_link_count = q.count - from ( - select count(*) count, c.controversy_dump_time_slices_id - from cd.medium_links c - group by c.controversy_dump_time_slices_id - ) q - where - q.controversy_dump_time_slices_id = cdts.controversy_dump_time_slices_id; - -update controversy_dump_time_slices set medium_link_count = 0 where medium_link_count is null; - -alter table controversy_dump_time_slices alter story_count set not null; -alter table controversy_dump_time_slices alter story_link_count set not null; -alter table controversy_dump_time_slices alter medium_count set not null; -alter table controversy_dump_time_slices alter medium_link_count set not null; - -commit; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4415-4416.sql b/apps/postgresql-server/schema/migrations/mediawords-4415-4416.sql deleted file mode 100644 index c2667a26d6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4415-4416.sql +++ /dev/null @@ -1,107 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4415 and 4416. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4415, and you would like to upgrade both the Media Cloud and the --- database to be at version 4416, import this SQL file: --- --- psql mediacloud < mediawords-4415-4416.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4416; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - -begin; - -create table cdts_files ( - cdts_files_id serial primary key, - controversy_dump_time_slices_id int not null references controversy_dump_time_slices on delete cascade, - file_name text, - file_content text -); - -create index cdts_files_cdts on cdts_files ( controversy_dump_time_slices_id ); - -create table cd_files ( - cd_files_id serial primary key, - controversy_dumps_id int not null references controversy_dumps on delete cascade, - file_name text, - file_content text -); - -create index cd_files_cd on cd_files ( controversy_dumps_id ); - -insert into cdts_files - ( controversy_dump_time_slices_id, file_name, file_content ) - select controversy_dump_time_slices_id, 'stories.csv', stories_csv - from controversy_dump_time_slices; - -insert into cdts_files - ( controversy_dump_time_slices_id, file_name, file_content ) - select controversy_dump_time_slices_id, 'story_links.csv', story_links_csv - from controversy_dump_time_slices; - -insert into cdts_files - ( controversy_dump_time_slices_id, file_name, file_content ) - select controversy_dump_time_slices_id, 'media.csv', media_csv - from controversy_dump_time_slices; - -insert into cdts_files - ( controversy_dump_time_slices_id, file_name, file_content ) - select controversy_dump_time_slices_id, 'medium_links.csv', medium_links_csv - from controversy_dump_time_slices; - -insert into cdts_files - ( controversy_dump_time_slices_id, file_name, file_content ) - select controversy_dump_time_slices_id, 'media.gexf', gexf - from controversy_dump_time_slices; - -insert into cd_files - ( controversy_dumps_id, file_name, file_content ) - select controversy_dumps_id, 'daily_counts.csv', daily_counts_csv - from controversy_dumps; - -insert into cd_files - ( controversy_dumps_id, file_name, file_content ) - select controversy_dumps_id, 'weekly_counts.csv', weekly_counts_csv - from controversy_dumps; - -alter table controversy_dump_time_slices drop stories_csv; -alter table controversy_dump_time_slices drop story_links_csv; -alter table controversy_dump_time_slices drop media_csv; -alter table controversy_dump_time_slices drop medium_links_csv; -alter table controversy_dump_time_slices drop gexf; - -alter table controversy_dumps drop daily_counts_csv; -alter table controversy_dumps drop weekly_counts_csv; - -commit; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4416-4417.sql b/apps/postgresql-server/schema/migrations/mediawords-4416-4417.sql deleted file mode 100644 index ceb88b7558..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4416-4417.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4416 and 4417. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4416, and you would like to upgrade both the Media Cloud and the --- database to be at version 4417, import this SQL file: --- --- psql mediacloud < mediawords-4416-4417.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4417; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -create index controversy_links_controversy on controversy_links ( controversies_id ); - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4417-4418.sql b/apps/postgresql-server/schema/migrations/mediawords-4417-4418.sql deleted file mode 100644 index 6e701519e2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4417-4418.sql +++ /dev/null @@ -1,119 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4417 and 4418. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4417, and you would like to upgrade both the Media Cloud and the --- database to be at version 4418, import this SQL file: --- --- psql mediacloud < mediawords-4417-4418.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4418; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -begin; - --- create a mirror of the stories table with the stories for each controversy. this is to make --- it much faster to query the stories associated with a given controversy, rather than querying the --- contested and bloated stories table. only inserts and updates on stories are triggered, because --- deleted cascading stories_id and controversies_id fields take care of deletes. -create table cd.live_stories ( - controversies_id int not null references controversies on delete cascade, - controversy_stories_id int not null references controversy_stories on delete cascade, - stories_id int not null references stories on delete cascade, - media_id int not null, - url varchar(1024) not null, - guid varchar(1024) not null, - title text not null, - description text null, - publish_date timestamp not null, - collect_date timestamp not null, - full_text_rss boolean not null default 'f', - language varchar(3) null -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset -); -create index live_story_controversy on cd.live_stories ( controversies_id ); -create unique index live_stories_story on cd.live_stories ( controversies_id, stories_id ); - -create function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into cd.live_stories - ( controversies_id, controversy_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language ) - select NEW.controversies_id, NEW.controversy_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language - from controversy_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.controversies_id = NEW.controversies_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create trigger controversy_stories_insert_live_story after insert on controversy_stories - for each row execute procedure insert_live_story(); - -create function update_live_story() returns trigger as $update_live_story$ - declare - controversy record; - begin - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - -create trigger stories_update_live_story after update on stories - for each row execute procedure update_live_story(); - -insert into cd.live_stories - ( controversies_id, controversy_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language ) - select cs.controversies_id, cs.controversy_stories_id, s.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language - from controversy_stories cs - join stories s on ( cs.stories_id = s.stories_id ); - -commit; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4418-4419.sql b/apps/postgresql-server/schema/migrations/mediawords-4418-4419.sql deleted file mode 100644 index 87a2d16ece..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4418-4419.sql +++ /dev/null @@ -1,60 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4418 and 4419. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4418, and you would like to upgrade both the Media Cloud and the --- database to be at version 4419, import this SQL file: --- --- psql mediacloud < mediawords-4418-4419.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER INDEX auth_users_roles_map_users_id_roles_id - RENAME TO auth_users_roles_map_auth_users_id_auth_roles_id; - -ALTER TABLE auth_users - RENAME users_id TO auth_users_id; - -ALTER TABLE auth_roles - RENAME roles_id TO auth_roles_id; - -ALTER TABLE auth_users_roles_map - RENAME auth_users_roles_map TO auth_users_roles_map_id; -ALTER TABLE auth_users_roles_map - RENAME users_id TO auth_users_id; -ALTER TABLE auth_users_roles_map - RENAME roles_id TO auth_roles_id; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4419; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4419-4420.sql b/apps/postgresql-server/schema/migrations/mediawords-4419-4420.sql deleted file mode 100644 index aa66482503..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4419-4420.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4419 and 4420. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4419, and you would like to upgrade both the Media Cloud and the --- database to be at version 4420, import this SQL file: --- --- psql mediacloud < mediawords-4419-4420.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -create table controversy_dump_tags ( - controversy_dump_tags_id serial primary key, - controversies_id int not null references controversies on delete cascade, - tags_id int not null references tags -); - -alter table controversy_dump_time_slices add tags_id int references tags; - -alter table cd.controversy_links_cross_media drop media_name; -alter table cd.controversy_links_cross_media drop ref_media_name; -alter table cd.stories drop description; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4420; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4420-4421.sql b/apps/postgresql-server/schema/migrations/mediawords-4420-4421.sql deleted file mode 100644 index 4d95b329d2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4420-4421.sql +++ /dev/null @@ -1,81 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4420 and 4421. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4420, and you would like to upgrade both the Media Cloud and the --- database to be at version 4421, import this SQL file: --- --- psql mediacloud < mediawords-4420-4421.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE story_sentences - ADD COLUMN last_updated timestamp with time zone NOT NULL DEFAULT now(); - -ALTER TABLE story_sentences - ALTER COLUMN last_updated DROP DEFAULT; - -CREATE OR REPLACE FUNCTION story_sentences_last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') then - - NEW.last_updated = now(); - ELSE - -- RAISE NOTICE 'NO path change % = %', OLD.path, NEW.path; - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER story_sentences_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences - FOR EACH ROW - EXECUTE PROCEDURE story_sentences_last_updated_trigger() ; - -create table controversy_ignore_redirects ( - controversy_ignore_redirects_id serial primary key, - url varchar( 1024 ) -); - -create index controversy_ignore_redirects_url on controversy_ignore_redirects ( url ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4421; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4421-4422.sql b/apps/postgresql-server/schema/migrations/mediawords-4421-4422.sql deleted file mode 100644 index 1f0ef6d121..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4421-4422.sql +++ /dev/null @@ -1,70 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4421 and 4422. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4421, and you would like to upgrade both the Media Cloud and the --- database to be at version 4422, import this SQL file: --- --- psql mediacloud < mediawords-4421-4422.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE media_edits - -- Don't reference the "media" table in the SQL diff because ALTER TABLE - -- will fill "media_id" with zeroes, and media.media_id = 0 might not exist. - -- - -- Also, make the default value of media_edits.media_id = 0 because at this - -- point we don't know which specific media was edited (someone has to - -- create those references by hand). - -- - -- Later, after creating manual references from media_edits.media_id to - -- media.media_id, one should ALTER this table further as such: - -- - -- ALTER TABLE media_edits ALTER COLUMN media_id DROP DEFAULT; - -- ALTER TABLE media_edits ADD CONSTRAINT media_edits_media_id_fkey - -- FOREIGN KEY (media_id) REFERENCES media(media_id) - -- ON DELETE NO ACTION ON UPDATE NO ACTION DEFERRABLE; - -- - ADD COLUMN media_id INT NOT NULL DEFAULT 0; - -CREATE INDEX media_edits_media_id ON media_edits (media_id); -CREATE INDEX media_edits_edited_field ON media_edits (edited_field); -CREATE INDEX media_edits_users_email ON media_edits (users_email); -CREATE INDEX story_edits_stories_id ON story_edits (stories_id); -CREATE INDEX story_edits_edited_field ON story_edits (edited_field); -CREATE INDEX story_edits_users_email ON story_edits (users_email); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4422; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4422-4423.sql b/apps/postgresql-server/schema/migrations/mediawords-4422-4423.sql deleted file mode 100644 index 3f83db5d61..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4422-4423.sql +++ /dev/null @@ -1,94 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4422 and 4423. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4422, and you would like to upgrade both the Media Cloud and the --- database to be at version 4423, import this SQL file: --- --- psql mediacloud < mediawords-4422-4423.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE media_sets_media_map - ADD COLUMN db_row_last_updated timestamp with time zone default now(); - -ALTER TABLE media_sets_media_map - ALTER COLUMN db_row_last_updated SET NOT NULL; - -ALTER TABLE stories - ADD COLUMN db_row_last_updated timestamp with time zone; - -ALTER TABLE story_sentences - ADD COLUMN db_row_last_updated timestamp with time zone; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4423; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE INDEX media_sets_media_map_db_row_last_updated ON media_sets_media_map ( db_row_last_updated ); - -CREATE INDEX stories_db_row_last_updated ON stories ( db_row_last_updated ); - -CREATE INDEX story_sentences_db_row_last_updated ON story_sentences ( db_row_last_updated ); - -CREATE TRIGGER media_sets_media_map_last_updated_trigger - BEFORE INSERT OR UPDATE ON media_sets_media_map - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - -CREATE TRIGGER stories_last_updated_trigger - BEFORE INSERT OR UPDATE ON stories - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - -CREATE TRIGGER story_sentences_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4423-4424.sql b/apps/postgresql-server/schema/migrations/mediawords-4423-4424.sql deleted file mode 100644 index 4e83d1e5d5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4423-4424.sql +++ /dev/null @@ -1,84 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4423 and 4424. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4423, and you would like to upgrade both the Media Cloud and the --- database to be at version 4424, import this SQL file: --- --- psql mediacloud < mediawords-4423-4424.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE media_tags_map - ADD COLUMN db_row_last_updated timestamp with time zone default now(); - -ALTER TABLE media_tags_map - ALTER COLUMN db_row_last_updated SET NOT NULL; - -create index media_tags_map_db_row_last_updated on media_tags_map ( db_row_last_updated ); - -ALTER TABLE stories_tags_map - ADD COLUMN db_row_last_updated timestamp with time zone default now(); - -ALTER TABLE stories_tags_map - ALTER COLUMN db_row_last_updated SET NOT NULL; - -CREATE TRIGGER media_tags_last_updated_trigger - BEFORE INSERT OR UPDATE ON media_tags_map - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - -CREATE TRIGGER stories_tags_map_last_updated_trigger - BEFORE INSERT OR UPDATE ON stories_tags_map - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4424; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4424-4425.sql b/apps/postgresql-server/schema/migrations/mediawords-4424-4425.sql deleted file mode 100644 index 6bd58e220d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4424-4425.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4424 and 4425. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4424, and you would like to upgrade both the Media Cloud and the --- database to be at version 4425, import this SQL file: --- --- psql mediacloud < mediawords-4424-4425.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4425; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - UPDATE story_sentences set db_row_last_updated = now() where stories_id = NEW.stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER stories_update_story_sentences_last_updated_trigger - AFTER UPDATE ON stories - FOR EACH ROW - EXECUTE PROCEDURE update_story_sentences_updated_time_trigger() ; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4425-4426.sql b/apps/postgresql-server/schema/migrations/mediawords-4425-4426.sql deleted file mode 100644 index 39b4c330bf..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4425-4426.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4425 and 4426. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4425, and you would like to upgrade both the Media Cloud and the --- database to be at version 4426, import this SQL file: --- --- psql mediacloud < mediawords-4425-4426.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4426; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - UPDATE stories set db_row_last_updated = now() where stories_id = NEW.stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER stories_tags_map_update_stories_last_updated_trigger - AFTER DELETE ON stories_tags_map - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_trigger(); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4426-4427.sql b/apps/postgresql-server/schema/migrations/mediawords-4426-4427.sql deleted file mode 100644 index 5e1057a791..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4426-4427.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4426 and 4427. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4426, and you would like to upgrade both the Media Cloud and the --- database to be at version 4427, import this SQL file: --- --- psql mediacloud < mediawords-4426-4427.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table cd.live_stories add db_row_last_updated timestamp with time zone; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into cd.live_stories - ( controversies_id, controversy_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language, - db_row_last_updated ) - select NEW.controversies_id, NEW.controversy_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language, - s.db_row_last_updated - from controversy_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.controversies_id = NEW.controversies_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4427; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4427-4428.sql b/apps/postgresql-server/schema/migrations/mediawords-4427-4428.sql deleted file mode 100644 index 2aaf46bf0b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4427-4428.sql +++ /dev/null @@ -1,101 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4427 and 4428. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4427, and you would like to upgrade both the Media Cloud and the --- database to be at version 4428, import this SQL file: --- --- psql mediacloud < mediawords-4427-4428.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - - --- --- Gearman job queue (jobs enqueued with enqueue_on_gearman()) --- - -CREATE TYPE gearman_job_queue_status AS ENUM ( - 'enqueued', -- Job is enqueued and waiting to be run - 'running', -- Job is currently running - 'finished', -- Job has finished successfully - 'failed' -- Job has failed -); - -CREATE TABLE gearman_job_queue ( - gearman_job_queue_id SERIAL PRIMARY KEY, - - -- Last status update time - last_modified TIMESTAMP NOT NULL DEFAULT LOCALTIMESTAMP, - - -- Gearman function name (e.g. "MediaWords::GearmanFunction::CM::DumpControversy") - function_name VARCHAR(255) NOT NULL, - - -- Gearman job handle (e.g. "H:tundra.local:8") - -- - -- This table expects all job handles to be unique, and Gearman would not - -- generate unique job handles if it is configured to store the job queue - -- in memory (as it does by default), so you *must* configure a persistent - -- queue storage. - -- For an instruction on how to store the Gearman job queue on PostgreSQL, - -- see doc/README.gearman.markdown. - job_handle VARCHAR(255) UNIQUE NOT NULL, - - -- Job status - status gearman_job_queue_status NOT NULL, - - -- Error message (if any) - error_message TEXT NULL -); - -CREATE INDEX gearman_job_queue_function_name ON gearman_job_queue (function_name); -CREATE UNIQUE INDEX gearman_job_queue_job_handle ON gearman_job_queue (job_handle); -CREATE INDEX gearman_job_queue_status ON gearman_job_queue (status); - --- Update "last_modified" on UPDATEs -CREATE FUNCTION gearman_job_queue_sync_lastmod() RETURNS trigger AS $$ -BEGIN - NEW.last_modified := NOW(); - RETURN NEW; -END; -$$ LANGUAGE PLPGSQL; - -CREATE TRIGGER gearman_job_queue_sync_lastmod - BEFORE UPDATE ON gearman_job_queue - FOR EACH ROW EXECUTE PROCEDURE gearman_job_queue_sync_lastmod(); - - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4428; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4428-4429.sql b/apps/postgresql-server/schema/migrations/mediawords-4428-4429.sql deleted file mode 100644 index e457178ed6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4428-4429.sql +++ /dev/null @@ -1,73 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4428 and 4429. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4428, and you would like to upgrade both the Media Cloud and the --- database to be at version 4429, import this SQL file: --- --- psql mediacloud < mediawords-4428-4429.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - - -ALTER TABLE gearman_job_queue - -- Unique Gearman job identifier that describes the job that is being run. - -- - -- In the Gearman::JobScheduler's case, this is a SHA256 of the serialized - -- Gearman function name and its parameters, e.g. - -- - -- sha256_hex("MediaWords::GearmanFunction::CM::DumpControversy({controversies_id => 1})") - -- = - -- "b9758abbd3811b0aaa53d0e97e188fcac54f58a876bb409b7395621411401ee8" - -- - -- Although "job_handle" above also serves as an unique identifier of the - -- specific job, and Gearman uses both at the same time to identify a job, - -- it provides no way to fetch the "unique job ID" (e.g. this SHA256 string) - -- by having a Gearman job handle (e.g. "H:tundra.local:8") and vice versa, - -- so we have to store it somewhere ourselves. - -- - -- The "unique job ID" is needed to check if the job with specific - -- parameters (e.g. a "dump controversy" job for the controversy ID) is - -- enqueued / running / failed. - -- - -- The unique job ID's length is limited to Gearman internal - -- GEARMAN_MAX_UNIQUE_SIZE which is set to 64 at the time of writing. - ADD COLUMN unique_job_id VARCHAR(64) NOT NULL; - -CREATE INDEX gearman_job_queue_unique_job_id ON gearman_job_queue (unique_job_id); - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4429; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4429-4430.sql b/apps/postgresql-server/schema/migrations/mediawords-4429-4430.sql deleted file mode 100644 index efb2cba606..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4429-4430.sql +++ /dev/null @@ -1,82 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4429 and 4430. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4429, and you would like to upgrade both the Media Cloud and the --- database to be at version 4430, import this SQL file: --- --- psql mediacloud < mediawords-4429-4430.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4430; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- Activity log --- - -CREATE TABLE activities ( - activities_id SERIAL PRIMARY KEY, - - -- Activity's name (e.g. "media_edit", "story_edit", etc.) - name VARCHAR(255) NOT NULL - CONSTRAINT activities_name_can_not_contain_spaces CHECK(name NOT LIKE '% %'), - - -- When did the activity happen - creation_date TIMESTAMP NOT NULL DEFAULT LOCALTIMESTAMP, - - -- User that executed the activity, either: - -- * user's email from "auth_users.email" (e.g. "foo@bar.baz.com", or - -- * username that initiated the action (e.g. "system:foo") - -- (store user's email instead of ID in case the user gets deleted) - user_identifier VARCHAR(255) NOT NULL, - - -- Indexed ID of the object that was modified in some way by the activity - -- (e.g. media's ID "media_edit" or story's ID in "story_edit") - object_id BIGINT NULL, - - -- User-provided reason explaining why the activity was made - reason TEXT NULL, - - -- Other free-form data describing the action in the JSON format - -- (e.g.: '{ "field": "name", "old_value": "Foo.", "new_value": "Bar." }') - -- FIXME: has potential to use 'JSON' type instead of 'TEXT' in - -- PostgreSQL 9.2+ - description_json TEXT NOT NULL DEFAULT '{ }' - -); - -CREATE INDEX activities_name ON activities (name); -CREATE INDEX activities_creation_date ON activities (creation_date); -CREATE INDEX activities_user_identifier ON activities (user_identifier); -CREATE INDEX activities_object_id ON activities (object_id); - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4430-4431.sql b/apps/postgresql-server/schema/migrations/mediawords-4430-4431.sql deleted file mode 100644 index ca1aebc76d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4430-4431.sql +++ /dev/null @@ -1,68 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4430 and 4431. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4430, and you would like to upgrade both the Media Cloud and the --- database to be at version 4431, import this SQL file: --- --- psql mediacloud < mediawords-4430-4431.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Generate random API token -CREATE OR REPLACE FUNCTION generate_api_token() RETURNS VARCHAR(64) LANGUAGE plpgsql AS $$ -DECLARE - token VARCHAR(64); -BEGIN - SELECT encode(digest(gen_random_bytes(256), 'sha256'), 'hex') INTO token; - RETURN token; -END; -$$; - --- Add "api_token" column (API tokens will be generated for old users) -ALTER TABLE auth_users - ADD COLUMN api_token VARCHAR(64) UNIQUE NOT NULL DEFAULT generate_api_token() CONSTRAINT api_token_64_characters CHECK(LENGTH(api_token) = 64); - --- --- Incorporate changes from the 4430->4431 diff from master --- - -ALTER TYPE download_state ADD value 'extractor_error'; - --- Fix downloads marked as errors when the problem was with the extractor -UPDATE downloads set state = 'extractor_error' where state='error' and type='content' and error_message is not null and error_message like 'extractor_error%'; - - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4431; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4431-4432.sql b/apps/postgresql-server/schema/migrations/mediawords-4431-4432.sql deleted file mode 100644 index 729bccffbc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4431-4432.sql +++ /dev/null @@ -1,94 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4431 and 4432. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4431, and you would like to upgrade both the Media Cloud and the --- database to be at version 4432, import this SQL file: --- --- psql mediacloud < mediawords-4431-4432.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP TRIGGER stories_update_story_sentences_last_updated_trigger ON stories; - -DROP TRIGGER stories_tags_map_update_stories_last_updated_trigger ON stories_tags_map; - -DROP TRIGGER IF EXISTS media_tags_map_update_stories_last_updated_trigger ON media_tags_map; -DROP TRIGGER IF EXISTS media_sets_media_map_update_stories_last_updated_trigger on media_sets_media_map; - -DROP FUNCTION update_stories_updated_time_trigger(); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4432; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - UPDATE stories set db_row_last_updated = now() where stories_id = OLD.stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_media_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - UPDATE stories set db_row_last_updated = now() where media_id = OLD.media_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER media_tags_map_update_stories_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON media_tags_map - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_by_media_id_trigger(); - -CREATE TRIGGER media_sets_media_map_update_stories_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON media_sets_media_map - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_by_media_id_trigger(); - -CREATE TRIGGER stories_update_story_sentences_last_updated_trigger - AFTER INSERT OR UPDATE ON stories - FOR EACH ROW - EXECUTE PROCEDURE update_story_sentences_updated_time_trigger() ; - -CREATE TRIGGER stories_tags_map_update_stories_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON stories_tags_map - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_by_stories_id_trigger(); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4432-4433.sql b/apps/postgresql-server/schema/migrations/mediawords-4432-4433.sql deleted file mode 100644 index 77e5d27c47..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4432-4433.sql +++ /dev/null @@ -1,92 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4432 and 4433. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4432, and you would like to upgrade both the Media Cloud and the --- database to be at version 4433, import this SQL file: --- --- psql mediacloud < mediawords-4432-4433.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4433; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_stories_id integer default null; - BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_media_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_media_id integer default null; - BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_media_id = NEW.media_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_media_id = OLD.media_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE media_id = reference_media_id; - - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4433-4434.sql b/apps/postgresql-server/schema/migrations/mediawords-4433-4434.sql deleted file mode 100644 index fe82a5c373..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4433-4434.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4433 and 4434. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4433, and you would like to upgrade both the Media Cloud and the --- database to be at version 4434, import this SQL file: --- --- psql mediacloud < mediawords-4433-4434.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE TABLE raw_downloads ( - raw_downloads_id SERIAL PRIMARY KEY, - downloads_id INTEGER NOT NULL REFERENCES downloads ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX raw_downloads_downloads_id ON raw_downloads (downloads_id); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4434; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4434-4435.sql b/apps/postgresql-server/schema/migrations/mediawords-4434-4435.sql deleted file mode 100644 index f2f729c89f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4434-4435.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4434 and 4435. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4434, and you would like to upgrade both the Media Cloud and the --- database to be at version 4435, import this SQL file: --- --- psql mediacloud < mediawords-4434-4435.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE EXTENSION IF NOT EXISTS pg_trgm; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4435; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4435-4436.sql b/apps/postgresql-server/schema/migrations/mediawords-4435-4436.sql deleted file mode 100644 index 2f698ee3d0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4435-4436.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4435 and 4436. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4435, and you would like to upgrade both the Media Cloud and the --- database to be at version 4436, import this SQL file: --- --- psql mediacloud < mediawords-4435-4436.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4436; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE INDEX media_name_trgm ON media USING gin (name gin_trgm_ops); - -CREATE INDEX media_url_trgm ON media USING gin (url gin_trgm_ops); - -CREATE INDEX dashboards_name_trgm ON dashboards USING gin (name gin_trgm_ops); - -CREATE INDEX media_sets_name_trgm ON media_sets USING gin (name gin_trgm_ops); - -CREATE INDEX media_sets_description_trgm ON media_sets USING gin (description gin_trgm_ops); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4436-4437.sql b/apps/postgresql-server/schema/migrations/mediawords-4436-4437.sql deleted file mode 100644 index 476b71f2fc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4436-4437.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4436 and 4437. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4436, and you would like to upgrade both the Media Cloud and the --- database to be at version 4437, import this SQL file: --- --- psql mediacloud < mediawords-4436-4437.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop index stories_title_pubdate; - -create index stories_title_hash on stories( md5( title ) ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4437; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4437-4438.sql b/apps/postgresql-server/schema/migrations/mediawords-4437-4438.sql deleted file mode 100644 index 66cab473e5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4437-4438.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4437 and 4438. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4437, and you would like to upgrade both the Media Cloud and the --- database to be at version 4438, import this SQL file: --- --- psql mediacloud < mediawords-4437-4438.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE processed_stories - ALTER COLUMN stories_id TYPE int; -- TYPE change - table: processed_stories - -- original: bigint not null references stories on delete cascade - -- new: int not null references stories on delete cascade - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4438; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4438-4439.sql b/apps/postgresql-server/schema/migrations/mediawords-4438-4439.sql deleted file mode 100644 index 371ddddd0d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4438-4439.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4438 and 4439. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4438, and you would like to upgrade both the Media Cloud and the --- database to be at version 4439, import this SQL file: --- --- psql mediacloud < mediawords-4438-4439.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table media_stats ( - media_stats_id serial primary key, - media_id int not null references media on delete cascade, - num_stories int not null, - num_sentences int not null, - mean_num_sentences int not null, - mean_text_length int not null, - num_stories_with_sentences int not null, - num_stories_with_text int not null, - stat_date date not null -); - -create index media_stats_medium on media_stats( media_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4439; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4439-4440.sql b/apps/postgresql-server/schema/migrations/mediawords-4439-4440.sql deleted file mode 100644 index ef1866c108..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4439-4440.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4439 and 4440. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4439, and you would like to upgrade both the Media Cloud and the --- database to be at version 4440, import this SQL file: --- --- psql mediacloud < mediawords-4439-4440.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4440; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -DROP TRIGGER IF EXISTS processed_stories_update_stories_last_updated_trigger ON processed_stories; - -CREATE TRIGGER processed_stories_update_stories_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON processed_stories - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_by_stories_id_trigger(); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4440-4441.sql b/apps/postgresql-server/schema/migrations/mediawords-4440-4441.sql deleted file mode 100644 index 5937f4820c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4440-4441.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4440 and 4441. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4440, and you would like to upgrade both the Media Cloud and the --- database to be at version 4441, import this SQL file: --- --- psql mediacloud < mediawords-4440-4441.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create index stories_publish_day on stories ( date_trunc( 'day', publish_date ) ); - -create index downloads_feed_download_time on downloads ( feeds_id, download_time ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4441; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4441-4442.sql b/apps/postgresql-server/schema/migrations/mediawords-4441-4442.sql deleted file mode 100644 index 6d106292cb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4441-4442.sql +++ /dev/null @@ -1,77 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4441 and 4442. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4441, and you would like to upgrade both the Media Cloud and the --- database to be at version 4442, import this SQL file: --- --- psql mediacloud < mediawords-4441-4442.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE raw_downloads - RENAME COLUMN downloads_id TO object_id; - -ALTER INDEX raw_downloads_downloads_id - RENAME TO raw_downloads_object_id; - - -CREATE TABLE story_sentences_tags_map ( - story_sentences_tags_map_id bigserial primary key, - story_sentences_id bigint not null references story_sentences on delete cascade, - tags_id int not null references tags on delete cascade, - db_row_last_updated timestamp with time zone NOT NULL -); - -CREATE INDEX story_sentences_tags_map_db_row_last_updated ON story_sentences_tags_map ( db_row_last_updated ); - -CREATE UNIQUE INDEX story_sentences_tags_map_story ON story_sentences_tags_map (story_sentences_id, tags_id); - -CREATE INDEX story_sentences_tags_map_tag ON story_sentences_tags_map (tags_id); - -CREATE INDEX story_sentences_tags_map_story_id ON story_sentences_tags_map USING btree (story_sentences_id); - -CREATE TRIGGER story_sentences_tags_map_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences_tags_map - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger() ; - -CREATE TRIGGER story_sentences_tags_map_update_story_sentences_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON story_sentences_tags_map - FOR EACH ROW - EXECUTE PROCEDURE update_stories_updated_time_by_stories_id_trigger(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4442; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4442-4443.sql b/apps/postgresql-server/schema/migrations/mediawords-4442-4443.sql deleted file mode 100644 index 822f362b3b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4442-4443.sql +++ /dev/null @@ -1,74 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4442 and 4443. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4442, and you would like to upgrade both the Media Cloud and the --- database to be at version 4443, import this SQL file: --- --- psql mediacloud < mediawords-4442-4443.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP TRIGGER story_sentences_tags_map_update_story_sentences_last_updated_trigger ON story_sentences_tags_map; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4443; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_story_sentences_id integer default null; - BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER story_sentences_tags_map_update_story_sentences_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON story_sentences_tags_map - FOR EACH ROW - EXECUTE PROCEDURE update_story_sentences_updated_time_by_story_sentences_id_trigger(); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4443-4444.sql b/apps/postgresql-server/schema/migrations/mediawords-4443-4444.sql deleted file mode 100644 index 10e3ae94c8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4443-4444.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4443 and 4444. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4443, and you would like to upgrade both the Media Cloud and the --- database to be at version 4444, import this SQL file: --- --- psql mediacloud < mediawords-4443-4444.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE EXTENSION IF NOT EXISTS pgcrypto; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4444; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4444-4445.sql b/apps/postgresql-server/schema/migrations/mediawords-4444-4445.sql deleted file mode 100644 index e634daefee..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4444-4445.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4444 and 4445. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4444, and you would like to upgrade both the Media Cloud and the --- database to be at version 4445, import this SQL file: --- --- psql mediacloud < mediawords-4444-4445.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -INSERT INTO auth_roles (role, description) VALUES - ('search', 'Access to the /search pages'); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4445; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4445-4446.sql b/apps/postgresql-server/schema/migrations/mediawords-4445-4446.sql deleted file mode 100644 index d67a6172ab..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4445-4446.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4445 and 4446. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4445, and you would like to upgrade both the Media Cloud and the --- database to be at version 4446, import this SQL file: --- --- psql mediacloud < mediawords-4445-4446.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE media - ADD COLUMN annotate_with_corenlp BOOLEAN NOT NULL DEFAULT(false); - -ALTER TABLE dashboards - ADD COLUMN public BOOLEAN NOT NULL DEFAULT(true); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4446; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4446-4447.sql b/apps/postgresql-server/schema/migrations/mediawords-4446-4447.sql deleted file mode 100644 index 5a643d295d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4446-4447.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4446 and 4447. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4446, and you would like to upgrade both the Media Cloud and the --- database to be at version 4447, import this SQL file: --- --- psql mediacloud < mediawords-4446-4447.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create table solr_imports ( - solr_imports_id serial primary key, - import_date timestamp not null, - full_import boolean not null default false -); - -create index solr_imports_date on solr_imports ( import_date ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4447; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4447-4448.sql b/apps/postgresql-server/schema/migrations/mediawords-4447-4448.sql deleted file mode 100644 index 45d1dac97d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4447-4448.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4447 and 4448. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4447, and you would like to upgrade both the Media Cloud and the --- database to be at version 4448, import this SQL file: --- --- psql mediacloud < mediawords-4447-4448.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table media_update_time_queue ( - media_id int not null references media on delete cascade, - db_row_last_updated timestamp with time zone not null -); - -create index media_update_time_queue_updated on media_update_time_queue ( db_row_last_updated ); - -drop trigger media_tags_map_update_stories_last_updated_trigger on media_tags_map; -drop trigger media_sets_media_map_update_stories_last_updated_trigger on media_sets_media_map; - -alter table controversies - add pattern text, - add solr_seed_query text, - add solr_seed_query_run boolean, - add description text; - -update controversies c - set - pattern = a.pattern, - solr_seed_query = '"CONTROVERSY CREATED BEFORE SOLR QUERY SUPPORT"', - solr_seed_query_run = 't', - description = name || ' in ' || a.media_set_names || ' from ' || a.start_date || ' to ' || a.end_date - from ( - select - ca.controversies_id, - min( qss.pattern ) pattern, - string_agg( ms.name, '; ' ) media_set_names, - min( q.start_date ) start_date, - min( q.end_date ) end_date - from controversies ca - join query_story_searches qss on ( ca.query_story_searches_id = qss.query_story_searches_id ) - join queries q on ( qss.queries_id = q.queries_id ) - join queries_media_sets_map qmsm on ( qmsm.queries_id = q.queries_id ) - join media_sets ms on ( ms.media_sets_id = qmsm.media_sets_id ) - group by ca.controversies_id - ) a - where a.controversies_id = c.controversies_id; - -alter table controversies - alter pattern set not null, - alter solr_seed_query set not null, - alter solr_seed_query_run set not null, - alter description set not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4448; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4448-4449.sql b/apps/postgresql-server/schema/migrations/mediawords-4448-4449.sql deleted file mode 100644 index aa28c20a70..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4448-4449.sql +++ /dev/null @@ -1,87 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4448 and 4449. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4448, and you would like to upgrade both the Media Cloud and the --- database to be at version 4449, import this SQL file: --- --- psql mediacloud < mediawords-4448-4449.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -BEGIN - - IF EXISTS ( - - SELECT 1 - FROM stories - INNER JOIN media ON stories.media_id = media.media_id - WHERE stories.stories_id = corenlp_stories_id - - -- We don't check if the story has been extracted here because the - -- CoreNLP worker might get to it sooner than the extractor (i.e. the - -- extractor might not be fast enough to set extracted = 't' before - -- CoreNLP annotation begins) - - -- Media is marked for CoreNLP annotation - AND media.annotate_with_corenlp = 't' - - -- English language stories only because they're the only ones - -- supported by CoreNLP at the time. - -- Stories with language field set to NULL are the ones fetched - -- before introduction of the multilanguage support, so they are - -- assumed to be in English - AND (stories.language = 'en' OR stories.language IS NULL) - - -- Story not yet marked as "processed" - AND NOT EXISTS ( - SELECT 1 - FROM processed_stories - WHERE stories.stories_id = processed_stories.stories_id - ) - - ) THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4449; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4449-4450.sql b/apps/postgresql-server/schema/migrations/mediawords-4449-4450.sql deleted file mode 100644 index ef50e57a18..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4449-4450.sql +++ /dev/null @@ -1,54 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4449 and 4450. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4449, and you would like to upgrade both the Media Cloud and the --- database to be at version 4450, import this SQL file: --- --- psql mediacloud < mediawords-4449-4450.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index auth_users_email on auth_users( email ); -create index auth_users_token on auth_users( api_token ); - -create table auth_user_ip_tokens ( - auth_user_ip_tokens_id serial primary key, - auth_users_id int not null references auth_users on delete cascade, - api_token varchar(64) unique not null default generate_api_token() constraint api_token_64_characters check( length( api_token ) = 64 ), - ip_address inet not null -); - -create index auth_user_ip_tokens_token on auth_user_ip_tokens ( api_token, ip_address ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4450; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4450-4451.sql b/apps/postgresql-server/schema/migrations/mediawords-4450-4451.sql deleted file mode 100644 index 12b892a75b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4450-4451.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4450 and 4451. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4450, and you would like to upgrade both the Media Cloud and the --- database to be at version 4451, import this SQL file: --- --- psql mediacloud < mediawords-4450-4451.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4451; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4451-4452.sql b/apps/postgresql-server/schema/migrations/mediawords-4451-4452.sql deleted file mode 100644 index 5eae105f0b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4451-4452.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4451 and 4452. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4451, and you would like to upgrade both the Media Cloud and the --- database to be at version 4452, import this SQL file: --- --- psql mediacloud < mediawords-4451-4452.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE tag_sets - ADD COLUMN label varchar(512), - ADD COLUMN description text; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4452; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4452-4453.sql b/apps/postgresql-server/schema/migrations/mediawords-4452-4453.sql deleted file mode 100644 index 0387f84d5d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4452-4453.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4452 and 4453. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4452, and you would like to upgrade both the Media Cloud and the --- database to be at version 4453, import this SQL file: --- --- psql mediacloud < mediawords-4452-4453.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE tags - ADD COLUMN label varchar(512), - ADD COLUMN description text; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4453; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4453-4454.sql b/apps/postgresql-server/schema/migrations/mediawords-4453-4454.sql deleted file mode 100644 index 4ad655b027..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4453-4454.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4453 and 4454. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4453, and you would like to upgrade both the Media Cloud and the --- database to be at version 4454, import this SQL file: --- --- psql mediacloud < mediawords-4453-4454.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE tag_sets - ADD COLUMN show_on_media_ boolean, - ADD COLUMN show_on_stories boolean; - -ALTER TABLE tags - ADD COLUMN show_on_media_ boolean, - ADD COLUMN show_on_stories boolean; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4454; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4454-4455.sql b/apps/postgresql-server/schema/migrations/mediawords-4454-4455.sql deleted file mode 100644 index 9d9a269901..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4454-4455.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4454 and 4455. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4454, and you would like to upgrade both the Media Cloud and the --- database to be at version 4455, import this SQL file: --- --- psql mediacloud < mediawords-4454-4455.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE tag_sets - DROP COLUMN show_on_media_, - ADD COLUMN show_on_media boolean; - -ALTER TABLE tags - DROP COLUMN show_on_media_, - ADD COLUMN show_on_media boolean; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4455; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4455-4456.sql b/apps/postgresql-server/schema/migrations/mediawords-4455-4456.sql deleted file mode 100644 index 108b2495cd..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4455-4456.sql +++ /dev/null @@ -1,210 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4455 and 4456. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4455, and you would like to upgrade both the Media Cloud and the --- database to be at version 4456, import this SQL file: --- --- psql mediacloud < mediawords-4455-4456.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- User requests (the ones that are configured to be logged) --- -CREATE TABLE auth_user_requests ( - - auth_user_requests_id SERIAL PRIMARY KEY, - - -- User's email (does *not* reference auth_users.email because the user - -- might be deleted) - email TEXT NOT NULL, - - -- Request path (e.g. "api/v2/stories/list") - request_path TEXT NOT NULL, - - -- When did the request happen? - request_timestamp TIMESTAMP NOT NULL DEFAULT LOCALTIMESTAMP, - - -- Number of "items" requested in a request - -- For example: - -- * a single request to "/api/v2/stories/list" would count as one item; - -- * a single request to "/search" would count as a single request plus the - -- number of stories if "csv=1" is specified, or just as a single request - -- if "csv=1" is not specified - requested_items_count INTEGER NOT NULL DEFAULT 1 - -); - -CREATE INDEX auth_user_requests_email ON auth_user_requests (email); -CREATE INDEX auth_user_requests_request_path ON auth_user_requests (request_path); - - --- --- User request daily counts --- -CREATE TABLE auth_user_request_daily_counts ( - - auth_user_request_daily_counts_id SERIAL PRIMARY KEY, - - -- User's email (does *not* reference auth_users.email because the user - -- might be deleted) - email TEXT NOT NULL, - - -- Day (request timestamp, date_truncated to a day) - day DATE NOT NULL, - - -- Number of requests - requests_count INTEGER NOT NULL, - - -- Number of requested items - requested_items_count INTEGER NOT NULL - -); - -CREATE INDEX auth_user_request_daily_counts_email ON auth_user_request_daily_counts (email); -CREATE INDEX auth_user_request_daily_counts_day ON auth_user_request_daily_counts (day); - - --- On each logged request, update "auth_user_request_daily_counts" table -CREATE OR REPLACE FUNCTION auth_user_requests_update_daily_counts() RETURNS trigger AS -$$ - -DECLARE - request_date DATE; - -BEGIN - - request_date := DATE_TRUNC('day', NEW.request_timestamp)::DATE; - - -- Try to UPDATE a previously INSERTed day - UPDATE auth_user_request_daily_counts - SET requests_count = requests_count + 1, - requested_items_count = requested_items_count + NEW.requested_items_count - WHERE email = NEW.email - AND day = request_date; - - IF FOUND THEN - RETURN NULL; - END IF; - - -- If UPDATE was not successful, do an INSERT (new day!) - INSERT INTO auth_user_request_daily_counts (email, day, requests_count, requested_items_count) - VALUES (NEW.email, request_date, 1, NEW.requested_items_count); - - RETURN NULL; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER auth_user_requests_update_daily_counts - AFTER INSERT ON auth_user_requests - FOR EACH ROW EXECUTE PROCEDURE auth_user_requests_update_daily_counts(); - --- User limits for logged + throttled controller actions -CREATE TABLE auth_user_limits ( - - auth_user_limits_id SERIAL NOT NULL, - - auth_users_id INTEGER NOT NULL UNIQUE REFERENCES auth_users(auth_users_id) - ON DELETE CASCADE ON UPDATE CASCADE DEFERRABLE, - - -- Request limit (0 or belonging to 'admin' / 'admin-readonly' group = no - -- limit) - weekly_requests_limit INTEGER NOT NULL DEFAULT 1000, - - -- Requested items (stories) limit (0 or belonging to 'admin' / - -- 'admin-readonly' group = no limit) - weekly_requested_items_limit INTEGER NOT NULL DEFAULT 20000 - -); - -CREATE UNIQUE INDEX auth_user_limits_auth_users_id ON auth_user_limits (auth_users_id); - --- Set the default limits for newly created users -CREATE OR REPLACE FUNCTION auth_users_set_default_limits() RETURNS trigger AS -$$ -BEGIN - - INSERT INTO auth_user_limits (auth_users_id) VALUES (NEW.auth_users_id); - RETURN NULL; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER auth_users_set_default_limits - AFTER INSERT ON auth_users - FOR EACH ROW EXECUTE PROCEDURE auth_users_set_default_limits(); - - --- Add helper function to find out weekly request / request items usage for a user -CREATE OR REPLACE FUNCTION auth_user_limits_weekly_usage(user_email TEXT) -RETURNS TABLE(email TEXT, weekly_requests_sum BIGINT, weekly_requested_items_sum BIGINT) AS -$$ - - SELECT auth_users.email, - COALESCE(SUM(auth_user_request_daily_counts.requests_count), 0) AS weekly_requests_sum, - COALESCE(SUM(auth_user_request_daily_counts.requested_items_count), 0) AS weekly_requested_items_sum - FROM auth_users - LEFT JOIN auth_user_request_daily_counts - ON auth_users.email = auth_user_request_daily_counts.email - AND auth_user_request_daily_counts.day > DATE_TRUNC('day', NOW())::date - INTERVAL '1 week' - WHERE auth_users.email = $1 - GROUP BY auth_users.email; - -$$ -LANGUAGE SQL; - - --- Set default limits to the previously created users by creating a temporary ON UPDATE trigger -CREATE TRIGGER auth_users_set_default_limits_for_previously_created_users - AFTER UPDATE ON auth_users - FOR EACH ROW EXECUTE PROCEDURE auth_users_set_default_limits(); - --- Run UPDATE trigger against all of the previously created users (that don't have their limit set yet) -UPDATE auth_users -SET auth_users_id = auth_users_id -WHERE NOT EXISTS ( - SELECT 1 - FROM auth_user_limits - WHERE auth_users_id = auth_users.auth_users_id -); - -DROP TRIGGER auth_users_set_default_limits_for_previously_created_users ON auth_users; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4456; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4456-4457.sql b/apps/postgresql-server/schema/migrations/mediawords-4456-4457.sql deleted file mode 100644 index 8bb63e6881..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4456-4457.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4456 and 4457. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4456, and you would like to upgrade both the Media Cloud and the --- database to be at version 4457, import this SQL file: --- --- psql mediacloud < mediawords-4456-4457.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE auth_users - ADD COLUMN non_public_api BOOLEAN NOT NULL DEFAULT false; - -UPDATE auth_users set non_public_api = 't'; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4457; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4457-4458.sql b/apps/postgresql-server/schema/migrations/mediawords-4457-4458.sql deleted file mode 100644 index 70fad347e9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4457-4458.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4457 and 4458. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4457, and you would like to upgrade both the Media Cloud and the --- database to be at version 4458, import this SQL file: --- --- psql mediacloud < mediawords-4457-4458.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table auth_registration_queue ( - auth_registration_queue_id serial primary key, - name text not null, - email text not null, - organization text not null, - motivation text not null, - approved boolean default false -); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4458; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4458-4459.sql b/apps/postgresql-server/schema/migrations/mediawords-4458-4459.sql deleted file mode 100644 index 874218226b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4458-4459.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4458 and 4459. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4458, and you would like to upgrade both the Media Cloud and the --- database to be at version 4459, import this SQL file: --- --- psql mediacloud < mediawords-4458-4459.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -insert into controversy_dates ( controversies_id, start_date, end_date ) - select c.controversies_id, q.start_date, q.end_date - from controversies c, - query_story_searches qss, - queries q - where - c.query_story_searches_id = qss.query_story_searches_id and - qss.queries_id = q.queries_id; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4459; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4459-4460.sql b/apps/postgresql-server/schema/migrations/mediawords-4459-4460.sql deleted file mode 100644 index 1c833b878d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4459-4460.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4459 and 4460. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4459, and you would like to upgrade both the Media Cloud and the --- database to be at version 4460, import this SQL file: --- --- psql mediacloud < mediawords-4459-4460.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- We no longer use PL/Perl and PL/PerlU so drop support for those in the --- database that's being migrated -DROP LANGUAGE IF EXISTS plperl; -DROP LANGUAGE IF EXISTS plperlu; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4460; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4460-4461.sql b/apps/postgresql-server/schema/migrations/mediawords-4460-4461.sql deleted file mode 100644 index a75a898984..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4460-4461.sql +++ /dev/null @@ -1,107 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4460 and 4461. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4460, and you would like to upgrade both the Media Cloud and the --- database to be at version 4461, import this SQL file: --- --- psql mediacloud < mediawords-4460-4461.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table solr_import_stories ( - stories_id int not null references stories on delete cascade -); - -create index solr_import_stories_story on solr_import_stories ( stories_id ); - -alter table media add db_row_last_updated timestamp with time zone; -create index media_db_row_last_updated on media( db_row_last_updated ); - -create temporary view mtm_last_updated as - select media_id, max( db_row_last_updated ) db_row_last_updated - from media_tags_map mtm group by mtm.media_id; - -create temporary view msmm_last_updated as - select media_id, max( db_row_last_updated ) db_row_last_updated - from media_sets_media_map msmm group by msmm.media_id; - -update media m set db_row_last_updated = mtm.db_row_last_updated - from mtm_last_updated mtm where m.media_id = mtm.media_id; - -update media m set db_row_last_updated = msmm.db_row_last_updated - from msmm_last_updated msmm - where m.media_id = msmm.media_id and - msmm.db_row_last_updated > m.db_row_last_updated; - -CREATE OR REPLACE FUNCTION update_media_last_updated () RETURNS trigger AS -$$ - DECLARE - BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - update media set db_row_last_updated = now() where media_id = NEW.media_id; - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - update media set db_row_last_updated = now() where media_id = OLD.media_id; - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - - -DROP TRIGGER IF EXISTS media_tags_map_last_updated_trigger on media_tags_map CASCADE; -drop trigger if exists media_tags_last_updated_trigger on media_tags_map; -DROP index media_tags_map_db_row_last_updated; - -alter table media_tags_map drop column db_row_last_updated CASCADE; - -DROP TRIGGER IF EXISTS media_sets_media_map_last_updated_trigger on media_sets_media_map CASCADE; - -DROP INDEX media_sets_media_map_db_row_last_updated ; - -alter table media_sets_media_map drop db_row_last_updated CASCADE; - -DROP TRIGGER IF EXISTS msmm_last_updated on media_sets_media_map CASCADE; -CREATE TRIGGER msmm_last_updated BEFORE INSERT OR UPDATE OR DELETE - ON media_sets_media_map FOR EACH ROW EXECUTE PROCEDURE update_media_last_updated() ; - -DROP TRIGGER IF EXISTS mtm_last_updated on media_tags_map CASCADE; -CREATE TRIGGER mtm_last_updated BEFORE INSERT OR UPDATE OR DELETE - ON media_tags_map FOR EACH ROW EXECUTE PROCEDURE update_media_last_updated() ; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4461; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4461-4462.sql b/apps/postgresql-server/schema/migrations/mediawords-4461-4462.sql deleted file mode 100644 index c017a39a50..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4461-4462.sql +++ /dev/null @@ -1,100 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4461 and 4462. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4461, and you would like to upgrade both the Media Cloud and the --- database to be at version 4462, import this SQL file: --- --- psql mediacloud < mediawords-4461-4462.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -delete from tags t using tag_sets ts where t.tag_sets_id = ts.tag_sets_id and ts.name = 'media_type'; -delete from tag_sets where name = 'media_type'; -insert into tag_sets ( name, label, description ) values ( 'media_type', 'Media Type', 'High level topology for media sources for use across a variety of different topics' ); - -create temporary table media_type_tags ( name text, label text, description text ); -insert into media_type_tags values - ( 'Independent Group', 'Ind. Group', 'An academic or nonprofit group that is not affiliated with the private sector or government, such as the Electronic Frontier Foundation or the Center for Democracy and Technology)' ), - ( 'Social Linking Site', 'Social Linking', 'A site that aggregates links based at least partially on user submissions and/or ranking, such as Reddit, Digg, Slashdot, MetaFilter, StumbleUpon, and other social news sites' ), - ( 'Blog', 'Blog', 'A web log, written by one or more individuals, that is not associated with a professional or advocacy organization or institution' ), - ( 'General Online News Media', 'General News', 'A site that is a mainstream media outlet, such as The New York Times and The Washington Post; an online-only news outlet, such as Slate, Salon, or the Huffington Post; or a citizen journalism or non-profit news outlet, such as Global Voices or ProPublica' ), - ( 'Issue Specific Campaign', 'Issue', 'A site specifically dedicated to campaigning for or against a single issue.' ), - ( 'News Aggregator', 'News Agg.', 'A site that contains little to no original content and compiles news from other sites, such as Yahoo News or Google News' ), - ( 'Tech Media', 'Tech Media', 'A site that focuses on technological news and information produced by a news organization, such as Arstechnica, Techdirt, or Wired.com' ), - ( 'Private Sector', 'Private Sec.', 'A non-news media for-profit actor, including, for instance, trade organizations, industry sites, and domain registrars' ), - ( 'Government', 'Government', 'A site associated with and run by a government-affiliated entity, such as the DOJ website, White House blog, or a U.S. Senator official website' ), - ( 'User-Generated Content Platform', 'User Gen.', 'A general communication and networking platform or tool, like Wikipedia, YouTube, Twitter, and Scribd, or a search engine like Google or speech platform like the Daily Kos' ); - -insert into tags ( tag_sets_id, tag, label, description ) - select ts.tag_sets_id, mtt.name, mtt.name, mtt.description - from tag_sets ts cross join media_type_tags mtt - where ts.name = 'media_type'; - -insert into media_tags_map ( media_id, tags_id ) - select cmc.media_id, t.tags_id - from - controversy_media_codes cmc - join controversies c on ( c.controversies_id = cmc.controversies_id ) - join tags t on ( substr( cmc.code, 1, 4 ) = substr( t.tag, 1, 4 ) ) - join tag_sets ts on ( t.tag_sets_id = ts.tag_sets_id ) - where - c.name = 'sopa' and - cmc.code_type = 'media_type' and - ts.name = 'media_type'; - -alter table cd.tags add label text; -alter table cd.tags add description text; - -alter table cd.tag_sets add label text; -alter table cd.tag_sets add description text; - -CREATE OR REPLACE FUNCTION update_media_last_updated () RETURNS trigger AS -$$ - DECLARE - BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - update media set db_row_last_updated = now() where media_id = NEW.media_id; - RETURN NEW; - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - update media set db_row_last_updated = now() where media_id = OLD.media_id; - RETURN OLD; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - -discard temp; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4462; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4462-4463.sql b/apps/postgresql-server/schema/migrations/mediawords-4462-4463.sql deleted file mode 100644 index 7cb219ecd8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4462-4463.sql +++ /dev/null @@ -1,91 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4462 and 4463. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4462, and you would like to upgrade both the Media Cloud and the --- database to be at version 4463, import this SQL file: --- --- psql mediacloud < mediawords-4462-4463.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view controversies_with_search_info; - -alter table controversy_dates add boundary boolean not null default 'false'; - -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - -create temporary table boundary_controversy_dates as - select controversy_dates_id - from ( - select controversy_dates_id, - rank() over ( - partition by controversies_id - order by ( end_date - start_date ) desc - ) date_range_rank - from controversy_dates - ) q - where q.date_range_rank = 1; - -update controversy_dates set boundary = 't' - where controversy_dates_id in ( select controversy_dates_id from boundary_controversy_dates ); - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - -create temporary table media_type_tags ( name text, label text, description text ); -insert into media_type_tags values - ( 'Not Typed', 'Not Typed', 'The medium has not yet been typed.' ), - ( 'Other', 'Other', 'The medium does not fit in any listed type.' ); - -insert into tags ( tag_sets_id, tag, label, description ) - select ts.tag_sets_id, mtt.name, mtt.name, mtt.description - from tag_sets ts cross join media_type_tags mtt - where ts.name = 'media_type'; - -discard temp; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4463; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4463-4464.sql b/apps/postgresql-server/schema/migrations/mediawords-4463-4464.sql deleted file mode 100644 index 99c50afc61..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4463-4464.sql +++ /dev/null @@ -1,82 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4463 and 4464. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4463, and you would like to upgrade both the Media Cloud and the --- database to be at version 4464, import this SQL file: --- --- psql mediacloud < mediawords-4463-4464.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- --- Returns true if the story can + should be annotated with CoreNLP --- -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -BEGIN - - -- Check "media.annotate_with_corenlp" - IF NOT EXISTS ( - - SELECT 1 - FROM stories - INNER JOIN media ON stories.media_id = media.media_id - WHERE stories.stories_id = corenlp_stories_id - AND media.annotate_with_corenlp = 't' - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because media is not set for annotation.', corenlp_stories_id; - RETURN FALSE; - - -- Check if story has sentences - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM story_sentences - WHERE stories_id = corenlp_stories_id - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it has no sentences.', corenlp_stories_id; - RETURN FALSE; - - -- Things are fine - ELSE - RETURN TRUE; - - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4464; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4464-4465.sql b/apps/postgresql-server/schema/migrations/mediawords-4464-4465.sql deleted file mode 100644 index 463b8cd44a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4464-4465.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4464 and 4465. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4464, and you would like to upgrade both the Media Cloud and the --- database to be at version 4465, import this SQL file: --- --- psql mediacloud < mediawords-4464-4465.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversies add controversy_tag_sets_id int references tag_sets; - -update controversies c set controversy_tag_sets_id = ts.tag_sets_id - from tag_sets ts - where ts.name = 'controversy_' || c.name; - -alter table controversies alter controversy_tag_sets_id set not null; - -alter table controversies add media_type_tag_sets_id int references tag_sets; - -create unique index controversies_tag_set on controversies( controversy_tag_sets_id ); -create unique index controversies_media_type_tag_set on controversies( media_type_tag_sets_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4465; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4465-4466.sql b/apps/postgresql-server/schema/migrations/mediawords-4465-4466.sql deleted file mode 100644 index 60c3121703..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4465-4466.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4465 and 4466. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4465, and you would like to upgrade both the Media Cloud and the --- database to be at version 4466, import this SQL file: --- --- psql mediacloud < mediawords-4465-4466.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create function insert_controversy_tag_set() returns trigger as $insert_controversy_tag_set$ - begin - insert into tag_sets ( name, label, description ) - select 'controversy_'||NEW.name, NEW.name||' controversy', 'Tag set for stories within the '||NEW.name||' controversy.'; - - select tag_sets_id into NEW.controversy_tag_sets_id from tag_sets where name = 'controversy_'||NEW.name; - - return NEW; - END; -$insert_controversy_tag_set$ LANGUAGE plpgsql; - -create trigger controversy_tag_set before insert on controversies - for each row execute procedure insert_controversy_tag_set(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4466; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4466-4467.sql b/apps/postgresql-server/schema/migrations/mediawords-4466-4467.sql deleted file mode 100644 index a5caab1e8e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4466-4467.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4466 and 4467. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4466, and you would like to upgrade both the Media Cloud and the --- database to be at version 4467, import this SQL file: --- --- psql mediacloud < mediawords-4466-4467.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view controversies_with_dates; -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4467; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4467-4468.sql b/apps/postgresql-server/schema/migrations/mediawords-4467-4468.sql deleted file mode 100644 index 0ef260425e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4467-4468.sql +++ /dev/null @@ -1,106 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4467 and 4468. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4467, and you would like to upgrade both the Media Cloud and the --- database to be at version 4468, import this SQL file: --- --- psql mediacloud < mediawords-4467-4468.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE downloads - DROP CONSTRAINT downloads_feed_id_valid; - -ALTER TABLE downloads - DROP CONSTRAINT downloads_story; - -DROP INDEX downloads_spider_urls; - -DROP INDEX downloads_spider_download_errors_to_clear; - -DROP INDEX downloads_queued_spider; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger () RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_story_sentences_id bigint default null; - BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -drop view controversies_with_dates; - -drop view if exists controversies_with_search_info; -drop table if exists controversy_query_story_searches_imported_stories_map; - -drop table if exists query_story_searches_stories_map; -alter table controversies drop column query_story_searches_id; -drop table if exists query_story_searches; - -ALTER TABLE downloads - ADD CONSTRAINT downloads_feed_id_valid check (feeds_id is not null); - -ALTER TABLE downloads - ADD CONSTRAINT downloads_story check (((type = 'feed') and stories_id is null) or (stories_id is not null)); - -ALTER TABLE downloads add constraint valid_download_type check( type NOT in ( 'spider_blog_home','spider_posting','spider_rss','spider_blog_friends_list','spider_validation_blog_home','spider_validation_rss','archival_only') ); - -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4468; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4468-4469.sql b/apps/postgresql-server/schema/migrations/mediawords-4468-4469.sql deleted file mode 100644 index a185a3b562..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4468-4469.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4468 and 4469. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4468, and you would like to upgrade both the Media Cloud and the --- database to be at version 4469, import this SQL file: --- --- psql mediacloud < mediawords-4468-4469.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversies alter solr_seed_query_run set default 'f'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4469; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4469-4470.sql b/apps/postgresql-server/schema/migrations/mediawords-4469-4470.sql deleted file mode 100644 index 737fa35680..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4469-4470.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4469 and 4470. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4469, and you would like to upgrade both the Media Cloud and the --- database to be at version 4470, import this SQL file: --- --- psql mediacloud < mediawords-4469-4470.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create index controversy_merged_stories_map_story on controversy_merged_stories_map ( target_stories_id ); -create index controversy_links_ref_story on controversy_links ( ref_stories_id ); -create index controversy_seed_urls_story on controversy_seed_urls ( stories_id ); -create index authors_stories_queue_story on authors_stories_queue( stories_id ); -create index story_subsets_processed_stories_map_processed_stories_id on story_subsets_processed_stories_map ( processed_stories_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4470; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4470-4471.sql b/apps/postgresql-server/schema/migrations/mediawords-4470-4471.sql deleted file mode 100644 index 80c261c014..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4470-4471.sql +++ /dev/null @@ -1,159 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4470 and 4471. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4470, and you would like to upgrade both the Media Cloud and the --- database to be at version 4471, import this SQL file: --- --- psql mediacloud < mediawords-4470-4471.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -alter table media_stats drop column mean_num_sentences; -alter table media_stats drop column mean_text_length; -alter table media_stats drop column num_stories_with_sentences; -alter table media_stats drop column num_stories_with_text; - --- update media stats table for new story. create the media / day row if needed. -create or replace function insert_story_media_stats() returns trigger as $insert_story_media_stats$ -begin - - insert into media_stats ( media_id, num_stories, num_sentences, stat_date ) - select NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - where not exists ( - select 1 from media_stats where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ) ); - - update media_stats set num_stories = num_stories + 1 - where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ); - - return NEW; -END; -$insert_story_media_stats$ LANGUAGE plpgsql; -create trigger stories_insert_story_media_stats after insert - on stories for each row execute procedure insert_story_media_stats(); - - --- update media stats table for updated story date -create function update_story_media_stats() returns trigger as $update_story_media_stats$ -declare - new_date date; - old_date date; -begin - - select date_trunc( 'day', NEW.publish_date ) into new_date; - select date_trunc( 'day', OLD.publish_date ) into old_date; - - IF ( new_date <> old_date ) THEN - update media_stats set num_stories = num_stories - 1 - where media_id = NEW.media_id and stat_date = old_date; - - insert into media_stats ( media_id, num_stories, num_sentences, stat_date ) - select NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - where not exists ( - select 1 from media_stats where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ) ); - - update media_stats set num_stories = num_stories + 1 - where media_id = NEW.media_id and stat_date = new_date; - END IF; - - return NEW; -END; -$update_story_media_stats$ LANGUAGE plpgsql; -create trigger stories_update_story_media_stats after update - on stories for each row execute procedure update_story_media_stats(); - - --- update media stats table for deleted story -create function delete_story_media_stats() returns trigger as $delete_story_media_stats$ -begin - - update media_stats set num_stories = num_stories - 1 - where media_id = OLD.media_id and stat_date = date_trunc( 'day', OLD.publish_date ); - - return NEW; -END; -$delete_story_media_stats$ LANGUAGE plpgsql; -create trigger story_delete_story_media_stats after delete - on stories for each row execute procedure delete_story_media_stats(); - - --- update media stats table for new story sentence. -create function insert_ss_media_stats() returns trigger as $$ -begin - update media_stats set num_sentences = num_sentences + 1 - where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ); - - return NEW; -END; -$$ LANGUAGE plpgsql; -create trigger ss_insert_story_media_stats after insert - on story_sentences for each row execute procedure insert_ss_media_stats(); - --- update media stats table for updated story_sentence date -create function update_ss_media_stats() returns trigger as $$ -declare - new_date date; - old_date date; -begin - - select date_trunc( 'day', NEW.publish_date ) into new_date; - select date_trunc( 'day', OLD.publish_date ) into old_date; - - IF ( new_date <> old_date ) THEN - update media_stats set num_sentences = num_sentences - 1 - where media_id = NEW.media_id and stat_date = old_date; - update media_stats set num_sentences = num_sentences + 1 - where media_id = NEW.media_id and stat_date = new_date; - END IF; - - return NEW; -END; -$$ LANGUAGE plpgsql; -create trigger ss_update_story_media_stats after update - on story_sentences for each row execute procedure update_ss_media_stats(); - --- update media stats table for deleted story sentence -create function delete_ss_media_stats() returns trigger as $$ -begin - - update media_stats set num_sentences = num_sentences - 1 - where media_id = OLD.media_id and stat_date = date_trunc( 'day', OLD.publish_date ); - - return NEW; -END; -$$ LANGUAGE plpgsql; -create trigger story_delete_ss_media_stats after delete - on story_sentences for each row execute procedure delete_ss_media_stats(); - - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4471; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4471-4472.sql b/apps/postgresql-server/schema/migrations/mediawords-4471-4472.sql deleted file mode 100644 index ceae4aad38..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4471-4472.sql +++ /dev/null @@ -1,154 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4471 and 4472. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4471, and you would like to upgrade both the Media Cloud and the --- database to be at version 4472, import this SQL file: --- --- psql mediacloud < mediawords-4471-4472.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE controversies - ADD COLUMN process_with_bitly BOOLEAN NOT NULL DEFAULT false; - -COMMENT ON COLUMN controversies.process_with_bitly - IS 'Enable processing controversy''s stories with Bit.ly; enqueue all new controversy stories for Bit.ly processing'; - --- Recreate view because otherwise it doesn't return the "process_with_bitly" column -DROP VIEW controversies_with_dates; -CREATE VIEW controversies_with_dates AS - SELECT c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - FROM - controversies c - JOIN controversy_dates cd ON ( c.controversies_id = cd.controversies_id ) - WHERE cd.boundary; - - --- Bit.ly stats --- (values can be NULL if Bit.ly is not enabled / configured for a controversy) -ALTER TABLE cd.story_link_counts - ADD COLUMN bitly_click_count INT NULL, - ADD COLUMN bitly_referrer_count INT NULL; - --- Bit.ly (aggregated) stats --- (values can be NULL if Bit.ly is not enabled / configured for a controversy) -ALTER TABLE cd.medium_link_counts - ADD COLUMN bitly_click_count INT NULL, - ADD COLUMN bitly_referrer_count INT NULL; - - --- Bit.ly stats for stories -CREATE TABLE cd.story_bitly_statistics ( - story_bitly_statistics_id SERIAL PRIMARY KEY, - stories_id INT NOT NULL UNIQUE REFERENCES public.stories ON DELETE CASCADE, - - -- Bit.ly stats - bitly_click_count INT NOT NULL, - bitly_referrer_count INT NOT NULL -); -CREATE UNIQUE INDEX story_bitly_statistics_stories_id - ON cd.story_bitly_statistics ( stories_id ); - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE FUNCTION cd.upsert_story_bitly_statistics ( - param_stories_id INT, - param_bitly_click_count INT, - param_bitly_referrer_count INT -) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE cd.story_bitly_statistics - SET bitly_click_count = param_bitly_click_count, - bitly_referrer_count = param_bitly_referrer_count - WHERE stories_id = param_stories_id; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO cd.story_bitly_statistics (stories_id, bitly_click_count, bitly_referrer_count) - VALUES (param_stories_id, param_bitly_click_count, param_bitly_referrer_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - --- Helper to test if all controversy's stories have aggregated Bit.ly stats already -CREATE FUNCTION cd.all_controversy_stories_have_bitly_statistics (param_controversies_id INT) RETURNS BOOL AS -$$ -DECLARE - controversy_exists BOOL; - stories_without_bitly_statistics INT; -BEGIN - - SELECT 1 INTO controversy_exists - FROM controversies - WHERE controversies_id = param_controversies_id - AND process_with_bitly = 't'; - IF NOT FOUND THEN - RAISE EXCEPTION 'Controversy % does not exist or is not set up for Bit.ly processing.', param_controversies_id; - RETURN FALSE; - END IF; - - SELECT COUNT(stories_id) INTO stories_without_bitly_statistics - FROM controversy_stories - WHERE controversies_id = param_controversies_id - AND stories_id NOT IN ( - SELECT stories_id - FROM cd.story_bitly_statistics - ) - GROUP BY controversies_id; - IF FOUND THEN - RAISE NOTICE 'Some stories (% of them) still don''t have aggregated Bit.ly statistics for controversy %.', stories_without_bitly_statistics, param_controversies_id; - RETURN FALSE; - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4472; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4472-4473.sql b/apps/postgresql-server/schema/migrations/mediawords-4472-4473.sql deleted file mode 100644 index 87b1d9406e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4472-4473.sql +++ /dev/null @@ -1,117 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4472 and 4473. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4472, and you would like to upgrade both the Media Cloud and the --- database to be at version 4473, import this SQL file: --- --- psql mediacloud < mediawords-4472-4473.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE cd.story_bitly_statistics - SET SCHEMA public; - -DROP FUNCTION cd.all_controversy_stories_have_bitly_statistics (INT); - - --- Recreate function because both the schema and the definition changes -DROP FUNCTION cd.upsert_story_bitly_statistics (INT, INT, INT); -CREATE FUNCTION upsert_story_bitly_statistics ( - param_stories_id INT, - param_bitly_click_count INT, - param_bitly_referrer_count INT -) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE story_bitly_statistics - SET bitly_click_count = param_bitly_click_count, - bitly_referrer_count = param_bitly_referrer_count - WHERE stories_id = param_stories_id; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO story_bitly_statistics (stories_id, bitly_click_count, bitly_referrer_count) - VALUES (param_stories_id, param_bitly_click_count, param_bitly_referrer_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - --- Helper to return a number of stories for which we don't have Bit.ly statistics yet -CREATE FUNCTION num_controversy_stories_without_bitly_statistics (param_controversies_id INT) RETURNS INT AS -$$ -DECLARE - controversy_exists BOOL; - num_stories_without_bitly_statistics INT; -BEGIN - - SELECT 1 INTO controversy_exists - FROM controversies - WHERE controversies_id = param_controversies_id - AND process_with_bitly = 't'; - IF NOT FOUND THEN - RAISE EXCEPTION 'Controversy % does not exist or is not set up for Bit.ly processing.', param_controversies_id; - RETURN FALSE; - END IF; - - SELECT COUNT(stories_id) INTO num_stories_without_bitly_statistics - FROM controversy_stories - WHERE controversies_id = param_controversies_id - AND stories_id NOT IN ( - SELECT stories_id - FROM story_bitly_statistics - ) - GROUP BY controversies_id; - IF NOT FOUND THEN - num_stories_without_bitly_statistics := 0; - END IF; - - RETURN num_stories_without_bitly_statistics; -END; -$$ -LANGUAGE plpgsql; - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4473; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4473-4474.sql b/apps/postgresql-server/schema/migrations/mediawords-4473-4474.sql deleted file mode 100644 index 58c558e2df..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4473-4474.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4473 and 4474. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4473, and you would like to upgrade both the Media Cloud and the --- database to be at version 4474, import this SQL file: --- --- psql mediacloud < mediawords-4473-4474.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table auth_users add created_date timestamp not null default now(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4474; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4474-4475.sql b/apps/postgresql-server/schema/migrations/mediawords-4474-4475.sql deleted file mode 100644 index 4902d4b256..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4474-4475.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4474 and 4475. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4474, and you would like to upgrade both the Media Cloud and the --- database to be at version 4475, import this SQL file: --- --- psql mediacloud < mediawords-4474-4475.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create table color_sets ( - color_sets_id serial primary key, - color varchar( 256 ) not null, - color_set varchar( 256 ) not null, - id varchar( 256 ) not null -); - -create index color_sets_set_id on color_sets ( color_set, id ); - --- prefill colors for partisan_code set so that liberal is blue and conservative is red -insert into color_sets ( color, color_set, id ) values ( 'c10032', 'partisan_code', 'partisan_2012_conservative' ); -insert into color_sets ( color, color_set, id ) values ( '00519b', 'partisan_code', 'partisan_2012_liberal' ); -insert into color_sets ( color, color_set, id ) values ( '009543', 'partisan_code', 'partisan_2012_libertarian' ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4475; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4475-4476.sql b/apps/postgresql-server/schema/migrations/mediawords-4475-4476.sql deleted file mode 100644 index a39dd19efe..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4475-4476.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4475 and 4476. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4475, and you would like to upgrade both the Media Cloud and the --- database to be at version 4476, import this SQL file: --- --- psql mediacloud < mediawords-4475-4476.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- stats for various externally dervied statistics about a story. keeping this separate for now --- from the bitly stats for simplicity sake during implementatino and testing -create table story_statistics ( - story_statistics_id serial primary key, - stories_id int not null references stories on delete cascade, - twitter_url_tweet_count int null, - twitter_url_tweet_count_error text null, - facebook_share_count int null, - facebook_share_count_error text null -); - -create unique index story_statistics_story on story_statistics ( stories_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4476; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4476-4477.sql b/apps/postgresql-server/schema/migrations/mediawords-4476-4477.sql deleted file mode 100644 index 264e9be6d3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4476-4477.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4476 and 4477. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4476, and you would like to upgrade both the Media Cloud and the --- database to be at version 4477, import this SQL file: --- --- psql mediacloud < mediawords-4476-4477.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table downloads alter download_time set default now(); -alter table stories alter collect_date set default now(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4477; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4477-4478.sql b/apps/postgresql-server/schema/migrations/mediawords-4477-4478.sql deleted file mode 100644 index ccb6b927ac..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4477-4478.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4477 and 4478. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4477, and you would like to upgrade both the Media Cloud and the --- database to be at version 4478, import this SQL file: --- --- psql mediacloud < mediawords-4477-4478.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table controversy_query_slices ( - controversy_query_slices_id serial primary key, - controversies_id int not null references controversies on delete cascade, - name varchar ( 1024 ) not null, - query text not null, - all_time_slices boolean not null -); - -alter table controversy_dump_time_slices - add controversy_query_slices_id int null references controversy_query_slices on delete set null; - -alter table controversy_dump_time_slices add is_shell boolean not null default false; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4478; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4478-4479.sql b/apps/postgresql-server/schema/migrations/mediawords-4478-4479.sql deleted file mode 100644 index 8ab35fc59e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4478-4479.sql +++ /dev/null @@ -1,97 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4478 and 4479. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4478, and you would like to upgrade both the Media Cloud and the --- database to be at version 4479, import this SQL file: --- --- psql mediacloud < mediawords-4478-4479.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -BEGIN - - -- FIXME this function is not really optimized for performance - - -- Check "media.annotate_with_corenlp" - IF NOT EXISTS ( - - SELECT 1 - FROM stories - INNER JOIN media ON stories.media_id = media.media_id - WHERE stories.stories_id = corenlp_stories_id - AND media.annotate_with_corenlp = 't' - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because media is not set for annotation.', corenlp_stories_id; - RETURN FALSE; - - -- Annotate English language stories only because they're the only ones - -- supported by CoreNLP at the time. - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM stories - - -- Stories with language field set to NULL are the ones fetched before - -- introduction of the multilanguage support, so they are assumed to be - -- English. - WHERE stories.language = 'en' OR stories.language IS NULL - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not in English.', corenlp_stories_id; - RETURN FALSE; - - -- Check if story has sentences - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM story_sentences - WHERE stories_id = corenlp_stories_id - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it has no sentences.', corenlp_stories_id; - RETURN FALSE; - - -- Things are fine - ELSE - RETURN TRUE; - - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4479; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4479-4480.sql b/apps/postgresql-server/schema/migrations/mediawords-4479-4480.sql deleted file mode 100644 index ce4e1ce166..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4479-4480.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4479 and 4480. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4479, and you would like to upgrade both the Media Cloud and the --- database to be at version 4480, import this SQL file: --- --- psql mediacloud < mediawords-4479-4480.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE story_statistics - RENAME COLUMN twitter_url_tweet_count_error TO twitter_api_error; -ALTER TABLE story_statistics - RENAME COLUMN facebook_share_count_error TO facebook_api_error; - -ALTER TABLE story_statistics - ADD COLUMN facebook_comment_count INT NULL, - ADD COLUMN twitter_api_collect_date TIMESTAMP NULL, - ADD COLUMN facebook_api_collect_date TIMESTAMP NULL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4480; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4480-4481.sql b/apps/postgresql-server/schema/migrations/mediawords-4480-4481.sql deleted file mode 100644 index 35944e4cd4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4480-4481.sql +++ /dev/null @@ -1,136 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4480 and 4481. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4480, and you would like to upgrade both the Media Cloud and the --- database to be at version 4481, import this SQL file: --- --- psql mediacloud < mediawords-4480-4481.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4481; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION story_triggers_enabled() RETURNS boolean LANGUAGE plpgsql AS $$ -BEGIN - - return current_setting('PRIVATE.use_story_triggers') = 'yes'; - EXCEPTION when undefined_object then - perform enable_story_triggers(); - return true; -END$$; - -CREATE OR REPLACE FUNCTION enable_story_triggers() RETURNS void LANGUAGE plpgsql AS $$ -DECLARE -BEGIN - perform set_config('PRIVATE.use_story_triggers', 'yes', false ); -END$$; - -CREATE OR REPLACE FUNCTION disable_story_triggers() RETURNS void LANGUAGE plpgsql AS $$ -DECLARE -BEGIN - perform set_config('PRIVATE.use_story_triggers', 'no', false ); -END$$; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - UPDATE story_sentences set db_row_last_updated = now() where stories_id = NEW.stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_stories_id integer default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_story_sentences_id bigint default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4481-4482.sql b/apps/postgresql-server/schema/migrations/mediawords-4481-4482.sql deleted file mode 100644 index 951ce2c714..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4481-4482.sql +++ /dev/null @@ -1,161 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4481 and 4482. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4481, and you would like to upgrade both the Media Cloud and the --- database to be at version 4482, import this SQL file: --- --- psql mediacloud < mediawords-4481-4482.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4482; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION insert_ss_media_stats() returns trigger as $$ -begin - - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - update media_stats set num_sentences = num_sentences + 1 - where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ); - - return NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION update_ss_media_stats() returns trigger as $$ -declare - new_date date; - old_date date; -begin - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - select date_trunc( 'day', NEW.publish_date ) into new_date; - select date_trunc( 'day', OLD.publish_date ) into old_date; - - IF ( new_date <> old_date ) THEN - update media_stats set num_sentences = num_sentences - 1 - where media_id = NEW.media_id and stat_date = old_date; - update media_stats set num_sentences = num_sentences + 1 - where media_id = NEW.media_id and stat_date = new_date; - END IF; - - return NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_ss_media_stats() returns trigger as $$ -begin - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - update media_stats set num_sentences = num_sentences - 1 - where media_id = OLD.media_id and stat_date = date_trunc( 'day', OLD.publish_date ); - - return NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION insert_story_media_stats() returns trigger as $insert_story_media_stats$ -begin - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - insert into media_stats ( media_id, num_stories, num_sentences, stat_date ) - select NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - where not exists ( - select 1 from media_stats where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ) ); - - update media_stats set num_stories = num_stories + 1 - where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ); - - return NEW; -END; -$insert_story_media_stats$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION update_story_media_stats() returns trigger as $update_story_media_stats$ -declare - new_date date; - old_date date; -begin - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - select date_trunc( 'day', NEW.publish_date ) into new_date; - select date_trunc( 'day', OLD.publish_date ) into old_date; - - IF ( new_date <> old_date ) THEN - update media_stats set num_stories = num_stories - 1 - where media_id = NEW.media_id and stat_date = old_date; - - insert into media_stats ( media_id, num_stories, num_sentences, stat_date ) - select NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - where not exists ( - select 1 from media_stats where media_id = NEW.media_id and stat_date = date_trunc( 'day', NEW.publish_date ) ); - - update media_stats set num_stories = num_stories + 1 - where media_id = NEW.media_id and stat_date = new_date; - - update story_sentences set publish_date = new_date where stories_id = OLD.stories_id; - END IF; - - return NEW; -END; -$update_story_media_stats$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_story_media_stats() returns trigger as $delete_story_media_stats$ -begin - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - update media_stats set num_stories = num_stories - 1 - where media_id = OLD.media_id and stat_date = date_trunc( 'day', OLD.publish_date ); - - return NEW; -END; -$delete_story_media_stats$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4482-4483.sql b/apps/postgresql-server/schema/migrations/mediawords-4482-4483.sql deleted file mode 100644 index 50571a2e5b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4482-4483.sql +++ /dev/null @@ -1,113 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4482 and 4483. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4482, and you would like to upgrade both the Media Cloud and the --- database to be at version 4483, import this SQL file: --- --- psql mediacloud < mediawords-4482-4483.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -BEGIN - - -- FIXME this function is not really optimized for performance - - -- Check "media.annotate_with_corenlp" - IF NOT EXISTS ( - - SELECT 1 - FROM stories - INNER JOIN media ON stories.media_id = media.media_id - WHERE stories.stories_id = corenlp_stories_id - AND media.annotate_with_corenlp = 't' - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because media is not set for annotation.', corenlp_stories_id; - RETURN FALSE; - - -- Check if the story is extracted - ELSEIF EXISTS ( - - SELECT 1 - FROM downloads - WHERE stories_id = corenlp_stories_id - AND type = 'content' - AND extracted = 'f' - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not extracted.', corenlp_stories_id; - RETURN FALSE; - - -- Annotate English language stories only because they're the only ones - -- supported by CoreNLP at the time. - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM stories - WHERE stories_id = corenlp_stories_id - - -- Stories with language field set to NULL are the ones fetched before - -- introduction of the multilanguage support, so they are assumed to be - -- English. - AND ( stories.language = 'en' OR stories.language IS NULL ) - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not in English.', corenlp_stories_id; - RETURN FALSE; - - -- Check if story has sentences - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM story_sentences - WHERE stories_id = corenlp_stories_id - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it has no sentences.', corenlp_stories_id; - RETURN FALSE; - - -- Things are fine - ELSE - RETURN TRUE; - - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4483; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4483-4484.sql b/apps/postgresql-server/schema/migrations/mediawords-4483-4484.sql deleted file mode 100644 index 83b65cba20..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4483-4484.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4483 and 4484. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4483, and you would like to upgrade both the Media Cloud and the --- database to be at version 4484, import this SQL file: --- --- psql mediacloud < mediawords-4483-4484.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4484; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - - IF ( story_triggers_enabled() ) AND ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_live_story() returns trigger as $update_live_story$ - begin - - IF NOT story_triggers_enabled() then - RETURN NEW; - END IF; - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4484-4485.sql b/apps/postgresql-server/schema/migrations/mediawords-4484-4485.sql deleted file mode 100644 index bafbc6c301..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4484-4485.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4484 and 4485. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4484, and you would like to upgrade both the Media Cloud and the --- database to be at version 4485, import this SQL file: --- --- psql mediacloud < mediawords-4484-4485.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4485; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - -- RAISE NOTICE 'BEGIN '; - - IF ( story_triggers_enabled() ) AND ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_live_story() returns trigger as $update_live_story$ - begin - - IF NOT story_triggers_enabled() then - RETURN NEW; - END IF; - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4485-4486.sql b/apps/postgresql-server/schema/migrations/mediawords-4485-4486.sql deleted file mode 100644 index e18f4714d8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4485-4486.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4485 and 4486. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4485, and you would like to upgrade both the Media Cloud and the --- database to be at version 4486, import this SQL file: --- --- psql mediacloud < mediawords-4485-4486.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE stories_tags_map - ALTER COLUMN db_row_last_updated SET DEFAULT now(); - -ALTER TABLE story_sentences_tags_map - ALTER COLUMN db_row_last_updated SET DEFAULT now(); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4486; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4486-4487.sql b/apps/postgresql-server/schema/migrations/mediawords-4486-4487.sql deleted file mode 100644 index d3b1bed98d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4486-4487.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4486 and 4487. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4486, and you would like to upgrade both the Media Cloud and the --- database to be at version 4487, import this SQL file: --- --- psql mediacloud < mediawords-4486-4487.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE TABLE auth_users_tag_sets_permissions ( - auth_users_tag_sets_permissions_id SERIAL PRIMARY KEY, - auth_users_id integer references auth_users NOT NULL, - tag_sets_id integer references tag_sets NOT NULL, - apply_tags boolean NOT NULL, - create_tags boolean NOT NULL, - edit_tag_set_descriptors boolean NOT NULL, - edit_tag_descriptors boolean NOT NULL -); - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4487; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE UNIQUE INDEX auth_users_tag_sets_permissions_auth_user_tag_set ON auth_users_tag_sets_permissions ( auth_users_id , tag_sets_id ); - -CREATE UNIQUE INDEX auth_users_tag_sets_permissions_auth_user ON auth_users_tag_sets_permissions ( auth_users_id ); - -CREATE UNIQUE INDEX auth_users_tag_sets_permissions_tag_sets ON auth_users_tag_sets_permissions ( tag_sets_id ); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4487-4488.sql b/apps/postgresql-server/schema/migrations/mediawords-4487-4488.sql deleted file mode 100644 index 72f5aa30b8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4487-4488.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4487 and 4488. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4487, and you would like to upgrade both the Media Cloud and the --- database to be at version 4488, import this SQL file: --- --- psql mediacloud < mediawords-4487-4488.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4488; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION story_triggers_enabled() RETURNS boolean LANGUAGE plpgsql AS $$ -BEGIN - - BEGIN - IF current_setting('PRIVATE.use_story_triggers') = '' THEN - perform enable_story_triggers(); - END IF; - EXCEPTION when undefined_object then - perform enable_story_triggers(); - - END; - - return true; - return current_setting('PRIVATE.use_story_triggers') = 'yes'; -END$$; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4488-4489.sql b/apps/postgresql-server/schema/migrations/mediawords-4488-4489.sql deleted file mode 100644 index c0eb41884e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4488-4489.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4488 and 4489. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4488, and you would like to upgrade both the Media Cloud and the --- database to be at version 4489, import this SQL file: --- --- psql mediacloud < mediawords-4488-4489.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP INDEX auth_users_tag_sets_permissions_auth_user; - -DROP INDEX auth_users_tag_sets_permissions_tag_sets; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4489; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE INDEX auth_users_tag_sets_permissions_auth_user ON auth_users_tag_sets_permissions ( auth_users_id ); - -CREATE INDEX auth_users_tag_sets_permissions_tag_sets ON auth_users_tag_sets_permissions ( tag_sets_id ); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4489-4490.sql b/apps/postgresql-server/schema/migrations/mediawords-4489-4490.sql deleted file mode 100644 index f6e8268ca1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4489-4490.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4489 and 4490. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4489, and you would like to upgrade both the Media Cloud and the --- database to be at version 4490, import this SQL file: --- --- psql mediacloud < mediawords-4489-4490.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE raw_downloads - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4490; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4490-4491.sql b/apps/postgresql-server/schema/migrations/mediawords-4490-4491.sql deleted file mode 100644 index 3fd123910c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4490-4491.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4490 and 4491. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4490, and you would like to upgrade both the Media Cloud and the --- database to be at version 4491, import this SQL file: --- --- psql mediacloud < mediawords-4490-4491.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE stories - ADD COLUMN disable_triggers boolean; - -ALTER TABLE story_sentences - ADD COLUMN disable_triggers boolean; - -ALTER TABLE processed_stories - ADD COLUMN disable_triggers boolean; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4491; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4491-4492.sql b/apps/postgresql-server/schema/migrations/mediawords-4491-4492.sql deleted file mode 100644 index cbe6fba0d8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4491-4492.sql +++ /dev/null @@ -1,163 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4491 and 4492. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4491, and you would like to upgrade both the Media Cloud and the --- database to be at version 4492, import this SQL file: --- --- psql mediacloud < mediawords-4491-4492.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4492; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - BEGIN - -- RAISE NOTICE 'BEGIN '; - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) AND NEW.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF ( story_triggers_enabled() ) AND ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_stories_id integer default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_story_sentences_id bigint default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4492-4493.sql b/apps/postgresql-server/schema/migrations/mediawords-4492-4493.sql deleted file mode 100644 index 4a59fcfd7f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4492-4493.sql +++ /dev/null @@ -1,197 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4492 and 4493. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4492, and you would like to upgrade both the Media Cloud and the --- database to be at version 4493, import this SQL file: --- --- psql mediacloud < mediawords-4492-4493.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4493; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - BEGIN - -- RAISE NOTICE 'BEGIN '; - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) AND NEW.disable_triggers THEN - RETURN NEW; - END IF; - END IF; - - IF ( story_triggers_enabled() ) AND ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') ) then - - NEW.db_row_last_updated = now(); - - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF NEW.disable_triggers THEN - RETURN NULL; - END IF; - - UPDATE story_sentences set db_row_last_updated = now() where stories_id = NEW.stories_id; - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_stories_id integer default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - IF table_with_trigger_column THEN - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id; - RETURN NULL; - ELSE - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id and (disable_triggers is NOT true); - RETURN NULL; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger() RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_story_sentences_id bigint default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - IF table_with_trigger_column THEN - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id; - RETURN NULL; - ELSE - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id and (disable_triggers is NOT true); - RETURN NULL; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4493-4494.sql b/apps/postgresql-server/schema/migrations/mediawords-4493-4494.sql deleted file mode 100644 index 0c30f58349..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4493-4494.sql +++ /dev/null @@ -1,77 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4493 and 4494. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4493, and you would like to upgrade both the Media Cloud and the --- database to be at version 4494, import this SQL file: --- --- psql mediacloud < mediawords-4493-4494.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- CoreNLP annotations --- -CREATE TABLE corenlp_annotations ( - corenlp_annotations_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX corenlp_annotations_object_id ON corenlp_annotations (object_id); - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE corenlp_annotations - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - --- --- Bit.ly processing results --- -CREATE TABLE bitly_processing_results ( - bitly_processing_results_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX bitly_processing_results_object_id ON bitly_processing_results (object_id); - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE bitly_processing_results - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4494; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4494-4495.sql b/apps/postgresql-server/schema/migrations/mediawords-4494-4495.sql deleted file mode 100644 index 8dba574936..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4494-4495.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4494 and 4495. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4494, and you would like to upgrade both the Media Cloud and the --- database to be at version 4495, import this SQL file: --- --- psql mediacloud < mediawords-4494-4495.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP FUNCTION purge_story_words(default_start_day date, default_end_day date); - -DROP TABLE story_sentence_words; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4495; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4495-4496.sql b/apps/postgresql-server/schema/migrations/mediawords-4495-4496.sql deleted file mode 100644 index 1f901729b1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4495-4496.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4495 and 4496. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4495, and you would like to upgrade both the Media Cloud and the --- database to be at version 4496, import this SQL file: --- --- psql mediacloud < mediawords-4495-4496.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP FUNCTION purge_daily_words_for_media_set(v_media_sets_id int, default_start_day date, default_end_day date); - -DROP VIEW daily_words_with_totals; - -DROP TABLE daily_words; - -DROP TABLE total_daily_words; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4496; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4496-4497.sql b/apps/postgresql-server/schema/migrations/mediawords-4496-4497.sql deleted file mode 100644 index 65942a0bd4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4496-4497.sql +++ /dev/null @@ -1,57 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4496 and 4497. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4496, and you would like to upgrade both the Media Cloud and the --- database to be at version 4497, import this SQL file: --- --- psql mediacloud < mediawords-4496-4497.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP VIEW top_500_weekly_words_normalized; - -DROP VIEW top_500_weekly_words_with_totals; - -DROP TABLE weekly_words; - -DROP TABLE top_500_weekly_words; - -DROP TABLE total_top_500_weekly_words; - -DROP TABLE total_weekly_words; - -DROP TABLE queries_top_weekly_words_json; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4497; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4497-4498.sql b/apps/postgresql-server/schema/migrations/mediawords-4497-4498.sql deleted file mode 100644 index 2e796f425e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4497-4498.sql +++ /dev/null @@ -1,81 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4497 and 4498. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4497, and you would like to upgrade both the Media Cloud and the --- database to be at version 4498, import this SQL file: --- --- psql mediacloud < mediawords-4497-4498.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP INDEX media_sets_cluster; - -ALTER TABLE media_sets - DROP CONSTRAINT dashboard_media_sets_type; - -ALTER TABLE media_sets - DROP COLUMN media_clusters_id; - -ALTER TABLE dashboard_media_sets - DROP COLUMN media_cluster_runs_id; - - -DROP TABLE media_cluster_words; - -DROP TABLE media_cluster_zscores; - -DROP TABLE media_clusters_media_map; - -DROP TABLE media_cluster_map_poles; - -DROP TABLE media_cluster_map_pole_similarities; - -DROP TABLE media_clusters; - -DROP TABLE media_cluster_maps; - -DROP TABLE media_cluster_links; - -DROP TABLE media_cluster_runs; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4498; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -ALTER TABLE media_sets - ADD CONSTRAINT dashboard_media_sets_type check ( ( ( set_type = 'medium' ) and ( media_id is not null ) ) - or - ( ( set_type = 'collection' ) and ( tags_id is not null ) ) - or - ( ( set_type = 'cluster' ) ) ); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4498-4499.sql b/apps/postgresql-server/schema/migrations/mediawords-4498-4499.sql deleted file mode 100644 index c612336517..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4498-4499.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4498 and 4499. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4498, and you would like to upgrade both the Media Cloud and the --- database to be at version 4499, import this SQL file: --- --- psql mediacloud < mediawords-4498-4499.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP TABLE authors_stories_map; - -DROP TABLE authors_stories_queue; - -DROP TABLE daily_author_words; - -DROP TABLE total_daily_author_words; - -DROP TABLE weekly_author_words; - -DROP TABLE top_500_weekly_author_words; - -DROP TABLE total_top_500_weekly_author_words; - -DROP TABLE authors; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4499; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4499-4500.sql b/apps/postgresql-server/schema/migrations/mediawords-4499-4500.sql deleted file mode 100644 index 6eb463e3a1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4499-4500.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4499 and 4500. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4499, and you would like to upgrade both the Media Cloud and the --- database to be at version 4500, import this SQL file: --- --- psql mediacloud < mediawords-4499-4500.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP TABLE popular_queries; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4500; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4500-4501.sql b/apps/postgresql-server/schema/migrations/mediawords-4500-4501.sql deleted file mode 100644 index 04aaf7e72a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4500-4501.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4500 and 4501. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4500, and you would like to upgrade both the Media Cloud and the --- database to be at version 4501, import this SQL file: --- --- psql mediacloud < mediawords-4500-4501.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP VIEW story_similarities_transitive; - -DROP TABLE queries_media_sets_map; - -DROP TABLE top_ten_tags_for_media; - -DROP TABLE daily_country_counts; - -DROP TABLE queries_dashboard_topics_map; - -DROP TABLE story_similarities; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4501; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4501-4502.sql b/apps/postgresql-server/schema/migrations/mediawords-4501-4502.sql deleted file mode 100644 index 32cf4b810c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4501-4502.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4501 and 4502. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4501, and you would like to upgrade both the Media Cloud and the --- database to be at version 4502, import this SQL file: --- --- psql mediacloud < mediawords-4501-4502.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -ALTER TABLE media add constraint media_dup_media_id_fkey_deferrable FOREIGN KEY (dup_media_id) REFERENCES media(media_id) ON DELETE SET NULL DEFERRABLE; -ALTER table media DROP CONSTRAINT media_dup_media_id_fkey; -ALTER TABLE media add constraint media_dup_media_id_fkey FOREIGN KEY (dup_media_id) REFERENCES media(media_id) ON DELETE SET NULL DEFERRABLE; -ALTER table media DROP CONSTRAINT media_dup_media_id_fkey_deferrable; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4502; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4502-4503.sql b/apps/postgresql-server/schema/migrations/mediawords-4502-4503.sql deleted file mode 100644 index 566253a5a2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4502-4503.sql +++ /dev/null @@ -1,130 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4502 and 4503. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4502, and you would like to upgrade both the Media Cloud and the --- database to be at version 4503, import this SQL file: --- --- psql mediacloud < mediawords-4502-4503.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE TABLE extra_corenlp_stories ( - extra_corenlp_stories_id SERIAL PRIMARY KEY, - stories_id INTEGER NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE -); -CREATE INDEX extra_corenlp_stories_stories_id ON extra_corenlp_stories (stories_id); - - --- --- Returns true if the story can + should be annotated with CoreNLP --- -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -BEGIN - - -- FIXME this function is not really optimized for performance - - -- Check "media.annotate_with_corenlp" and "extra_corenlp_stories" - IF NOT EXISTS ( - - SELECT 1 - FROM stories - INNER JOIN media ON stories.media_id = media.media_id - WHERE stories.stories_id = corenlp_stories_id - AND media.annotate_with_corenlp = 't' - - ) AND NOT EXISTS ( - - SELECT 1 - FROM extra_corenlp_stories - WHERE extra_corenlp_stories.stories_id = corenlp_stories_id - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not enabled for annotation.', corenlp_stories_id; - RETURN FALSE; - - -- Check if the story is extracted - ELSEIF EXISTS ( - - SELECT 1 - FROM downloads - WHERE stories_id = corenlp_stories_id - AND type = 'content' - AND extracted = 'f' - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not extracted.', corenlp_stories_id; - RETURN FALSE; - - -- Annotate English language stories only because they're the only ones - -- supported by CoreNLP at the time. - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM stories - WHERE stories_id = corenlp_stories_id - - -- Stories with language field set to NULL are the ones fetched before - -- introduction of the multilanguage support, so they are assumed to be - -- English. - AND ( stories.language = 'en' OR stories.language IS NULL ) - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it is not in English.', corenlp_stories_id; - RETURN FALSE; - - -- Check if story has sentences - ELSEIF NOT EXISTS ( - - SELECT 1 - FROM story_sentences - WHERE stories_id = corenlp_stories_id - - ) THEN - RAISE NOTICE 'Story % is not annotatable with CoreNLP because it has no sentences.', corenlp_stories_id; - RETURN FALSE; - - -- Things are fine - ELSE - RETURN TRUE; - - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4503; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4503-4504.sql b/apps/postgresql-server/schema/migrations/mediawords-4503-4504.sql deleted file mode 100644 index aba75b2b27..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4503-4504.sql +++ /dev/null @@ -1,81 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4503 and 4504. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4503, and you would like to upgrade both the Media Cloud and the --- database to be at version 4504, import this SQL file: --- --- psql mediacloud < mediawords-4503-4504.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- On each logged request, update "auth_user_request_daily_counts" table -CREATE OR REPLACE FUNCTION auth_user_requests_update_daily_counts() RETURNS trigger AS -$$ - -DECLARE - request_date DATE; - -BEGIN - - -- Try to prevent deadlocks - LOCK TABLE auth_user_request_daily_counts IN SHARE ROW EXCLUSIVE MODE; - - request_date := DATE_TRUNC('day', NEW.request_timestamp)::DATE; - - WITH upsert AS ( - -- Try to UPDATE a previously INSERTed day - UPDATE auth_user_request_daily_counts - SET requests_count = requests_count + 1, - requested_items_count = requested_items_count + NEW.requested_items_count - WHERE email = NEW.email - AND day = request_date - RETURNING * - ) - INSERT INTO auth_user_request_daily_counts (email, day, requests_count, requested_items_count) - SELECT NEW.email, request_date, 1, NEW.requested_items_count - WHERE NOT EXISTS ( - SELECT * - FROM upsert - ); - - RETURN NULL; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4504; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4504-4505.sql b/apps/postgresql-server/schema/migrations/mediawords-4504-4505.sql deleted file mode 100644 index 6c098a5a9c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4504-4505.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4504 and 4505. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4504, and you would like to upgrade both the Media Cloud and the --- database to be at version 4505, import this SQL file: --- --- psql mediacloud < mediawords-4504-4505.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4505; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4505-4506.sql b/apps/postgresql-server/schema/migrations/mediawords-4505-4506.sql deleted file mode 100644 index e771c4cb92..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4505-4506.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4505 and 4506. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4505, and you would like to upgrade both the Media Cloud and the --- database to be at version 4506, import this SQL file: --- --- psql mediacloud < mediawords-4505-4506.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -INSERT INTO auth_roles (role, description) VALUES - ('cm-readonly', 'Controversy mapper; excludes media and story editing'); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4506; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4506-4507.sql b/apps/postgresql-server/schema/migrations/mediawords-4506-4507.sql deleted file mode 100644 index f1ac50d437..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4506-4507.sql +++ /dev/null @@ -1,214 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4506 and 4507. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4506, and you would like to upgrade both the Media Cloud and the --- database to be at version 4507, import this SQL file: --- --- psql mediacloud < mediawords-4506-4507.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Dropping temporarily; will recreate afterwards -drop view media_with_media_types; - --- Dropping to recreate with a different list of columns -DROP VIEW media_with_collections; - -ALTER TABLE public.media - DROP COLUMN feeds_added; -ALTER TABLE cd.media - DROP COLUMN feeds_added; - --- Recreating with a different list of columns -CREATE VIEW media_with_collections AS - SELECT t.tag, - m.media_id, - m.url, - m.name, - m.moderated, - m.moderation_notes, - m.full_text_rss - FROM media m, - tags t, - tag_sets ts, - media_tags_map mtm - WHERE ts.name::text = 'collection'::text - AND ts.tag_sets_id = t.tag_sets_id - AND mtm.tags_id = t.tags_id - AND mtm.media_id = m.media_id - ORDER BY m.media_id; - --- Recreating temporarily dropped views -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - - -CREATE OR REPLACE FUNCTION media_has_active_syndicated_feeds(param_media_id INT) RETURNS boolean AS $$ -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - - SELECT 1 - FROM media - WHERE media_id = param_media_id - - ) THEN - RAISE EXCEPTION 'Media % does not exist.', param_media_id; - RETURN FALSE; - END IF; - - -- Check if media has feeds - IF EXISTS ( - - SELECT 1 - FROM feeds - WHERE media_id = param_media_id - AND feed_status = 'active' - - -- Website might introduce RSS feeds later - AND feed_type = 'syndicated' - - ) THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- Media feed rescraping state -CREATE TABLE media_rescraping ( - media_id int NOT NULL UNIQUE REFERENCES media ON DELETE CASCADE, - - -- Disable periodic rescraping? - disable BOOLEAN NOT NULL DEFAULT 'f', - - -- Timestamp of last rescrape; NULL means that media was never scraped at all - last_rescrape_time TIMESTAMP WITH TIME ZONE NULL -); - -CREATE UNIQUE INDEX media_rescraping_media_id on media_rescraping(media_id); -CREATE INDEX media_rescraping_last_rescrape_time on media_rescraping(last_rescrape_time); - --- Insert new rows to "media_rescraping" for each new row in "media" -CREATE OR REPLACE FUNCTION media_rescraping_add_initial_state_trigger() RETURNS trigger AS -$$ - BEGIN - INSERT INTO media_rescraping (media_id, disable, last_rescrape_time) - VALUES (NEW.media_id, 'f', NULL); - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER media_rescraping_add_initial_state_trigger - AFTER INSERT ON media - FOR EACH ROW EXECUTE PROCEDURE media_rescraping_add_initial_state_trigger(); - - --- Set the initial rescraping state for all existing media types -INSERT INTO media_rescraping (media_id, disable, last_rescrape_time) - SELECT media_id, - 'f', - -- Span across 1 year so that all media doesn't get rescraped at the same time - (NOW() - RANDOM() * (NOW() - (NOW() - INTERVAL '1 year'))) - FROM media - WHERE NOT EXISTS ( - SELECT 1 - FROM media_rescraping - WHERE media_rescraping.media_id = media.media_id - ); - - --- Feeds for media item that were found after (re)scraping -CREATE TABLE feeds_after_rescraping ( - feeds_after_rescraping_id SERIAL PRIMARY KEY, - media_id INT NOT NULL REFERENCES media ON DELETE CASCADE, - name VARCHAR(512) NOT NULL, - url VARCHAR(1024) NOT NULL, - feed_type feed_feed_type NOT NULL DEFAULT 'syndicated' -); -CREATE INDEX feeds_after_rescraping_media_id ON feeds_after_rescraping(media_id); -CREATE INDEX feeds_after_rescraping_name ON feeds_after_rescraping(name); -CREATE UNIQUE INDEX feeds_after_rescraping_url ON feeds_after_rescraping(url, media_id); - - --- Feed is "stale" (hasn't provided a new story in some time) --- Not to be confused with "stale feeds" in extractor! -CREATE OR REPLACE FUNCTION feed_is_stale(param_feeds_id INT) RETURNS boolean AS $$ -BEGIN - - -- Check if feed exists at all - IF NOT EXISTS ( - SELECT 1 - FROM feeds - WHERE feeds.feeds_id = param_feeds_id - ) THEN - RAISE EXCEPTION 'Feed % does not exist.', param_feeds_id; - RETURN FALSE; - END IF; - - -- Check if feed is active - IF EXISTS ( - SELECT 1 - FROM feeds - WHERE feeds.feeds_id = param_feeds_id - AND ( - feeds.last_new_story_time IS NULL - OR feeds.last_new_story_time < NOW() - INTERVAL '6 months' - ) - ) THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4507; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4507-4508.sql b/apps/postgresql-server/schema/migrations/mediawords-4507-4508.sql deleted file mode 100644 index aa00bbfaef..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4507-4508.sql +++ /dev/null @@ -1,62 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4507 and 4508. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4507, and you would like to upgrade both the Media Cloud and the --- database to be at version 4508, import this SQL file: --- --- psql mediacloud < mediawords-4507-4508.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- "media.content_delay" should have been added in "mediawords-4504-4505.sql" --- schema migration file, but it wasn't there --- --- Additionally, the column is live on the production database, so we test if --- it's there before trying to add it - -DO $$ - BEGIN - ALTER TABLE media - -- Delay content downloads for this media source this many hours - ADD COLUMN content_delay int; - EXCEPTION - WHEN duplicate_column THEN - RAISE NOTICE 'Column "media.content_delay" already exists.'; - END -$$; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4508; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4508-4509.sql b/apps/postgresql-server/schema/migrations/mediawords-4508-4509.sql deleted file mode 100644 index 7f49bf2cda..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4508-4509.sql +++ /dev/null @@ -1,54 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4508 and 4509. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4508, and you would like to upgrade both the Media Cloud and the --- database to be at version 4509, import this SQL file: --- --- psql mediacloud < mediawords-4508-4509.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversies drop column if exists query_story_searches_id cascade; - -drop view if exists controversies_with_dates; - -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4509; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4509-4510.sql b/apps/postgresql-server/schema/migrations/mediawords-4509-4510.sql deleted file mode 100644 index 1502acc1b8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4509-4510.sql +++ /dev/null @@ -1,115 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4509 and 4510. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4509, and you would like to upgrade both the Media Cloud and the --- database to be at version 4510, import this SQL file: --- --- psql mediacloud < mediawords-4509-4510.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Helper to find corrupted sequences (the ones in which the primary key's sequence value > MAX(primary_key)) -CREATE OR REPLACE FUNCTION find_corrupted_sequences() -RETURNS TABLE(tablename VARCHAR, maxid BIGINT, sequenceval BIGINT) -AS $BODY$ -DECLARE - r RECORD; -BEGIN - - SET client_min_messages TO WARNING; - DROP TABLE IF EXISTS temp_corrupted_sequences; - CREATE TEMPORARY TABLE temp_corrupted_sequences ( - tablename VARCHAR NOT NULL UNIQUE, - maxid BIGINT, - sequenceval BIGINT - ) ON COMMIT DROP; - SET client_min_messages TO NOTICE; - - FOR r IN ( - - -- Get all tables, their primary keys and serial sequence names - SELECT t.relname AS tablename, - primarykey AS idcolumn, - pg_get_serial_sequence(t.relname, primarykey) AS serialsequence - FROM pg_constraint AS c - JOIN pg_class AS t ON c.conrelid = t.oid - JOIN pg_namespace nsp ON nsp.oid = t.relnamespace - JOIN ( - SELECT a.attname AS primarykey, - i.indrelid - FROM pg_index AS i - JOIN pg_attribute AS a - ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) - WHERE i.indisprimary - ) AS pkey ON pkey.indrelid = t.relname::regclass - WHERE conname LIKE '%_pkey' - AND nsp.nspname = 'public' - AND t.relname NOT IN ( - 'story_similarities_100_short', - 'url_discovery_counts' - ) - ORDER BY t.relname - - ) - LOOP - - -- Filter out the tables that have their max ID bigger than the last - -- sequence value - EXECUTE ' - INSERT INTO temp_corrupted_sequences - SELECT tablename, - maxid, - sequenceval - FROM ( - SELECT ''' || r.tablename || ''' AS tablename, - MAX(' || r.idcolumn || ') AS maxid, - ( SELECT last_value FROM ' || r.serialsequence || ') AS sequenceval - FROM ' || r.tablename || ' - ) AS id_and_sequence - WHERE maxid > sequenceval - '; - - END LOOP; - - RETURN QUERY SELECT * FROM temp_corrupted_sequences ORDER BY tablename; - -END -$BODY$ -LANGUAGE 'plpgsql'; - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4510; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4510-4511.sql b/apps/postgresql-server/schema/migrations/mediawords-4510-4511.sql deleted file mode 100644 index 07318cdd5a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4510-4511.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4509 and 4510. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4509, and you would like to upgrade both the Media Cloud and the --- database to be at version 4510, import this SQL file: --- --- psql mediacloud < mediawords-4509-4510.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversy_seed_urls add publish_date text; -alter table controversy_seed_urls add title text; -alter table controversy_seed_urls add guid text; -alter table controversy_seed_urls add content text; -alter table controversies add max_iterations int not null default 15; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4511; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4511-4512.sql b/apps/postgresql-server/schema/migrations/mediawords-4511-4512.sql deleted file mode 100644 index cd09f3a26e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4511-4512.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4511 and 4512. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4511, and you would like to upgrade both the Media Cloud and the --- database to be at version 4512, import this SQL file: --- --- psql mediacloud < mediawords-4511-4512.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table solr_imports add num_stories bigint; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4512; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4512-4513.sql b/apps/postgresql-server/schema/migrations/mediawords-4512-4513.sql deleted file mode 100644 index f225a50841..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4512-4513.sql +++ /dev/null @@ -1,198 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4512 and 4513. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4512, and you would like to upgrade both the Media Cloud and the --- database to be at version 4513, import this SQL file: --- --- psql mediacloud < mediawords-4512-4513.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Copy of "feeds" table from yesterday; used for generating reports for rescraping efforts -CREATE TABLE feeds_from_yesterday ( - feeds_id INT NOT NULL, - media_id INT NOT NULL, - name VARCHAR(512) NOT NULL, - url VARCHAR(1024) NOT NULL, - feed_type feed_feed_type NOT NULL, - feed_status feed_feed_status NOT NULL -); - -CREATE INDEX feeds_from_yesterday_feeds_id ON feeds_from_yesterday(feeds_id); -CREATE INDEX feeds_from_yesterday_media_id ON feeds_from_yesterday(media_id); -CREATE INDEX feeds_from_yesterday_name ON feeds_from_yesterday(name); -CREATE UNIQUE INDEX feeds_from_yesterday_url ON feeds_from_yesterday(url, media_id); - --- --- Update "feeds_from_yesterday" with a new set of feeds --- -CREATE OR REPLACE FUNCTION update_feeds_from_yesterday() RETURNS VOID AS $$ -BEGIN - - TRUNCATE TABLE feeds_from_yesterday; - INSERT INTO feeds_from_yesterday (feeds_id, media_id, name, url, feed_type, feed_status) - SELECT feeds_id, media_id, name, url, feed_type, feed_status - FROM feeds; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- Print out a diff between "feeds" and "feeds_from_yesterday" --- -CREATE OR REPLACE FUNCTION rescraping_changes() RETURNS VOID AS -$$ -DECLARE - r_media RECORD; - r_feed RECORD; -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - SELECT 1 - FROM feeds_from_yesterday - ) THEN - RAISE EXCEPTION '"feeds_from_yesterday" table is empty.'; - END IF; - - RAISE NOTICE 'Changes between "feeds" and "feeds_from_yesterday":'; - RAISE NOTICE ''; - - FOR r_media IN - SELECT * - FROM media - WHERE media_id IN ( - SELECT DISTINCT media_id - FROM ( - SELECT feeds_id, media_id, feed_type, feed_status, name, url FROM feeds_from_yesterday - EXCEPT - SELECT feeds_id, media_id, feed_type, feed_status, name, url FROM feeds - ) AS modified_feeds - ) - ORDER BY media_id - LOOP - RAISE NOTICE 'MODIFIED media: media_id=%, name="%", url="%"', - r_media.media_id, - r_media.name, - r_media.url; - - FOR r_feed IN - SELECT * - FROM feeds - WHERE media_id = r_media.media_id - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds_from_yesterday - ) - LOOP - RAISE NOTICE ' ADDED feed: feeds_id=%, feed_type=%, feed_status=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.feed_status, - r_feed.name, - r_feed.url; - END LOOP; - - -- Feeds shouldn't get deleted but we're checking anyways - FOR r_feed IN - SELECT * - FROM feeds_from_yesterday - WHERE media_id = r_media.media_id - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds - ) - LOOP - RAISE NOTICE ' DELETED feed: feeds_id=%, feed_type=%, feed_status=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.feed_status, - r_feed.name, - r_feed.url; - END LOOP; - - FOR r_feed IN - SELECT feeds_before.feeds_id, - - feeds_before.name AS before_name, - feeds_before.url AS before_url, - feeds_before.feed_type AS before_feed_type, - feeds_before.feed_status AS before_feed_status, - - feeds_after.name AS after_name, - feeds_after.url AS after_url, - feeds_after.feed_type AS after_feed_type, - feeds_after.feed_status AS after_feed_status - - FROM feeds_from_yesterday AS feeds_before - INNER JOIN feeds AS feeds_after ON ( - feeds_before.feeds_id = feeds_after.feeds_id - AND ( - feeds_before.name != feeds_after.name - OR feeds_before.url != feeds_after.url - OR feeds_before.feed_type != feeds_after.feed_type - OR feeds_before.feed_status != feeds_after.feed_status - ) - ) - - WHERE feeds_before.media_id = r_media.media_id - - ORDER BY feeds_before.feeds_id - LOOP - RAISE NOTICE ' MODIFIED feed: feeds_id=%', r_feed.feeds_id; - RAISE NOTICE ' BEFORE: feed_type=%, feed_status=%, name="%", url="%"', - r_feed.before_feed_type, - r_feed.before_feed_status, - r_feed.before_name, - r_feed.before_url; - RAISE NOTICE ' AFTER: feed_type=%, feed_status=%, name="%", url="%"', - r_feed.after_feed_type, - r_feed.after_feed_status, - r_feed.after_name, - r_feed.after_url; - END LOOP; - - RAISE NOTICE ''; - - END LOOP; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4513; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4513-4514.sql b/apps/postgresql-server/schema/migrations/mediawords-4513-4514.sql deleted file mode 100644 index 79b42857a5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4513-4514.sql +++ /dev/null @@ -1,250 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4513 and 4514. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4513, and you would like to upgrade both the Media Cloud and the --- database to be at version 4514, import this SQL file: --- --- psql mediacloud < mediawords-4513-4514.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Print out a diff between "feeds" and "feeds_from_yesterday" --- -CREATE OR REPLACE FUNCTION rescraping_changes() RETURNS VOID AS -$$ -DECLARE - r_count RECORD; - r_media RECORD; - r_feed RECORD; -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - SELECT 1 - FROM feeds_from_yesterday - ) THEN - RAISE EXCEPTION '"feeds_from_yesterday" table is empty.'; - END IF; - - -- Fill temp. tables with changes to print out later - CREATE TEMPORARY TABLE rescraping_changes_media ON COMMIT DROP AS - SELECT * - FROM media - WHERE media_id IN ( - SELECT DISTINCT media_id - FROM ( - -- Don't compare "name" because it's insignificant - ( - SELECT feeds_id, media_id, feed_type, feed_status, url FROM feeds_from_yesterday - EXCEPT - SELECT feeds_id, media_id, feed_type, feed_status, url FROM feeds - ) UNION ALL ( - SELECT feeds_id, media_id, feed_type, feed_status, url FROM feeds - EXCEPT - SELECT feeds_id, media_id, feed_type, feed_status, url FROM feeds_from_yesterday - ) - ) AS modified_feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_added ON COMMIT DROP AS - SELECT * - FROM feeds - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds_from_yesterday - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_deleted ON COMMIT DROP AS - SELECT * - FROM feeds_from_yesterday - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_modified ON COMMIT DROP AS - SELECT feeds_before.media_id, - feeds_before.feeds_id, - - feeds_before.name AS before_name, - feeds_before.url AS before_url, - feeds_before.feed_type AS before_feed_type, - feeds_before.feed_status AS before_feed_status, - - feeds_after.name AS after_name, - feeds_after.url AS after_url, - feeds_after.feed_type AS after_feed_type, - feeds_after.feed_status AS after_feed_status - - FROM feeds_from_yesterday AS feeds_before - INNER JOIN feeds AS feeds_after ON ( - feeds_before.feeds_id = feeds_after.feeds_id - AND ( - -- Don't compare "name" because it's insignificant - feeds_before.url != feeds_after.url - OR feeds_before.feed_type != feeds_after.feed_type - OR feeds_before.feed_status != feeds_after.feed_status - ) - ) - - WHERE feeds_before.media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ); - - -- Print out changes - RAISE NOTICE 'Changes between "feeds" and "feeds_from_yesterday":'; - RAISE NOTICE ''; - - SELECT COUNT(1) AS media_count INTO r_count FROM rescraping_changes_media; - RAISE NOTICE '* Modified media: %', r_count.media_count; - SELECT COUNT(1) AS feeds_added_count INTO r_count FROM rescraping_changes_feeds_added; - RAISE NOTICE '* Added feeds: %', r_count.feeds_added_count; - SELECT COUNT(1) AS feeds_deleted_count INTO r_count FROM rescraping_changes_feeds_deleted; - RAISE NOTICE '* Deleted feeds: %', r_count.feeds_deleted_count; - SELECT COUNT(1) AS feeds_modified_count INTO r_count FROM rescraping_changes_feeds_modified; - RAISE NOTICE '* Modified feeds: %', r_count.feeds_modified_count; - RAISE NOTICE ''; - - FOR r_media IN - SELECT *, - - -- Prioritize US MSM media - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND tag_sets.name = 'collection' - AND tags.tag = 'ap_english_us_top25_20100110' - ) AS belongs_to_us_msm, - - -- Prioritize media with "show_on_media" - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND ( - tag_sets.show_on_media - OR tags.show_on_media - ) - ) AS show_on_media - - FROM rescraping_changes_media - - ORDER BY belongs_to_us_msm DESC, - show_on_media DESC, - media_id - LOOP - RAISE NOTICE 'MODIFIED media: media_id=%, name="%", url="%"', - r_media.media_id, - r_media.name, - r_media.url; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_added - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' ADDED feed: feeds_id=%, feed_type=%, feed_status=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.feed_status, - r_feed.name, - r_feed.url; - END LOOP; - - -- Feeds shouldn't get deleted but we're checking anyways - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_deleted - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' DELETED feed: feeds_id=%, feed_type=%, feed_status=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.feed_status, - r_feed.name, - r_feed.url; - END LOOP; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_modified - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' MODIFIED feed: feeds_id=%', r_feed.feeds_id; - RAISE NOTICE ' BEFORE: feed_type=%, feed_status=%, name="%", url="%"', - r_feed.before_feed_type, - r_feed.before_feed_status, - r_feed.before_name, - r_feed.before_url; - RAISE NOTICE ' AFTER: feed_type=%, feed_status=%, name="%", url="%"', - r_feed.after_feed_type, - r_feed.after_feed_status, - r_feed.after_name, - r_feed.after_url; - END LOOP; - - RAISE NOTICE ''; - - END LOOP; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4514; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4514-4515.sql b/apps/postgresql-server/schema/migrations/mediawords-4514-4515.sql deleted file mode 100644 index df9911a2dd..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4514-4515.sql +++ /dev/null @@ -1,63 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4514 and 4515. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4514, and you would like to upgrade both the Media Cloud and the --- database to be at version 4515, import this SQL file: --- --- psql mediacloud < mediawords-4514-4515.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4515; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4515-4516.sql b/apps/postgresql-server/schema/migrations/mediawords-4515-4516.sql deleted file mode 100644 index 44d04223f4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4515-4516.sql +++ /dev/null @@ -1,69 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4515 and 4516. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4515, and you would like to upgrade both the Media Cloud and the --- database to be at version 4516, import this SQL file: --- --- psql mediacloud < mediawords-4515-4516.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- stats for deprecated Twitter share counts -create table story_statistics_twitter ( - story_statistics_id serial primary key, - stories_id int not null references stories on delete cascade, - - twitter_url_tweet_count int null, - twitter_api_collect_date timestamp null, - twitter_api_error text null -); - -create unique index story_statistics_twitter_story on story_statistics_twitter ( stories_id ); - - --- Migrate Twitter stats collected so far to the new table -INSERT INTO story_statistics_twitter (stories_id, twitter_url_tweet_count, twitter_api_collect_date, twitter_api_error) - SELECT stories_id, twitter_url_tweet_count, twitter_api_collect_date, twitter_api_error - FROM story_statistics; - - -ALTER TABLE story_statistics - DROP COLUMN twitter_url_tweet_count, - DROP COLUMN twitter_api_collect_date, - DROP COLUMN twitter_api_error; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4516; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4516-4517.sql b/apps/postgresql-server/schema/migrations/mediawords-4516-4517.sql deleted file mode 100644 index e3907e055a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4516-4517.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4516 and 4517. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4516, and you would like to upgrade both the Media Cloud and the --- database to be at version 4517, import this SQL file: --- --- psql mediacloud < mediawords-4516-4517.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -DROP INDEX IF EXISTS stories_db_row_last_updated; -DROP INDEX IF EXISTS downloads_file_status; -DROP INDEX IF EXISTS downloads_relative_path; -DROP INDEX IF EXISTS downloads_type; -DROP INDEX IF EXISTS downloads_host_state_priority; -DROP INDEX IF EXISTS downloads_feed_state; -DROP INDEX IF EXISTS downloads_url; -DROP INDEX IF EXISTS downloads_for_extractor_trainer; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4517; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4517-4518.sql b/apps/postgresql-server/schema/migrations/mediawords-4517-4518.sql deleted file mode 100644 index 331defcbc6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4517-4518.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4517 and 4518. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4517, and you would like to upgrade both the Media Cloud and the --- database to be at version 4518, import this SQL file: --- --- psql mediacloud < mediawords-4517-4518.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION update_feeds_from_yesterday() RETURNS VOID AS $$ -BEGIN - - DELETE FROM feeds_from_yesterday; - INSERT INTO feeds_from_yesterday (feeds_id, media_id, name, url, feed_type, feed_status) - SELECT feeds_id, media_id, name, url, feed_type, feed_status - FROM feeds; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4518; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4518-4519.sql b/apps/postgresql-server/schema/migrations/mediawords-4518-4519.sql deleted file mode 100644 index 38c233f7ef..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4518-4519.sql +++ /dev/null @@ -1,61 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4518 and 4519. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4518, and you would like to upgrade both the Media Cloud and the --- database to be at version 4519, import this SQL file: --- --- psql mediacloud < mediawords-4518-4519.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table story_sentences add is_dup boolean null; - --- we have to do this in a function to create the partial index on a constant value, --- which you cannot do with a simple 'create index ... where publish_date > now()' -create or replace function create_initial_story_sentences_dup() RETURNS boolean as $$ -declare - one_month_ago date; -begin - select now() - interval '1 month' into one_month_ago; - - raise notice 'date: %', one_month_ago; - - execute 'create index story_sentences_dup on story_sentences( md5( sentence ) ) ' || - 'where week_start_date( publish_date::date ) > ''' || one_month_ago || '''::date'; - - return true; -END; -$$ LANGUAGE plpgsql; - -select create_initial_story_sentences_dup(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4519; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4519-4520.sql b/apps/postgresql-server/schema/migrations/mediawords-4519-4520.sql deleted file mode 100644 index 02d791dcb1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4519-4520.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4519 and 4520. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4519, and you would like to upgrade both the Media Cloud and the --- database to be at version 4520, import this SQL file: --- --- psql mediacloud < mediawords-4519-4520.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop table story_sentence_counts cascade; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4520; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4520-4521.sql b/apps/postgresql-server/schema/migrations/mediawords-4520-4521.sql deleted file mode 100644 index ee7f13f89c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4520-4521.sql +++ /dev/null @@ -1,95 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4520 and 4521. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4520, and you would like to upgrade both the Media Cloud and the --- database to be at version 4521, import this SQL file: --- --- psql mediacloud < mediawords-4520-4521.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION IF EXISTS upsert_story_bitly_statistics(INT, INT, INT); -CREATE OR REPLACE FUNCTION upsert_story_bitly_statistics(param_stories_id INT, param_bitly_click_count INT) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE story_bitly_statistics - SET bitly_click_count = param_bitly_click_count - WHERE stories_id = param_stories_id; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO story_bitly_statistics (stories_id, bitly_click_count) - VALUES (param_stories_id, param_bitly_click_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - --- Copy the referrer counts to a legacy table -create table story_statistics_bitly_referrers ( - story_statistics_id serial primary key, - stories_id int not null references stories on delete cascade, - - bitly_referrer_count int null -); - -create unique index story_statistics_bitly_referrers_story on story_statistics_bitly_referrers ( stories_id ); - -INSERT INTO story_statistics_bitly_referrers (stories_id, bitly_referrer_count) - SELECT stories_id, bitly_referrer_count - FROM story_bitly_statistics; - - -ALTER TABLE story_bitly_statistics - DROP COLUMN bitly_referrer_count; - -ALTER TABLE cd.story_link_counts - DROP COLUMN bitly_referrer_count; - -ALTER TABLE cd.medium_link_counts - DROP COLUMN bitly_referrer_count; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4521; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4521-4522.sql b/apps/postgresql-server/schema/migrations/mediawords-4521-4522.sql deleted file mode 100644 index 66605fdeb4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4521-4522.sql +++ /dev/null @@ -1,361 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4521 and 4522. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4521, and you would like to upgrade both the Media Cloud and the --- database to be at version 4522, import this SQL file: --- --- psql mediacloud < mediawords-4521-4522.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE bitly_processing_results - ADD COLUMN collect_date TIMESTAMP NULL DEFAULT NOW(); - --- Set to NULL for all the current data because we don't know the exact collection date -UPDATE bitly_processing_results - SET collect_date = NULL; - - --- --- Bit.ly total story click counts --- - --- "Master" table (no indexes, no foreign keys as they'll be ineffective) -CREATE TABLE bitly_clicks_total ( - bitly_clicks_id BIGSERIAL NOT NULL, - stories_id INT NOT NULL, - - click_count INT NOT NULL -); - --- Automatic Bit.ly total click count partitioning to stories_id chunks of 1m rows -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - chunk_size CONSTANT INT := 1000000; -- 1m stories in a chunk - to_char_format CONSTANT TEXT := '000000'; -- Up to 1m of chunks, suffixed as "_000001", ..., "_999999" - - stories_id_chunk_number INT; -- millions part of stories_id (e.g. 30 for stories_id = 30,000,000) - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT NEW.stories_id / chunk_size INTO stories_id_chunk_number; - SELECT 'bitly_clicks_total_' || trim(leading ' ' from to_char(stories_id_chunk_number, to_char_format)) - INTO target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((NEW.stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_unique - UNIQUE (stories_id) - - ) INHERITS (bitly_clicks_total); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_total' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER bitly_clicks_total_partition_by_stories_id_insert_trigger - BEFORE INSERT ON bitly_clicks_total - FOR EACH ROW EXECUTE PROCEDURE bitly_clicks_total_partition_by_stories_id_insert_trigger(); - - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_total ( - param_stories_id INT, - param_click_count INT -) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE bitly_clicks_total - SET click_count = param_click_count - WHERE stories_id = param_stories_id; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO bitly_clicks_total (stories_id, click_count) - VALUES (param_stories_id, param_click_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - --- --- Bit.ly daily story click counts --- - --- "Master" table (no indexes, no foreign keys as they'll be ineffective) -CREATE TABLE bitly_clicks_daily ( - bitly_clicks_id BIGSERIAL NOT NULL, - stories_id INT NOT NULL, - - day DATE NOT NULL, - click_count INT NOT NULL -); - --- Automatic Bit.ly daily click count partitioning to stories_id chunks of 1m rows -CREATE OR REPLACE FUNCTION bitly_clicks_daily_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_daily_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - chunk_size CONSTANT INT := 1000000; -- 1m stories in a chunk - to_char_format CONSTANT TEXT := '000000'; -- Up to 1m of chunks, suffixed as "_000001", ..., "_999999" - - stories_id_chunk_number INT; -- millions part of stories_id (e.g. 30 for stories_id = 30,000,000) - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT NEW.stories_id / chunk_size INTO stories_id_chunk_number; - SELECT 'bitly_clicks_daily_' || trim(leading ' ' from to_char(stories_id_chunk_number, to_char_format)) - INTO target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((NEW.stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_day_unique - UNIQUE (stories_id, day) - - ) INHERITS (bitly_clicks_daily); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_daily' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER bitly_clicks_daily_partition_by_stories_id_insert_trigger - BEFORE INSERT ON bitly_clicks_daily - FOR EACH ROW EXECUTE PROCEDURE bitly_clicks_daily_partition_by_stories_id_insert_trigger(); - - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_daily ( - param_stories_id INT, - param_day DATE, - param_click_count INT -) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE bitly_clicks_daily - SET click_count = param_click_count - WHERE stories_id = param_stories_id - AND day = param_day; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO bitly_clicks_daily (stories_id, day, click_count) - VALUES (param_stories_id, param_day, param_click_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - --- --- Bit.ly processing schedule --- -CREATE TABLE bitly_processing_schedule ( - bitly_processing_schedule_id BIGSERIAL NOT NULL, - stories_id INT NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - fetch_at TIMESTAMP NOT NULL -); - -CREATE INDEX bitly_processing_schedule_stories_id - ON bitly_processing_schedule (stories_id); -CREATE INDEX bitly_processing_schedule_fetch_at - ON bitly_processing_schedule (fetch_at); - - --- Helper to return a number of stories for which we don't have Bit.ly statistics yet -DROP FUNCTION num_controversy_stories_without_bitly_statistics(INT); -CREATE FUNCTION num_controversy_stories_without_bitly_statistics (param_controversies_id INT) RETURNS INT AS -$$ -DECLARE - controversy_exists BOOL; - num_stories_without_bitly_statistics INT; -BEGIN - - SELECT 1 INTO controversy_exists - FROM controversies - WHERE controversies_id = param_controversies_id - AND process_with_bitly = 't'; - IF NOT FOUND THEN - RAISE EXCEPTION 'Controversy % does not exist or is not set up for Bit.ly processing.', param_controversies_id; - RETURN FALSE; - END IF; - - SELECT COUNT(stories_id) INTO num_stories_without_bitly_statistics - FROM controversy_stories - WHERE controversies_id = param_controversies_id - AND stories_id NOT IN ( - SELECT stories_id - FROM bitly_clicks_total - ) - GROUP BY controversies_id; - IF NOT FOUND THEN - num_stories_without_bitly_statistics := 0; - END IF; - - RETURN num_stories_without_bitly_statistics; -END; -$$ -LANGUAGE plpgsql; - - --- Migrate old data to the new partitioned table infrastructure and drop it afterwards --- (might take up ~6 minutes or so) -DROP FUNCTION IF EXISTS upsert_story_bitly_statistics(INT, INT); -DROP FUNCTION IF EXISTS upsert_story_bitly_statistics(INT, INT, INT); - -INSERT INTO bitly_clicks_total (stories_id, click_count) - SELECT stories_id, bitly_click_count - FROM story_bitly_statistics; - -DROP TABLE story_bitly_statistics; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4522; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4522-4523.sql b/apps/postgresql-server/schema/migrations/mediawords-4522-4523.sql deleted file mode 100644 index aab0044d4e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4522-4523.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4522 and 4523. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4522, and you would like to upgrade both the Media Cloud and the --- database to be at version 4523, import this SQL file: --- --- psql mediacloud < mediawords-4522-4523.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4523; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4523-4524.sql b/apps/postgresql-server/schema/migrations/mediawords-4523-4524.sql deleted file mode 100644 index 519dbdfb96..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4523-4524.sql +++ /dev/null @@ -1,88 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4523 and 4524. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4523, and you would like to upgrade both the Media Cloud and the --- database to be at version 4524, import this SQL file: --- --- psql mediacloud < mediawords-4523-4524.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table cd.live_stories add ap_syndicated boolean null; - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into cd.live_stories - ( controversies_id, controversy_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language, - db_row_last_updated, ap_syndicated ) - select NEW.controversies_id, NEW.controversy_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language, - s.db_row_last_updated, s.ap_syndicated - from controversy_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.controversies_id = NEW.controversies_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - IF NOT story_triggers_enabled() then - RETURN NEW; - END IF; - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated, - ap_syndicated = NEW.ap_syndicated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4524; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4524-4525.sql b/apps/postgresql-server/schema/migrations/mediawords-4524-4525.sql deleted file mode 100644 index 35c60543a4..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4524-4525.sql +++ /dev/null @@ -1,74 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4524 and 4525. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4524, and you would like to upgrade both the Media Cloud and the --- database to be at version 4525, import this SQL file: --- --- psql mediacloud < mediawords-4524-4525.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION update_live_story() returns trigger as $update_live_story$ - begin - - IF NOT story_triggers_enabled() then - RETURN NEW; - END IF; - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated, - ap_syndicated = ( - SELECT ap_syndicated - FROM stories - WHERE stories_id = NEW.stories_id - ) - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4525; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4525-4526.sql b/apps/postgresql-server/schema/migrations/mediawords-4525-4526.sql deleted file mode 100644 index d910584b42..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4525-4526.sql +++ /dev/null @@ -1,66 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4525 and 4526. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4525, and you would like to upgrade both the Media Cloud and the --- database to be at version 4526, import this SQL file: --- --- psql mediacloud < mediawords-4525-4526.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - IF NOT story_triggers_enabled() then - RETURN NEW; - END IF; - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated, - ap_syndicated = NEW.ap_syndicated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4526; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4526-4527.sql b/apps/postgresql-server/schema/migrations/mediawords-4526-4527.sql deleted file mode 100644 index 194c3a581a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4526-4527.sql +++ /dev/null @@ -1,63 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4526 and 4527. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4526, and you would like to upgrade both the Media Cloud and the --- database to be at version 4527, import this SQL file: --- --- psql mediacloud < mediawords-4526-4527.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated, - ap_syndicated = NEW.ap_syndicated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4527; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4527-4528.sql b/apps/postgresql-server/schema/migrations/mediawords-4527-4528.sql deleted file mode 100644 index b14c74ff0b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4527-4528.sql +++ /dev/null @@ -1,93 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4527 and 4528. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4527, and you would like to upgrade both the Media Cloud and the --- database to be at version 4528, import this SQL file: --- --- psql mediacloud < mediawords-4527-4528.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- - -alter table stories drop column if exists ap_syndicated; -alter table cd.live_stories drop column if exists ap_syndicated; - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into cd.live_stories - ( controversies_id, controversy_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language, - db_row_last_updated ) - select NEW.controversies_id, NEW.controversy_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language, - s.db_row_last_updated - from controversy_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.controversies_id = NEW.controversies_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update cd.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - -create table stories_ap_syndicated ( - stories_ap_syndicated_id serial primary key, - stories_id int not null references stories on delete cascade, - ap_syndicated boolean not null -); - -create unique index stories_ap_syndicated_story on stories_ap_syndicated ( stories_id ); - --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4528; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4528-4529.sql b/apps/postgresql-server/schema/migrations/mediawords-4528-4529.sql deleted file mode 100644 index c563f53480..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4528-4529.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4528 and 4529. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4528, and you would like to upgrade both the Media Cloud and the --- database to be at version 4529, import this SQL file: --- --- psql mediacloud < mediawords-4528-4529.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE solr_import_stories - RENAME TO solr_import_extra_stories; - -ALTER INDEX solr_import_stories_story - RENAME TO solr_import_extra_stories_story; - -INSERT INTO solr_import_extra_stories (stories_id) - SELECT stories_id - FROM bitly_clicks_total; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4529; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4529-4530.sql b/apps/postgresql-server/schema/migrations/mediawords-4529-4530.sql deleted file mode 100644 index 9fd70a9472..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4529-4530.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4529 and 4530. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4529, and you would like to upgrade both the Media Cloud and the --- database to be at version 4530, import this SQL file: --- --- psql mediacloud < mediawords-4529-4530.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index controversy_stories_controversy on controversy_stories( controversies_id ); - - -- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4530; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4530-4531.sql b/apps/postgresql-server/schema/migrations/mediawords-4530-4531.sql deleted file mode 100644 index 9b113cf6b5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4530-4531.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4530 and 4531. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4530, and you would like to upgrade both the Media Cloud and the --- database to be at version 4531, import this SQL file: --- --- psql mediacloud < mediawords-4530-4531.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP VIEW IF EXISTS media_adjusted_extractor_training_downloads_count; -DROP VIEW IF EXISTS media_extractor_training_downloads_count_adjustments; -DROP VIEW IF EXISTS media_extractor_training_downloads_count; -DROP VIEW IF EXISTS yahoo_top_political_2008_media; -DROP VIEW IF EXISTS technorati_top_political_2008_media; - - -DROP TABLE extractor_training_lines; -DROP TABLE extracted_lines; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4531; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4531-4532.sql b/apps/postgresql-server/schema/migrations/mediawords-4531-4532.sql deleted file mode 100644 index 926b651ec2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4531-4532.sql +++ /dev/null @@ -1,73 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4531 and 4532. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4531, and you would like to upgrade both the Media Cloud and the --- database to be at version 4532, import this SQL file: --- --- psql mediacloud < mediawords-4531-4532.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DO $$ -BEGIN - IF NOT EXISTS ( - - -- No raw_downloads.object_id => downloads.downloads_id foreign key? - SELECT 1 - FROM information_schema.table_constraints tc - INNER JOIN information_schema.constraint_column_usage ccu - USING (constraint_catalog, constraint_schema, constraint_name) - INNER JOIN information_schema.key_column_usage kcu - USING (constraint_catalog, constraint_schema, constraint_name) - WHERE constraint_type = 'FOREIGN KEY' - AND tc.table_name = 'raw_downloads' - AND kcu.column_name = 'object_id' - AND ccu.table_name = 'downloads' - AND ccu.column_name = 'downloads_id' - - ) THEN - - -- Re-add foreign key - ALTER TABLE raw_downloads - ADD CONSTRAINT raw_downloads_downloads_id_fkey - FOREIGN KEY (object_id) REFERENCES downloads(downloads_id) ON DELETE CASCADE; - - END IF; -END; -$$; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4532; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4532-4533.sql b/apps/postgresql-server/schema/migrations/mediawords-4532-4533.sql deleted file mode 100644 index 21a842399b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4532-4533.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4532 and 4533. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4532, and you would like to upgrade both the Media Cloud and the --- database to be at version 4533, import this SQL file: --- --- psql mediacloud < mediawords-4532-4533.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- list of stories that have been imported from feedly -create table scraped_stories ( - scraped_stories_id serial primary key, - stories_id int not null references stories on delete cascade, - import_module text not null -); - -create index scraped_stories_story on scraped_stories ( stories_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4533; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4533-4534.sql b/apps/postgresql-server/schema/migrations/mediawords-4533-4534.sql deleted file mode 100644 index a0649fba00..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4533-4534.sql +++ /dev/null @@ -1,60 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4533 and 4534. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4533, and you would like to upgrade both the Media Cloud and the --- database to be at version 4534, import this SQL file: --- --- psql mediacloud < mediawords-4533-4534.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- dates on which feeds have been scraped with MediaWords::ImportStories and the module used for scraping -create table scraped_feeds ( - scraped_feeds_id serial primary key, - feeds_id int not null references feeds on delete cascade, - scrape_date timestamp not null default now(), - import_module text not null -); - -create index scraped_feeds_feed on scraped_feeds ( feeds_id ); - -create view feedly_unscraped_feeds as - select f.* - from feeds f - left join scraped_feeds sf on - ( f.feeds_id = sf.feeds_id and sf.import_module = 'MediaWords::ImportStories::Feedly' ) - where - f.feed_type = 'syndicated' and - f.feed_status = 'active' and - sf.feeds_id is null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4534; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4534-4535.sql b/apps/postgresql-server/schema/migrations/mediawords-4534-4535.sql deleted file mode 100644 index b901500021..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4534-4535.sql +++ /dev/null @@ -1,225 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4534 and 4535. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4534, and you would like to upgrade both the Media Cloud and the --- database to be at version 4535, import this SQL file: --- --- psql mediacloud < mediawords-4534-4535.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- Returns true if the date is greater than the latest import date in solr_imports -CREATE OR REPLACE FUNCTION before_last_solr_import(db_row_last_updated timestamp with time zone) RETURNS boolean AS $$ -BEGIN - RETURN ( ( db_row_last_updated is null ) OR - ( db_row_last_updated < ( select max( import_date ) from solr_imports ) ) ); -END; -$$ -LANGUAGE 'plpgsql' -; - -CREATE OR REPLACE FUNCTION update_media_last_updated () RETURNS trigger AS -$$ - DECLARE - BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - update media set db_row_last_updated = now() - where media_id = NEW.media_id and before_last_solr_import( db_row_last_updated ); - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - update media set db_row_last_updated = now() - where media_id = OLD.media_id and before_last_solr_import( db_row_last_updated ); - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger () RETURNS trigger AS -$$ - DECLARE - path_change boolean; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF NEW.disable_triggers THEN - RETURN NULL; - END IF; - - UPDATE story_sentences set db_row_last_updated = now() - where stories_id = NEW.stories_id and before_last_solr_import( db_row_last_updated ); - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger () RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_stories_id integer default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - IF table_with_trigger_column THEN - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id - and before_last_solr_import( db_row_last_updated ); - RETURN NULL; - ELSE - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id and (disable_triggers is NOT true) - and before_last_solr_import( db_row_last_updated ); - RETURN NULL; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger () RETURNS trigger AS -$$ - DECLARE - path_change boolean; - table_with_trigger_column boolean default false; - reference_story_sentences_id bigint default null; - BEGIN - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF NOT story_triggers_enabled() THEN - RETURN NULL; - END IF; - - IF TG_TABLE_NAME in ( 'processed_stories', 'stories', 'story_sentences') THEN - table_with_trigger_column = true; - ELSE - table_with_trigger_column = false; - END IF; - - IF table_with_trigger_column THEN - IF TG_OP = 'INSERT' AND NEW.disable_triggers THEN - RETURN NULL; - ELSEIF ( ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') ) AND OLD.disable_triggers THEN - RETURN NULL; - END IF; - END IF; - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_story_sentences_id = NEW.story_sentences_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_story_sentences_id = OLD.story_sentences_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - IF table_with_trigger_column THEN - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id - and before_last_solr_import( db_row_last_updated ); - RETURN NULL; - ELSE - UPDATE story_sentences - SET db_row_last_updated = now() - WHERE story_sentences_id = reference_story_sentences_id and (disable_triggers is NOT true) - and before_last_solr_import( db_row_last_updated ); - RETURN NULL; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_media_id_trigger () RETURNS trigger AS -$$ - DECLARE - path_change boolean; - reference_media_id integer default null; - BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_media_id = EWEW.media_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_media_id = OLD.media_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories SET db_row_last_updated = now() - WHERE media_id = reference_media_id and before_last_solr_import( db_row_last_updated ); - - RETURN NULL; - END; -$$ -LANGUAGE 'plpgsql'; - - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4535; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4535-4536.sql b/apps/postgresql-server/schema/migrations/mediawords-4535-4536.sql deleted file mode 100644 index 250fac09f3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4535-4536.sql +++ /dev/null @@ -1,83 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4535 and 4536. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4535, and you would like to upgrade both the Media Cloud and the --- database to be at version 4536, import this SQL file: --- --- psql mediacloud < mediawords-4535-4536.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index media_annotate on media ( annotate_with_corenlp, media_id ); -create index live_stories_story_solo on cd.live_stories ( stories_id ); - --- --- Returns true if the story can + should be annotated with CoreNLP --- -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) -RETURNS boolean AS $$ -DECLARE - story record; -BEGIN - - SELECT stories_id, media_id, language INTO story from stories where stories_id = corenlp_stories_id; - - IF NOT ( story.language = 'en' or story.language is null ) THEN - - RETURN FALSE; - - ELSEIF NOT EXISTS ( - - SELECT 1 FROM media WHERE media.annotate_with_corenlp = 't' and media_id = story.media_id - - ) AND NOT EXISTS ( - - SELECT 1 FROM extra_corenlp_stories WHERE extra_corenlp_stories.stories_id = corenlp_stories_id - - ) THEN - - RETURN FALSE; - - ELSEIF NOT EXISTS ( SELECT 1 FROM story_sentences WHERE stories_id = corenlp_stories_id ) THEN - - RETURN FALSE; - - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4536; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4536-4537.sql b/apps/postgresql-server/schema/migrations/mediawords-4536-4537.sql deleted file mode 100644 index 567f5a8e48..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4536-4537.sql +++ /dev/null @@ -1,62 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4536 and 4537. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4536, and you would like to upgrade both the Media Cloud and the --- database to be at version 4537, import this SQL file: --- --- psql mediacloud < mediawords-4536-4537.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION update_media_last_updated () RETURNS trigger AS -$$ - DECLARE - BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - update media set db_row_last_updated = now() - where media_id = NEW.media_id; - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - update media set db_row_last_updated = now() - where media_id = OLD.media_id; - END IF; - - RETURN NEW; - END; -$$ -LANGUAGE 'plpgsql'; - -drop function if exists update_stories_updated_time_by_media_id_trigger (); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4537; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4537-4538.sql b/apps/postgresql-server/schema/migrations/mediawords-4537-4538.sql deleted file mode 100644 index 85542d2258..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4537-4538.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4537 and 4538. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4537, and you would like to upgrade both the Media Cloud and the --- database to be at version 4538, import this SQL file: --- --- psql mediacloud < mediawords-4537-4538.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE IF EXISTS gearman_job_queue; -DROP FUNCTION IF EXISTS gearman_job_queue_sync_lastmod(); - -COMMENT ON COLUMN controversies.process_with_bitly - IS 'Enable processing controversy''s stories with Bit.ly; add all new controversy stories to Bit.ly processing queue'; - - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4538; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4538-4539.sql b/apps/postgresql-server/schema/migrations/mediawords-4538-4539.sql deleted file mode 100644 index 07511b3045..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4538-4539.sql +++ /dev/null @@ -1,65 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4538 and 4539. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4538, and you would like to upgrade both the Media Cloud and the --- database to be at version 4539, import this SQL file: --- --- psql mediacloud < mediawords-4538-4539.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION update_media_last_updated () RETURNS trigger AS -$$ - DECLARE - BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - update media set db_row_last_updated = now() - where media_id = NEW.media_id; - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - update media set db_row_last_updated = now() - where media_id = OLD.media_id; - END IF; - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - RETURN NEW; - ELSE - RETURN OLD; - END IF; - END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4539; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4539-4540.sql b/apps/postgresql-server/schema/migrations/mediawords-4539-4540.sql deleted file mode 100644 index 51743096a6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4539-4540.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4539 and 4540. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4539, and you would like to upgrade both the Media Cloud and the --- database to be at version 4540, import this SQL file: --- --- psql mediacloud < mediawords-4539-4540.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table feeds add skip_bitly_processing boolean; - -update feeds set skip_bitly_processing = true where name like 'MediaWords::ImportStories%'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4540; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4540-4541.sql b/apps/postgresql-server/schema/migrations/mediawords-4540-4541.sql deleted file mode 100644 index cb84fd73b1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4540-4541.sql +++ /dev/null @@ -1,88 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4540 and 4541. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4540, and you would like to upgrade both the Media Cloud and the --- database to be at version 4541, import this SQL file: --- --- psql mediacloud < mediawords-4540-4541.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- To be recreated later on -DROP VIEW media_with_media_types; - - -DROP TABLE extra_corenlp_stories; - -DROP INDEX media_annotate; - -ALTER TABLE media - DROP COLUMN annotate_with_corenlp; - -CREATE OR REPLACE FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT) RETURNS boolean AS $$ -DECLARE - story record; -BEGIN - - SELECT stories_id, media_id, language INTO story from stories where stories_id = corenlp_stories_id; - - IF NOT ( story.language = 'en' or story.language is null ) THEN - RETURN FALSE; - - ELSEIF NOT EXISTS ( SELECT 1 FROM story_sentences WHERE stories_id = corenlp_stories_id ) THEN - RETURN FALSE; - - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE 'plpgsql'; - - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4541; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4541-4542.sql b/apps/postgresql-server/schema/migrations/mediawords-4541-4542.sql deleted file mode 100644 index 68529550bb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4541-4542.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4541 and 4542. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4541, and you would like to upgrade both the Media Cloud and the --- database to be at version 4542, import this SQL file: --- --- psql mediacloud < mediawords-4541-4542.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table controversy_dead_links ( - controversy_dead_links_id serial primary key, - controversies_id int not null, - stories_id int not null, - url text not null -); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4542; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4542-4543.sql b/apps/postgresql-server/schema/migrations/mediawords-4542-4543.sql deleted file mode 100644 index 6dbaca7ed6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4542-4543.sql +++ /dev/null @@ -1,69 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4542 and 4543. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4542, and you would like to upgrade both the Media Cloud and the --- database to be at version 4543, import this SQL file: --- --- psql mediacloud < mediawords-4542-4543.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Stories without Readability tag --- -CREATE TABLE IF NOT EXISTS stories_without_readability_tag ( - stories_id BIGINT NOT NULL REFERENCES stories (stories_id) -); -CREATE INDEX stories_without_readability_tag_stories_id - ON stories_without_readability_tag (stories_id); - --- Fill in the table manually with: --- --- INSERT INTO scratch.stories_without_readability_tag (stories_id) --- SELECT stories.stories_id --- FROM stories --- LEFT JOIN stories_tags_map --- ON stories.stories_id = stories_tags_map.stories_id - --- -- "extractor_version:readability-lxml-0.3.0.5" --- AND stories_tags_map.tags_id = 8929188 - --- -- No Readability tag --- WHERE stories_tags_map.tags_id IS NULL --- ; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4543; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4543-4544.sql b/apps/postgresql-server/schema/migrations/mediawords-4543-4544.sql deleted file mode 100644 index d00e45fde0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4543-4544.sql +++ /dev/null @@ -1,80 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4543 and 4544. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4543, and you would like to upgrade both the Media Cloud and the --- database to be at version 4544, import this SQL file: --- --- psql mediacloud < mediawords-4543-4544.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Will recreate afterwards -DROP VIEW media_with_media_types; - - -DROP FUNCTION IF EXISTS purge_story_sentences(date, date); - -DROP FUNCTION IF EXISTS media_set_sw_data_retention_dates(int, date, date); - -DROP FUNCTION media_set_retains_sw_data_for_date(int, date, date, date); - -DROP VIEW media_sets_explict_sw_data_dates; - - -SET search_path = cd, pg_catalog; -ALTER TABLE media - DROP COLUMN sw_data_start_date, - DROP COLUMN sw_data_end_date; - - -SET search_path = public, pg_catalog; -ALTER TABLE media - DROP COLUMN sw_data_start_date, - DROP COLUMN sw_data_end_date; - - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4544; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4544-4545.sql b/apps/postgresql-server/schema/migrations/mediawords-4544-4545.sql deleted file mode 100644 index 8e722ade3d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4544-4545.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4544 and 4545. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4544, and you would like to upgrade both the Media Cloud and the --- database to be at version 4545, import this SQL file: --- --- psql mediacloud < mediawords-4544-4545.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Leftover type -DROP TYPE IF EXISTS gearman_job_queue_status; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4545; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4545-4546.sql b/apps/postgresql-server/schema/migrations/mediawords-4545-4546.sql deleted file mode 100644 index 701f31955c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4545-4546.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4545 and 4546. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4545, and you would like to upgrade both the Media Cloud and the --- database to be at version 4546, import this SQL file: --- --- psql mediacloud < mediawords-4545-4546.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP SCHEMA IF EXISTS stories_tags_map_media_sub_tables; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4546; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4546-4547.sql b/apps/postgresql-server/schema/migrations/mediawords-4546-4547.sql deleted file mode 100644 index e893731fab..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4546-4547.sql +++ /dev/null @@ -1,76 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4546 and 4547. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4546, and you would like to upgrade both the Media Cloud and the --- database to be at version 4547, import this SQL file: --- --- psql mediacloud < mediawords-4546-4547.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP VIEW media_sets_tt2_locale_format; - -DROP VIEW dashboard_topics_tt2_locale_format; - -DROP TABLE IF EXISTS controversy_query_story_searches_imported_stories_map; - -DROP TABLE IF EXISTS query_story_searches_stories_map; - -DROP TABLE IF EXISTS query_story_searches; - -DROP TABLE queries_country_counts_json; - -DROP TABLE queries; - -DROP TABLE dashboard_media_sets; - -DROP TABLE story_subsets_processed_stories_map; - -DROP TABLE story_subsets; - -DROP TABLE media_sets_media_map; - -DROP TABLE media_sets; - -DROP TABLE dashboard_topics; - -DROP TABLE dashboards; - - -DELETE FROM auth_roles WHERE role = 'query-create'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4547; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4547-4548.sql b/apps/postgresql-server/schema/migrations/mediawords-4547-4548.sql deleted file mode 100644 index f626554c51..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4547-4548.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4547 and 4548. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4547, and you would like to upgrade both the Media Cloud and the --- database to be at version 4548, import this SQL file: --- --- psql mediacloud < mediawords-4547-4548.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION loop_forever(); - -DROP FUNCTION add_query_version(new_query_version_enum_string character varying); - -DROP FUNCTION show_stat_activity(); - -DROP FUNCTION cat(text, text); - -DROP FUNCTION cancel_pg_process(cancel_pid integer); - -DROP VIEW story_extracted_texts; - -DROP VIEW media_feed_counts; - -DROP TABLE url_discovery_counts; - -DROP TABLE extractor_results_cache; - -DROP TABLE feedless_stories; - -DROP SEQUENCE IF EXISTS extractor_results_cache_extractor_results_cache_id_seq; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4548; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4548-4549.sql b/apps/postgresql-server/schema/migrations/mediawords-4548-4549.sql deleted file mode 100644 index d1313c76b5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4548-4549.sql +++ /dev/null @@ -1,105 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4548 and 4549. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4548, and you would like to upgrade both the Media Cloud and the --- database to be at version 4549, import this SQL file: --- --- psql mediacloud < mediawords-4548-4549.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Will recreate right after -DROP VIEW IF EXISTS daily_stats; -DROP VIEW IF EXISTS downloads_sites; -DROP VIEW IF EXISTS downloads_media; -DROP VIEW IF EXISTS downloads_non_media; -DROP VIEW IF EXISTS downloads_to_be_extracted; -DROP VIEW IF EXISTS downloads_with_error_in_past_day; -DROP VIEW IF EXISTS downloads_in_past_day; - - -DROP TRIGGER IF EXISTS download_relative_file_path_trigger ON downloads; - -DROP FUNCTION IF EXISTS get_relative_file_path("path" text); - -DROP FUNCTION IF EXISTS download_relative_file_path_trigger(); - -DROP INDEX IF EXISTS file_status_downloads_time_new_format; - -DROP INDEX IF EXISTS relative_file_paths_new_format_to_verify; - -DROP INDEX IF EXISTS relative_file_paths_to_verify; - - -ALTER TABLE downloads - DROP COLUMN old_download_time, - DROP COLUMN old_state, - DROP COLUMN file_status, - DROP COLUMN relative_file_path; - - -create view downloads_media as select d.*, f.media_id as _media_id from downloads d, feeds f where d.feeds_id = f.feeds_id; -create view downloads_non_media as select d.* from downloads d where d.feeds_id is null; -CREATE VIEW downloads_sites as select site_from_host( host ) as site, * from downloads_media; -CREATE VIEW downloads_to_be_extracted as select * from downloads where extracted = 'f' and state = 'success' and type = 'content'; -CREATE VIEW downloads_in_past_day as select * from downloads where download_time > now() - interval '1 day'; -CREATE VIEW downloads_with_error_in_past_day as select * from downloads_in_past_day where state = 'error'; - -CREATE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4549; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4549-4550.sql b/apps/postgresql-server/schema/migrations/mediawords-4549-4550.sql deleted file mode 100644 index 67d0571c18..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4549-4550.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4549 and 4550. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4549, and you would like to upgrade both the Media Cloud and the --- database to be at version 4550, import this SQL file: --- --- psql mediacloud < mediawords-4549-4550.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION IF EXISTS enum.enum_add(CHARACTER VARYING, CHARACTER VARYING); -DROP FUNCTION IF EXISTS enum.enum_del(CHARACTER VARYING, CHARACTER VARYING); -DROP SCHEMA IF EXISTS enum; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4550; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4550-4551.sql b/apps/postgresql-server/schema/migrations/mediawords-4550-4551.sql deleted file mode 100644 index cffa147a2b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4550-4551.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4550 and 4551. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4550, and you would like to upgrade both the Media Cloud and the --- database to be at version 4551, import this SQL file: --- --- psql mediacloud < mediawords-4550-4551.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Remove old stopword tables -DROP TABLE IF EXISTS stopwords_tiny; -DROP TABLE IF EXISTS stopwords_short; -DROP TABLE IF EXISTS stopwords_long; - -DROP TABLE IF EXISTS stopword_stems_tiny; -DROP TABLE IF EXISTS stopword_stems_short; -DROP TABLE IF EXISTS stopword_stems_long; - -DROP FUNCTION IF EXISTS is_stop_stem(TEXT, TEXT, TEXT); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4551; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4551-4552.sql b/apps/postgresql-server/schema/migrations/mediawords-4551-4552.sql deleted file mode 100644 index 110f5f4f69..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4551-4552.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4551 and 4552. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4551, and you would like to upgrade both the Media Cloud and the --- database to be at version 4552, import this SQL file: --- --- psql mediacloud < mediawords-4551-4552.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TYPE IF EXISTS query_version_enum; -DROP TYPE IF EXISTS url_discovery_status_type; -DROP TYPE IF EXISTS download_file_status; -DROP TYPE IF EXISTS authors_stories_queue_type; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4552; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4552-4553.sql b/apps/postgresql-server/schema/migrations/mediawords-4552-4553.sql deleted file mode 100644 index 4637f59e66..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4552-4553.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4552 and 4553. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4552, and you would like to upgrade both the Media Cloud and the --- database to be at version 4553, import this SQL file: --- --- psql mediacloud < mediawords-4552-4553.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DELETE FROM database_variables WHERE name = 'LAST_STORY_SENTENCES_ID_PROCESSED'; -DELETE FROM database_variables WHERE name = 'LAST_REIMPORTED_STORY_SENTENCES_ID'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4553; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4553-4554.sql b/apps/postgresql-server/schema/migrations/mediawords-4553-4554.sql deleted file mode 100644 index d901b50dcb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4553-4554.sql +++ /dev/null @@ -1,77 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4552 and 4553. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4552, and you would like to upgrade both the Media Cloud and the --- database to be at version 4553, import this SQL file: --- --- psql mediacloud < mediawords-4552-4553.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversies add has_been_spidered boolean not null default false; -alter table controversies add has_been_dumped boolean not null default false; - -alter table controversies add state text not null default 'created but not queued'; -alter table controversies add error_message text null; - -alter table controversy_dumps add state text not null default 'queued'; -alter table controversy_dumps add error_message text null; - -update controversies c set has_been_spidered = true where not exists ( - select 1 from controversy_stories cs - where iteration < 15 and link_mined = false and c.controversies_id = cs.controversies_id -); - -update controversies set state = 'spidering completed' where has_been_spidered; -update controversies set state = 'unknown' where state != 'spidering completed'; - -update controversy_dumps set state = 'completed' where exists ( - select 1 from controversy_dump_time_slices cdts - where cdts.controversy_dumps_id = cdts.controversy_dumps_id and cdts.period = 'overall' -); - -update controversies c set has_been_dumped = true where exists ( - select 1 from controversy_dumps cd where c.controversies_id = cd.controversies_id and cd.state = 'completed' -); - -drop view controversies_with_dates; -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4554; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4554-4555.sql b/apps/postgresql-server/schema/migrations/mediawords-4554-4555.sql deleted file mode 100644 index 9dd3c6efb8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4554-4555.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4554 and 4555. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4554, and you would like to upgrade both the Media Cloud and the --- database to be at version 4555, import this SQL file: --- --- psql mediacloud < mediawords-4554-4555.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table cd.story_link_counts add facebook_share_count int null; - -update cd.story_link_counts slc set facebook_share_count = ss.facebook_share_count - from story_statistics ss where ss.stories_id = slc.stories_id; - -drop view controversies_with_dates; -alter table controversies drop column process_with_bitly; -create view controversies_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - controversies c - join controversy_dates cd on ( c.controversies_id = cd.controversies_id ) - where - cd.boundary; - - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4555; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4555-4556.sql b/apps/postgresql-server/schema/migrations/mediawords-4555-4556.sql deleted file mode 100644 index 39e9aef0fd..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4555-4556.sql +++ /dev/null @@ -1,65 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4555 and 4556. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4555, and you would like to upgrade both the Media Cloud and the --- database to be at version 4556, import this SQL file: --- --- psql mediacloud < mediawords-4555-4556.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Create index if it doesn't exist already --- --- Should be removed after migrating to PostgreSQL 9.5 because it supports --- CREATE INDEX IF NOT EXISTS natively. -CREATE OR REPLACE FUNCTION create_index_if_not_exists(schema_name TEXT, table_name TEXT, index_name TEXT, index_sql TEXT) -RETURNS void AS $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname = index_name - AND n.nspname = schema_name - ) THEN - EXECUTE 'CREATE INDEX ' || index_name || ' ON ' || schema_name || '.' || table_name || ' ' || index_sql; - END IF; -END -$$ -LANGUAGE plpgsql VOLATILE; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4556; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4556-4557.sql b/apps/postgresql-server/schema/migrations/mediawords-4556-4557.sql deleted file mode 100644 index 71a5d47d32..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4556-4557.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4556 and 4557. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4556, and you would like to upgrade both the Media Cloud and the --- database to be at version 4557, import this SQL file: --- --- psql mediacloud < medimawords-4556-4557.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Returns first 64 bits (16 characters) of MD5 hash --- --- Useful for reducing index sizes (e.g. in story_sentences.sentence) where --- 64 bits of entropy is enough. -CREATE OR REPLACE FUNCTION half_md5(string TEXT) RETURNS bytea AS $$ - SELECT SUBSTRING(digest(string, 'md5'::text), 0, 9); -$$ LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4557; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4557-4558.sql b/apps/postgresql-server/schema/migrations/mediawords-4557-4558.sql deleted file mode 100644 index cec5d7ed30..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4557-4558.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4557 and 4558. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4557, and you would like to upgrade both the Media Cloud and the --- database to be at version 4558, import this SQL file: --- --- psql mediacloud < mediawords-4557-4558.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Might already exist on production -SELECT create_index_if_not_exists( - 'public', - 'story_sentences', - 'story_sentences_sentence_half_md5', - '(half_md5(sentence))' -); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4558; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4558-4559.sql b/apps/postgresql-server/schema/migrations/mediawords-4558-4559.sql deleted file mode 100644 index e6fede9a43..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4558-4559.sql +++ /dev/null @@ -1,57 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4558 and 4559. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4558, and you would like to upgrade both the Media Cloud and the --- database to be at version 4559, import this SQL file: --- --- psql mediacloud < mediawords-4558-4559.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Stories from failed Bit.ly RabbitMQ queue --- (RabbitMQ failed reindexing a huge queue so we had to recover stories in --- that queue manually. Story IDs from this table are to be gradually moved to --- Bit.ly processing schedule.) --- -CREATE TABLE IF NOT EXISTS stories_from_failed_bitly_rabbitmq_queue ( - stories_id BIGINT NOT NULL REFERENCES stories (stories_id) -); -CREATE INDEX stories_from_failed_bitly_rabbitmq_queue_stories_id - ON stories_from_failed_bitly_rabbitmq_queue (stories_id); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4559; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4559-4560.sql b/apps/postgresql-server/schema/migrations/mediawords-4559-4560.sql deleted file mode 100644 index f379784825..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4559-4560.sql +++ /dev/null @@ -1,111 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4559 and 4560. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4559, and you would like to upgrade both the Media Cloud and the --- database to be at version 4560, import this SQL file: --- --- psql mediacloud < mediawords-4559-4560.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table cd.story_link_counts add media_inlink_count int; -alter table cd.medium_link_counts add media_inlink_count int; -alter table cd.medium_link_counts add sum_media_inlink_count int; - -update cd.story_link_counts set media_inlink_count = 0; -update cd.medium_link_counts set media_inlink_count = 0, sum_media_inlink_count = 0; - -create temporary table story_media_links as - SELECT - count(*) source_media_link_count, - s.media_id source_media_id, - sl.ref_stories_id ref_stories_id, - sl.controversy_dump_time_slices_id - FROM cd.story_links sl - join controversy_dump_time_slices cdts using ( controversy_dump_time_slices_id ) - join cd.stories s on ( s.stories_id = sl.source_stories_id and s.controversy_dumps_id = cdts.controversy_dumps_id ) - group by s.media_id, sl.ref_stories_id, sl.controversy_dump_time_slices_id; - -create temporary table story_media_link_counts as - select - count(*) media_inlink_count, - sml.ref_stories_id stories_id, - sml.controversy_dump_time_slices_id - from - story_media_links sml - group by sml.ref_stories_id, sml.controversy_dump_time_slices_id; - -update cd.story_link_counts slc set media_inlink_count = smlc.media_inlink_count - from story_media_link_counts smlc - where - slc.stories_id = smlc.stories_id and - slc.controversy_dump_time_slices_id = smlc.controversy_dump_time_slices_id; - -create temporary table medium_media_link_counts as - select - count(*) media_inlink_count, - ml.ref_media_id as media_id, - ml.controversy_dump_time_slices_id - from - cd.medium_links ml - group by ml.ref_media_id, ml.controversy_dump_time_slices_id; - -create temporary table medium_sum_media_link_counts as - select - sum( media_inlink_count ) sum_media_inlink_count, - s.media_id, - smlc.controversy_dump_time_slices_id - from - story_media_link_counts smlc - join controversy_dump_time_slices using ( controversy_dump_time_slices_id ) - join cd.stories s using ( stories_id, controversy_dumps_id ) - group by s.media_id, smlc.controversy_dump_time_slices_id; - - -update cd.medium_link_counts mlc set - media_inlink_count = mmlc.media_inlink_count, - sum_media_inlink_count = msmlc.sum_media_inlink_count - from - medium_media_link_counts mmlc, - medium_sum_media_link_counts msmlc - where - mmlc.media_id = mlc.media_id and - mmlc.controversy_dump_time_slices_id = mlc.controversy_dump_time_slices_id and - msmlc.media_id = mlc.media_id and - msmlc.controversy_dump_time_slices_id = mlc.controversy_dump_time_slices_id; - - -alter table cd.story_link_counts alter media_inlink_count set not null; -alter table cd.medium_link_counts alter media_inlink_count set not null; -alter table cd.medium_link_counts alter sum_media_inlink_count set not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4560; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4560-4561.sql b/apps/postgresql-server/schema/migrations/mediawords-4560-4561.sql deleted file mode 100644 index 2e8bf110b0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4560-4561.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4560 and 4561. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4560, and you would like to upgrade both the Media Cloud and the --- database to be at version 4561, import this SQL file: --- --- psql mediacloud < mediawords-4560-4561.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE stories_from_failed_bitly_rabbitmq_queue; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4561; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4561-4562.sql b/apps/postgresql-server/schema/migrations/mediawords-4561-4562.sql deleted file mode 100644 index a5aeda3716..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4561-4562.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4561 and 4562. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4561, and you would like to upgrade both the Media Cloud and the --- database to be at version 4562, import this SQL file: --- --- psql mediacloud < mediawords-4561-4562.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION create_initial_story_sentences_dup(); - --- Drop indexes created by create_initial_story_sentences_dup() or index_story_sentence_dups.pl -DROP INDEX IF EXISTS story_sentences_dup; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4562; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4562-4563.sql b/apps/postgresql-server/schema/migrations/mediawords-4562-4563.sql deleted file mode 100644 index 5c4372a82e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4562-4563.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4562 and 4563. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4562, and you would like to upgrade both the Media Cloud and the --- database to be at version 4563, import this SQL file: --- --- psql mediacloud < mediawords-4562-4563.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION IF EXISTS create_index_if_not_exists(TEXT, TEXT, TEXT, TEXT); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4563; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4563-4564.sql b/apps/postgresql-server/schema/migrations/mediawords-4563-4564.sql deleted file mode 100644 index 16d6de02c6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4563-4564.sql +++ /dev/null @@ -1,319 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4563 and 4564. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4563, and you would like to upgrade both the Media Cloud and the --- database to be at version 4564, import this SQL file: --- --- psql mediacloud < mediawords-4563-4564.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table controversies rename to topics; -alter table topics rename controversies_id to topics_id; -alter table topics rename controversy_tag_sets_id to topic_tag_sets_id; - -alter index controversies_name rename to topics_name; -alter index controversies_tag_set rename to topics_tag_set; -alter index controversies_media_type_tag_set rename to topics_media_type_tag_set; - -drop trigger controversy_tag_set on topics; -drop function insert_controversy_tag_set(); - -create function insert_topic_tag_set() returns trigger as $insert_topic_tag_set$ - begin - insert into tag_sets ( name, label, description ) - select 'topic_'||NEW.name, NEW.name||' topic', 'Tag set for stories within the '||NEW.name||' topic.'; - - select tag_sets_id into NEW.topic_tag_sets_id from tag_sets where name = 'topic_'||NEW.name; - - return NEW; - END; -$insert_topic_tag_set$ LANGUAGE plpgsql; - -create trigger topic_tag_set before insert on topics - for each row execute procedure insert_topic_tag_set(); - -alter table controversy_dates rename to topic_dates; - -alter table topic_dates rename controversy_dates_id to topic_dates_id; -alter table topic_dates rename controversies_id to topics_id; - -drop view controversies_with_dates; - -create or replace view topics_with_dates as - select c.*, - to_char( cd.start_date, 'YYYY-MM-DD' ) start_date, - to_char( cd.end_date, 'YYYY-MM-DD' ) end_date - from - topics c - join topic_dates cd on ( c.topics_id = cd.topics_id ) - where - cd.boundary; - -drop table controversy_dump_tags; - -alter table controversy_media_codes rename to topic_media_codes; -alter table topic_media_codes rename controversies_id to topics_id; - -alter table controversy_merged_stories_map rename to topic_merged_stories_map; - -alter index controversy_merged_stories_map_source rename to topic_merged_stories_map_source; -alter index controversy_merged_stories_map_story rename to topic_merged_stories_map_story; - -alter table controversy_stories rename to topic_stories; -alter table topic_stories rename controversy_stories_id to topic_stories_id; -alter table topic_stories rename controversies_id to topics_id; - -alter index controversy_stories_sc rename to topic_stories_sc; -alter index controversy_stories_controversy rename to topic_stories_topic; - -alter table controversy_dead_links rename to topic_dead_links; -alter table topic_dead_links rename controversy_dead_links_id to topic_dead_links_id; -alter table topic_dead_links rename controversies_id to topics_id; - -alter table controversy_links rename to topic_links; -alter table topic_links rename controversy_links_id to topic_links_id; -alter table topic_links rename controversies_id to topics_id; - -alter table topic_links rename constraint - controversy_links_controversy_story_stories_id to topic_links_topic_story_stories_id; - -alter index controversy_links_scr rename to topic_links_scr; -alter index controversy_links_controversy rename to topic_links_topic; -alter index controversy_links_ref_story rename to topic_links_ref_story; - -drop view controversy_links_cross_media; - -create or replace view topic_links_cross_media AS - SELECT s.stories_id, - sm.name AS media_name, - r.stories_id AS ref_stories_id, - rm.name AS ref_media_name, - cl.url AS url, - cs.topics_id, - cl.topic_links_id - FROM media sm, - media rm, - topic_links cl, - stories s, - stories r, - topic_stories cs - WHERE cl.ref_stories_id != cl.stories_id - AND s.stories_id = cl.stories_id - AND cl.ref_stories_id = r.stories_id - AND s.media_id != r.media_id - AND sm.media_id = s.media_id - AND rm.media_id = r.media_id - AND cs.stories_id = cl.ref_stories_id - AND cs.topics_id = cl.topics_id; - -alter table controversy_seed_urls rename to topic_seed_urls; - -alter table topic_seed_urls rename controversy_seed_urls_id to topic_seed_urls_id; -alter table topic_seed_urls rename controversies_id to topics_id; - -alter index controversy_seed_urls_controversy rename to topic_seed_urls_topic; -alter index controversy_seed_urls_url rename to topic_seed_urls_url; -alter index controversy_seed_urls_story rename to topic_seed_urls_story; - -alter table controversy_ignore_redirects rename to topic_ignore_redirects; - -alter table topic_ignore_redirects rename controversy_ignore_redirects_id to topic_ignore_redirects_id; - -alter index controversy_ignore_redirects_url rename to topic_ignore_redirects_url; - -alter table controversy_query_slices rename to foci; - -alter table foci rename controversy_query_slices_id to foci_id; -alter table foci rename controversies_id to topics_id; -alter table foci rename all_time_slices to all_timespans; - -alter table controversy_dumps rename to snapshots; - -alter table snapshots rename controversy_dumps_id to snapshots_id; -alter table snapshots rename controversies_id to topics_id; -alter table snapshots rename dump_date to snapshot_date; - -alter index controversy_dumps_controversy rename to snapshots_topic; - -alter table controversy_dump_time_slices rename to timespans; - -alter table timespans rename controversy_dump_time_slices_id to timespans_id; -alter table timespans rename controversy_dumps_id to snapshots_id; -alter table timespans rename controversy_query_slices_id to foci_id; - -alter index controversy_dump_time_slices_dump rename to timespans_dump; - -alter table cdts_files rename to timespan_files; - -alter table timespan_files rename cdts_files_id to timespan_files_id; -alter table timespan_files rename controversy_dump_time_slices_id to timespans_id; - -alter index cdts_files_cdts rename to timespan_files_timespan; - -alter table cd_files rename controversy_dumps_id to snapshots_id; - -drop function num_controversy_stories_without_bitly_statistics(param_topics_id INT); - -create or replace function num_topic_stories_without_bitly_statistics (param_topics_id INT) RETURNS INT AS -$$ -DECLARE - topic_exists BOOL; - num_stories_without_bitly_statistics INT; -BEGIN - - SELECT 1 INTO topic_exists - FROM topics - WHERE topics_id = param_topics_id; - IF NOT FOUND THEN - RAISE EXCEPTION 'topic % does not exist or is not set up for Bit.ly processing.', param_topics_id; - RETURN FALSE; - END IF; - - SELECT COUNT(stories_id) INTO num_stories_without_bitly_statistics - FROM topic_stories - WHERE topics_id = param_topics_id - AND stories_id NOT IN ( - SELECT stories_id - FROM bitly_clicks_total - ) - GROUP BY topics_id; - IF NOT FOUND THEN - num_stories_without_bitly_statistics := 0; - END IF; - - RETURN num_stories_without_bitly_statistics; -END; -$$ -LANGUAGE plpgsql; - -alter table cd.controversy_stories rename to topic_stories; -alter table cd.topic_stories rename controversy_dumps_id to snapshots_id; -alter table cd.topic_stories rename controversy_stories_id to topic_stories_id; -alter table cd.topic_stories rename controversies_id to topics_id; - -alter index cd.controversy_stories_id rename to topic_stories_id; - -alter table cd.controversy_links_cross_media rename to topic_links_cross_media; -alter table cd.topic_links_cross_media rename controversy_dumps_id to snapshots_id; -alter table cd.topic_links_cross_media rename controversy_links_id to topic_links_id; -alter table cd.topic_links_cross_media rename controversies_id to topics_id; - -alter index cd.controversy_links_story rename to topic_links_story; -alter index cd.controversy_links_ref rename to topic_links_ref; - -alter table cd.controversy_media_codes rename to topic_media_codes; -alter table cd.topic_media_codes rename controversy_dumps_id to snapshots_id; -alter table cd.topic_media_codes rename controversies_id to topics_id; - -alter index cd.controversy_media_codes_medium rename to topic_media_codes_medium; - -alter table cd.media rename controversy_dumps_id to snapshots_id; -alter table cd.media_tags_map rename controversy_dumps_id to snapshots_id; -alter table cd.stories rename controversy_dumps_id to snapshots_id; -alter table cd.stories_tags_map rename controversy_dumps_id to snapshots_id; -alter table cd.tags rename controversy_dumps_id to snapshots_id; -alter table cd.tag_sets rename controversy_dumps_id to snapshots_id; -alter table cd.daily_date_counts rename controversy_dumps_id to snapshots_id; -alter table cd.weekly_date_counts rename controversy_dumps_id to snapshots_id; - -alter table cd.story_links rename controversy_dump_time_slices_id to timespans_id; -alter table cd.story_link_counts rename controversy_dump_time_slices_id to timespans_id; -alter table cd.medium_link_counts rename controversy_dump_time_slices_id to timespans_id; -alter table cd.medium_links rename controversy_dump_time_slices_id to timespans_id; - -alter table cd.live_stories rename controversies_id to topics_id; -alter table cd.live_stories rename controversy_stories_id to topic_stories_id; - -alter index cd.live_story_controversy rename to live_story_topic; - -drop table cd.word_counts; - -alter trigger controversy_stories_insert_live_story on topic_stories rename to topic_stories_insert_live_story; - -drop table if exists controversy_query_story_searches_imported_stories_map; - -update auth_roles set role = 'tm', description = 'Topic mapper; includes media and story editing' where role = 'cm'; -update auth_roles set role = 'tm-readonly', description = 'Topic mapper; excludes media and story editing' where role = 'cm-readonly'; - -alter type cd_period_type rename to snap_period_type; - -alter table cd_files rename to snap_files; - -alter table snap_files rename cd_files_id to snap_files_id; - -alter schema cd rename to snap; - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into snap.live_stories - ( topics_id, topic_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language, - db_row_last_updated ) - select NEW.topics_id, NEW.topic_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language, - s.db_row_last_updated - from topic_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.topics_id = NEW.topics_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update snap.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - -update feeds set name = 'Spider Feed' where name = 'Controversy Spider Feed'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4564; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4564-4565.sql b/apps/postgresql-server/schema/migrations/mediawords-4564-4565.sql deleted file mode 100644 index 89e11b4080..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4564-4565.sql +++ /dev/null @@ -1,50 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4564 and 4565. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4564, and you would like to upgrade both the Media Cloud and the --- database to be at version 4565, import this SQL file: --- --- psql mediacloud < mediawords-4564-4565.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table api_links ( - api_links_id bigserial primary key, - path text not null, - params_json text not null, - next_link_id bigint null references api_links on delete set null deferrable, - previous_link_id bigint null references api_links on delete set null deferrable -); - -create unique index api_links_params on api_links ( path, md5( params_json ) ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4565; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4565-4566.sql b/apps/postgresql-server/schema/migrations/mediawords-4565-4566.sql deleted file mode 100644 index 0469e948da..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4565-4566.sql +++ /dev/null @@ -1,281 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4565 and 4566. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4565, and you would like to upgrade both the Media Cloud and the --- database to be at version 4566, import this SQL file: --- --- psql mediacloud < mediawords-4565-4566.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION bitly_partition_chunk_size() -RETURNS integer AS $$ -BEGIN - RETURN 1000000; -END; $$ -LANGUAGE plpgsql IMMUTABLE; - -create or replace function bitly_get_partition_name( stories_id int, table_name text ) -returns text as $$ -declare - to_char_format CONSTANT TEXT := '000000'; -- Up to 1m of chunks, suffixed as "_000001", ..., "_999999" - - stories_id_chunk_number int; - - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_daily_000001") - -begin - select stories_id / bitly_partition_chunk_size() INTO stories_id_chunk_number; - raise notice 'stories_id: %', stories_id; - select table_name || '_' || trim(leading ' ' from to_char(stories_id_chunk_number, to_char_format)) - into target_table_name; - - raise notice 'table name: %', target_table_name; - - return target_table_name; -END; -$$ -LANGUAGE plpgsql; - --- Automatic Bit.ly total click count partitioning to stories_id chunks of 1m rows -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - select bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_total' ) into target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / bitly_partition_chunk_size() ) * bitly_partition_chunk_size() INTO stories_id_start; - SELECT ((NEW.stories_id / bitly_partition_chunk_size()) + 1) * bitly_partition_chunk_size() INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_unique - UNIQUE (stories_id) - - ) INHERITS (bitly_clicks_total); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_total' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_total ( - param_stories_id INT, - param_click_count INT -) RETURNS VOID AS -$$ -DECLARE - partition_name text; -BEGIN - - select bitly_get_partition_name( param_stories_id, 'bitly_clicks_total' ) into partition_name; - - LOOP - EXECUTE ' - UPDATE ' || partition_name || ' - SET click_count = param_click_count - WHERE stories_id = param_stories_id - AND day = param_day'; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - EXECUTE ' - INSERT INTO ' || partition_name || ' (stories_id, click_count) - VALUES (param_stories_id, param_click_count)'; - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION bitly_clicks_daily_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_daily_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - select bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_daily' ) into target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / bitly_partition_chunk_size() ) * bitly_partition_chunk_size() INTO stories_id_start; - SELECT ((NEW.stories_id / bitly_partition_chunk_size() ) + 1) * bitly_partition_chunk_size() INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_day_unique - UNIQUE (stories_id, day) - - ) INHERITS (bitly_clicks_daily); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_daily' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_daily ( - param_stories_id INT, - param_day DATE, - param_click_count INT -) RETURNS VOID AS -$$ -DECLARE - partition_name text; -BEGIN - - select bitly_get_partition_name( param_stories_id, 'bitly_clicks_daily' ) into partition_name; - - LOOP - EXECUTE ' - UPDATE ' || partition_name || ' - SET click_count = param_click_count - WHERE stories_id = param_stories_id - AND day = param_day'; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - EXECUTE ' - INSERT INTO ' || partition_name || ' (stories_id, day, click_count) - VALUES (param_stories_id, param_day, param_click_count)'; - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4566; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4566-4567.sql b/apps/postgresql-server/schema/migrations/mediawords-4566-4567.sql deleted file mode 100644 index 1385f30b00..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4566-4567.sql +++ /dev/null @@ -1,282 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4566 and 4567. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4566, and you would like to upgrade both the Media Cloud and the --- database to be at version 4567, import this SQL file: --- --- psql mediacloud < mediawords-4566-4567.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE OR REPLACE FUNCTION bitly_partition_chunk_size() -RETURNS integer AS $$ -BEGIN - RETURN 1000000; -END; $$ -LANGUAGE plpgsql IMMUTABLE; - -create or replace function bitly_get_partition_name( stories_id int, table_name text ) -returns text as $$ -declare - to_char_format CONSTANT TEXT := '000000'; -- Up to 1m of chunks, suffixed as "_000001", ..., "_999999" - - stories_id_chunk_number int; - - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_daily_000001") - -begin - select stories_id / bitly_partition_chunk_size() INTO stories_id_chunk_number; - - select table_name || '_' || trim(leading ' ' from to_char(stories_id_chunk_number, to_char_format)) - into target_table_name; - - return target_table_name; -END; -$$ -LANGUAGE plpgsql; - --- Automatic Bit.ly total click count partitioning to stories_id chunks of 1m rows -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - select bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_total' ) into target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / bitly_partition_chunk_size() ) * bitly_partition_chunk_size() INTO stories_id_start; - SELECT ((NEW.stories_id / bitly_partition_chunk_size()) + 1) * bitly_partition_chunk_size() INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_unique - UNIQUE (stories_id) - - ) INHERITS (bitly_clicks_total); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_total' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_total ( - param_stories_id INT, - param_click_count INT -) RETURNS VOID AS -$$ -DECLARE - partition_name text; - update_count int; -BEGIN - - select bitly_get_partition_name( param_stories_id, 'bitly_clicks_total' ) into partition_name; - - LOOP - EXECUTE ' - UPDATE ' || partition_name || ' - SET click_count = ' || param_click_count || ' - WHERE stories_id = ' || param_stories_id; - get diagnostics update_count = ROW_COUNT; - IF update_count > 0 THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - EXECUTE ' - INSERT INTO ' || partition_name || ' (stories_id, click_count) - VALUES (' || param_stories_id || ', ' || param_click_count || ')'; - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION bitly_clicks_daily_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_daily_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - select bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_daily' ) into target_table_name; - - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = current_schema() - AND table_name = target_table_name - ) THEN - - SELECT (NEW.stories_id / bitly_partition_chunk_size() ) * bitly_partition_chunk_size() INTO stories_id_start; - SELECT ((NEW.stories_id / bitly_partition_chunk_size() ) + 1) * bitly_partition_chunk_size() INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_day_unique - UNIQUE (stories_id, day) - - ) INHERITS (bitly_clicks_daily); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_daily' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.*; - ' USING NEW; - - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - --- Helper to INSERT / UPDATE story's Bit.ly statistics -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_daily ( - param_stories_id INT, - param_day DATE, - param_click_count INT -) RETURNS VOID AS -$$ -DECLARE - partition_name text; - update_count int; -BEGIN - - select bitly_get_partition_name( param_stories_id, 'bitly_clicks_daily' ) into partition_name; - - LOOP - EXECUTE ' - UPDATE ' || partition_name || ' - SET click_count = ' || param_click_count || ' - WHERE stories_id = ' || param_stories_id || ' - AND day = ''' || param_day || ''''; - get diagnostics update_count = ROW_COUNT; - IF update_count > 0 THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - EXECUTE ' - INSERT INTO ' || partition_name || ' (stories_id, day, click_count) - VALUES ( ' || param_stories_id || ', ''' || param_day || ''', ' || param_click_count || ')'; - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4567; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4567-4568.sql b/apps/postgresql-server/schema/migrations/mediawords-4567-4568.sql deleted file mode 100644 index d5c848ea8b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4567-4568.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4567 and 4568. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4567, and you would like to upgrade both the Media Cloud and the --- database to be at version 4568, import this SQL file: --- --- psql mediacloud < mediawords-4567-4568.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE bitly_clicks_daily; - -DROP FUNCTION upsert_bitly_clicks_daily(param_stories_id INT, param_day DATE, param_click_count INT); - -DROP FUNCTION bitly_clicks_daily_partition_by_stories_id_insert_trigger(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4568; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4568-4569.sql b/apps/postgresql-server/schema/migrations/mediawords-4568-4569.sql deleted file mode 100644 index e365dd3d49..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4568-4569.sql +++ /dev/null @@ -1,141 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4568 and 4569. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4568, and you would like to upgrade both the Media Cloud and the --- database to be at version 4569, import this SQL file: --- --- psql mediacloud < mediawords-4568-4569.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION table_exists(target_table_name VARCHAR) RETURNS BOOLEAN AS $$ -BEGIN - RETURN EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = CURRENT_SCHEMA() - AND table_name = target_table_name - ); -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") -BEGIN - SELECT bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_total' ) INTO target_table_name; - EXECUTE 'INSERT INTO ' || target_table_name || ' SELECT $1.*;' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION bitly_clicks_total_create_partitions() RETURNS VOID AS -$$ -DECLARE - chunk_size INT; - max_stories_id BIGINT; - partition_stories_id BIGINT; - - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT bitly_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT bitly_get_partition_name( partition_stories_id, 'bitly_clicks_total' ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_unique - UNIQUE (stories_id) - - ) INHERITS (bitly_clicks_total); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_total' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - --- Create initial partitions for empty database -SELECT bitly_clicks_total_create_partitions(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4569; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4569-4570.sql b/apps/postgresql-server/schema/migrations/mediawords-4569-4570.sql deleted file mode 100644 index d1ed47c82a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4569-4570.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4569 and 4570. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4569, and you would like to upgrade both the Media Cloud and the --- database to be at version 4570, import this SQL file: --- --- psql mediacloud < mediawords-4569-4570.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION create_missing_partitions() RETURNS VOID AS -$$ -BEGIN - -- "bitly_clicks_total" table - RAISE NOTICE 'Creating partitions in "bitly_clicks_total" table...'; - PERFORM bitly_clicks_total_create_partitions(); -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4570; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4570-4571.sql b/apps/postgresql-server/schema/migrations/mediawords-4570-4571.sql deleted file mode 100644 index 94038cd9a0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4570-4571.sql +++ /dev/null @@ -1,206 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4570 and 4571. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4570, and you would like to upgrade both the Media Cloud and the --- database to be at version 4571, import this SQL file: --- --- psql mediacloud < mediawords-4570-4571.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION bitly_partition_chunk_size() RETURNS integer AS $$ -BEGIN - RETURN 100 * 1000 * 1000; -- 100m rows in each partition -END; $$ -LANGUAGE plpgsql IMMUTABLE; - - -CREATE OR REPLACE FUNCTION bitly_get_partition_name(stories_id INT, table_name TEXT) RETURNS TEXT AS $$ -DECLARE - to_char_format CONSTANT TEXT := '00'; -- Up to 100 partitions, suffixed as "_00", "_01" ..., "_99" - -- (having more of them is not feasible) - stories_id_chunk_number INT; - - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") -BEGIN - SELECT stories_id / bitly_partition_chunk_size() INTO stories_id_chunk_number; - - SELECT table_name || '_' || trim(leading ' ' FROM to_char(stories_id_chunk_number, to_char_format)) - INTO target_table_name; - - RETURN target_table_name; -END; -$$ -LANGUAGE plpgsql; - - --- Migrate old 1m partitions to the new 100m ones -DO $$ -DECLARE - final_table_name CONSTANT TEXT := 'bitly_clicks_total'; - temp_table_name CONSTANT TEXT := 'temp_bitly_clicks_total'; - - chunk_size INT; - max_stories_id BIGINT; - - target_temp_table_name TEXT; -- partition table name (e.g. "temp_bitly_clicks_total_000001") - target_final_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT bitly_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - RAISE NOTICE 'Will create partitions for storing up to % stories.', max_stories_id; - - RAISE NOTICE 'Creating temporary master table...'; - EXECUTE 'DROP TABLE IF EXISTS ' || temp_table_name || ' CASCADE;'; - EXECUTE ' - CREATE TABLE ' || temp_table_name || ' ( - bitly_clicks_id BIGSERIAL NOT NULL, - stories_id INT NOT NULL, - click_count INT NOT NULL - ); - '; - - RAISE NOTICE 'Creating temporary partitions...'; - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - - SELECT bitly_get_partition_name( partition_stories_id, temp_table_name ) INTO target_temp_table_name; - - RAISE NOTICE 'Creating partition "%" for story ID %', target_temp_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE 'DROP TABLE IF EXISTS ' || target_temp_table_name || ';'; - EXECUTE ' - CREATE TABLE ' || target_temp_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_temp_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_temp_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || ''') - - ) INHERITS (' || temp_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = final_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_temp_table_name || ' OWNER TO ' || target_table_owner || ';'; - - RAISE NOTICE 'Copying stories [%; %) from old table...', stories_id_start, stories_id_end; - EXECUTE ' - INSERT INTO ' || target_temp_table_name || ' (stories_id, click_count) - SELECT stories_id, click_count FROM ' || final_table_name || ' - WHERE stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''; - '; - - RAISE NOTICE 'Creating unique index on "stories_id"...'; - EXECUTE ' - CREATE UNIQUE INDEX ' || target_temp_table_name || '_stories_id_unique - ON ' || target_temp_table_name || ' (stories_id); - '; - - RAISE NOTICE 'Creating foreign key to "stories"...'; - EXECUTE ' - ALTER TABLE ' || target_temp_table_name || ' - ADD CONSTRAINT ' || target_temp_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id); - '; - - END LOOP; - - RAISE NOTICE 'Dropping main table...'; - EXECUTE 'DROP TABLE ' || final_table_name || ' CASCADE'; - - RAISE NOTICE 'Renaming temporary table to main table...'; - EXECUTE 'ALTER TABLE ' || temp_table_name || ' RENAME TO ' || final_table_name || ';'; - EXECUTE ' - ALTER SEQUENCE ' || temp_table_name || '_bitly_clicks_id_seq - RENAME TO ' || final_table_name || '_bitly_clicks_id_seq; - '; - - RAISE NOTICE 'Renaming partitions...'; - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - - SELECT bitly_get_partition_name( partition_stories_id, temp_table_name ) INTO target_temp_table_name; - SELECT bitly_get_partition_name( partition_stories_id, final_table_name ) INTO target_final_table_name; - - RAISE NOTICE 'Renaming partition "%" to "%"...', target_temp_table_name, target_final_table_name; - EXECUTE ' - ALTER TABLE ' || target_temp_table_name || ' - RENAME CONSTRAINT ' || target_temp_table_name || '_stories_id - TO ' || target_final_table_name || '_stories_id; - '; - EXECUTE ' - ALTER TABLE ' || target_temp_table_name || ' - RENAME CONSTRAINT ' || target_temp_table_name || '_stories_id_fkey - TO ' || target_final_table_name || '_stories_id_fkey; - '; - EXECUTE ' - ALTER INDEX ' || target_temp_table_name || '_pkey - RENAME TO ' || target_final_table_name || '_pkey; - '; - EXECUTE ' - ALTER INDEX ' || target_temp_table_name || '_stories_id_unique - RENAME TO ' || target_final_table_name || '_stories_id_unique; - '; - EXECUTE 'ALTER TABLE ' || target_temp_table_name || ' RENAME TO ' || target_final_table_name || ';'; - - END LOOP; - -END$$; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4571; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4571-4572.sql b/apps/postgresql-server/schema/migrations/mediawords-4571-4572.sql deleted file mode 100644 index 30e3f8dfd3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4571-4572.sql +++ /dev/null @@ -1,70 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4571 and 4572. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4571, and you would like to upgrade both the Media Cloud and the --- database to be at version 4572, import this SQL file: --- --- psql mediacloud < mediawords-4571-4572.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION upsert_bitly_clicks_total(param_stories_id INT, param_click_count INT) RETURNS VOID AS -$$ -BEGIN - LOOP - -- Try UPDATing - UPDATE bitly_clicks_total - SET click_count = param_click_count - WHERE stories_id = param_stories_id; - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO bitly_clicks_total (stories_id, click_count) - VALUES (param_stories_id, param_click_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4572; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4572-4573.sql b/apps/postgresql-server/schema/migrations/mediawords-4572-4573.sql deleted file mode 100644 index 7b0f2b8525..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4572-4573.sql +++ /dev/null @@ -1,119 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4572 and 4573. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4572, and you would like to upgrade both the Media Cloud and the --- database to be at version 4573, import this SQL file: --- --- psql mediacloud < mediawords-4572-4573.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4573; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - -create type focal_technique_type as enum ( 'Boolean Query' ); - -create table focal_set_definitions ( - focal_set_definitions_id serial primary key, - topics_id int not null references topics on delete cascade, - name text not null, - description text null, - focal_technique focal_technique_type not null -); - -create unique index focal_set_definitions_topic_name on focal_set_definitions ( topics_id, name ); - -create table focus_definitions ( - focus_definitions_id serial primary key, - focal_set_definitions_id int not null references focal_set_definitions on delete cascade, - name text not null, - description text null, - arguments json not null -); - -create unique index focus_definition_set_name on focus_definitions ( focal_set_definitions_id, name ); - -create table focal_sets ( - focal_sets_id serial primary key, - snapshots_id int not null references snapshots, - name text not null, - description text null, - focal_technique focal_technique_type not null -); - -create unique index focal_set_snapshot on focal_sets ( snapshots_id, name ); - -alter table foci alter name type text; -alter table foci drop column all_timespans; -alter table foci add focal_sets_id int references focal_sets on delete cascade; -alter table foci add description text null; - -alter table foci rename query to arguments; -update foci set arguments = '{ "query": ' || to_json( arguments ) || ' }'; -alter table foci alter arguments type json using arguments::json; - -create unique index foci_set_name on foci ( focal_sets_id, name ); - --- remove on delete set null from timespans.foci_id -alter table timespans rename foci_id to foci_id_tmp; -alter table timespans add foci_id int null references foci; -update timespans set foci_id = foci_id_tmp; -alter table timespans drop column foci_id_tmp; - -insert into focal_set_definitions ( topics_id, name, focal_technique ) - select - t.topics_id, 'Queries', 'Boolean Query' - from topics t - where - exists ( select 1 from foci f where f.topics_id = t.topics_id ); - -insert into focus_definitions ( focal_set_definitions_id, name, arguments ) - select - fsd.focal_set_definitions_id, f.name, f.arguments - from focal_set_definitions fsd - join foci f using ( topics_id ); - -insert into focal_sets ( snapshots_id, name, focal_technique ) - select - q.snapshots_id, fsd.name, fsd.focal_technique - from focal_set_definitions fsd - join ( select distinct topics_id, snapshots_id from timespans t join foci f using ( foci_id ) ) q using ( topics_id ); - -update foci f set focal_sets_id = fs.focal_sets_id - from focal_sets fs join snapshots s using ( snapshots_id ) - where f.topics_id = s.topics_id; - -alter table foci alter focal_sets_id set not null; -alter table foci drop column topics_id; - -delete from timespans where is_shell; - -alter table timespans drop column is_shell; diff --git a/apps/postgresql-server/schema/migrations/mediawords-4573-4574.sql b/apps/postgresql-server/schema/migrations/mediawords-4573-4574.sql deleted file mode 100644 index c2ab7aa0c3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4573-4574.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4573 and 4574. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4573, and you would like to upgrade both the Media Cloud and the --- database to be at version 4574, import this SQL file: --- --- psql mediacloud < mediawords-4573-4574.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create view controversies as select topics_id controversies_id, * from topics; -create view controversy_dumps as - select snapshots_id controversy_dumps_id, topics_id controversies_id, snapshot_date dump_date, * from snapshots; -create view controversy_dump_time_slices as - select timespans_id controversy_dump_time_slices_id, snapshots_id controversy_dumps_id, foci_id controversy_query_slices_id, * - from timespans; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4574; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4574-4575.sql b/apps/postgresql-server/schema/migrations/mediawords-4574-4575.sql deleted file mode 100644 index 20a46db9bc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4574-4575.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4574 and 4575. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4574, and you would like to upgrade both the Media Cloud and the --- database to be at version 4575, import this SQL file: --- --- psql mediacloud < mediawords-4574-4575.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Recreate the trigger in case it doesn't exist -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -BEGIN - SELECT bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_total' ) INTO target_table_name; - EXECUTE 'INSERT INTO ' || target_table_name || ' SELECT $1.*;' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -DROP TRIGGER IF EXISTS bitly_clicks_total_partition_by_stories_id_insert_trigger ON bitly_clicks_total; -CREATE TRIGGER bitly_clicks_total_partition_by_stories_id_insert_trigger - BEFORE INSERT ON bitly_clicks_total - FOR EACH ROW EXECUTE PROCEDURE bitly_clicks_total_partition_by_stories_id_insert_trigger(); - - --- Move data that errorneously got into the master table to partitions -CREATE TEMPORARY TABLE temp_bitly_clicks_total_master_table ( - stories_id BIGINT NOT NULL, - click_count INT NOT NULL -); -INSERT INTO temp_bitly_clicks_total_master_table (stories_id, click_count) - SELECT stories_id, click_count - FROM ONLY bitly_clicks_total; -- ONLY the master table, not partitions -TRUNCATE ONLY bitly_clicks_total; -- ONLY the master table, not partitions --- VACUUM FULL ANALYZE bitly_clicks_total; -- Free up used space - --- In case some click counts are to be UPDATEd -DELETE FROM bitly_clicks_total WHERE stories_id IN ( - SELECT stories_id - FROM temp_bitly_clicks_total_master_table -); - -INSERT INTO bitly_clicks_total (stories_id, click_count) - SELECT DISTINCT stories_id, click_count - FROM temp_bitly_clicks_total_master_table; - -DROP TABLE temp_bitly_clicks_total_master_table; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4575; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4575-4576.sql b/apps/postgresql-server/schema/migrations/mediawords-4575-4576.sql deleted file mode 100644 index c410461840..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4575-4576.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4575 and 4576. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4575, and you would like to upgrade both the Media Cloud and the --- database to be at version 4576, import this SQL file: --- --- psql mediacloud < mediawords-4575-4576.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4576; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE INDEX tags_label ON tags (label); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4576-4577.sql b/apps/postgresql-server/schema/migrations/mediawords-4576-4577.sql deleted file mode 100644 index 0ba3b61d37..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4576-4577.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4576 and 4577. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4576, and you would like to upgrade both the Media Cloud and the --- database to be at version 4577, import this SQL file: --- --- psql mediacloud < mediawords-4576-4577.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- cached extractor results for extraction jobs with use_cache set to true -create table cached_extractor_results( - cached_extractor_results_id bigserial primary key, - extracted_html text, - extracted_text text, - downloads_id bigint -); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4577; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4577-4578.sql b/apps/postgresql-server/schema/migrations/mediawords-4577-4578.sql deleted file mode 100644 index ec810a32a9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4577-4578.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4577 and 4578. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4577, and you would like to upgrade both the Media Cloud and the --- database to be at version 4578, import this SQL file: --- --- psql mediacloud < mediawords-4577-4578.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- it's better to have a few duplicates than deal with locking issues, so we don't try to make this unique -create index cached_extractor_results_downloads_id on cached_extractor_results( downloads_id ); - -alter table cached_extractor_results alter downloads_id set not null; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4578; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4578-4579.sql b/apps/postgresql-server/schema/migrations/mediawords-4578-4579.sql deleted file mode 100644 index 05110094dd..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4578-4579.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4578 and 4579. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4578, and you would like to upgrade both the Media Cloud and the --- database to be at version 4579, import this SQL file: --- --- psql mediacloud < mediawords-4578-4579.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- keep track of performance of the topic spider -create table topic_spider_metrics ( - topic_spider_metrics_id serial primary key, - topics_id int references topics on delete cascade, - iteration int not null, - links_processed int not null, - elapsed_time int not null, - processed_date timestamp not null default now() -); - -create index topic_spider_metrics_topic on topic_spider_metrics( topics_id ); -create index topic_spider_metrics_dat on topic_spider_metrics( processed_date ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4579; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4579-4580.sql b/apps/postgresql-server/schema/migrations/mediawords-4579-4580.sql deleted file mode 100644 index cd81e40e3e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4579-4580.sql +++ /dev/null @@ -1,180 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4579 and 4580. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4579, and you would like to upgrade both the Media Cloud and the --- database to be at version 4580, import this SQL file: --- --- psql mediacloud < mediawords-4579-4580.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION bitly_clicks_total_create_partitions() RETURNS VOID AS -$$ -DECLARE - chunk_size INT; - max_stories_id BIGINT; - partition_stories_id BIGINT; - - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT bitly_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT bitly_get_partition_name( partition_stories_id, 'bitly_clicks_total' ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - -- Primary key - CONSTRAINT ' || target_table_name || '_pkey - PRIMARY KEY (bitly_clicks_id), - - -- Partition by stories_id - CONSTRAINT ' || target_table_name || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || target_table_name || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || target_table_name || '_stories_id_unique - UNIQUE (stories_id) - - ) INHERITS (bitly_clicks_total); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'bitly_clicks_total' - AND t.table_schema = CURRENT_SCHEMA() - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION find_corrupted_sequences() RETURNS TABLE(tablename VARCHAR, maxid BIGINT, sequenceval BIGINT) -AS $BODY$ -DECLARE - r RECORD; -BEGIN - - SET client_min_messages TO WARNING; - DROP TABLE IF EXISTS temp_corrupted_sequences; - CREATE TEMPORARY TABLE temp_corrupted_sequences ( - tablename VARCHAR NOT NULL UNIQUE, - maxid BIGINT, - sequenceval BIGINT - ) ON COMMIT DROP; - SET client_min_messages TO NOTICE; - - FOR r IN ( - - -- Get all tables, their primary keys and serial sequence names - SELECT t.relname AS tablename, - primarykey AS idcolumn, - pg_get_serial_sequence(t.relname, primarykey) AS serialsequence - FROM pg_constraint AS c - JOIN pg_class AS t ON c.conrelid = t.oid - JOIN pg_namespace nsp ON nsp.oid = t.relnamespace - JOIN ( - SELECT a.attname AS primarykey, - i.indrelid - FROM pg_index AS i - JOIN pg_attribute AS a - ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) - WHERE i.indisprimary - ) AS pkey ON pkey.indrelid = t.relname::regclass - WHERE conname LIKE '%_pkey' - AND nsp.nspname = CURRENT_SCHEMA() - AND t.relname NOT IN ( - 'story_similarities_100_short', - 'url_discovery_counts' - ) - ORDER BY t.relname - - ) - LOOP - - -- Filter out the tables that have their max ID bigger than the last - -- sequence value - EXECUTE ' - INSERT INTO temp_corrupted_sequences - SELECT tablename, - maxid, - sequenceval - FROM ( - SELECT ''' || r.tablename || ''' AS tablename, - MAX(' || r.idcolumn || ') AS maxid, - ( SELECT last_value FROM ' || r.serialsequence || ') AS sequenceval - FROM ' || r.tablename || ' - ) AS id_and_sequence - WHERE maxid > sequenceval - '; - - END LOOP; - - RETURN QUERY SELECT * FROM temp_corrupted_sequences ORDER BY tablename; - -END -$BODY$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4580; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4580-4581.sql b/apps/postgresql-server/schema/migrations/mediawords-4580-4581.sql deleted file mode 100644 index 93e5e39ae5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4580-4581.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4580 and 4581. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4580, and you would like to upgrade both the Media Cloud and the --- database to be at version 4581, import this SQL file: --- --- psql mediacloud < mediawords-4580-4581.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table media add last_solr_import_date timestamp with time zone not null default now(); - -update media set last_solr_import_date = dv.value::timestamp from database_variables dv where dv.name = 'last_media_solr_import'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4581; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4581-4582.sql b/apps/postgresql-server/schema/migrations/mediawords-4581-4582.sql deleted file mode 100644 index 5cb040008d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4581-4582.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4581 and 4582. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4581, and you would like to upgrade both the Media Cloud and the --- database to be at version 4582, import this SQL file: --- --- psql mediacloud < mediawords-4581-4582.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE auth_user_requests; -DROP FUNCTION auth_user_requests_update_daily_counts(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4582; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4582-4583.sql b/apps/postgresql-server/schema/migrations/mediawords-4582-4583.sql deleted file mode 100644 index 3180981725..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4582-4583.sql +++ /dev/null @@ -1,93 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4582 and 4583. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4582, and you would like to upgrade both the Media Cloud and the --- database to be at version 4583, import this SQL file: --- --- psql mediacloud < mediawords-4582-4583.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Remove duplicates that could have been created due to race conditions and whatnot -DELETE FROM auth_user_request_daily_counts -WHERE auth_user_request_daily_counts_id IN ( - SELECT auth_user_request_daily_counts_id - FROM ( - SELECT auth_user_request_daily_counts_id, - ROW_NUMBER() OVER (partition BY email, day ORDER BY auth_user_request_daily_counts_id) AS row_number - FROM auth_user_request_daily_counts) AS auth_user_request_daily_counts - WHERE auth_user_request_daily_counts.row_number > 1); - - -DROP INDEX IF EXISTS auth_user_request_daily_counts_email; -DROP INDEX IF EXISTS auth_user_request_daily_counts_day; - -CREATE OR REPLACE FUNCTION upsert_auth_user_request_daily_counts(param_email TEXT, param_requested_items_count INT) RETURNS VOID AS -$$ -DECLARE - request_date DATE; -BEGIN - request_date := DATE_TRUNC('day', LOCALTIMESTAMP)::DATE; - - LOOP - -- Try UPDATing - UPDATE auth_user_request_daily_counts - SET requests_count = requests_count + 1, - requested_items_count = requested_items_count + param_requested_items_count - WHERE email = param_email - AND day = request_date; - - IF FOUND THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - INSERT INTO auth_user_request_daily_counts (email, day, requests_count, requested_items_count) - VALUES (param_email, request_date, 1, param_requested_items_count); - RETURN; - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4583; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE UNIQUE INDEX auth_user_request_daily_counts_email_day ON auth_user_request_daily_counts (email, day); - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4583-4584.sql b/apps/postgresql-server/schema/migrations/mediawords-4583-4584.sql deleted file mode 100644 index 32b954bb29..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4583-4584.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4583 and 4584. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4583, and you would like to upgrade both the Media Cloud and the --- database to be at version 4584, import this SQL file: --- --- psql mediacloud < mediawords-4583-4584.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Rename "cd" to "snap" which somehow didn't happen in 4563-4564 migration -DO $$ -BEGIN - IF EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'cd') THEN - ALTER SCHEMA cd RENAME TO snap; - END IF; -END$$; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4584; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4584-4585.sql b/apps/postgresql-server/schema/migrations/mediawords-4584-4585.sql deleted file mode 100644 index 5d258a84cf..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4584-4585.sql +++ /dev/null @@ -1,98 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4584 and 4585. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4584, and you would like to upgrade both the Media Cloud and the --- database to be at version 4585, import this SQL file: --- --- psql mediacloud < mediawords-4584-4585.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP VIEW topics_with_dates; -CREATE VIEW topics_with_dates AS - select c.*, - to_char( td.start_date, 'YYYY-MM-DD' ) start_date, - to_char( td.end_date, 'YYYY-MM-DD' ) end_date - from - topics c - join topic_dates td on ( c.topics_id = td.topics_id ) - where - td.boundary; - - -CREATE OR REPLACE FUNCTION insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into snap.live_stories - ( topics_id, topic_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language, - db_row_last_updated ) - select NEW.topics_id, NEW.topic_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language, - s.db_row_last_updated - from topic_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.topics_id = NEW.topics_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION update_live_story() returns trigger as $update_live_story$ - begin - - update snap.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language, - db_row_last_updated = NEW.db_row_last_updated - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4585; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4585-4586.sql b/apps/postgresql-server/schema/migrations/mediawords-4585-4586.sql deleted file mode 100644 index efae53615c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4585-4586.sql +++ /dev/null @@ -1,69 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4585 and 4586. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4585, and you would like to upgrade both the Media Cloud and the --- database to be at version 4586, import this SQL file: --- --- psql mediacloud < mediawords-4585-4586.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP VIEW media_with_media_types; -- will recreate right afterwards - -ALTER TABLE media - DROP COLUMN use_pager, - DROP COLUMN unpaged_stories; - -CREATE VIEW media_with_media_types AS - SELECT m.*, mtm.tags_id media_type_tags_id, t.label media_type - FROM - media m - LEFT JOIN ( - tags t - JOIN tag_sets ts ON ( ts.tag_sets_id = t.tag_sets_id AND ts.name = 'media_type' ) - JOIN media_tags_map mtm ON ( mtm.tags_id = t.tags_id ) - ) ON ( m.media_id = mtm.media_id ); - - -SET search_path = snap, pg_catalog; - -ALTER TABLE media - DROP COLUMN use_pager, - DROP COLUMN unpaged_stories; - - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4586; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4586-4587.sql b/apps/postgresql-server/schema/migrations/mediawords-4586-4587.sql deleted file mode 100644 index 394555a516..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4586-4587.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4586 and 4587. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4586, and you would like to upgrade both the Media Cloud and the --- database to be at version 4587, import this SQL file: --- --- psql mediacloud < mediawords-4586-4587.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- ALTER TYPE ... ADD VALUE doesn't work in a transaction or a multi-line --- query, so the new enum value gets added in Schema.pm manually. - ---ALTER TYPE feed_feed_type ADD VALUE 'univision'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4587; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4587-4588.sql b/apps/postgresql-server/schema/migrations/mediawords-4587-4588.sql deleted file mode 100644 index c9fa3e53c0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4587-4588.sql +++ /dev/null @@ -1,68 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4587 and 4588. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4587, and you would like to upgrade both the Media Cloud and the --- database to be at version 4588, import this SQL file: --- --- psql mediacloud < mediawords-4587-4588.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - - -SET search_path = public, pg_catalog; - -DROP VIEW media_with_media_types; -- will recreate right afterwards - -ALTER TABLE media - DROP COLUMN extract_author; - -CREATE VIEW media_with_media_types AS - SELECT m.*, mtm.tags_id media_type_tags_id, t.label media_type - FROM - media m - LEFT JOIN ( - tags t - JOIN tag_sets ts ON ( ts.tag_sets_id = t.tag_sets_id AND ts.name = 'media_type' ) - JOIN media_tags_map mtm ON ( mtm.tags_id = t.tags_id ) - ) ON ( m.media_id = mtm.media_id ); - - -SET search_path = snap, pg_catalog; - -ALTER TABLE media - DROP COLUMN extract_author; - - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4588; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4588-4589.sql b/apps/postgresql-server/schema/migrations/mediawords-4588-4589.sql deleted file mode 100644 index b2d8fc93e8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4588-4589.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4588 and 4589. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4588, and you would like to upgrade both the Media Cloud and the --- database to be at version 4589, import this SQL file: --- --- psql mediacloud < mediawords-4588-4589.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add is_public boolean not null default false; - -create type topic_permission AS ENUM ( 'read', 'write', 'admin' ); - --- per user permissions for topics -create table topic_permissions ( - topic_permissions_id serial primary key, - topics_id int not null references topics on delete cascade, - auth_users_id int not null references auth_users on delete cascade, - permission topic_permission not null -); - -create index topic_permissions_topic on topic_permissions( topics_id ); -create unique index topic_permissions_user on topic_permissions( auth_users_id, topics_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4589; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4589-4590.sql b/apps/postgresql-server/schema/migrations/mediawords-4589-4590.sql deleted file mode 100644 index 22478122bb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4589-4590.sql +++ /dev/null @@ -1,78 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4589 and 4590. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4589, and you would like to upgrade both the Media Cloud and the --- database to be at version 4590, import this SQL file: --- --- psql mediacloud < mediawords-4589-4590.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- topics table with auth_users_id and user_permission fields that indicate the permission level for --- the user for the topic. permissions in decreasing order are admin, write, read, none. users with --- the admin role have admin permission for every topic. users with admin-readonly role have at least --- read access to every topic. all users have read access to every is_public topic. otherwise, the --- topic_permissions tableis used, with 'none' for no topic_permission. -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4590; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4590-4591.sql b/apps/postgresql-server/schema/migrations/mediawords-4590-4591.sql deleted file mode 100644 index 4108c63ea1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4590-4591.sql +++ /dev/null @@ -1,117 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4590 and 4591. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4590, and you would like to upgrade both the Media Cloud and the --- database to be at version 4591, import this SQL file: --- --- psql mediacloud < mediawords-4590-4591.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add ch_monitor_id bigint null; -alter table topics add twitter_parent_topics_id int null references topics on delete set null; -alter table topics add import_twitter_urls boolean not null default false; - -alter table topic_dead_links alter stories_id drop not null; - -alter table snap.story_link_counts add simple_tweet_count int null; -alter table snap.story_link_counts add normalized_tweet_count int null; - -alter table snap.medium_link_counts add facebook_share_count int null; -alter table snap.medium_link_counts add simple_tweet_count int null; -alter table snap.medium_link_counts add normalized_tweet_count int null; - -alter table timespans add tweet_count int; -update timespans set tweet_count = 0; -alter table timespans alter tweet_count set not null; - --- list of tweet counts and fetching statuses for each day of each topic -create table topic_tweet_days ( - topic_tweet_days_id serial primary key, - topics_id int not null references topics on delete cascade, - day date not null, - tweet_count int not null, - num_ch_tweets int not null, - tweets_fetched boolean not null default false -); - -create unique index topic_tweet_days_td on topic_tweet_days ( topics_id, day ); - --- list of tweets associated with a given topic -create table topic_tweets ( - topic_tweets_id serial primary key, - topic_tweet_days_id int not null references topic_tweet_days on delete cascade, - data json not null, - tweet_id varchar(256) not null, - content text not null, - publish_date timestamp not null, - twitter_user varchar( 1024 ) not null -); - -create unique index topic_tweets_id on topic_tweets( topic_tweet_days_id, tweet_id ); -create index topic_tweet_topic_user on topic_tweets( topic_tweet_days_id, twitter_user ); - --- urls parsed from topic tweets and imported into topic_seed_urls -create table topic_tweet_urls ( - topic_tweet_urls_id serial primary key, - topic_tweets_id int not null references topic_tweets on delete cascade, - url varchar (1024) not null -); - -create index topic_tweet_urls_url on topic_tweet_urls ( url ); -create unique index topic_tweet_urls_tt on topic_tweet_urls ( topic_tweets_id, url ); - --- view that joins together the related topic_tweets, topic_tweet_days, topic_tweet_urls, and topic_seed_urls tables --- tables for convenient querying of topic twitter url data -create view topic_tweet_full_urls as - select distinct - t.topics_id parent_topics_id, twt.topics_id twitter_topics_id, - tt.topic_tweets_id, tt.content, tt.publish_date, tt.twitter_user, - ttd.day, ttd.tweet_count, ttd.num_ch_tweets, ttd.tweets_fetched, - ttu.url, tsu.stories_id - from - topics t - join topics twt on ( t.topics_id = twt.twitter_parent_topics_id ) - join topic_tweet_days ttd on ( t.topics_id = ttd.topics_id ) - join topic_tweets tt using ( topic_tweet_days_id ) - join topic_tweet_urls ttu using ( topic_tweets_id ) - left join topic_seed_urls tsu - on ( tsu.topics_id in ( twt.twitter_parent_topics_id, twt.topics_id ) and ttu.url = tsu.url ); - -create table snap.timespan_tweets ( - topic_tweets_id int not null references topic_tweets on delete cascade, - timespans_id int not null references timespans on delete cascade -); - -create unique index snap_timespan_tweets_u on snap.timespan_tweets( timespans_id, topic_tweets_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4591; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4591-4592.sql b/apps/postgresql-server/schema/migrations/mediawords-4591-4592.sql deleted file mode 100644 index 45fd53a271..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4591-4592.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4591 and 4592. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4591, and you would like to upgrade both the Media Cloud and the --- database to be at version 4592, import this SQL file: --- --- psql mediacloud < mediawords-4591-4592.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter index snap.story_link_counts_story rename to story_link_counts_ts; -create index story_link_counts_story on snap.story_link_counts( stories_id ); - -alter table snapshots add searchable boolean not null default false; - -update snapshots set searchable = true; - -create index snapshots_searchable on snapshots ( searchable ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4592; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4592-4593.sql b/apps/postgresql-server/schema/migrations/mediawords-4592-4593.sql deleted file mode 100644 index 63e8b04212..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4592-4593.sql +++ /dev/null @@ -1,88 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4592 and 4593. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4592, and you would like to upgrade both the Media Cloud and the --- database to be at version 4593, import this SQL file: --- --- psql mediacloud < mediawords-4592-4593.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop index topic_tweet_urls_tt; -create index topic_tweet_urls_tt on topic_tweet_urls( topic_tweets_id, url ); - -drop view topics_with_dates; -create view topics_with_dates as - select c.*, - to_char( td.start_date, 'YYYY-MM-DD' ) start_date, - to_char( td.end_date, 'YYYY-MM-DD' ) end_date - from - topics c - join topic_dates td on ( c.topics_id = td.topics_id ) - where - td.boundary; - - -drop view topics_with_user_permission; -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4593; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4593-4594.sql b/apps/postgresql-server/schema/migrations/mediawords-4593-4594.sql deleted file mode 100644 index f50d151382..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4593-4594.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4593 and 4594. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4593, and you would like to upgrade both the Media Cloud and the --- database to be at version 4594, import this SQL file: --- --- psql mediacloud < mediawords-4593-4594.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -drop index story_sentences_publish_day; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4594; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4594-4595.sql b/apps/postgresql-server/schema/migrations/mediawords-4594-4595.sql deleted file mode 100644 index c6ed486842..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4594-4595.sql +++ /dev/null @@ -1,62 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4594 and 4595. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4594, and you would like to upgrade both the Media Cloud and the --- database to be at version 4595, import this SQL file: --- --- psql mediacloud < mediawords-4594-4595.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- ALTER TYPE ... ADD VALUE doesn't work in a transaction or a multi-line --- query, so the new enum value gets added in Schema.pm manually. - ---ALTER TYPE feed_feed_type ADD VALUE 'superglue'; - - ---- Superglue (TV) stories metadata --> -CREATE TABLE stories_superglue_metadata ( - stories_superglue_metadata_id SERIAL PRIMARY KEY, - stories_id INT NOT NULL REFERENCES stories ON DELETE CASCADE, - thumbnail_url VARCHAR NOT NULL, - segment_duration NUMERIC NOT NULL -); - -CREATE UNIQUE INDEX stories_superglue_metadata_stories_id - ON stories_superglue_metadata (stories_id); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4595; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4595-4596.sql b/apps/postgresql-server/schema/migrations/mediawords-4595-4596.sql deleted file mode 100644 index da30256bc0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4595-4596.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4595 and 4596. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4595, and you would like to upgrade both the Media Cloud and the --- database to be at version 4596, import this SQL file: --- --- psql mediacloud < mediawords-4595-4596.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': - --- notes for internal media cloud consumption (eg. 'added this for yochai') -alter table media add editor_notes text null; - --- notes for public consumption (eg. 'leading dissident paper in anatarctica') -alter table media add public_notes text null; - --- if true, indicates that media cloud closely monitors the health of this source -alter table media add is_monitored boolean not null default false; - --- if true, users can expect tags and associations in this tag set not to change in major ways -alter table tags add is_static boolean not null default false; - -insert into auth_users_roles_map ( auth_users_id, auth_roles_id ) - select auth_users_id, auth_roles_id - from auth_users u - join auth_roles r on ( r.role = 'admin-readonly' ) - where - u.non_public_api and - not exists ( - select 1 - from auth_users_roles_map m - join auth_roles mr using ( auth_roles_id ) - where - m.auth_users_id = u.auth_users_id and - r.role in ( 'admin', 'admin-readonly' ) - ); - -alter table auth_users drop column non_public_api; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4596; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4596-4597.sql b/apps/postgresql-server/schema/migrations/mediawords-4596-4597.sql deleted file mode 100644 index eac5ffa82c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4596-4597.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4596 and 4597. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4596, and you would like to upgrade both the Media Cloud and the --- database to be at version 4597, import this SQL file: --- --- psql mediacloud < mediawords-4596-4597.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop index story_sentences_language; -drop index story_sentences_media_id; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4597; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4597-4598.sql b/apps/postgresql-server/schema/migrations/mediawords-4597-4598.sql deleted file mode 100644 index 8953c162b1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4597-4598.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4597 and 4598. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4597, and you would like to upgrade both the Media Cloud and the --- database to be at version 4598, import this SQL file: --- --- psql mediacloud < mediawords-4597-4598.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table snap.tweet_stories ( - snapshots_id int not null references snapshots on delete cascade, - topic_tweets_id int not null references topic_tweets on delete cascade, - publish_date date not null, - twitter_user varchar( 1024 ) not null, - stories_id int not null, - media_id int not null, - num_ch_tweets int not null, - tweet_count int not null -); - -create index snap_tweet_stories on snap.tweet_stories ( snapshots_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4598; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4598-4599.sql b/apps/postgresql-server/schema/migrations/mediawords-4598-4599.sql deleted file mode 100644 index f7c778edd9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4598-4599.sql +++ /dev/null @@ -1,84 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4598 and 4599. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4598, and you would like to upgrade both the Media Cloud and the --- database to be at version 4599, import this SQL file: --- --- psql mediacloud < mediawords-4598-4599.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE stories_superglue_metadata - ADD COLUMN video_url VARCHAR NOT NULL DEFAULT ''; - --- Copy story (video) URLs to metadata table -UPDATE stories_superglue_metadata -SET video_url = superglue_stories.url -FROM ( - SELECT stories_id, - url - FROM stories - WHERE stories_id IN ( - SELECT stories_id - FROM feeds_stories_map - WHERE feeds_id IN ( - SELECT feeds_id - FROM feeds - WHERE feed_type = 'superglue' - ) - ) -) AS superglue_stories -WHERE stories_superglue_metadata.stories_id = superglue_stories.stories_id; - --- Remove URLs (set to GUID) from "stories" table -UPDATE stories -SET url = guid -WHERE stories_id IN ( - SELECT stories_id - FROM feeds_stories_map - WHERE feeds_id IN ( - SELECT feeds_id - FROM feeds - WHERE feed_type = 'superglue' - ) -); - -ALTER TABLE stories_superglue_metadata - ALTER COLUMN video_url DROP DEFAULT; - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4599; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4599-4600.sql b/apps/postgresql-server/schema/migrations/mediawords-4599-4600.sql deleted file mode 100644 index 1fd1ddf0ee..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4599-4600.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4599 and 4600. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4599, and you would like to upgrade both the Media Cloud and the --- database to be at version 4600, import this SQL file: --- --- psql mediacloud < mediawords-4599-4600.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index tags_show_on_media on tags ( show_on_media ); -create index tags_show_on_stories on tags ( show_on_stories ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4600; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4600-4601.sql b/apps/postgresql-server/schema/migrations/mediawords-4600-4601.sql deleted file mode 100644 index e468a4a845..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4600-4601.sql +++ /dev/null @@ -1,94 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4600 and 4601. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4600, and you would like to upgrade both the Media Cloud and the --- database to be at version 4601, import this SQL file: --- --- psql mediacloud < mediawords-4600-4601.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -create table media_stats_weekly ( - media_id int not null references media on delete cascade, - stories_rank int not null, - num_stories numeric not null, - sentences_rank int not null, - num_sentences numeric not null, - stat_week date not null -); - -create index media_stats_weekly_medium on media_stats_weekly ( media_id ); - -create table media_expected_volume ( - media_id int not null references media on delete cascade, - start_date date not null, - end_date date not null, - expected_stories numeric not null, - expected_sentences numeric not null -); - -create index media_expected_volume_medium on media_expected_volume ( media_id ); - -create table media_coverage_gaps ( - media_id int not null references media on delete cascade, - stat_week date not null, - num_stories numeric not null, - expected_stories numeric not null, - num_sentences numeric not null, - expected_sentences numeric not null -); - -create index media_coverage_gaps_medium on media_coverage_gaps ( media_id ); - -create table media_health ( - media_id int not null references media on delete cascade, - num_stories numeric not null, - num_stories_y numeric not null, - num_stories_w numeric not null, - num_stories_90 numeric not null, - num_sentences numeric not null, - num_sentences_y numeric not null, - num_sentences_w numeric not null, - num_sentences_90 numeric not null, - is_healthy boolean not null default false, - has_active_feed boolean not null default true, - start_date date not null, - end_date date not null, - expected_sentences numeric not null, - expected_stories numeric not null, - coverage_gaps int not null -); - -create index media_health_medium on media_health ( media_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4601; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4601-4602.sql b/apps/postgresql-server/schema/migrations/mediawords-4601-4602.sql deleted file mode 100644 index 2929f22f3d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4601-4602.sql +++ /dev/null @@ -1,60 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4601 and 4602. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4601, and you would like to upgrade both the Media Cloud and the --- database to be at version 4602, import this SQL file: --- --- psql mediacloud < mediawords-4601-4602.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table media_suggestions ( - media_suggestions_id serial primary key, - name text, - url text not null, - feed_url text, - reason text, - auth_users_id int references auth_users on delete set null, - date_submitted timestamp not null default now() -); - -create index media_suggestions_date on media_suggestions ( date_submitted ); - -create table media_suggestions_tags_map ( - media_suggestions_id int references media_suggestions on delete cascade, - tags_id int references tags on delete cascade -); - -create index media_suggestions_tags_map_ms on media_suggestions_tags_map ( media_suggestions_id ); -create index media_suggestions_tags_map_tag on media_suggestions_tags_map ( tags_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4602; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4602-4603.sql b/apps/postgresql-server/schema/migrations/mediawords-4602-4603.sql deleted file mode 100644 index 40cb3e195b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4602-4603.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4602 and 4603. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4602, and you would like to upgrade both the Media Cloud and the --- database to be at version 4603, import this SQL file: --- --- psql mediacloud < mediawords-4602-4603.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table mediacloud_stats ( - stats_date date not null default now(), - daily_downloads bigint not null, - daily_stories bigint not null, - active_crawled_media bigint not null, - active_crawled_feeds bigint not null, - total_stories bigint not null, - total_downloads bigint not null, - total_sentences bigint not null -); - -alter table media add primary_language varchar( 4 ) null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4603; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4603-4604.sql b/apps/postgresql-server/schema/migrations/mediawords-4603-4604.sql deleted file mode 100644 index 67daa1f7fc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4603-4604.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4603 and 4604. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4603, and you would like to upgrade both the Media Cloud and the --- database to be at version 4604, import this SQL file: --- --- psql mediacloud < mediawords-4603-4604.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- job states as implemented in mediawords.job.StatefulJobBroker -create table job_states ( - job_states_id serial primary key, - - --MediaWords::Job::* class implementing the job - class varchar( 1024 ) not null, - - -- short class specific state - state varchar( 1024 ) not null, - - -- optional longer message describing the state, such as a stack trace for an error - message text, - - -- last time this job state was updated - last_updated timestamp not null default now(), - - -- details about the job - args json not null, - priority text not null, - - -- the hostname and process_id of the running process - hostname text not null, - process_id int not null -); - -create index job_states_class_date on job_states( class, last_updated ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4604; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4604-4605.sql b/apps/postgresql-server/schema/migrations/mediawords-4604-4605.sql deleted file mode 100644 index 25bd82454c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4604-4605.sql +++ /dev/null @@ -1,118 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4604 and 4605. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4604, and you would like to upgrade both the Media Cloud and the --- database to be at version 4605, import this SQL file: --- --- psql mediacloud < mediawords-4604-4605.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view topic_tweet_full_urls; -create view topic_tweet_full_urls as - select distinct - t.topics_id, - tt.topic_tweets_id, tt.content, tt.publish_date, tt.twitter_user, - ttd.day, ttd.tweet_count, ttd.num_ch_tweets, ttd.tweets_fetched, - ttu.url, tsu.stories_id - from - topics t - join topic_tweet_days ttd on ( t.topics_id = ttd.topics_id ) - join topic_tweets tt using ( topic_tweet_days_id ) - join topic_tweet_urls ttu using ( topic_tweets_id ) - left join topic_seed_urls tsu - on ( tsu.topics_id = t.topics_id and ttu.url = tsu.url ); - -drop view controversies; -drop view topics_with_user_permission; -drop view topics_with_dates; - -alter table topics add twitter_topics_id int null references topics on delete set null; - -update topics a set twitter_topics_id = b.topics_id - from topics b - where - a.topics_id = b.twitter_parent_topics_id; - -update topics a set ch_monitor_id = b.ch_monitor_id - from topics b - where - a.twitter_parent_topics_id = b.topics_id; - -update topics set ch_monitor_id = null where twitter_topics_id is not null; - -alter table topics drop twitter_parent_topics_id; - -create view controversies as select topics_id controversies_id, * from topics; - -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - -create view topics_with_dates as - select c.*, - to_char( td.start_date, 'YYYY-MM-DD' ) start_date, - to_char( td.end_date, 'YYYY-MM-DD' ) end_date - from - topics c - join topic_dates td on ( c.topics_id = td.topics_id ) - where - td.boundary; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4605; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4605-4606.sql b/apps/postgresql-server/schema/migrations/mediawords-4605-4606.sql deleted file mode 100644 index 257225764b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4605-4606.sql +++ /dev/null @@ -1,70 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4605 and 4606. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4605, and you would like to upgrade both the Media Cloud and the --- database to be at version 4606, import this SQL file: --- --- psql mediacloud < mediawords-4605-4606.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add start_date date; -alter table topics add end_date date; - -update topics t set start_date = td.start_date, end_date = td.end_date - from topic_dates td - where - t.topics_id = td.topics_id and - td.boundary; - -alter table topics alter start_date set not null; -alter table topics alter end_date set not null; - -drop view topics_with_dates; -drop table if exists snapshot_tags; - -create table topics_media_map ( - topics_id int not null references topics on delete cascade, - media_id int not null references media on delete cascade -); - -create index topics_media_map_topic on topics_media_map ( topics_id ); - -create table topics_media_tags_map ( - topics_id int not null references topics on delete cascade, - tags_id int not null references tags on delete cascade -); - -create index topics_media_tags_map_topic on topics_media_tags_map ( topics_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4606; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4606-4607.sql b/apps/postgresql-server/schema/migrations/mediawords-4606-4607.sql deleted file mode 100644 index a6f803dad2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4606-4607.sql +++ /dev/null @@ -1,94 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4606 and 4607. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4606, and you would like to upgrade both the Media Cloud and the --- database to be at version 4607, import this SQL file: --- --- psql mediacloud < mediawords-4606-4607.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view topics_with_user_permission; -drop view controversies; - -alter table topics drop column if exists topic_tag_sets_id; -alter table topics drop has_been_spidered; -alter table topics drop has_been_dumped; -alter table topics rename error_message to message; - -update topics set state = 'queued' where state = 'created but not queued'; -update topics set state = 'completed' where state = 'ready'; -update topics set state = 'running', message = state where length( state ) > 25; - -alter table snapshots rename error_message to message; - -update snapshots set state = 'error' where state like '%failed'; -update snapshots set state = 'error', message = state where state <> 'completed'; - -drop trigger topic_tag_set on topics; -drop function insert_topic_tag_set(); - -create view controversies as select topics_id controversies_id, * from topics; - -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4607; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4607-4608.sql b/apps/postgresql-server/schema/migrations/mediawords-4607-4608.sql deleted file mode 100644 index cabbcce79a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4607-4608.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4607 and 4608. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4607, and you would like to upgrade both the Media Cloud and the --- database to be at version 4608, import this SQL file: --- --- psql mediacloud < mediawords-4607-4608.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create type bot_policy_type AS ENUM ( 'all', 'no bots', 'only bots'); -alter table snapshots add bot_policy bot_policy_type null; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4608; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4608-4609.sql b/apps/postgresql-server/schema/migrations/mediawords-4608-4609.sql deleted file mode 100644 index c285fd63c3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4608-4609.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4608 and 4609. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4608, and you would like to upgrade both the Media Cloud and the --- database to be at version 4609, import this SQL file: --- --- psql mediacloud < mediawords-4608-4609.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE stories_without_readability_tag; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4609; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4609-4610.sql b/apps/postgresql-server/schema/migrations/mediawords-4609-4610.sql deleted file mode 100644 index 7c952d0cae..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4609-4610.sql +++ /dev/null @@ -1,79 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4609 and 4610. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4609, and you would like to upgrade both the Media Cloud and the --- database to be at version 4610, import this SQL file: --- --- psql mediacloud < mediawords-4609-4610.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view controversies; -drop view topics_with_user_permission; - -alter table topics drop column if exists topic_tag_sets_id; - -create view controversies as select topics_id controversies_id, * from topics; - -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4610; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4610-4611.sql b/apps/postgresql-server/schema/migrations/mediawords-4610-4611.sql deleted file mode 100644 index 82da832e80..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4610-4611.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4610 and 4611. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4610, and you would like to upgrade both the Media Cloud and the --- database to be at version 4611, import this SQL file: --- --- psql mediacloud < mediawords-4610-4611.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create view pending_job_states as select * from job_states where state in ( 'running', 'queued' ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4611; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4611-4612.sql b/apps/postgresql-server/schema/migrations/mediawords-4611-4612.sql deleted file mode 100644 index ceb9811960..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4611-4612.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4611 and 4612. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4611, and you would like to upgrade both the Media Cloud and the --- database to be at version 4612, import this SQL file: --- --- psql mediacloud < mediawords-4611-4612.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -alter table media_health add media_health_id serial primary key; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4612; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4612-4613.sql b/apps/postgresql-server/schema/migrations/mediawords-4612-4613.sql deleted file mode 100644 index 0ef5984f7d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4612-4613.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4612 and 4613. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4612, and you would like to upgrade both the Media Cloud and the --- database to be at version 4613, import this SQL file: --- --- psql mediacloud < mediawords-4612-4613.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create type topics_job_queue_type AS ENUM ( 'mc', 'public' ); - -alter table topics add job_queue topics_job_queue_type; -update topics set job_queue = 'mc'; -alter table topics alter job_queue set not null; - -alter table topics add max_stories int null; - -create temporary table topic_num_stories as - select t.topics_id, max( ts.story_count ) num_stories - from topics t - join snapshots s using ( topics_id ) - join timespans ts using ( snapshots_id ) - group by t.topics_id; - -update topics set max_stories = 200000; -update topics t set max_stories = tns.num_stories * 2 - from topic_num_stories tns - where - t.topics_id = tns.topics_id and - tns.num_stories > 100000; - -alter table topics alter max_stories set not null; - -alter table topics add max_stories_reached boolean not null default false; - -alter table auth_users add max_topic_stories int not null default 100000; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4613; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4613-4614.sql b/apps/postgresql-server/schema/migrations/mediawords-4613-4614.sql deleted file mode 100644 index 6856529a17..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4613-4614.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4613 and 4614. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4613, and you would like to upgrade both the Media Cloud and the --- database to be at version 4614, import this SQL file: --- --- psql mediacloud < mediawords-4613-4614.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics drop max_stories_reached; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4614; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4614-4615.sql b/apps/postgresql-server/schema/migrations/mediawords-4614-4615.sql deleted file mode 100644 index fb8db344a8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4614-4615.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4614 and 4615. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4614, and you would like to upgrade both the Media Cloud and the --- database to be at version 4615, import this SQL file: --- --- psql mediacloud < mediawords-4614-4615.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table mediacloud_stats add mediacloud_stats_id serial primary key; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4615; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4615-4616.sql b/apps/postgresql-server/schema/migrations/mediawords-4615-4616.sql deleted file mode 100644 index 96bebcb499..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4615-4616.sql +++ /dev/null @@ -1,104 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4615 and 4616. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4615, and you would like to upgrade both the Media Cloud and the --- database to be at version 4616, import this SQL file: --- --- psql mediacloud < mediawords-4615-4616.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- definition of bipolar comparisons for retweeter polarization scores -create table retweeter_scores ( - retweeter_scores_id serial primary key, - topics_id int not null references topics on delete cascade, - group_a_id int null, - group_b_id int null, - name text not null, - state text not null default 'created but not queued', - message text null -); - --- group retweeters together so that we an compare, for example, sanders/warren retweeters to cruz/kasich retweeters -create table retweeter_groups ( - retweeter_groups_id serial primary key, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - name text not null -); - -alter table retweeter_scores add constraint retweeter_scores_group_a - foreign key ( group_a_id ) references retweeter_groups on delete cascade; -alter table retweeter_scores add constraint retweeter_scores_group_b - foreign key ( group_b_id ) references retweeter_groups on delete cascade; - --- list of twitter users within a given topic that have retweeted the given user -create table retweeters ( - retweeters_id serial primary key, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - twitter_user varchar(1024) not null, - retweeted_user varchar(1024) not null -); - -create unique index retweeters_user on retweeters( retweeter_scores_id, twitter_user, retweeted_user ); - -create table retweeter_groups_users_map ( - retweeter_groups_id int not null references retweeter_groups on delete cascade, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - retweeted_user varchar(1024) not null -); - --- count of shares by retweeters for each retweeted_user in retweeters -create table retweeter_stories ( - retweeter_shares_id serial primary key, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - stories_id int not null references stories on delete cascade, - retweeted_user varchar(1024) not null, - share_count int not null -); - -create unique index retweeter_stories_psu - on retweeter_stories ( retweeter_scores_id, stories_id, retweeted_user ); - --- polarization scores for media within a topic for the given retweeter_scoresdefinition -create table retweeter_media ( - retweeter_media_id serial primary key, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - media_id int not null references media on delete cascade, - group_a_count int not null, - group_b_count int not null, - group_a_count_n float not null, - score float not null -); - -create unique index retweeter_media_score on retweeter_media ( retweeter_scores_id, media_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4616; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4616-4617.sql b/apps/postgresql-server/schema/migrations/mediawords-4616-4617.sql deleted file mode 100644 index a8bedbfd66..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4616-4617.sql +++ /dev/null @@ -1,101 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4616 and 4617. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4616, and you would like to upgrade both the Media Cloud and the --- database to be at version 4617, import this SQL file: --- --- psql mediacloud < mediawords-4616-4617.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- assume default num_partitions of 5 - -alter table retweeter_scores add num_partitions int; -update retweeter_scores set num_partitions = 5; -alter table retweeter_scores alter num_partitions set not null; - -alter table retweeter_media add partition float; -update retweeter_media set partition = floor( ( score + 1 ) * 2.4999 ); -alter table retweeter_media alter partition set not null; - -create table retweeter_partition_matrix ( - retweeter_partition_matrix_id serial primary key, - retweeter_scores_id int not null references retweeter_scores on delete cascade, - retweeter_groups_id int not null references retweeter_groups on delete cascade, - group_name text not null, - share_count int not null, - group_proportion float not null, - partition int not null -); - -create index retweeter_partition_matrix_score on retweeter_partition_matrix ( retweeter_scores_id ); - -insert into retweeter_partition_matrix - ( retweeter_scores_id, share_count, group_proportion, partition, retweeter_groups_id, group_name ) - with rpm as ( - select - rs.retweeter_scores_id, - sum( rs.share_count ) share_count, - rm.partition, - rg.retweeter_groups_id, - rg.name group_name - from retweeter_stories rs - join retweeter_groups_users_map rgum - on ( rgum.retweeted_user = rs.retweeted_user and - rs.retweeter_scores_id = rgum.retweeter_scores_id ) - join retweeter_groups rg using ( retweeter_groups_id ) - join stories s using ( stories_id ) - join retweeter_media rm - on ( s.media_id = rm.media_id and rm.retweeter_scores_id = rs.retweeter_scores_id ) - group by rs.retweeter_scores_id, rg.retweeter_groups_id, rm.partition - ), - - rpm_totals as ( - select - sum( share_count ) group_share_count, - retweeter_groups_id - from rpm - group by retweeter_groups_id - ) - - select - rpm.retweeter_scores_id, - rpm.share_count, - ( rpm.share_count::float / rpm_totals.group_share_count::float )::float group_proprtion, - rpm.partition, - rpm.retweeter_groups_id, - rpm.group_name - from rpm - join rpm_totals using ( retweeter_groups_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4617; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4617-4618.sql b/apps/postgresql-server/schema/migrations/mediawords-4617-4618.sql deleted file mode 100644 index e5f9aa2b8a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4617-4618.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4617 and 4618. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4617, and you would like to upgrade both the Media Cloud and the --- database to be at version 4618, import this SQL file: --- --- psql mediacloud < mediawords-4617-4618.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- log of all stories import into solr, with the import date -create table solr_imported_stories ( - stories_id int not null references stories on delete cascade, - import_date timestamp not null -); - -create index solr_imported_stories_story on solr_imported_stories ( stories_id ); -create index solr_imported_stories_day on solr_imported_stories ( date_trunc( 'day', import_date ) ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4618; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4618-4619.sql b/apps/postgresql-server/schema/migrations/mediawords-4618-4619.sql deleted file mode 100644 index 03a5fa2cc9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4618-4619.sql +++ /dev/null @@ -1,55 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4618 and 4619. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4618, and you would like to upgrade both the Media Cloud and the --- database to be at version 4619, import this SQL file: --- --- psql mediacloud < mediawords-4618-4619.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view media_with_media_types; - -alter table media drop primary_language; - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4619; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4619-4620.sql b/apps/postgresql-server/schema/migrations/mediawords-4619-4620.sql deleted file mode 100644 index d3085f75d7..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4619-4620.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4619 and 4620. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4619, and you would like to upgrade both the Media Cloud and the --- database to be at version 4620, import this SQL file: --- --- psql mediacloud < mediawords-4619-4620.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER FUNCTION generate_api_token() - RENAME TO generate_api_key; - -ALTER INDEX auth_users_token - RENAME TO auth_users_api_key; - -ALTER INDEX auth_user_ip_tokens_token - RENAME TO auth_user_ip_tokens_api_key_ip_address; - -ALTER TABLE auth_users - RENAME COLUMN api_token TO api_key; - -ALTER TABLE auth_user_ip_tokens - RENAME TO auth_user_ip_address_api_keys; - -ALTER TABLE auth_user_ip_address_api_keys - RENAME COLUMN auth_user_ip_tokens_id TO auth_user_ip_address_api_keys_id; -ALTER TABLE auth_user_ip_address_api_keys - RENAME COLUMN api_token TO api_key; - -ALTER INDEX auth_user_ip_tokens_api_key_ip_address - RENAME TO auth_user_ip_address_api_keys_api_key_ip_address; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4620; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4620-4621.sql b/apps/postgresql-server/schema/migrations/mediawords-4620-4621.sql deleted file mode 100644 index 07d6473b34..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4620-4621.sql +++ /dev/null @@ -1,84 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4620 and 4621. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4620, and you would like to upgrade both the Media Cloud and the --- database to be at version 4621, import this SQL file: --- --- psql mediacloud < mediawords-4620-4621.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -DROP INDEX auth_user_ip_address_api_keys_api_key_ip_address; -DROP INDEX auth_users_api_key; - -ALTER TABLE auth_user_ip_address_api_keys - RENAME TO auth_user_api_keys; - -ALTER TABLE auth_user_api_keys - ALTER COLUMN ip_address DROP NOT NULL; - --- Copy existing non-IP limited API keys to "auth_user_api_keys" -INSERT INTO auth_user_api_keys (auth_users_id, api_key, ip_address) - SELECT auth_users_id, api_key, NULL - FROM auth_users; - -ALTER TABLE auth_users - DROP COLUMN api_key; - -CREATE UNIQUE INDEX auth_user_api_keys_api_key_ip_address - ON auth_user_api_keys (api_key, ip_address); - --- Autogenerate non-IP limited API key -CREATE OR REPLACE FUNCTION auth_user_api_keys_add_non_ip_limited_api_key() RETURNS trigger AS -$$ -BEGIN - - INSERT INTO auth_user_api_keys (auth_users_id, api_key, ip_address) - VALUES ( - NEW.auth_users_id, - DEFAULT, -- Autogenerated API key - NULL -- Not limited by IP address - ); - RETURN NULL; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE TRIGGER auth_user_api_keys_add_non_ip_limited_api_key - AFTER INSERT ON auth_users - FOR EACH ROW EXECUTE PROCEDURE auth_user_api_keys_add_non_ip_limited_api_key(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4621; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4621-4622.sql b/apps/postgresql-server/schema/migrations/mediawords-4621-4622.sql deleted file mode 100644 index 2b65a92c22..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4621-4622.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4621 and 4622. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4621, and you would like to upgrade both the Media Cloud and the --- database to be at version 4622, import this SQL file: --- --- psql mediacloud < mediawords-4621-4622.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE auth_registration_queue; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4622; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4622-4623.sql b/apps/postgresql-server/schema/migrations/mediawords-4622-4623.sql deleted file mode 100644 index 8d2094ae14..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4622-4623.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4622 and 4623. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4622, and you would like to upgrade both the Media Cloud and the --- database to be at version 4623, import this SQL file: --- --- psql mediacloud < mediawords-4622-4623.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE TABLE auth_users_subscribe_to_newsletter ( - auth_users_subscribe_to_newsletter_id SERIAL PRIMARY KEY, - auth_users_id INTEGER NOT NULL REFERENCES auth_users (auth_users_id) ON DELETE CASCADE -); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4623; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4623-4624.sql b/apps/postgresql-server/schema/migrations/mediawords-4623-4624.sql deleted file mode 100644 index 56bbfb7594..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4623-4624.sql +++ /dev/null @@ -1,198 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4623 and 4624. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4623, and you would like to upgrade both the Media Cloud and the --- database to be at version 4624, import this SQL file: --- --- psql mediacloud < mediawords-4623-4624.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Schema to hold object caches --- - -CREATE SCHEMA cache; - -CREATE OR REPLACE LANGUAGE plpgsql; - - --- Upsert helper to INSERT or UPDATE an object to object cache -CREATE OR REPLACE FUNCTION cache.upsert_cache_object ( - param_table_name VARCHAR, - param_object_id BIGINT, - param_raw_data BYTEA -) RETURNS VOID AS -$$ -DECLARE - _cache_object_found INT; -BEGIN - - -- Unsure how to pass BYTEA objects as EXECUTE parameters - CREATE TEMPORARY TABLE cache_object_to_upsert ( - object_id BIGINT NOT NULL, - raw_data BYTEA NOT NULL - ) ON COMMIT DROP; - INSERT INTO cache_object_to_upsert (object_id, raw_data) - VALUES (param_object_id, param_raw_data); - - LOOP - -- Try UPDATing - EXECUTE ' - UPDATE ' || param_table_name || ' - SET raw_data = cache_object_to_upsert.raw_data - FROM ( - SELECT object_id, raw_data - FROM cache_object_to_upsert - ) AS cache_object_to_upsert - WHERE ' || param_table_name || '.object_id = cache_object_to_upsert.object_id - RETURNING * - ' INTO _cache_object_found; - - IF _cache_object_found IS NOT NULL THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - - EXECUTE ' - INSERT INTO ' || param_table_name || ' (object_id, raw_data) - SELECT object_id, raw_data FROM cache_object_to_upsert - '; - - RETURN; - - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - --- Trigger to update "db_row_last_updated" for cache tables -CREATE OR REPLACE FUNCTION cache.update_cache_db_row_last_updated() -RETURNS TRIGGER AS $$ -BEGIN - NEW.db_row_last_updated = NOW(); - RETURN NEW; -END; -$$ LANGUAGE 'plpgsql'; - - --- Helper to purge object caches -CREATE OR REPLACE FUNCTION cache.purge_object_caches() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Purging "s3_raw_downloads_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_raw_downloads_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''7 days''; - '; - - RAISE NOTICE 'Purging "s3_bitly_processing_results_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_bitly_processing_results_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''7 days''; - '; - -END; -$$ -LANGUAGE plpgsql; - - --- --- Raw downloads from S3 cache --- - -CREATE UNLOGGED TABLE cache.s3_raw_downloads_cache ( - s3_raw_downloads_cache_id SERIAL PRIMARY KEY, - object_id BIGINT NOT NULL - REFERENCES public.downloads (downloads_id) - ON DELETE CASCADE, - - -- Will be used to purge old cache objects; - -- don't forget to update cache.purge_object_caches() - db_row_last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX s3_raw_downloads_cache_object_id - ON cache.s3_raw_downloads_cache (object_id); -CREATE INDEX s3_raw_downloads_cache_db_row_last_updated - ON cache.s3_raw_downloads_cache (db_row_last_updated); - -ALTER TABLE cache.s3_raw_downloads_cache - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - -CREATE TRIGGER s3_raw_downloads_cache_db_row_last_updated_trigger - BEFORE INSERT OR UPDATE ON cache.s3_raw_downloads_cache - FOR EACH ROW EXECUTE PROCEDURE cache.update_cache_db_row_last_updated(); - - --- --- Raw Bit.ly processing results from S3 cache --- - -CREATE UNLOGGED TABLE cache.s3_bitly_processing_results_cache ( - s3_bitly_processing_results_cache_id SERIAL PRIMARY KEY, - object_id BIGINT NOT NULL - REFERENCES public.stories (stories_id) - ON DELETE CASCADE, - - -- Will be used to purge old cache objects; - -- don't forget to update cache.purge_object_caches() - db_row_last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX s3_bitly_processing_results_cache_object_id - ON cache.s3_bitly_processing_results_cache (object_id); -CREATE INDEX s3_bitly_processing_results_cache_db_row_last_updated - ON cache.s3_bitly_processing_results_cache (db_row_last_updated); - -ALTER TABLE cache.s3_bitly_processing_results_cache - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - -CREATE TRIGGER s3_bitly_processing_results_cache_db_row_last_updated_trigger - BEFORE INSERT OR UPDATE ON cache.s3_bitly_processing_results_cache - FOR EACH ROW EXECUTE PROCEDURE cache.update_cache_db_row_last_updated(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4624; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4624-4625.sql b/apps/postgresql-server/schema/migrations/mediawords-4624-4625.sql deleted file mode 100644 index cb3b945e62..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4624-4625.sql +++ /dev/null @@ -1,66 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4624 and 4625. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4624, and you would like to upgrade both the Media Cloud and the --- database to be at version 4625, import this SQL file: --- --- psql mediacloud < mediawords-4624-4625.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Helper to purge object caches -CREATE OR REPLACE FUNCTION cache.purge_object_caches() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Purging "s3_raw_downloads_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_raw_downloads_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''3 days''; - '; - - RAISE NOTICE 'Purging "s3_bitly_processing_results_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_bitly_processing_results_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''3 days''; - '; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4625; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4625-4626.sql b/apps/postgresql-server/schema/migrations/mediawords-4625-4626.sql deleted file mode 100644 index d7ada6863b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4625-4626.sql +++ /dev/null @@ -1,87 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4625 and 4626. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4625, and you would like to upgrade both the Media Cloud and the --- database to be at version 4626, import this SQL file: --- --- psql mediacloud < mediawords-4625-4626.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Upsert helper to INSERT or UPDATE an object to object cache -CREATE OR REPLACE FUNCTION cache.upsert_cache_object ( - param_table_name VARCHAR, - param_object_id BIGINT, - param_raw_data BYTEA -) RETURNS VOID AS -$$ -DECLARE - _cache_object_found INT; -BEGIN - - LOOP - -- Try UPDATing - EXECUTE ' - UPDATE ' || param_table_name || ' - SET raw_data = $2 - WHERE object_id = $1 - RETURNING * - ' INTO _cache_object_found - USING param_object_id, param_raw_data; - - IF _cache_object_found IS NOT NULL THEN RETURN; END IF; - - -- Nothing to UPDATE, try to INSERT a new record - BEGIN - - EXECUTE ' - INSERT INTO ' || param_table_name || ' (object_id, raw_data) - VALUES ($1, $2) - ' USING param_object_id, param_raw_data; - - RETURN; - - EXCEPTION WHEN UNIQUE_VIOLATION THEN - -- If someone else INSERTs the same key concurrently, - -- we will get a unique-key failure. In that case, do - -- nothing and loop to try the UPDATE again. - END; - END LOOP; -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4626; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4626-4627.sql b/apps/postgresql-server/schema/migrations/mediawords-4626-4627.sql deleted file mode 100644 index d1461b68cb..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4626-4627.sql +++ /dev/null @@ -1,63 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4626 and 4627. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4626, and you would like to upgrade both the Media Cloud and the --- database to be at version 4627, import this SQL file: --- --- psql mediacloud < mediawords-4626-4627.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION upsert_bitly_clicks_total(param_stories_id INT, param_click_count INT); - -CREATE OR REPLACE FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "bitly_clicks_total_000001") -BEGIN - SELECT bitly_get_partition_name( NEW.stories_id, 'bitly_clicks_total' ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id) DO UPDATE - SET click_count = EXCLUDED.click_count - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4627; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4627-4628.sql b/apps/postgresql-server/schema/migrations/mediawords-4627-4628.sql deleted file mode 100644 index e82026bc0c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4627-4628.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4627 and 4628. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4627, and you would like to upgrade both the Media Cloud and the --- database to be at version 4628, import this SQL file: --- --- psql mediacloud < mediawords-4627-4628.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION upsert_auth_user_request_daily_counts(param_email TEXT, param_requested_items_count INT); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4628; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4628-4629.sql b/apps/postgresql-server/schema/migrations/mediawords-4628-4629.sql deleted file mode 100644 index 9bd98e43df..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4628-4629.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4628 and 4629. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4628, and you would like to upgrade both the Media Cloud and the --- database to be at version 4629, import this SQL file: --- --- psql mediacloud < mediawords-4628-4629.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION cache.upsert_cache_object(param_table_name VARCHAR, param_object_id BIGINT, param_raw_data BYTEA); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4629; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4629-4630.sql b/apps/postgresql-server/schema/migrations/mediawords-4629-4630.sql deleted file mode 100644 index e6ad1fddad..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4629-4630.sql +++ /dev/null @@ -1,58 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4629 and 4630. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4629, and you would like to upgrade both the Media Cloud and the --- database to be at version 4630, import this SQL file: --- --- psql mediacloud < mediawords-4629-4630.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE auth_user_limits - ALTER COLUMN weekly_requests_limit SET DEFAULT 10000, - ALTER COLUMN weekly_requested_items_limit SET DEFAULT 100000; - - --- Hike up the older limits -UPDATE auth_user_limits -SET weekly_requests_limit = DEFAULT -WHERE weekly_requests_limit < 10000; - -UPDATE auth_user_limits -SET weekly_requested_items_limit = DEFAULT -WHERE weekly_requested_items_limit < 100000; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4630; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4630-4631.sql b/apps/postgresql-server/schema/migrations/mediawords-4630-4631.sql deleted file mode 100644 index 1820847fd9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4630-4631.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4630 and 4631. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4630, and you would like to upgrade both the Media Cloud and the --- database to be at version 4631, import this SQL file: --- --- psql mediacloud < mediawords-4630-4631.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add is_logogram boolean not null default false; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4631; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4631-4632.sql b/apps/postgresql-server/schema/migrations/mediawords-4631-4632.sql deleted file mode 100644 index 6a59187c47..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4631-4632.sql +++ /dev/null @@ -1,85 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4631 and 4632. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4631, and you would like to upgrade both the Media Cloud and the --- database to be at version 4632, import this SQL file: --- --- psql mediacloud < mediawords-4631-4632.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Returns true if the story can + should be annotated with CLIFF --- -CREATE OR REPLACE FUNCTION story_is_annotatable_with_cliff(cliff_stories_id INT) -RETURNS boolean AS $$ -DECLARE - story record; -BEGIN - - SELECT stories_id, media_id, language INTO story from stories where stories_id = cliff_stories_id; - - IF NOT ( story.language = 'en' or story.language is null ) THEN - RETURN FALSE; - - ELSEIF NOT EXISTS ( SELECT 1 FROM story_sentences WHERE stories_id = cliff_stories_id ) THEN - RETURN FALSE; - - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- CLIFF annotations --- -CREATE TABLE cliff_annotations ( - cliff_annotations_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX cliff_annotations_object_id ON cliff_annotations (object_id); - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE cliff_annotations - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4632; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4632-4633.sql b/apps/postgresql-server/schema/migrations/mediawords-4632-4633.sql deleted file mode 100644 index 56382a1e5c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4632-4633.sql +++ /dev/null @@ -1,85 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4632 and 4633. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4632, and you would like to upgrade both the Media Cloud and the --- database to be at version 4633, import this SQL file: --- --- psql mediacloud < mediawords-4632-4633.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Returns true if the story can + should be annotated with NYTLabels --- -CREATE OR REPLACE FUNCTION story_is_annotatable_with_nytlabels(nytlabels_stories_id INT) -RETURNS boolean AS $$ -DECLARE - story record; -BEGIN - - SELECT stories_id, media_id, language INTO story from stories where stories_id = nytlabels_stories_id; - - IF NOT ( story.language = 'en' or story.language is null ) THEN - RETURN FALSE; - - ELSEIF NOT EXISTS ( SELECT 1 FROM story_sentences WHERE stories_id = nytlabels_stories_id ) THEN - RETURN FALSE; - - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- NYTLabels annotations --- -CREATE TABLE nytlabels_annotations ( - nytlabels_annotations_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX nytlabels_annotations_object_id ON nytlabels_annotations (object_id); - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE nytlabels_annotations - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4633; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4633-4634.sql b/apps/postgresql-server/schema/migrations/mediawords-4633-4634.sql deleted file mode 100644 index 316d09561e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4633-4634.sql +++ /dev/null @@ -1,72 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4633 and 4634. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4633, and you would like to upgrade both the Media Cloud and the --- database to be at version 4634, import this SQL file: --- --- psql mediacloud < mediawords-4633-4634.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION story_is_annotatable_with_corenlp(corenlp_stories_id INT); - -DROP FUNCTION story_is_annotatable_with_cliff(cliff_stories_id INT); - -DROP FUNCTION story_is_annotatable_with_nytlabels(nytlabels_stories_id INT); - - -CREATE OR REPLACE FUNCTION story_is_english_and_has_sentences(param_stories_id INT) RETURNS boolean AS $$ -DECLARE - story record; -BEGIN - - SELECT stories_id, media_id, language INTO story from stories where stories_id = param_stories_id; - - IF NOT ( story.language = 'en' or story.language is null ) THEN - RETURN FALSE; - - ELSEIF NOT EXISTS ( SELECT 1 FROM story_sentences WHERE stories_id = param_stories_id ) THEN - RETURN FALSE; - - END IF; - - RETURN TRUE; - -END; -$$ -LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4634; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4634-4635.sql b/apps/postgresql-server/schema/migrations/mediawords-4634-4635.sql deleted file mode 100644 index 7427f9a40b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4634-4635.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4634 and 4635. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4634, and you would like to upgrade both the Media Cloud and the --- database to be at version 4635, import this SQL file: --- --- psql mediacloud < mediawords-4634-4635.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE corenlp_annotations; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4635; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4635-4636.sql b/apps/postgresql-server/schema/migrations/mediawords-4635-4636.sql deleted file mode 100644 index 865903ff25..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4635-4636.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4635 and 4636. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4635, and you would like to upgrade both the Media Cloud and the --- database to be at version 4636, import this SQL file: --- --- psql mediacloud < mediawords-4635-4636.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE IF EXISTS story_sentences_tags_map; - --- Remove orphan tags --- DELETE FROM tags --- WHERE NOT EXISTS (SELECT 1 FROM feeds_tags_map WHERE tags.tags_id = feeds_tags_map.tags_id) --- AND NOT EXISTS (SELECT 1 FROM media_tags_map WHERE tags.tags_id = media_tags_map.tags_id) --- AND NOT EXISTS (SELECT 1 FROM stories_tags_map WHERE tags.tags_id = stories_tags_map.tags_id) --- AND NOT EXISTS (SELECT 1 FROM media_suggestions_tags_map WHERE tags.tags_id = media_suggestions_tags_map.tags_id) --- ; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4636; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4636-4637.sql b/apps/postgresql-server/schema/migrations/mediawords-4636-4637.sql deleted file mode 100644 index b3a245ad5f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4636-4637.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4636 and 4637. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4636, and you would like to upgrade both the Media Cloud and the --- database to be at version 4637, import this SQL file: --- --- psql mediacloud < mediawords-4636-4637.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP INDEX IF EXISTS auth_users_email; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4637; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4637-4638.sql b/apps/postgresql-server/schema/migrations/mediawords-4637-4638.sql deleted file mode 100644 index bc9b12416b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4637-4638.sql +++ /dev/null @@ -1,111 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4637 and 4638. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4637, and you would like to upgrade both the Media Cloud and the --- database to be at version 4638, import this SQL file: --- --- psql mediacloud < mediawords-4637-4638.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Delete duplicate users (the non-lowercase emails) -DELETE FROM auth_user_request_daily_counts -WHERE email IN ( - SELECT email - FROM auth_users - WHERE LOWER(email) IN ( - - -- Users with duplicate emails - SELECT LOWER(email) - FROM auth_users - GROUP BY LOWER(email) - HAVING COUNT(*) > 1 - ) - - -- Emails that are not lowercase - AND email::text != lower(email)::text -); - -DELETE FROM auth_users -WHERE email IN ( - SELECT email - FROM auth_users - WHERE LOWER(email) IN ( - - -- Users with duplicate emails - SELECT LOWER(email) - FROM auth_users - GROUP BY LOWER(email) - HAVING COUNT(*) > 1 - ) - - -- Emails that are not lowercase - AND email::text != lower(email)::text -); - - -CREATE EXTENSION IF NOT EXISTS citext SCHEMA public; -CREATE EXTENSION IF NOT EXISTS citext SCHEMA snap; - -DROP FUNCTION auth_user_limits_weekly_usage(user_email TEXT); - -ALTER TABLE auth_users - ALTER COLUMN email TYPE CITEXT; - -ALTER TABLE auth_user_request_daily_counts - ALTER COLUMN email TYPE CITEXT; - -ALTER TABLE activities - ALTER COLUMN user_identifier TYPE CITEXT; - - -CREATE OR REPLACE FUNCTION auth_user_limits_weekly_usage(user_email CITEXT) RETURNS TABLE(email CITEXT, weekly_requests_sum BIGINT, weekly_requested_items_sum BIGINT) AS -$$ - - SELECT auth_users.email, - COALESCE(SUM(auth_user_request_daily_counts.requests_count), 0) AS weekly_requests_sum, - COALESCE(SUM(auth_user_request_daily_counts.requested_items_count), 0) AS weekly_requested_items_sum - FROM auth_users - LEFT JOIN auth_user_request_daily_counts - ON auth_users.email = auth_user_request_daily_counts.email - AND auth_user_request_daily_counts.day > DATE_TRUNC('day', NOW())::date - INTERVAL '1 week' - WHERE auth_users.email = $1 - GROUP BY auth_users.email; - -$$ -LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4638; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4638-4639.sql b/apps/postgresql-server/schema/migrations/mediawords-4638-4639.sql deleted file mode 100644 index 00aff9833c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4638-4639.sql +++ /dev/null @@ -1,292 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4638 and 4639. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4638, and you would like to upgrade both the Media Cloud and the --- database to be at version 4639, import this SQL file: --- --- psql mediacloud < mediawords-4638-4639.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Will recreate later -DROP VIEW IF EXISTS daily_stats; -DROP VIEW IF EXISTS stories_collected_in_past_day; - - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS $$ - -DECLARE - path_change boolean; - -BEGIN - IF (TG_OP = 'UPDATE') OR (TG_OP = 'INSERT') then - NEW.db_row_last_updated = NOW(); - END IF; - - RETURN NEW; -END; - -$$ LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger() RETURNS trigger AS $$ - -DECLARE - path_change boolean; - -BEGIN - UPDATE story_sentences - SET db_row_last_updated = NOW() - WHERE stories_id = NEW.stories_id - AND before_last_solr_import( db_row_last_updated ); - - RETURN NULL; -END; - -$$ LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS $$ - -DECLARE - path_change boolean; - reference_stories_id integer default null; - -BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id - AND before_last_solr_import( db_row_last_updated ); - - RETURN NULL; - -END; - -$$ LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION insert_ss_media_stats() RETURNS trigger AS $$ -BEGIN - - UPDATE media_stats - SET num_sentences = num_sentences + 1 - WHERE media_id = NEW.media_id - AND stat_date = date_trunc( 'day', NEW.publish_date ); - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION update_ss_media_stats() RETURNS trigger AS $$ - -DECLARE - new_date DATE; - old_date DATE; - -BEGIN - SELECT date_trunc( 'day', NEW.publish_date ) INTO new_date; - SELECT date_trunc( 'day', OLD.publish_date ) INTO old_date; - - IF ( new_date != old_date ) THEN - - UPDATE media_stats - SET num_sentences = num_sentences - 1 - WHERE media_id = NEW.media_id - AND stat_date = old_date; - - UPDATE media_stats - SET num_sentences = num_sentences + 1 - WHERE media_id = NEW.media_id - AND stat_date = new_date; - - END IF; - - RETURN NEW; -END; - -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_ss_media_stats() RETURNS trigger AS $$ -BEGIN - - UPDATE media_stats - SET num_sentences = num_sentences - 1 - WHERE media_id = OLD.media_id - AND stat_date = date_trunc( 'day', OLD.publish_date ); - - RETURN NEW; - -END; - -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION insert_story_media_stats() RETURNS trigger AS $$ -BEGIN - - INSERT INTO media_stats ( media_id, num_stories, num_sentences, stat_date ) - SELECT NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - WHERE NOT EXISTS ( - SELECT 1 - FROM media_stats - WHERE media_id = NEW.media_id - AND stat_date = date_trunc( 'day', NEW.publish_date ) - ); - - UPDATE media_stats - SET num_stories = num_stories + 1 - WHERE media_id = NEW.media_id - AND stat_date = date_trunc( 'day', NEW.publish_date ); - - RETURN NEW; - -END; - -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION update_story_media_stats() RETURNS trigger AS $$ - -DECLARE - new_date DATE; - old_date DATE; - -BEGIN - - SELECT date_trunc( 'day', NEW.publish_date ) INTO new_date; - SELECT date_trunc( 'day', OLD.publish_date ) INTO old_date; - - IF ( new_date != old_date ) THEN - - UPDATE media_stats - SET num_stories = num_stories - 1 - WHERE media_id = NEW.media_id - AND stat_date = old_date; - - INSERT INTO media_stats ( media_id, num_stories, num_sentences, stat_date ) - SELECT NEW.media_id, 0, 0, date_trunc( 'day', NEW.publish_date ) - WHERE NOT EXISTS ( - SELECT 1 - FROM media_stats - WHERE media_id = NEW.media_id - AND stat_date = date_trunc( 'day', NEW.publish_date ) - ); - - UPDATE media_stats - SET num_stories = num_stories + 1 - WHERE media_id = NEW.media_id - AND stat_date = new_date; - - UPDATE story_sentences - SET publish_date = new_date - WHERE stories_id = OLD.stories_id; - - END IF; - - RETURN NEW; -END; - -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_story_media_stats() RETURNS trigger AS $$ -BEGIN - - UPDATE media_stats - SET num_stories = num_stories - 1 - WHERE media_id = OLD.media_id - AND stat_date = date_trunc( 'day', OLD.publish_date ); - - RETURN NEW; - -END; -$$ LANGUAGE plpgsql; - - -DROP FUNCTION story_triggers_enabled(); - -DROP FUNCTION enable_story_triggers(); - -DROP FUNCTION disable_story_triggers(); - --- Not used -DROP FUNCTION update_story_sentences_updated_time_by_story_sentences_id_trigger(); - - -ALTER TABLE stories - DROP COLUMN disable_triggers; - -ALTER TABLE story_sentences - DROP COLUMN disable_triggers; - -ALTER TABLE processed_stories - DROP COLUMN disable_triggers; - - --- Recreate views -CREATE VIEW stories_collected_in_past_day AS - SELECT * - FROM stories - WHERE collect_date > now() - interval '1 day'; - -CREATE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4639; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4639-4640.sql b/apps/postgresql-server/schema/migrations/mediawords-4639-4640.sql deleted file mode 100644 index a91d943383..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4639-4640.sql +++ /dev/null @@ -1,83 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4639 and 4640. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4639, and you would like to upgrade both the Media Cloud and the --- database to be at version 4640, import this SQL file: --- --- psql mediacloud < mediawords-4639-4640.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- keep track of per domain web requests so that we can throttle them using mediawords.util.web.user_agent.throttled. --- this is unlogged because we don't care about anything more than about 10 seconds old. we don't have a primary --- key because we want it just to be a fast table for temporary storage. -create unlogged table domain_web_requests ( - domain text not null, - request_time timestamp not null default now() -); - -create index domain_web_requests_domain on domain_web_requests ( domain ); - --- return false if there is a request for the given domain within the last domain_timeout_arg seconds. otherwise --- return true and insert a row into domain_web_request for the domain. this function does not lock the table and --- so may allow some parallel requests through. -create or replace function get_domain_web_requests_lock( domain_arg text, domain_timeout_arg int ) returns boolean as $$ -begin - --- we don't want this table to grow forever or to have to manage it externally, so just truncate about every --- 1 million requests. only do this if there are more than 1000 rows in the table so that unit tests will not --- randomly fail. -if ( select random() * 1000000 ) < 1 then - if exists ( select 1 from domain_web_requests offset 1000 ) then - truncate table domain_web_requests; - end if; -end if; - -if exists ( - select * - from domain_web_requests - where - domain = domain_arg and - extract( epoch from now() - request_time ) < domain_timeout_arg - ) then - - return false; -end if; - -delete from domain_web_requests where domain = domain_arg; -insert into domain_web_requests (domain) select domain_arg; - -return true; -end -$$ language plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4640; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4640-4641.sql b/apps/postgresql-server/schema/migrations/mediawords-4640-4641.sql deleted file mode 100644 index bb6ef3e315..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4640-4641.sql +++ /dev/null @@ -1,70 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4640 and 4641. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4640, and you would like to upgrade both the Media Cloud and the --- database to be at version 4641, import this SQL file: --- --- psql mediacloud < mediawords-4640-4641.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION table_exists(target_table_name VARCHAR) RETURNS BOOLEAN AS $$ -DECLARE - schema_position INT; - schema VARCHAR; -BEGIN - - SELECT POSITION('.' IN target_table_name) INTO schema_position; - - -- "." at string index 0 would return position 1 - IF schema_position = 0 THEN - schema := CURRENT_SCHEMA(); - ELSE - schema := SUBSTRING(target_table_name FROM 1 FOR schema_position - 1); - target_table_name := SUBSTRING(target_table_name FROM schema_position + 1); - END IF; - - RETURN EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = schema - AND table_name = target_table_name - ); - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4641; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4641-4642.sql b/apps/postgresql-server/schema/migrations/mediawords-4641-4642.sql deleted file mode 100644 index bf9f28357a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4641-4642.sql +++ /dev/null @@ -1,195 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4641 and 4642. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4641, and you would like to upgrade both the Media Cloud and the --- database to be at version 4642, import this SQL file: --- --- psql mediacloud < mediawords-4641-4642.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE stories_tags_map RENAME TO stories_tags_map_old; - - -CREATE OR REPLACE FUNCTION stories_tags_map_partition_chunk_size() -RETURNS BIGINT AS $$ -BEGIN - RETURN 100 * 1000 * 1000; -- 100m stories in each partition -END; $$ -LANGUAGE plpgsql IMMUTABLE; - --- "Master" table (no indexes, no foreign keys as they'll be ineffective) -CREATE TABLE stories_tags_map ( - stories_tags_map_id BIGSERIAL NOT NULL, - stories_id INT NOT NULL, - tags_id INT NOT NULL, - db_row_last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - -CREATE TRIGGER stories_tags_map_last_updated_trigger - BEFORE INSERT OR UPDATE ON stories_tags_map - FOR EACH ROW EXECUTE PROCEDURE last_updated_trigger(); - -CREATE TRIGGER stories_tags_map_update_stories_last_updated_trigger - AFTER INSERT OR UPDATE OR DELETE ON stories_tags_map - FOR EACH ROW EXECUTE PROCEDURE update_stories_updated_time_by_stories_id_trigger(); - - -CREATE OR REPLACE FUNCTION stories_tags_map_get_partition_name(stories_id BIGINT, table_name TEXT) -RETURNS TEXT AS $$ -DECLARE - to_char_format CONSTANT TEXT := '00'; -- Up to 100 partitions, suffixed as "_00", "_01" ..., "_99" - -- (having more of them is not feasible) - stories_id_chunk_number INT; - - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT stories_id / stories_tags_map_partition_chunk_size() INTO stories_id_chunk_number; - - SELECT table_name || '_' || trim(leading ' ' FROM to_char(stories_id_chunk_number, to_char_format)) - INTO target_table_name; - - RETURN target_table_name; -END; -$$ -LANGUAGE plpgsql; - - --- Create missing stories_tags_map partitions -CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() -RETURNS VOID AS -$$ -DECLARE - chunk_size INT; - max_stories_id BIGINT; - partition_stories_id BIGINT; - - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT stories_tags_map_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT stories_tags_map_get_partition_name( partition_stories_id, 'stories_tags_map' ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (stories_tags_map_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL, - - -- Foreign key to tags.tags_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_tags_id_fkey - FOREIGN KEY (tags_id) REFERENCES tags (tags_id) MATCH FULL, - - -- Unique duplets - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_tags_id_unique - UNIQUE (stories_id, tags_id) - - ) INHERITS (stories_tags_map); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'stories_tags_map' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - --- Create initial partitions for empty database -SELECT stories_tags_map_create_partitions(); - - --- Upsert row into correct partition -CREATE OR REPLACE FUNCTION stories_tags_map_partition_by_stories_id_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT stories_tags_map_get_partition_name( NEW.stories_id, 'stories_tags_map' ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER stories_tags_map_partition_by_stories_id_insert_trigger - BEFORE INSERT ON stories_tags_map - FOR EACH ROW EXECUTE PROCEDURE stories_tags_map_partition_by_stories_id_insert_trigger(); - - -INSERT INTO stories_tags_map (stories_id, tags_id, db_row_last_updated) -SELECT stories_id, tags_id, db_row_last_updated FROM stories_tags_map_old; - -DROP TABLE stories_tags_map_old; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4642; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4642-4643.sql b/apps/postgresql-server/schema/migrations/mediawords-4642-4643.sql deleted file mode 100644 index e63c14571e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4642-4643.sql +++ /dev/null @@ -1,122 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4642 and 4643. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4642, and you would like to upgrade both the Media Cloud and the --- database to be at version 4643, import this SQL file: --- --- psql mediacloud < mediawords-4642-4643.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE UNIQUE INDEX stories_tags_map_pkey ON stories_tags_map (stories_tags_map_id); -ALTER TABLE stories_tags_map ADD PRIMARY KEY USING INDEX stories_tags_map_pkey; - - --- Create missing stories_tags_map partitions -CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() -RETURNS VOID AS -$$ -DECLARE - chunk_size INT; - max_stories_id BIGINT; - partition_stories_id BIGINT; - - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") - target_table_owner TEXT; -- partition table owner (e.g. "mediaclouduser") - - stories_id_start INT; -- stories_id chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_end INT; -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) -BEGIN - - SELECT stories_tags_map_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT stories_tags_map_get_partition_name( partition_stories_id, 'stories_tags_map' ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (stories_tags_map_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE, - - -- Foreign key to tags.tags_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_tags_id_fkey - FOREIGN KEY (tags_id) REFERENCES tags (tags_id) MATCH FULL ON DELETE CASCADE, - - -- Unique duplets - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_tags_id_unique - UNIQUE (stories_id, tags_id) - - ) INHERITS (stories_tags_map); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = 'stories_tags_map' - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - END IF; - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4643; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4643-4644.sql b/apps/postgresql-server/schema/migrations/mediawords-4643-4644.sql deleted file mode 100644 index d30a3eeaaa..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4643-4644.sql +++ /dev/null @@ -1,82 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4642 and 4643. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4642, and you would like to upgrade both the Media Cloud and the --- database to be at version 4643, import this SQL file: --- --- psql mediacloud < mediawords-4642-4643.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topic_stories add link_mine_error text; - --- update media stats table for deleted story sentence -CREATE FUNCTION update_media_db_row_last_updated() RETURNS trigger AS $$ -BEGIN - NEW.db_row_last_updated = now(); - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -create trigger update_media_db_row_last_updated before update or insert - on media for each row execute procedure update_media_db_row_last_updated(); - ---- allow lookup of media by mediawords.util.url.normalize_url_lossy. --- the data in this table is accessed and kept up to date by topics_base.media.lookup_medium_by_url() -create table media_normalized_urls ( - media_normalized_urls_id serial primary key, - media_id int not null references media, - normalized_url varchar(1024) not null, - db_row_last_updated timestamp not null default now(), - - -- assigned the value of mediawords.util.url.normalize_url_lossy_version() - normalize_url_lossy_version int not null -); - -create unique index media_normalized_urls_medium on media_normalized_urls(normalize_url_lossy_version, media_id); -create index media_normalized_urls_url on media_normalized_urls(normalized_url); - -create table topic_fetch_urls( - topic_fetch_urls_id bigserial primary key, - topics_id int not null references topics on delete cascade, - url text not null, - code int, - fetch_date timestamp, - state text not null, - message text, - stories_id int references stories on delete cascade, - assume_match boolean not null default false -); - -create index topic_fetch_urls_pending on topic_fetch_urls(topics_id) where state = 'pending'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4644; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4644-4645.sql b/apps/postgresql-server/schema/migrations/mediawords-4644-4645.sql deleted file mode 100644 index cf3bf3a496..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4644-4645.sql +++ /dev/null @@ -1,41 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4644 and 4645. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4644, and you would like to upgrade both the Media Cloud and the --- database to be at version 4645, import this SQL file: --- --- psql mediacloud < mediawords-4644-4645.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -create index media_normalized_urls_db_row_last_updated on media_normalized_urls(db_row_last_updated); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4645; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4645-4646.sql b/apps/postgresql-server/schema/migrations/mediawords-4645-4646.sql deleted file mode 100644 index 33455b252c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4645-4646.sql +++ /dev/null @@ -1,64 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4645 and 4646. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4645, and you would like to upgrade both the Media Cloud and the --- database to be at version 4646, import this SQL file: --- --- psql mediacloud < mediawords-4645-4646.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- --- --- SimilarWeb metrics --- -CREATE TABLE similarweb_metrics ( - similarweb_metrics_id SERIAL PRIMARY KEY, - domain VARCHAR(1024) NOT NULL, - month DATE, - visits INTEGER, - update_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - UNIQUE (domain, month) -); - - --- --- Unnormalized table --- -CREATE TABLE similarweb_media_metrics ( - similarweb_media_metrics_id SERIAL PRIMARY KEY, - media_id INTEGER UNIQUE NOT NULL references media, - similarweb_domain VARCHAR(1024) NOT NULL, - domain_exact_match BOOLEAN NOT NULL, - monthly_audience INTEGER NOT NULL, - update_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4646; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4646-4647.sql b/apps/postgresql-server/schema/migrations/mediawords-4646-4647.sql deleted file mode 100644 index 72a718a678..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4646-4647.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4646 and 4647. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4646, and you would like to upgrade both the Media Cloud and the --- database to be at version 4647, import this SQL file: --- --- psql mediacloud < mediawords-4646-4647.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topic_fetch_urls add topic_links_id int references topic_links on delete cascade; - -create index topic_fetch_urls_url on topic_fetch_urls(md5(url)); - -update topic_links set link_spidered = 't' where ref_stories_id is not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4647; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4647-4648.sql b/apps/postgresql-server/schema/migrations/mediawords-4647-4648.sql deleted file mode 100644 index ad40d904f0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4647-4648.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4647 and 4648. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4647, and you would like to upgrade both the Media Cloud and the --- database to be at version 4648, import this SQL file: --- --- psql mediacloud < mediawords-4647-4648.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index topic_fetch_urls_link on topic_fetch_urls(topic_links_id); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4648; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4648-4649.sql b/apps/postgresql-server/schema/migrations/mediawords-4648-4649.sql deleted file mode 100644 index e589fa9d1d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4648-4649.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4648 and 4649. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4648, and you would like to upgrade both the Media Cloud and the --- database to be at version 4649, import this SQL file: --- --- psql mediacloud < mediawords-4648-4649.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index live_stories_topic_story on snap.live_stories ( topic_stories_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4649; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4649-4650.sql b/apps/postgresql-server/schema/migrations/mediawords-4649-4650.sql deleted file mode 100644 index 7ea60d519d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4649-4650.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4649 and 4650. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4649, and you would like to upgrade both the Media Cloud and the --- database to be at version 4650, import this SQL file: --- --- psql mediacloud < mediawords-4649-4650.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table media_tags_map add tagged_date date null; -alter table media_tags_map alter tagged_date set default now(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4650; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4650-4651.sql b/apps/postgresql-server/schema/migrations/mediawords-4650-4651.sql deleted file mode 100644 index 4ea7c278f5..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4650-4651.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4650 and 4651. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4650, and you would like to upgrade both the Media Cloud and the --- database to be at version 4651, import this SQL file: --- --- psql mediacloud < mediawords-4650-4651.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create type retweeter_scores_match_type AS ENUM ( 'retweet', 'regex' ); -alter table retweeter_scores add match_type retweeter_scores_match_type not null default 'retweet'; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4651; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4651-4652.sql b/apps/postgresql-server/schema/migrations/mediawords-4651-4652.sql deleted file mode 100644 index 7910a679ce..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4651-4652.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4651 and 4652. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4651, and you would like to upgrade both the Media Cloud and the --- database to be at version 4652, import this SQL file: --- --- psql mediacloud < mediawords-4651-4652.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TABLE similarweb_metrics - DROP CONSTRAINT similarweb_metrics_domain_month_key; - -CREATE UNIQUE INDEX similarweb_metrics_domain_month - ON similarweb_metrics (domain, month); - -ALTER TABLE similarweb_metrics - ALTER COLUMN visits TYPE BIGINT USING visits::bigint; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4652; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4652-4653.sql b/apps/postgresql-server/schema/migrations/mediawords-4652-4653.sql deleted file mode 100644 index a5d5644cb7..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4652-4653.sql +++ /dev/null @@ -1,71 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4652 and 4653. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4652, and you would like to upgrade both the Media Cloud and the --- database to be at version 4653, import this SQL file: --- --- psql mediacloud < mediawords-4652-4653.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Snapshot word2vec models --- -CREATE TABLE snap.word2vec_models ( - word2vec_models_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL REFERENCES snapshots (snapshots_id) ON DELETE CASCADE, - creation_date TIMESTAMP NOT NULL DEFAULT NOW() -); - --- We'll need to find the latest word2vec model -CREATE INDEX snap_word2vec_models_object_id_creation_date ON snap.word2vec_models (object_id, creation_date); - -CREATE TABLE snap.word2vec_models_data ( - word2vec_models_data_id SERIAL PRIMARY KEY, - object_id INTEGER NOT NULL - REFERENCES snap.word2vec_models (word2vec_models_id) - ON DELETE CASCADE, - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX snap_word2vec_models_data_object_id ON snap.word2vec_models_data (object_id); - --- Don't (attempt to) compress BLOBs in "raw_data" because they're going to be --- compressed already -ALTER TABLE snap.word2vec_models_data - ALTER COLUMN raw_data SET STORAGE EXTERNAL; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4653; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4653-4654.sql b/apps/postgresql-server/schema/migrations/mediawords-4653-4654.sql deleted file mode 100644 index 77cc1044b8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4653-4654.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4653 and 4654. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4653, and you would like to upgrade both the Media Cloud and the --- database to be at version 4654, import this SQL file: --- --- psql mediacloud < mediawords-4653-4654.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index tags_fts on tags using gin(to_tsvector('english'::regconfig, (tag::text || ' '::text) || label::text)); - -drop index tags_tag_1; -drop index tags_tag_2; -drop index tags_tag_3; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4654; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4654-4655.sql b/apps/postgresql-server/schema/migrations/mediawords-4654-4655.sql deleted file mode 100644 index c7d9db3d0f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4654-4655.sql +++ /dev/null @@ -1,105 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4654 and 4655. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4654, and you would like to upgrade both the Media Cloud and the --- database to be at version 4655, import this SQL file: --- --- psql mediacloud < mediawords-4654-4655.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE feeds DROP COLUMN skip_bitly_processing; - -DROP TABLE IF EXISTS story_statistics_bitly_referrers; - -DROP TABLE bitly_clicks_total CASCADE; - -DROP FUNCTION bitly_partition_chunk_size(); - -DROP FUNCTION bitly_get_partition_name(INT, TEXT); - -DROP FUNCTION bitly_clicks_total_partition_by_stories_id_insert_trigger(); - -DROP FUNCTION bitly_clicks_total_create_partitions(); - -DROP TABLE bitly_processing_schedule; - -DROP FUNCTION num_topic_stories_without_bitly_statistics(INT); - -ALTER TABLE snap.story_link_counts DROP COLUMN bitly_click_count; - -ALTER TABLE snap.medium_link_counts DROP COLUMN bitly_click_count; - -DROP TABLE bitly_processing_results; - -DROP TABLE cache.s3_bitly_processing_results_cache; - -DROP TABLE IF EXISTS bitly_processed_stories; - - --- Create missing partitions for partitioned tables -CREATE OR REPLACE FUNCTION create_missing_partitions() -RETURNS VOID AS -$$ -BEGIN - - -- "stories_tags_map" table - RAISE NOTICE 'Creating partitions in "stories_tags_map" table...'; - PERFORM stories_tags_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- Helper to purge object caches -CREATE OR REPLACE FUNCTION cache.purge_object_caches() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Purging "s3_raw_downloads_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_raw_downloads_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''3 days''; - '; - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4655; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4655-4656.sql b/apps/postgresql-server/schema/migrations/mediawords-4655-4656.sql deleted file mode 100644 index 0e7f457463..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4655-4656.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4655 and 4656. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4655, and you would like to upgrade both the Media Cloud and the --- database to be at version 4656, import this SQL file: --- --- psql mediacloud < mediawords-4655-4656.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop trigger ss_insert_story_media_stats on story_sentences; -drop trigger ss_update_story_media_stats on story_sentences; -drop trigger story_delete_ss_media_stats on story_sentences; - -drop function insert_ss_media_stats(); -drop function update_ss_media_stats(); -drop function delete_ss_media_stats(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4656; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4656-4657.sql b/apps/postgresql-server/schema/migrations/mediawords-4656-4657.sql deleted file mode 100644 index 3d2be9699a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4656-4657.sql +++ /dev/null @@ -1,65 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4656 and 4657. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4656, and you would like to upgrade both the Media Cloud and the --- database to be at version 4657, import this SQL file: --- --- psql mediacloud < mediawords-4656-4657.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Returns first 64 bits (16 characters) of MD5 hash --- --- Useful for reducing index sizes (e.g. in story_sentences.sentence) where --- 64 bits of entropy is enough. -CREATE OR REPLACE FUNCTION half_md5(string TEXT) RETURNS bytea AS $$ - -- pgcrypto's functions are being referred with public schema prefix to make pg_upgrade work - SELECT SUBSTRING(public.digest(string, 'md5'::text), 0, 9); -$$ LANGUAGE SQL; - - --- Generate random API key -CREATE OR REPLACE FUNCTION generate_api_key() RETURNS VARCHAR(64) LANGUAGE plpgsql AS $$ -DECLARE - api_key VARCHAR(64); -BEGIN - -- pgcrypto's functions are being referred with public schema prefix to make pg_upgrade work - SELECT encode(public.digest(public.gen_random_bytes(256), 'sha256'), 'hex') INTO api_key; - RETURN api_key; -END; -$$; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4657; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4657-4658.sql b/apps/postgresql-server/schema/migrations/mediawords-4657-4658.sql deleted file mode 100644 index 7600a190ab..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4657-4658.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4657 and 4658. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4657, and you would like to upgrade both the Media Cloud and the --- database to be at version 4658, import this SQL file: --- --- psql mediacloud < mediawords-4657-4658.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add is_story_index_ready boolean not null default true; -update topics set is_story_index_ready = false; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4658; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4658-4659.sql b/apps/postgresql-server/schema/migrations/mediawords-4658-4659.sql deleted file mode 100644 index 8a8afc405a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4658-4659.sql +++ /dev/null @@ -1,201 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4658 and 4659. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4658, and you would like to upgrade both the Media Cloud and the --- database to be at version 4659, import this SQL file: --- --- psql mediacloud < mediawords-4658-4659.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION stories_tags_map_partition_chunk_size(); - -DROP FUNCTION IF EXISTS stories_tags_map_get_partition_name(stories_id INT, table_name TEXT); -DROP FUNCTION IF EXISTS stories_tags_map_get_partition_name(stories_id BIGINT, table_name TEXT); - - -CREATE OR REPLACE FUNCTION stories_partition_chunk_size() RETURNS BIGINT AS $$ -BEGIN - RETURN 100 * 1000 * 1000; -- 100m stories in each partition -END; $$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION stories_partition_name(base_table_name TEXT, stories_id INT) RETURNS TEXT AS $$ -DECLARE - - -- Up to 100 partitions, suffixed as "_00", "_01" ..., "_99" - -- (having more of them is not feasible) - to_char_format CONSTANT TEXT := '00'; - - -- Partition table name (e.g. "stories_tags_map_01") - table_name TEXT; - - stories_id_chunk_number INT; - -BEGIN - SELECT stories_id / stories_partition_chunk_size() INTO stories_id_chunk_number; - - SELECT base_table_name || '_' || TRIM(leading ' ' FROM TO_CHAR(stories_id_chunk_number, to_char_format)) - INTO table_name; - - RETURN table_name; -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION stories_create_partitions(base_table_name TEXT) RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_stories_id INT; - partition_stories_id INT; - - -- Partition table name (e.g. "stories_tags_map_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "stories_id" chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_start BIGINT; - - -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) - stories_id_end BIGINT; -BEGIN - - SELECT stories_partition_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT stories_partition_name( base_table_name, partition_stories_id ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (' || base_table_name || '_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE - - ) INHERITS (' || base_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT stories_create_partitions('stories_tags_map')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - -- Add extra foreign keys / constraints to the newly created partitions - EXECUTE ' - ALTER TABLE ' || partition || ' - - -- Foreign key to tags.tags_id - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_tags_id_fkey - FOREIGN KEY (tags_id) REFERENCES tags (tags_id) MATCH FULL ON DELETE CASCADE, - - -- Unique duplets - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_stories_id_tags_id_unique - UNIQUE (stories_id, tags_id); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION stories_tags_map_partition_by_stories_id_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT stories_partition_name( 'stories_tags_map', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4659; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4659-4660.sql b/apps/postgresql-server/schema/migrations/mediawords-4659-4660.sql deleted file mode 100644 index 381c7fc82a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4659-4660.sql +++ /dev/null @@ -1,70 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4659 and 4660. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4659, and you would like to upgrade both the Media Cloud and the --- database to be at version 4660, import this SQL file: --- --- psql mediacloud < mediawords-4659-4660.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TRIGGER stories_tags_map_partition_by_stories_id_insert_trigger ON stories_tags_map; - -DROP FUNCTION stories_tags_map_partition_by_stories_id_insert_trigger(); - - -CREATE OR REPLACE FUNCTION stories_tags_map_partition_upsert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT stories_partition_name( 'stories_tags_map', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER stories_tags_map_partition_upsert_trigger - BEFORE INSERT ON stories_tags_map - FOR EACH ROW - EXECUTE PROCEDURE stories_tags_map_partition_upsert_trigger(); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4660; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4660-4661.sql b/apps/postgresql-server/schema/migrations/mediawords-4660-4661.sql deleted file mode 100644 index dba15b6590..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4660-4661.sql +++ /dev/null @@ -1,98 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4660 and 4661. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4660, and you would like to upgrade both the Media Cloud and the --- database to be at version 4661, import this SQL file: --- --- psql mediacloud < mediawords-4660-4661.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS $$ - -BEGIN - IF (TG_OP = 'UPDATE') OR (TG_OP = 'INSERT') then - NEW.db_row_last_updated = NOW(); - END IF; - - RETURN NEW; -END; - -$$ LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION update_story_sentences_updated_time_trigger() RETURNS trigger AS $$ - -BEGIN - UPDATE story_sentences - SET db_row_last_updated = NOW() - WHERE stories_id = NEW.stories_id - AND before_last_solr_import( db_row_last_updated ); - - RETURN NULL; -END; - -$$ LANGUAGE 'plpgsql'; - - -CREATE OR REPLACE FUNCTION update_stories_updated_time_by_stories_id_trigger() RETURNS trigger AS $$ - -DECLARE - reference_stories_id integer default null; - -BEGIN - - IF TG_OP = 'INSERT' THEN - -- The "old" record doesn't exist - reference_stories_id = NEW.stories_id; - ELSIF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'DELETE') THEN - reference_stories_id = OLD.stories_id; - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - END IF; - - UPDATE stories - SET db_row_last_updated = now() - WHERE stories_id = reference_stories_id - AND before_last_solr_import( db_row_last_updated ); - - RETURN NULL; - -END; - -$$ LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4661; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4661-4662.sql b/apps/postgresql-server/schema/migrations/mediawords-4661-4662.sql deleted file mode 100644 index e9c99043b2..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4661-4662.sql +++ /dev/null @@ -1,82 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4661 and 4662. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4661, and you would like to upgrade both the Media Cloud and the --- database to be at version 4662, import this SQL file: --- --- psql mediacloud < mediawords-4661-4662.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - -DROP TRIGGER stories_last_updated_trigger ON stories; -DROP TRIGGER stories_update_story_sentences_last_updated_trigger ON stories; -DROP TRIGGER story_sentences_last_updated_trigger ON story_sentences; - - -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS $$ - -BEGIN - NEW.db_row_last_updated = NOW(); - RETURN NEW; -END; - -$$ LANGUAGE 'plpgsql'; - - -CREATE TRIGGER stories_last_updated_trigger - BEFORE INSERT OR UPDATE ON stories - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger(); - -CREATE TRIGGER stories_update_story_sentences_last_updated_trigger - AFTER INSERT OR UPDATE ON stories - FOR EACH ROW - EXECUTE PROCEDURE update_story_sentences_updated_time_trigger(); - -CREATE TRIGGER story_sentences_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger(); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4662; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4662-4663.sql b/apps/postgresql-server/schema/migrations/mediawords-4662-4663.sql deleted file mode 100644 index d1e511930e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4662-4663.sql +++ /dev/null @@ -1,68 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4662 and 4663. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4662, and you would like to upgrade both the Media Cloud and the --- database to be at version 4663, import this SQL file: --- --- psql mediacloud < mediawords-4662-4663.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Update "db_row_last_updated" column to trigger Solr (re)imports for given --- row; no update gets done if "db_row_last_updated" is set explicitly in --- INSERT / UPDATE (e.g. when copying between tables) -CREATE OR REPLACE FUNCTION last_updated_trigger() RETURNS trigger AS $$ - -BEGIN - - IF TG_OP = 'INSERT' THEN - IF NEW.db_row_last_updated IS NULL THEN - NEW.db_row_last_updated = NOW(); - END IF; - - ELSIF TG_OP = 'UPDATE' THEN - IF NEW.db_row_last_updated = OLD.db_row_last_updated THEN - NEW.db_row_last_updated = NOW(); - END IF; - END IF; - - RETURN NEW; - -END; - -$$ LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4663; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4663-4664.sql b/apps/postgresql-server/schema/migrations/mediawords-4663-4664.sql deleted file mode 100644 index b372392829..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4663-4664.sql +++ /dev/null @@ -1,347 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4663 and 4664. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4663, and you would like to upgrade both the Media Cloud and the --- database to be at version 4664, import this SQL file: --- --- psql mediacloud < mediawords-4663-4664.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - -ALTER TABLE story_sentences - RENAME TO story_sentences_nonpartitioned; - -ALTER TABLE story_sentences_nonpartitioned - RENAME COLUMN story_sentences_id TO story_sentences_nonpartitioned_id; - -ALTER INDEX story_sentences_story - RENAME TO story_sentences_nonpartitioned_story; -ALTER INDEX story_sentences_db_row_last_updated - RENAME TO story_sentences_nonpartitioned_db_row_last_updated; -ALTER INDEX story_sentences_sentence_half_md5 - RENAME TO story_sentences_nonpartitioned_sentence_half_md5; -ALTER TRIGGER story_sentences_last_updated_trigger ON story_sentences_nonpartitioned - RENAME TO story_sentences_nonpartitioned_last_updated_trigger; - - --- "Master" table (no indexes, no foreign keys as they'll be ineffective) -CREATE TABLE story_sentences_partitioned ( - story_sentences_partitioned_id BIGSERIAL PRIMARY KEY NOT NULL, - stories_id INT NOT NULL, - sentence_number INT NOT NULL, - sentence TEXT NOT NULL, - media_id INT NOT NULL, - publish_date TIMESTAMP NOT NULL, - - -- Time this row was last updated - db_row_last_updated TIMESTAMP WITH TIME ZONE, - - -- 2- or 3-character ISO 690 language code; empty if unknown, NULL if unset - language VARCHAR(3) NULL, - - -- Set to 'true' for every sentence for which a duplicate sentence was - -- found in a future story (even though that duplicate sentence wasn't - -- added to the table) - -- - -- "We only use is_dup in the topic spidering, but I think it is critical - -- there. It is there because the first time I tried to run a spider on a - -- broadly popular topic, it was unusable because of the amount of - -- irrelevant content. When I dug in, I found that stories were getting - -- included because of matches on boilerplate content that was getting - -- duped out of most stories but not the first time it appeared. So I added - -- the check to remove stories that match on a dup sentence, even if it is - -- the dup sentence, and things cleaned up." - is_dup BOOLEAN NULL -); - -CREATE TRIGGER story_sentences_partitioned_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences_partitioned - FOR EACH ROW EXECUTE PROCEDURE last_updated_trigger(); - - --- Make the partitioned table continue the sequence where the non-partitioned --- table left off -SELECT setval( - pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id'), - (SELECT MAX(story_sentences_nonpartitioned_id) FROM story_sentences_nonpartitioned) -); - - --- Create missing "story_sentences_partitioned" partitions -CREATE OR REPLACE FUNCTION story_sentences_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT stories_create_partitions('story_sentences_partitioned')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_media_id_fkey - FOREIGN KEY (media_id) REFERENCES media (media_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_stories_id_sentence_number - ON ' || partition || ' (stories_id, sentence_number); - - CREATE INDEX ' || partition || '_db_row_last_updated - ON ' || partition || ' (db_row_last_updated); - - CREATE INDEX ' || partition || '_sentence_media_week - ON ' || partition || ' (half_md5(sentence), media_id, week_start_date(publish_date::date)); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - --- Create initial "story_sentences_partitioned" partitions for empty database -SELECT story_sentences_create_partitions(); - - --- View that joins the non-partitioned and partitioned tables while the data is --- being migrated -CREATE OR REPLACE VIEW story_sentences AS - - SELECT * - FROM ( - SELECT - story_sentences_partitioned_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM story_sentences_partitioned - - UNION ALL - - SELECT - story_sentences_nonpartitioned_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM story_sentences_nonpartitioned - - ) AS ss; - - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW story_sentences - ALTER COLUMN story_sentences_id - SET DEFAULT nextval(pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id')) + 1; - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "story_sentences" view -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -DECLARE - target_table_name TEXT; -- partition table name (e.g. "story_sentences_01") - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- All new INSERTs go to partitioned table only - SELECT stories_partition_name( 'story_sentences_partitioned', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- UPDATE on both tables - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - UPDATE story_sentences_nonpartitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- DELETE from both tables - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER story_sentences_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON story_sentences - FOR EACH ROW EXECUTE PROCEDURE story_sentences_view_insert_update_delete(); - - --- Copy a chunk of sentences from a non-partitioned "story_sentences" to a --- partitioned one; call this repeatedly to migrate all the data to the partitioned table -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(chunk_size INT) -RETURNS VOID AS $$ -BEGIN - - RAISE NOTICE 'Copying % rows to the partitioned table...', chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH rows_to_move AS ( - DELETE FROM story_sentences_nonpartitioned - WHERE story_sentences_nonpartitioned_id IN ( - SELECT story_sentences_nonpartitioned_id - FROM story_sentences_nonpartitioned - LIMIT chunk_size - ) - RETURNING story_sentences_nonpartitioned.* - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM rows_to_move; - - RAISE NOTICE 'Done copying % rows to the partitioned table.', chunk_size; - -END; -$$ -LANGUAGE plpgsql; - - --- Migrate a huge chunk of sentences to the partitioned table --- --- This should help upgrade the dev environments by copying *all* sentences in --- small test datasets while not blocking the migration in production due to a --- small chunk size. -SELECT copy_chunk_of_nonpartitioned_sentences_to_partitions(100000); - - -CREATE OR REPLACE FUNCTION create_missing_partitions() RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "stories_tags_map" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_partitioned" table...'; - PERFORM story_sentences_create_partitions(); - - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4664; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4664-4665.sql b/apps/postgresql-server/schema/migrations/mediawords-4664-4665.sql deleted file mode 100644 index f3650df02b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4664-4665.sql +++ /dev/null @@ -1,120 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4664 and 4665. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4664, and you would like to upgrade both the Media Cloud and the --- database to be at version 4665, import this SQL file: --- --- psql mediacloud < mediawords-4664-4665.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(INT); - --- Copy a chunk of story sentences from a non-partitioned "story_sentences" to a --- partitioned one; call this repeatedly to migrate all the data to the partitioned table -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT) -RETURNS VOID AS $$ -BEGIN - - RAISE NOTICE 'Copying sentences of up to % stories to the partitioned table...', story_chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH deleted_rows AS ( - - -- Fetch and delete sentences of selected stories - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id IN ( - - -- Start with fetching a bunch of stories to copy between tables to - -- ensure that all of every story's sentences get copied in a single - -- chunk so that they could get deduplicated - SELECT stories_id - FROM story_sentences_nonpartitioned - LIMIT story_chunk_size - - ) - RETURNING story_sentences_nonpartitioned.* - - ), - - deduplicated_rows AS ( - - -- Deduplicate sentences (nonpartitioned table has weird duplicates) - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - RAISE NOTICE 'Done copying sentences of up to % stories to the partitioned table.', story_chunk_size; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4665; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4665-4666.sql b/apps/postgresql-server/schema/migrations/mediawords-4665-4666.sql deleted file mode 100644 index 99ebfb34c9..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4665-4666.sql +++ /dev/null @@ -1,123 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4665 and 4666. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4665, and you would like to upgrade both the Media Cloud and the --- database to be at version 4666, import this SQL file: --- --- psql mediacloud < mediawords-4665-4666.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Copy a chunk of story sentences from a non-partitioned "story_sentences" to a --- partitioned one; call this repeatedly to migrate all the data to the partitioned table -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT) -RETURNS VOID AS $$ -BEGIN - - RAISE NOTICE 'Copying sentences of up to % stories to the partitioned table...', story_chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH deleted_rows AS ( - - -- Fetch and delete sentences of selected stories - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id IN ( - - -- Start with fetching a bunch of stories to copy between tables to - -- ensure that all of every story's sentences get copied in a single - -- chunk so that they could get deduplicated - SELECT stories_id - FROM story_sentences_nonpartitioned - - -- Follow the insertion order to copy to the oldest (and less busy) - -- partitions first - ORDER BY stories_id - - LIMIT story_chunk_size - - ) - RETURNING story_sentences_nonpartitioned.* - - ), - - deduplicated_rows AS ( - - -- Deduplicate sentences (nonpartitioned table has weird duplicates) - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - RAISE NOTICE 'Done copying sentences of up to % stories to the partitioned table.', story_chunk_size; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4666; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4666-4667.sql b/apps/postgresql-server/schema/migrations/mediawords-4666-4667.sql deleted file mode 100644 index dd831b116b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4666-4667.sql +++ /dev/null @@ -1,123 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4666 and 4667. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4666, and you would like to upgrade both the Media Cloud and the --- database to be at version 4667, import this SQL file: --- --- psql mediacloud < mediawords-4666-4667.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Copy a chunk of story sentences from a non-partitioned "story_sentences" to a --- partitioned one; call this repeatedly to migrate all the data to the partitioned table -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT) -RETURNS VOID AS $$ -BEGIN - - RAISE NOTICE 'Copying sentences of up to % stories to the partitioned table...', story_chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH deleted_rows AS ( - - -- Fetch and delete sentences of selected stories - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id IN ( - - -- Start with fetching a bunch of stories to copy between tables to - -- ensure that all of every story's sentences get copied in a single - -- chunk so that they could get deduplicated - SELECT DISTINCT stories_id - FROM story_sentences_nonpartitioned - - -- Follow the insertion order to copy to the oldest (and less busy) - -- partitions first - ORDER BY stories_id - - LIMIT story_chunk_size - - ) - RETURNING story_sentences_nonpartitioned.* - - ), - - deduplicated_rows AS ( - - -- Deduplicate sentences (nonpartitioned table has weird duplicates) - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - RAISE NOTICE 'Done copying sentences of up to % stories to the partitioned table.', story_chunk_size; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4667; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4667-4668.sql b/apps/postgresql-server/schema/migrations/mediawords-4667-4668.sql deleted file mode 100644 index 2c3348fd41..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4667-4668.sql +++ /dev/null @@ -1,137 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4667 and 4668. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4667, and you would like to upgrade both the Media Cloud and the --- database to be at version 4668, import this SQL file: --- --- psql mediacloud < mediawords-4667-4668.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT) -RETURNS VOID AS $$ -BEGIN - - RAISE NOTICE 'Copying sentences of up to % stories to the partitioned table...', story_chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH deleted_rows AS ( - - -- Fetch and delete sentences of selected stories - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id IN ( - - -- Pick unique story IDs from the returned resultset - SELECT DISTINCT stories_id - FROM ( - - -- "SELECT DISTINCT stories_ID ... ORDER BY stories_id" from - -- the non-partitioned table to copy them to the partitioned - -- one worked fine at first but then got superslow. My guess is - -- that it's because of the index bloat: the oldest story IDs - -- got removed from the table (their tuples were marked as - -- "deleted"), so after a while the database was struggling to - -- get through all the dead rows to get to the next chunk of - -- the live ones. - -- - -- "SELECT DISTINCT" without the "ORDER BY" has a similar - -- effect, probably because it uses the very same index. At - -- least for now, the most effective strategy seems to do a - -- sequential scan with a LIMIT on the table, collect an - -- approximate amount of sentences for the given story count to - -- copy, and then DISTINCT them as a separate step. - SELECT stories_id - FROM story_sentences_nonpartitioned - - -- Assume that a single story has 10 sentences + add some leeway - LIMIT story_chunk_size * 15 - ) AS stories_and_sentences - - ) - RETURNING story_sentences_nonpartitioned.* - - ), - - deduplicated_rows AS ( - - -- Deduplicate sentences: nonpartitioned table has weird duplicates, - -- and the new index insists on (stories_id, sentence_number) - -- uniqueness (which is a logical assumption to make) - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - RAISE NOTICE 'Done copying sentences of up to % stories to the partitioned table.', story_chunk_size; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4668; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4668-4669.sql b/apps/postgresql-server/schema/migrations/mediawords-4668-4669.sql deleted file mode 100644 index bf2d59a488..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4668-4669.sql +++ /dev/null @@ -1,151 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4668 and 4669. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4668, and you would like to upgrade both the Media Cloud and the --- database to be at version 4669, import this SQL file: --- --- psql mediacloud < mediawords-4668-4669.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "story_sentences" view -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- All new INSERTs go to partitioned table only - - -- FIXME restore back to the version that uses stories_partition_name() - - IF (NEW.stories_id >= 0 AND NEW.stories_id < 100000000) THEN - INSERT INTO story_sentences_partitioned_00 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 100000000 AND NEW.stories_id < 200000000) THEN - INSERT INTO story_sentences_partitioned_01 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 200000000 AND NEW.stories_id < 300000000) THEN - INSERT INTO story_sentences_partitioned_02 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 300000000 AND NEW.stories_id < 400000000) THEN - INSERT INTO story_sentences_partitioned_03 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 400000000 AND NEW.stories_id < 500000000) THEN - INSERT INTO story_sentences_partitioned_04 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 500000000 AND NEW.stories_id < 600000000) THEN - INSERT INTO story_sentences_partitioned_05 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 600000000 AND NEW.stories_id < 700000000) THEN - INSERT INTO story_sentences_partitioned_06 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 700000000 AND NEW.stories_id < 800000000) THEN - INSERT INTO story_sentences_partitioned_07 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 800000000 AND NEW.stories_id < 900000000) THEN - INSERT INTO story_sentences_partitioned_08 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 900000000 AND NEW.stories_id < 1000000000) THEN - INSERT INTO story_sentences_partitioned_09 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 1000000000 AND NEW.stories_id < 1100000000) THEN - INSERT INTO story_sentences_partitioned_10 VALUES (NEW.*); - - ELSIF (NEW.stories_id >= 1100000000 AND NEW.stories_id < 1200000000) THEN - INSERT INTO story_sentences_partitioned_11 VALUES (NEW.*); - - ELSE - RAISE EXCEPTION 'stories_id out of range: %', NEW.stories_id; - - END IF; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- UPDATE on both tables - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - UPDATE story_sentences_nonpartitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- DELETE from both tables - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4669; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4669-4670.sql b/apps/postgresql-server/schema/migrations/mediawords-4669-4670.sql deleted file mode 100644 index da74b53549..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4669-4670.sql +++ /dev/null @@ -1,143 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4669 and 4670. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4669, and you would like to upgrade both the Media Cloud and the --- database to be at version 4670, import this SQL file: --- --- psql mediacloud < mediawords-4669-4670.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT) -RETURNS VOID AS $$ - -DECLARE - copied_sentence_count INT; - -BEGIN - - RAISE NOTICE 'Copying sentences of up to % stories to the partitioned table...', story_chunk_size; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - WITH deleted_rows AS ( - - -- Fetch and delete sentences of selected stories - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id IN ( - - -- Pick unique story IDs from the returned resultset - SELECT DISTINCT stories_id - FROM ( - - -- "SELECT DISTINCT stories_ID ... ORDER BY stories_id" from - -- the non-partitioned table to copy them to the partitioned - -- one worked fine at first but then got superslow. My guess is - -- that it's because of the index bloat: the oldest story IDs - -- got removed from the table (their tuples were marked as - -- "deleted"), so after a while the database was struggling to - -- get through all the dead rows to get to the next chunk of - -- the live ones. - -- - -- "SELECT DISTINCT" without the "ORDER BY" has a similar - -- effect, probably because it uses the very same index. At - -- least for now, the most effective strategy seems to do a - -- sequential scan with a LIMIT on the table, collect an - -- approximate amount of sentences for the given story count to - -- copy, and then DISTINCT them as a separate step. - SELECT stories_id - FROM story_sentences_nonpartitioned - - -- Assume that a single story has 10 sentences + add some leeway - LIMIT story_chunk_size * 15 - ) AS stories_and_sentences - - ) - RETURNING story_sentences_nonpartitioned.* - - ), - - deduplicated_rows AS ( - - -- Deduplicate sentences: nonpartitioned table has weird duplicates, - -- and the new index insists on (stories_id, sentence_number) - -- uniqueness (which is a logical assumption to make) - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - - ) - - -- INSERT into view to hit the partitioning trigger - INSERT INTO story_sentences ( - story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - GET DIAGNOSTICS copied_sentence_count = ROW_COUNT; - - RAISE NOTICE 'Copied % sentences to the partitioned table.', copied_sentence_count; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4670; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4670-4671.sql b/apps/postgresql-server/schema/migrations/mediawords-4670-4671.sql deleted file mode 100644 index a96a6ad926..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4670-4671.sql +++ /dev/null @@ -1,116 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4670 and 4671. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4670, and you would like to upgrade both the Media Cloud and the --- database to be at version 4671, import this SQL file: --- --- psql mediacloud < mediawords-4670-4671.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "story_sentences" view -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -DECLARE - target_table_name TEXT; -- partition table name (e.g. "story_sentences_01") - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- All new INSERTs go to partitioned table only - SELECT stories_partition_name( 'story_sentences_partitioned', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- UPDATE on both tables - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - UPDATE story_sentences_nonpartitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- DELETE from both tables - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4671; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4671-4672.sql b/apps/postgresql-server/schema/migrations/mediawords-4671-4672.sql deleted file mode 100644 index 1ca5442e3d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4671-4672.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4671 and 4672. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4671, and you would like to upgrade both the Media Cloud and the --- database to be at version 4672, import this SQL file: --- --- psql mediacloud < mediawords-4671-4672.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TRIGGER IF EXISTS ss_insert_story_media_stats ON story_sentences_nonpartitioned; -DROP TRIGGER IF EXISTS ss_update_story_media_stats ON story_sentences_nonpartitioned; -DROP TRIGGER IF EXISTS story_delete_ss_media_stats ON story_sentences_nonpartitioned; - -DROP FUNCTION IF EXISTS insert_ss_media_stats(); -DROP FUNCTION IF EXISTS update_ss_media_stats(); -DROP FUNCTION IF EXISTS delete_ss_media_stats(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4672; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4672-4673.sql b/apps/postgresql-server/schema/migrations/mediawords-4672-4673.sql deleted file mode 100644 index f0e71078c0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4672-4673.sql +++ /dev/null @@ -1,185 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4672 and 4673. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4672, and you would like to upgrade both the Media Cloud and the --- database to be at version 4673, import this SQL file: --- --- psql mediacloud < mediawords-4672-4673.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(story_chunk_size INT); - --- Copy a chunk of story sentences from a non-partitioned "story_sentences" to a --- partitioned one: --- --- * Expects starting and ending stories_id instead of a chunk size in order to --- avoid index bloat that would happen when copying sentences in sequential --- chunks --- * Copies directly to partitions to skip (slow) INSERT triggers on --- "story_sentences" view --- * Disables all triggers while copying to skip updating db_row_last_updated --- --- Returns number of rows that were copied. --- --- Call this repeatedly to migrate all the data to the partitioned table. -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(start_stories_id INT, end_stories_id INT) -RETURNS INT AS $$ - -DECLARE - copied_sentence_count INT; - - -- Partition table names for both stories_id bounds - start_stories_id_table_name TEXT; - end_stories_id_table_name TEXT; - -BEGIN - - IF NOT (start_stories_id < end_stories_id) THEN - RAISE EXCEPTION '"end_stories_id" must be bigger than "start_stories_id".'; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', start_stories_id) - INTO start_stories_id_table_name; - IF NOT (table_exists(start_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "start_stories_id" = % does not exist.', - start_stories_id_table_name, start_stories_id; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', end_stories_id) - INTO end_stories_id_table_name; - IF NOT (table_exists(end_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "end_stories_id" = % does not exist.', - end_stories_id_table_name, end_stories_id; - END IF; - - IF NOT (start_stories_id_table_name = end_stories_id_table_name) THEN - RAISE EXCEPTION - '"start_stories_id" = % and "end_stories_id" = % must be within the same partition.', - start_stories_id, end_stories_id; - END IF; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - RAISE NOTICE - 'Copying sentences of stories_id BETWEEN % AND % to the partitioned table...', - start_stories_id, end_stories_id; - - -- Disable all triggers to avoid hitting last_updated_trigger() -- the - -- copied rows don't need their db_row_last_updated to be updated - SET session_replication_role = REPLICA; - - BEGIN - - EXECUTE ' - - -- Fetch and delete sentences within bounds - WITH deleted_rows AS ( - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id BETWEEN ' || start_stories_id || ' AND ' || end_stories_id || ' - RETURNING story_sentences_nonpartitioned.* - ), - - -- Deduplicate sentences: nonpartitioned table has weird duplicates, - -- and the new index insists on (stories_id, sentence_number) - -- uniqueness (which is a logical assumption to make) - -- - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want. - deduplicated_rows AS ( - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - ) - - -- INSERT directly into the partition to circumvent slow insertion - -- trigger on "story_sentences" view - INSERT INTO ' || start_stories_id_table_name || ' ( - story_sentences_partitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - '; - - GET DIAGNOSTICS copied_sentence_count = ROW_COUNT; - - EXCEPTION WHEN others THEN - - -- Reenable all triggers - SET session_replication_role = DEFAULT; - - RAISE EXCEPTION '% %', SQLERRM, SQLSTATE; - - END; - - -- Reenable all triggers - SET session_replication_role = DEFAULT; - - RAISE NOTICE - 'Finished copying sentences of stories_id BETWEEN % AND % to the partitioned table, copied % sentences.', - start_stories_id, end_stories_id, copied_sentence_count; - - RETURN copied_sentence_count; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4673; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4673-4674.sql b/apps/postgresql-server/schema/migrations/mediawords-4673-4674.sql deleted file mode 100644 index 4bc4356101..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4673-4674.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4673 and 4674. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4673, and you would like to upgrade both the Media Cloud and the --- database to be at version 4674, import this SQL file: --- --- psql mediacloud < mediawords-4673-4674.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE INDEX auth_users_created_day ON auth_users (date_trunc('day', created_date)); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4674; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4674-4675.sql b/apps/postgresql-server/schema/migrations/mediawords-4674-4675.sql deleted file mode 100644 index 4616703109..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4674-4675.sql +++ /dev/null @@ -1,149 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4674 and 4675. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4674, and you would like to upgrade both the Media Cloud and the --- database to be at version 4675, import this SQL file: --- --- psql mediacloud < mediawords-4674-4675.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4675; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(start_stories_id INT, end_stories_id INT) RETURNS INT AS $$ - -DECLARE - copied_sentence_count INT; - - -- Partition table names for both stories_id bounds - start_stories_id_table_name TEXT; - end_stories_id_table_name TEXT; - -BEGIN - - IF NOT (start_stories_id < end_stories_id) THEN - RAISE EXCEPTION '"end_stories_id" must be bigger than "start_stories_id".'; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', start_stories_id) - INTO start_stories_id_table_name; - IF NOT (table_exists(start_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "start_stories_id" = % does not exist.', - start_stories_id_table_name, start_stories_id; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', end_stories_id) - INTO end_stories_id_table_name; - IF NOT (table_exists(end_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "end_stories_id" = % does not exist.', - end_stories_id_table_name, end_stories_id; - END IF; - - IF NOT (start_stories_id_table_name = end_stories_id_table_name) THEN - RAISE EXCEPTION - '"start_stories_id" = % and "end_stories_id" = % must be within the same partition.', - start_stories_id, end_stories_id; - END IF; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - RAISE NOTICE - 'Copying sentences of stories_id BETWEEN % AND % to the partitioned table...', - start_stories_id, end_stories_id; - - EXECUTE ' - - -- Fetch and delete sentences within bounds - WITH deleted_rows AS ( - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id BETWEEN ' || start_stories_id || ' AND ' || end_stories_id || ' - RETURNING story_sentences_nonpartitioned.* - ), - - -- Deduplicate sentences: nonpartitioned table has weird duplicates, - -- and the new index insists on (stories_id, sentence_number) - -- uniqueness (which is a logical assumption to make) - -- - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want. - deduplicated_rows AS ( - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - ) - - -- INSERT directly into the partition to circumvent slow insertion - -- trigger on "story_sentences" view - INSERT INTO ' || start_stories_id_table_name || ' ( - story_sentences_partitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - db_row_last_updated, - language, - is_dup - FROM deduplicated_rows; - - '; - - GET DIAGNOSTICS copied_sentence_count = ROW_COUNT; - - RAISE NOTICE - 'Finished copying sentences of stories_id BETWEEN % AND % to the partitioned table, copied % sentences.', - start_stories_id, end_stories_id, copied_sentence_count; - - RETURN copied_sentence_count; - -END; -$$ -LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4675-4676.sql b/apps/postgresql-server/schema/migrations/mediawords-4675-4676.sql deleted file mode 100644 index 892e2dcd94..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4675-4676.sql +++ /dev/null @@ -1,146 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4675 and 4676. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4675, and you would like to upgrade both the Media Cloud and the --- database to be at version 4676, import this SQL file: --- --- psql mediacloud < mediawords-4675-4676.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TRIGGER story_sentences_partitioned_last_updated_trigger ON story_sentences_partitioned; - - -CREATE TRIGGER story_sentences_partitioned_00_last_updated_trigger - BEFORE INSERT OR UPDATE ON story_sentences_partitioned - FOR EACH ROW - EXECUTE PROCEDURE last_updated_trigger(); - - --- Note: "INSERT ... RETURNING *" doesn't work with the trigger, please use --- "story_sentences" view instead -CREATE OR REPLACE FUNCTION story_sentences_partitioned_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT stories_partition_name('story_sentences_partitioned', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER story_sentences_partitioned_01_insert_trigger - BEFORE INSERT ON story_sentences_partitioned - FOR EACH ROW - EXECUTE PROCEDURE story_sentences_partitioned_insert_trigger(); - - -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -DECLARE - target_table_name TEXT; -- partition table name (e.g. "story_sentences_01") - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- All new INSERTs go to partitioned table only. - -- - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition and update db_row_last_updated. - INSERT INTO story_sentences_partitioned SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- UPDATE on both tables - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - UPDATE story_sentences_nonpartitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - db_row_last_updated = NEW.db_row_last_updated, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- DELETE from both tables - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4676; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4676-4677.sql b/apps/postgresql-server/schema/migrations/mediawords-4676-4677.sql deleted file mode 100644 index 6d4b6ce799..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4676-4677.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4676 and 4677. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4676, and you would like to upgrade both the Media Cloud and the --- database to be at version 4677, import this SQL file: --- --- psql mediacloud < mediawords-4676-4677.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE IF EXISTS stories_without_readability_tag; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4677; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4677-4678.sql b/apps/postgresql-server/schema/migrations/mediawords-4677-4678.sql deleted file mode 100644 index 34fb4997a6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4677-4678.sql +++ /dev/null @@ -1,84 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4677 and 4678. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4677, and you would like to upgrade both the Media Cloud and the --- database to be at version 4678, import this SQL file: --- --- psql mediacloud < mediawords-4677-4678.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- To be recreated later -DROP VIEW media_with_collections; -DROP VIEW media_with_media_types; - -DROP INDEX media_moderated; - -ALTER TABLE media - DROP COLUMN moderated, - DROP COLUMN moderation_notes; - -ALTER TABLE snap.media - DROP COLUMN moderated, - DROP COLUMN moderation_notes; - -CREATE VIEW media_with_collections AS - SELECT t.tag, - m.media_id, - m.url, - m.name, - m.full_text_rss - FROM media m, - tags t, - tag_sets ts, - media_tags_map mtm - WHERE ts.name::text = 'collection'::text - AND ts.tag_sets_id = t.tag_sets_id - AND mtm.tags_id = t.tags_id - AND mtm.media_id = m.media_id - ORDER BY m.media_id; - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4678; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4678-4679.sql b/apps/postgresql-server/schema/migrations/mediawords-4678-4679.sql deleted file mode 100644 index 899fd2585e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4678-4679.sql +++ /dev/null @@ -1,329 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4678 and 4679. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4678, and you would like to upgrade both the Media Cloud and the --- database to be at version 4679, import this SQL file: --- --- psql mediacloud < mediawords-4678-4679.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- To be recreated later -DROP VIEW feedly_unscraped_feeds; - --- NULLs allowed and no DEFAULT value to minimize AccessExclusiveLock duration -ALTER TABLE feeds ADD COLUMN active BOOLEAN NULL; -ALTER TABLE feeds_from_yesterday ADD COLUMN active BOOLEAN NULL; - --- Treat all feeds with "inactive" / "skipped" feed_status as inactive -UPDATE feeds SET active = CASE WHEN feed_status = 'active' THEN true ELSE false END; -UPDATE feeds_from_yesterday SET active = CASE WHEN feed_status = 'active' THEN true ELSE false END; - --- Readd NOT NULL -ALTER TABLE feeds ALTER COLUMN active SET NOT NULL; -ALTER TABLE feeds_from_yesterday ALTER COLUMN active SET NOT NULL; - --- Readd DEFAULT value ("feeds_from_yesterday" doesn't have it because it stores rows copied from "feeds") -ALTER TABLE feeds ALTER COLUMN active SET DEFAULT 't'; - --- DROP feed_status -ALTER TABLE feeds DROP COLUMN feed_status; -ALTER TABLE feeds_from_yesterday DROP COLUMN feed_status; - --- DROP the now-unused type -DROP TYPE feed_feed_status; - -CREATE OR REPLACE FUNCTION media_has_active_syndicated_feeds(param_media_id INT) RETURNS boolean AS $$ -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - - SELECT 1 - FROM media - WHERE media_id = param_media_id - - ) THEN - RAISE EXCEPTION 'Media % does not exist.', param_media_id; - RETURN FALSE; - END IF; - - -- Check if media has feeds - IF EXISTS ( - - SELECT 1 - FROM feeds - WHERE media_id = param_media_id - AND active = 't' - - -- Website might introduce RSS feeds later - AND feed_type = 'syndicated' - - ) THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_feeds_from_yesterday() RETURNS VOID AS $$ -BEGIN - - DELETE FROM feeds_from_yesterday; - INSERT INTO feeds_from_yesterday (feeds_id, media_id, name, url, feed_type, active) - SELECT feeds_id, media_id, name, url, feed_type, active - FROM feeds; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION rescraping_changes() RETURNS VOID AS -$$ -DECLARE - r_count RECORD; - r_media RECORD; - r_feed RECORD; -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - SELECT 1 - FROM feeds_from_yesterday - ) THEN - RAISE EXCEPTION '"feeds_from_yesterday" table is empty.'; - END IF; - - -- Fill temp. tables with changes to print out later - CREATE TEMPORARY TABLE rescraping_changes_media ON COMMIT DROP AS - SELECT * - FROM media - WHERE media_id IN ( - SELECT DISTINCT media_id - FROM ( - -- Don't compare "name" because it's insignificant - ( - SELECT feeds_id, media_id, feed_type, active, url FROM feeds_from_yesterday - EXCEPT - SELECT feeds_id, media_id, feed_type, active, url FROM feeds - ) UNION ALL ( - SELECT feeds_id, media_id, feed_type, active, url FROM feeds - EXCEPT - SELECT feeds_id, media_id, feed_type, active, url FROM feeds_from_yesterday - ) - ) AS modified_feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_added ON COMMIT DROP AS - SELECT * - FROM feeds - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds_from_yesterday - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_deleted ON COMMIT DROP AS - SELECT * - FROM feeds_from_yesterday - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_modified ON COMMIT DROP AS - SELECT feeds_before.media_id, - feeds_before.feeds_id, - - feeds_before.name AS before_name, - feeds_before.url AS before_url, - feeds_before.feed_type AS before_feed_type, - feeds_before.active AS before_active, - - feeds_after.name AS after_name, - feeds_after.url AS after_url, - feeds_after.feed_type AS after_feed_type, - feeds_after.active AS after_active - - FROM feeds_from_yesterday AS feeds_before - INNER JOIN feeds AS feeds_after ON ( - feeds_before.feeds_id = feeds_after.feeds_id - AND ( - -- Don't compare "name" because it's insignificant - feeds_before.url != feeds_after.url - OR feeds_before.feed_type != feeds_after.feed_type - OR feeds_before.active != feeds_after.active - ) - ) - - WHERE feeds_before.media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ); - - -- Print out changes - RAISE NOTICE 'Changes between "feeds" and "feeds_from_yesterday":'; - RAISE NOTICE ''; - - SELECT COUNT(1) AS media_count INTO r_count FROM rescraping_changes_media; - RAISE NOTICE '* Modified media: %', r_count.media_count; - SELECT COUNT(1) AS feeds_added_count INTO r_count FROM rescraping_changes_feeds_added; - RAISE NOTICE '* Added feeds: %', r_count.feeds_added_count; - SELECT COUNT(1) AS feeds_deleted_count INTO r_count FROM rescraping_changes_feeds_deleted; - RAISE NOTICE '* Deleted feeds: %', r_count.feeds_deleted_count; - SELECT COUNT(1) AS feeds_modified_count INTO r_count FROM rescraping_changes_feeds_modified; - RAISE NOTICE '* Modified feeds: %', r_count.feeds_modified_count; - RAISE NOTICE ''; - - FOR r_media IN - SELECT *, - - -- Prioritize US MSM media - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND tag_sets.name = 'collection' - AND tags.tag = 'ap_english_us_top25_20100110' - ) AS belongs_to_us_msm, - - -- Prioritize media with "show_on_media" - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND ( - tag_sets.show_on_media - OR tags.show_on_media - ) - ) AS show_on_media - - FROM rescraping_changes_media - - ORDER BY belongs_to_us_msm DESC, - show_on_media DESC, - media_id - LOOP - RAISE NOTICE 'MODIFIED media: media_id=%, name="%", url="%"', - r_media.media_id, - r_media.name, - r_media.url; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_added - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' ADDED feed: feeds_id=%, feed_type=%, active=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.active, - r_feed.name, - r_feed.url; - END LOOP; - - -- Feeds shouldn't get deleted but we're checking anyways - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_deleted - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' DELETED feed: feeds_id=%, feed_type=%, active=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.feed_type, - r_feed.active, - r_feed.name, - r_feed.url; - END LOOP; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_modified - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' MODIFIED feed: feeds_id=%', r_feed.feeds_id; - RAISE NOTICE ' BEFORE: feed_type=%, active=%, name="%", url="%"', - r_feed.before_feed_type, - r_feed.before_active, - r_feed.before_name, - r_feed.before_url; - RAISE NOTICE ' AFTER: feed_type=%, active=%, name="%", url="%"', - r_feed.after_feed_type, - r_feed.after_active, - r_feed.after_name, - r_feed.after_url; - END LOOP; - - RAISE NOTICE ''; - - END LOOP; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE VIEW feedly_unscraped_feeds AS - select f.* - from feeds f - left join scraped_feeds sf on - ( f.feeds_id = sf.feeds_id and sf.import_module = 'MediaWords::ImportStories::Feedly' ) - where - f.feed_type = 'syndicated' and - f.active = 't' and - sf.feeds_id is null; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4679; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4679-4680.sql b/apps/postgresql-server/schema/migrations/mediawords-4679-4680.sql deleted file mode 100644 index f974bc7b80..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4679-4680.sql +++ /dev/null @@ -1,311 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4679 and 4680. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4679, and you would like to upgrade both the Media Cloud and the --- database to be at version 4680, import this SQL file: --- --- psql mediacloud < mediawords-4679-4680.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- To be recreated later -DROP VIEW feedly_unscraped_feeds; - -ALTER TYPE feed_feed_type RENAME TO feed_type; -ALTER TABLE feeds RENAME COLUMN feed_type TO type; -ALTER TABLE feeds_from_yesterday RENAME COLUMN feed_type TO type; - -CREATE OR REPLACE FUNCTION media_has_active_syndicated_feeds(param_media_id INT) RETURNS boolean AS $$ -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - - SELECT 1 - FROM media - WHERE media_id = param_media_id - - ) THEN - RAISE EXCEPTION 'Media % does not exist.', param_media_id; - RETURN FALSE; - END IF; - - -- Check if media has feeds - IF EXISTS ( - - SELECT 1 - FROM feeds - WHERE media_id = param_media_id - AND active = 't' - - -- Website might introduce RSS feeds later - AND "type" = 'syndicated' - - ) THEN - RETURN TRUE; - ELSE - RETURN FALSE; - END IF; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION update_feeds_from_yesterday() RETURNS VOID AS $$ -BEGIN - - DELETE FROM feeds_from_yesterday; - INSERT INTO feeds_from_yesterday (feeds_id, media_id, name, url, type, active) - SELECT feeds_id, media_id, name, url, type, active - FROM feeds; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE OR REPLACE FUNCTION rescraping_changes() RETURNS VOID AS -$$ -DECLARE - r_count RECORD; - r_media RECORD; - r_feed RECORD; -BEGIN - - -- Check if media exists - IF NOT EXISTS ( - SELECT 1 - FROM feeds_from_yesterday - ) THEN - RAISE EXCEPTION '"feeds_from_yesterday" table is empty.'; - END IF; - - -- Fill temp. tables with changes to print out later - CREATE TEMPORARY TABLE rescraping_changes_media ON COMMIT DROP AS - SELECT * - FROM media - WHERE media_id IN ( - SELECT DISTINCT media_id - FROM ( - -- Don't compare "name" because it's insignificant - ( - SELECT feeds_id, media_id, type, active, url FROM feeds_from_yesterday - EXCEPT - SELECT feeds_id, media_id, type, active, url FROM feeds - ) UNION ALL ( - SELECT feeds_id, media_id, type, active, url FROM feeds - EXCEPT - SELECT feeds_id, media_id, type, active, url FROM feeds_from_yesterday - ) - ) AS modified_feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_added ON COMMIT DROP AS - SELECT * - FROM feeds - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds_from_yesterday - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_deleted ON COMMIT DROP AS - SELECT * - FROM feeds_from_yesterday - WHERE media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ) - AND feeds_id NOT IN ( - SELECT feeds_id - FROM feeds - ); - - CREATE TEMPORARY TABLE rescraping_changes_feeds_modified ON COMMIT DROP AS - SELECT feeds_before.media_id, - feeds_before.feeds_id, - - feeds_before.name AS before_name, - feeds_before.url AS before_url, - feeds_before.type AS before_type, - feeds_before.active AS before_active, - - feeds_after.name AS after_name, - feeds_after.url AS after_url, - feeds_after.type AS after_type, - feeds_after.active AS after_active - - FROM feeds_from_yesterday AS feeds_before - INNER JOIN feeds AS feeds_after ON ( - feeds_before.feeds_id = feeds_after.feeds_id - AND ( - -- Don't compare "name" because it's insignificant - feeds_before.url != feeds_after.url - OR feeds_before.type != feeds_after.type - OR feeds_before.active != feeds_after.active - ) - ) - - WHERE feeds_before.media_id IN ( - SELECT media_id - FROM rescraping_changes_media - ); - - -- Print out changes - RAISE NOTICE 'Changes between "feeds" and "feeds_from_yesterday":'; - RAISE NOTICE ''; - - SELECT COUNT(1) AS media_count INTO r_count FROM rescraping_changes_media; - RAISE NOTICE '* Modified media: %', r_count.media_count; - SELECT COUNT(1) AS feeds_added_count INTO r_count FROM rescraping_changes_feeds_added; - RAISE NOTICE '* Added feeds: %', r_count.feeds_added_count; - SELECT COUNT(1) AS feeds_deleted_count INTO r_count FROM rescraping_changes_feeds_deleted; - RAISE NOTICE '* Deleted feeds: %', r_count.feeds_deleted_count; - SELECT COUNT(1) AS feeds_modified_count INTO r_count FROM rescraping_changes_feeds_modified; - RAISE NOTICE '* Modified feeds: %', r_count.feeds_modified_count; - RAISE NOTICE ''; - - FOR r_media IN - SELECT *, - - -- Prioritize US MSM media - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND tag_sets.name = 'collection' - AND tags.tag = 'ap_english_us_top25_20100110' - ) AS belongs_to_us_msm, - - -- Prioritize media with "show_on_media" - EXISTS ( - SELECT 1 - FROM tags AS tags - INNER JOIN media_tags_map - ON tags.tags_id = media_tags_map.tags_id - INNER JOIN tag_sets - ON tags.tag_sets_id = tag_sets.tag_sets_id - WHERE media_tags_map.media_id = rescraping_changes_media.media_id - AND ( - tag_sets.show_on_media - OR tags.show_on_media - ) - ) AS show_on_media - - FROM rescraping_changes_media - - ORDER BY belongs_to_us_msm DESC, - show_on_media DESC, - media_id - LOOP - RAISE NOTICE 'MODIFIED media: media_id=%, name="%", url="%"', - r_media.media_id, - r_media.name, - r_media.url; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_added - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' ADDED feed: feeds_id=%, type=%, active=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.type, - r_feed.active, - r_feed.name, - r_feed.url; - END LOOP; - - -- Feeds shouldn't get deleted but we're checking anyways - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_deleted - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' DELETED feed: feeds_id=%, type=%, active=%, name="%", url="%"', - r_feed.feeds_id, - r_feed.type, - r_feed.active, - r_feed.name, - r_feed.url; - END LOOP; - - FOR r_feed IN - SELECT * - FROM rescraping_changes_feeds_modified - WHERE media_id = r_media.media_id - ORDER BY feeds_id - LOOP - RAISE NOTICE ' MODIFIED feed: feeds_id=%', r_feed.feeds_id; - RAISE NOTICE ' BEFORE: type=%, active=%, name="%", url="%"', - r_feed.before_type, - r_feed.before_active, - r_feed.before_name, - r_feed.before_url; - RAISE NOTICE ' AFTER: type=%, active=%, name="%", url="%"', - r_feed.after_type, - r_feed.after_active, - r_feed.after_name, - r_feed.after_url; - END LOOP; - - RAISE NOTICE ''; - - END LOOP; - -END; -$$ -LANGUAGE 'plpgsql'; - -CREATE VIEW feedly_unscraped_feeds AS - select f.* - from feeds f - left join scraped_feeds sf on - ( f.feeds_id = sf.feeds_id and sf.import_module = 'MediaWords::ImportStories::Feedly' ) - where - f.type = 'syndicated' and - f.active = 't' and - sf.feeds_id is null; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4680; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4680-4681.sql b/apps/postgresql-server/schema/migrations/mediawords-4680-4681.sql deleted file mode 100644 index e1f5a764b3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4680-4681.sql +++ /dev/null @@ -1,184 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4680 and 4681. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4680, and you would like to upgrade both the Media Cloud and the --- database to be at version 4681, import this SQL file: --- --- psql mediacloud < mediawords-4680-4681.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- To be recreated later -DROP VIEW media_with_media_types; -DROP VIEW feedly_unscraped_feeds; - - -DROP VIEW media_with_collections; -DROP VIEW downloads_sites; - -DROP INDEX media_name_trgm; -DROP INDEX media_url_trgm; -DROP INDEX feeds_reparse; -DROP INDEX downloads_in_old_format; -DROP INDEX downloads_sites_pending; -DROP INDEX downloads_sites_downloads_id_pending; - -DROP TABLE media_rss_full_text_detection_data; - -ALTER TABLE media DROP COLUMN last_solr_import_date; -ALTER TABLE feeds DROP COLUMN reparse; - -DROP FUNCTION site_from_host(varchar); - - --- Remove obsolete tags -DELETE FROM tags -WHERE (tag, label, description) IN ( - ( - 'Not Typed', - 'Not Typed', - 'The medium has not yet been typed.' - ), - ( - 'Other', - 'Other', - 'The medium does not fit in any listed type.' - ), - ( - 'Independent Group', - 'Ind. Group', - - -- Single multiline string - 'An academic or nonprofit group that is not affiliated with the private sector or government, ' - 'such as the Electronic Frontier Foundation or the Center for Democracy and Technology)' - ), - ( - 'Social Linking Site', - 'Social Linking', - - -- Single multiline string - 'A site that aggregates links based at least partially on user submissions and/or ranking, ' - 'such as Reddit, Digg, Slashdot, MetaFilter, StumbleUpon, and other social news sites' - ), - ( - 'Blog', - 'Blog', - - -- Single multiline string - 'A web log, written by one or more individuals, that is not associated with a professional ' - 'or advocacy organization or institution' - ), - ( - 'General Online News Media', - 'General News', - - -- Single multiline string - 'A site that is a mainstream media outlet, such as The New York Times and The Washington Post; ' - 'an online-only news outlet, such as Slate, Salon, or the Huffington Post; ' - 'or a citizen journalism or non-profit news outlet, such as Global Voices or ProPublica' - ), - ( - 'Issue Specific Campaign', - 'Issue', - 'A site specifically dedicated to campaigning for or against a single issue.' - ), - ( - 'News Aggregator', - 'News Agg.', - - -- Single multiline string - 'A site that contains little to no original content and compiles news from other sites, ' - 'such as Yahoo News or Google News' - ), - ( - 'Tech Media', - 'Tech Media', - - -- Single multiline string - 'A site that focuses on technological news and information produced by a news organization, ' - 'such as Arstechnica, Techdirt, or Wired.com' - ), - ( - 'Private Sector', - 'Private Sec.', - - -- Single multiline string - 'A non-news media for-profit actor, including, for instance, trade organizations, industry ' - 'sites, and domain registrars' - ), - ( - 'Government', - 'Government', - - -- Single multiline string - 'A site associated with and run by a government-affiliated entity, such as the DOJ website, ' - 'White House blog, or a U.S. Senator official website' - ), - ( - 'User-Generated Content Platform', - 'User Gen.', - - -- Single multiline string - 'A general communication and networking platform or tool, like Wikipedia, YouTube, Twitter, ' - 'and Scribd, or a search engine like Google or speech platform like the Daily Kos' - ) -); - -DELETE FROM tags WHERE tag_sets_id IN (SELECT tag_sets_id FROM tag_sets WHERE name = 'media_type'); -DELETE FROM tag_sets WHERE name = 'media_type'; - - -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - -create view feedly_unscraped_feeds as - select f.* - from feeds f - left join scraped_feeds sf on - ( f.feeds_id = sf.feeds_id and sf.import_module = 'MediaWords::ImportStories::Feedly' ) - where - f.type = 'syndicated' and - f.active = 't' and - sf.feeds_id is null; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4681; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4681-4682.sql b/apps/postgresql-server/schema/migrations/mediawords-4681-4682.sql deleted file mode 100644 index d37c77cc9f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4681-4682.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4681 and 4682. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4681, and you would like to upgrade both the Media Cloud and the --- database to be at version 4682, import this SQL file: --- --- psql mediacloud < mediawords-4681-4682.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION IF EXISTS find_corrupted_sequences(); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4682; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4682-4683.sql b/apps/postgresql-server/schema/migrations/mediawords-4682-4683.sql deleted file mode 100644 index cdc38dc28a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4682-4683.sql +++ /dev/null @@ -1,473 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4682 and 4683. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4682, and you would like to upgrade both the Media Cloud and the --- database to be at version 4683, import this SQL file: --- --- psql mediacloud < mediawords-4682-4683.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop trigger mtm_last_updated on media_tags_map; -drop trigger stories_last_updated_trigger on stories; -drop trigger stories_update_story_sentences_last_updated_trigger on stories; -drop trigger stories_tags_map_last_updated_trigger on stories_tags_map; -drop trigger stories_tags_map_update_stories_last_updated_trigger on stories_tags_map; -drop trigger story_sentences_nonpartitioned_last_updated_trigger on story_sentences_nonpartitioned; -drop trigger story_sentences_partitioned_00_last_updated_trigger on story_sentences_partitioned; -drop trigger processed_stories_update_stories_last_updated_trigger on processed_stories; -drop trigger update_media_db_row_last_updated on media; - -drop function before_last_solr_import(timestamp with time zone); -drop function last_updated_trigger(); -drop function update_story_sentences_updated_time_trigger(); -drop function update_stories_updated_time_by_stories_id_trigger(); - -alter table media add normalized_url varchar(1024) null; - -drop view media_with_media_types; -alter table media drop column db_row_last_updated; -create view media_with_media_types as - select m.*, mtm.tags_id media_type_tags_id, t.label media_type - from - media m - left join ( - tags t - join tag_sets ts on ( ts.tag_sets_id = t.tag_sets_id and ts.name = 'media_type' ) - join media_tags_map mtm on ( mtm.tags_id = t.tags_id ) - ) on ( m.media_id = mtm.media_id ); - -create index media_normalized_url on media(normalized_url); - -drop function update_media_db_row_last_updated(); -drop table media_normalized_urls; -drop table media_update_time_queue; - -drop view daily_stats; -drop view stories_collected_in_past_day; - -alter table stories drop column db_row_last_updated; - - -CREATE VIEW stories_collected_in_past_day AS - SELECT * - FROM stories - WHERE collect_date > now() - interval '1 day'; - -CREATE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - - -create function insert_solr_import_story() returns trigger as $insert_solr_import_story$ -DECLARE - - queue_stories_id INT; - -BEGIN - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - select NEW.stories_id into queue_stories_id; - ELSE - select OLD.stories_id into queue_stories_id; - END IF; - - insert into solr_import_stories ( stories_id ) - select queue_stories_id - where exists ( - select 1 from processed_stories where stories_id = queue_stories_id - ); - - IF ( TG_OP = 'UPDATE' ) OR (TG_OP = 'INSERT') THEN - RETURN NEW; - ELSE - RETURN OLD; - END IF; - -END; - -$insert_solr_import_story$ LANGUAGE plpgsql; - -create trigger stories_insert_solr_import_story after insert or update or delete - on stories for each row execute procedure insert_solr_import_story(); - -alter table stories_tags_map drop column db_row_last_updated; - -create trigger stm_insert_solr_import_story before insert or update or delete - on stories_tags_map for each row execute procedure insert_solr_import_story(); - -drop view story_sentences; - -alter table story_sentences_partitioned drop column db_row_last_updated; - --- recreate this function without db_row_last_updated function -CREATE OR REPLACE FUNCTION story_sentences_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT stories_create_partitions('story_sentences_partitioned')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_media_id_fkey - FOREIGN KEY (media_id) REFERENCES media (media_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_stories_id_sentence_number - ON ' || partition || ' (stories_id, sentence_number); - - CREATE INDEX ' || partition || '_sentence_media_week - ON ' || partition || ' (half_md5(sentence), media_id, week_start_date(publish_date::date)); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE FUNCTION edit_ss() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT stories_create_partitions('story_sentences_partitioned')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE 'DROP INDEX ' || partition || '_db_row_last_updated;'; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - -select edit_ss(); - -drop function edit_ss(); - --- replace with version without db_row_last_updated -CREATE OR REPLACE VIEW story_sentences AS - - SELECT * - FROM ( - SELECT - story_sentences_partitioned_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - FROM story_sentences_partitioned - - UNION ALL - - SELECT - story_sentences_nonpartitioned_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - FROM story_sentences_nonpartitioned - - ) AS ss; - --- replace with version without db_row_last_updated -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -DECLARE - target_table_name TEXT; -- partition table name (e.g. "story_sentences_01") - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- All new INSERTs go to partitioned table only. - -- - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO story_sentences_partitioned SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- UPDATE on both tables - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - UPDATE story_sentences_nonpartitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- DELETE from both tables - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER story_sentences_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON story_sentences - FOR EACH ROW EXECUTE PROCEDURE story_sentences_view_insert_update_delete(); - -ALTER VIEW story_sentences - ALTER COLUMN story_sentences_id - SET DEFAULT nextval(pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id')) + 1; - -CREATE OR REPLACE FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(start_stories_id INT, end_stories_id INT) -RETURNS INT AS $$ - -DECLARE - copied_sentence_count INT; - - -- Partition table names for both stories_id bounds - start_stories_id_table_name TEXT; - end_stories_id_table_name TEXT; - -BEGIN - - IF NOT (start_stories_id < end_stories_id) THEN - RAISE EXCEPTION '"end_stories_id" must be bigger than "start_stories_id".'; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', start_stories_id) - INTO start_stories_id_table_name; - IF NOT (table_exists(start_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "start_stories_id" = % does not exist.', - start_stories_id_table_name, start_stories_id; - END IF; - - SELECT stories_partition_name('story_sentences_partitioned', end_stories_id) - INTO end_stories_id_table_name; - IF NOT (table_exists(end_stories_id_table_name)) THEN - RAISE EXCEPTION - 'Table "%" for "end_stories_id" = % does not exist.', - end_stories_id_table_name, end_stories_id; - END IF; - - IF NOT (start_stories_id_table_name = end_stories_id_table_name) THEN - RAISE EXCEPTION - '"start_stories_id" = % and "end_stories_id" = % must be within the same partition.', - start_stories_id, end_stories_id; - END IF; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - RAISE NOTICE - 'Copying sentences of stories_id BETWEEN % AND % to the partitioned table...', - start_stories_id, end_stories_id; - - EXECUTE ' - - -- Fetch and delete sentences within bounds - WITH deleted_rows AS ( - DELETE FROM story_sentences_nonpartitioned - WHERE stories_id BETWEEN ' || start_stories_id || ' AND ' || end_stories_id || ' - RETURNING story_sentences_nonpartitioned.* - ), - - -- Deduplicate sentences: nonpartitioned table has weird duplicates, - -- and the new index insists on (stories_id, sentence_number) - -- uniqueness (which is a logical assumption to make) - -- - -- Assume that the sentence with the biggest story_sentences_id is the - -- newest one and so is the one that we want. - deduplicated_rows AS ( - SELECT DISTINCT ON (stories_id, sentence_number) * - FROM deleted_rows - ORDER BY stories_id, sentence_number, story_sentences_nonpartitioned_id DESC - ) - - -- INSERT directly into the partition to circumvent slow insertion - -- trigger on "story_sentences" view - INSERT INTO ' || start_stories_id_table_name || ' ( - story_sentences_partitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - ) - SELECT - story_sentences_nonpartitioned_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - FROM deduplicated_rows; - - '; - - GET DIAGNOSTICS copied_sentence_count = ROW_COUNT; - - RAISE NOTICE - 'Finished copying sentences of stories_id BETWEEN % AND % to the partitioned table, copied % sentences.', - start_stories_id, end_stories_id, copied_sentence_count; - - RETURN copied_sentence_count; - -END; -$$ -LANGUAGE plpgsql; - -alter table solr_import_extra_stories rename to solr_import_stories; - -alter table snap.live_stories drop column db_row_last_updated; - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into snap.live_stories - ( topics_id, topic_stories_id, stories_id, media_id, url, guid, title, description, - publish_date, collect_date, full_text_rss, language ) - select NEW.topics_id, NEW.topic_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.description, s.publish_date, s.collect_date, s.full_text_rss, s.language - from topic_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.topics_id = NEW.topics_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update snap.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - - -create trigger ps_insert_solr_import_story after insert or update or delete - on processed_stories for each row execute procedure insert_solr_import_story(); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4683; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4683-4684.sql b/apps/postgresql-server/schema/migrations/mediawords-4683-4684.sql deleted file mode 100644 index 41c7522a62..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4683-4684.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4683 and 4684. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4683, and you would like to upgrade both the Media Cloud and the --- database to be at version 4684, import this SQL file: --- --- psql mediacloud < mediawords-4683-4684.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE media DROP COLUMN IF EXISTS db_row_last_updated; -ALTER TABLE story_sentences_nonpartitioned DROP COLUMN IF EXISTS db_row_last_updated; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4684; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4684-4685.sql b/apps/postgresql-server/schema/migrations/mediawords-4684-4685.sql deleted file mode 100644 index 14e542353b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4684-4685.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4684 and 4685. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4684, and you would like to upgrade both the Media Cloud and the --- database to be at version 4685, import this SQL file: --- --- psql mediacloud < mediawords-4684-4685.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE feeds_after_rescraping - RENAME COLUMN feed_type TO type; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4685; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4685-4686.sql b/apps/postgresql-server/schema/migrations/mediawords-4685-4686.sql deleted file mode 100644 index 7474ef8481..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4685-4686.sql +++ /dev/null @@ -1,162 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4685 and 4686. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4685, and you would like to upgrade both the Media Cloud and the --- database to be at version 4686, import this SQL file: --- --- psql mediacloud < mediawords-4685-4686.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Will recreate later -DROP VIEW feedly_unscraped_feeds; - - -DROP TABLE stories_superglue_metadata; - - -CREATE TEMPORARY TABLE temp_superglue_feeds AS - SELECT feeds_id - FROM feeds - WHERE type = 'superglue'; - -CREATE TEMPORARY TABLE temp_superglue_stories AS - SELECT stories_id - FROM feeds_stories_map - WHERE feeds_id IN (SELECT feeds_id FROM temp_superglue_feeds); - --- Faster deletes and foreign key matching -CREATE INDEX retweeter_stories_stories_id ON retweeter_stories (stories_id); - -DELETE FROM cliff_annotations -WHERE object_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM nytlabels_annotations -WHERE object_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM topic_links -WHERE ref_stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM topic_merged_stories_map -WHERE source_stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM topic_merged_stories_map -WHERE target_stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM topic_seed_urls -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM topic_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM snap.live_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM processed_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM retweeter_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM scraped_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM solr_import_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM solr_imported_stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM stories_ap_syndicated -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM stories_tags_map -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM story_sentences -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM story_statistics -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM story_statistics_twitter -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM stories -WHERE stories_id IN (SELECT stories_id FROM temp_superglue_stories); - -DELETE FROM downloads -WHERE feeds_id IN (SELECT feeds_id FROM temp_superglue_feeds); - -DELETE FROM feeds -WHERE feeds_id IN (SELECT feeds_id FROM temp_superglue_feeds); - --- No longer need it -DROP INDEX retweeter_stories_stories_id; - -DROP TABLE temp_superglue_stories; -DROP TABLE temp_superglue_feeds; - - --- Only way to get rid of valid enum type value is to recreate the enum -CREATE TYPE feed_type_new AS ENUM ('syndicated', 'web_page', 'univision'); - -ALTER TABLE feeds ALTER COLUMN type DROP DEFAULT; -ALTER TABLE feeds ALTER COLUMN type TYPE feed_type_new USING (type::text::feed_type_new); -ALTER TABLE feeds ALTER COLUMN type SET DEFAULT 'syndicated'; - -ALTER TABLE feeds_after_rescraping ALTER COLUMN type DROP DEFAULT; -ALTER TABLE feeds_after_rescraping ALTER COLUMN type TYPE feed_type_new USING (type::text::feed_type_new); -ALTER TABLE feeds_after_rescraping ALTER COLUMN type SET DEFAULT 'syndicated'; - -ALTER TABLE feeds_from_yesterday ALTER COLUMN type DROP DEFAULT; -ALTER TABLE feeds_from_yesterday ALTER COLUMN type TYPE feed_type_new USING (type::text::feed_type_new); -ALTER TABLE feeds_from_yesterday ALTER COLUMN type SET DEFAULT 'syndicated'; - -DROP TYPE feed_type; -ALTER TYPE feed_type_new RENAME TO feed_type; - - -CREATE VIEW feedly_unscraped_feeds AS - SELECT f.* - FROM feeds AS f - LEFT JOIN scraped_feeds AS sf - ON f.feeds_id = sf.feeds_id - AND sf.import_module = 'MediaWords::ImportStories::Feedly' - WHERE f.type = 'syndicated' - AND f.active = 't' - AND sf.feeds_id IS NULL; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4686; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4686-4687.sql b/apps/postgresql-server/schema/migrations/mediawords-4686-4687.sql deleted file mode 100644 index eb86f5235b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4686-4687.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4686 and 4687. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4686, and you would like to upgrade both the Media Cloud and the --- database to be at version 4687, import this SQL file: --- --- psql mediacloud < mediawords-4686-4687.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- --- track self liks and all links for a given domain within a given topic -create table topic_domains ( - topic_domains_id serial primary key, - topics_id int not null, - domain text not null, - self_links int not null default 0, - all_links int not null default 0 -); - -create unique index domain_topic_domain on topic_domains (topics_id, md5(domain)); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4687; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4687-4688.sql b/apps/postgresql-server/schema/migrations/mediawords-4687-4688.sql deleted file mode 100644 index feea79018f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4687-4688.sql +++ /dev/null @@ -1,125 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4687 and 4688. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4687, and you would like to upgrade both the Media Cloud and the --- database to be at version 4688, import this SQL file: --- --- psql mediacloud < mediawords-4687-4688.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'story_sentences'; - - -DROP FUNCTION copy_chunk_of_nonpartitioned_sentences_to_partitions(start_stories_id INT, end_stories_id INT); - - -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ - -DECLARE - target_table_name TEXT; -- partition table name (e.g. "story_sentences_01") - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO story_sentences_partitioned SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -DROP VIEW story_sentences; - -CREATE VIEW story_sentences AS - SELECT - story_sentences_partitioned_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - FROM story_sentences_partitioned; - -CREATE TRIGGER story_sentences_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON story_sentences - FOR EACH ROW EXECUTE PROCEDURE story_sentences_view_insert_update_delete(); - -ALTER VIEW story_sentences - ALTER COLUMN story_sentences_id - SET DEFAULT nextval(pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id')) + 1; - - -TRUNCATE TABLE story_sentences_nonpartitioned; -DROP TABLE story_sentences_nonpartitioned; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4688; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4688-4689.sql b/apps/postgresql-server/schema/migrations/mediawords-4688-4689.sql deleted file mode 100644 index ec1b7cd5bc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4688-4689.sql +++ /dev/null @@ -1,244 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4688 and 4689. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4688, and you would like to upgrade both the Media Cloud and the --- database to be at version 4689, import this SQL file: --- --- psql mediacloud < mediawords-4688-4689.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION stories_partition_chunk_size(); -DROP FUNCTION stories_partition_name(base_table_name TEXT, stories_id INT); -DROP FUNCTION stories_create_partitions(base_table_name TEXT); - - -CREATE OR REPLACE FUNCTION partition_by_stories_id_chunk_size() RETURNS BIGINT AS $$ -BEGIN - RETURN 100 * 1000 * 1000; -- 100m stories in each partition -END; $$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION partition_by_stories_id_partition_name(base_table_name TEXT, stories_id INT) RETURNS TEXT AS $$ -DECLARE - - -- Up to 100 partitions, suffixed as "_00", "_01" ..., "_99" - -- (having more of them is not feasible) - to_char_format CONSTANT TEXT := '00'; - - -- Partition table name (e.g. "stories_tags_map_01") - table_name TEXT; - - stories_id_chunk_number INT; - -BEGIN - SELECT stories_id / partition_by_stories_id_chunk_size() INTO stories_id_chunk_number; - - SELECT base_table_name || '_' || TRIM(leading ' ' FROM TO_CHAR(stories_id_chunk_number, to_char_format)) - INTO table_name; - - RETURN table_name; -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION partition_by_stories_id_create_partitions(base_table_name TEXT) RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_stories_id INT; - partition_stories_id INT; - - -- Partition table name (e.g. "stories_tags_map_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "stories_id" chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_start BIGINT; - - -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) - stories_id_end BIGINT; -BEGIN - - SELECT partition_by_stories_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT partition_by_stories_id_partition_name( base_table_name, partition_stories_id ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (' || base_table_name || '_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE - - ) INHERITS (' || base_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('stories_tags_map')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - -- Add extra foreign keys / constraints to the newly created partitions - EXECUTE ' - ALTER TABLE ' || partition || ' - - -- Foreign key to tags.tags_id - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_tags_id_fkey - FOREIGN KEY (tags_id) REFERENCES tags (tags_id) MATCH FULL ON DELETE CASCADE, - - -- Unique duplets - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_stories_id_tags_id_unique - UNIQUE (stories_id, tags_id); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION stories_tags_map_partition_upsert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( 'stories_tags_map', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION story_sentences_partitioned_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name('story_sentences_partitioned', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION story_sentences_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('story_sentences_partitioned')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_media_id_fkey - FOREIGN KEY (media_id) REFERENCES media (media_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_stories_id_sentence_number - ON ' || partition || ' (stories_id, sentence_number); - - CREATE INDEX ' || partition || '_sentence_media_week - ON ' || partition || ' (half_md5(sentence), media_id, week_start_date(publish_date::date)); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4689; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4689-4690.sql b/apps/postgresql-server/schema/migrations/mediawords-4689-4690.sql deleted file mode 100644 index b5b567b62b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4689-4690.sql +++ /dev/null @@ -1,86 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4689 and 4690. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4689, and you would like to upgrade both the Media Cloud and the --- database to be at version 4690, import this SQL file: --- --- psql mediacloud < mediawords-4689-4690.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO story_sentences_partitioned SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE story_sentences_partitioned - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM story_sentences_partitioned - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4690; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4690-4691.sql b/apps/postgresql-server/schema/migrations/mediawords-4690-4691.sql deleted file mode 100644 index cbecbeb773..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4690-4691.sql +++ /dev/null @@ -1,217 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4690 and 4691. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4690, and you would like to upgrade both the Media Cloud and the --- database to be at version 4691, import this SQL file: --- --- psql mediacloud < mediawords-4690-4691.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Rename unpartitioned table --- -ALTER TABLE feeds_stories_map - RENAME TO feeds_stories_map_unpartitioned; - - --- --- Create partitions --- - --- "Master" table (no indexes, no foreign keys as they'll be ineffective) -CREATE TABLE feeds_stories_map_partitioned ( - - -- PRIMARY KEY on master table needed for database handler's primary_key_column() method to work - feeds_stories_map_partitioned_id BIGSERIAL PRIMARY KEY NOT NULL, - - feeds_id INT NOT NULL, - stories_id INT NOT NULL -); - --- Note: "INSERT ... RETURNING *" doesn't work with the trigger, please use --- "feeds_stories_map" view instead -CREATE OR REPLACE FUNCTION feeds_stories_map_partitioned_insert_trigger() -RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "feeds_stories_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name('feeds_stories_map_partitioned', NEW.stories_id ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER feeds_stories_map_partitioned_insert_trigger - BEFORE INSERT ON feeds_stories_map_partitioned - FOR EACH ROW EXECUTE PROCEDURE feeds_stories_map_partitioned_insert_trigger(); - - --- Create missing "feeds_stories_map_partitioned" partitions -CREATE OR REPLACE FUNCTION feeds_stories_map_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('feeds_stories_map_partitioned')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_feeds_id_fkey - FOREIGN KEY (feeds_id) REFERENCES feeds (feeds_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_feeds_id_stories_id - ON ' || partition || ' (feeds_id, stories_id); - - CREATE INDEX ' || partition || '_stories_id - ON ' || partition || ' (stories_id); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - --- Create initial "feeds_stories_map_partitioned" partitions for empty database -SELECT feeds_stories_map_create_partitions(); - - --- Proxy view to "feeds_stories_map_partitioned" to make RETURNING work -CREATE OR REPLACE VIEW feeds_stories_map AS - - SELECT - feeds_stories_map_partitioned_id AS feeds_stories_map_id, - feeds_id, - stories_id - FROM feeds_stories_map_partitioned; - - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW feeds_stories_map - ALTER COLUMN feeds_stories_map_id - SET DEFAULT nextval(pg_get_serial_sequence('feeds_stories_map_partitioned', 'feeds_stories_map_partitioned_id')) + 1; - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "feeds_stories_map" view -CREATE OR REPLACE FUNCTION feeds_stories_map_view_insert_update_delete() RETURNS trigger AS $$ - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO feeds_stories_map_partitioned SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE feeds_stories_map_partitioned - SET feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id - WHERE feeds_id = OLD.feeds_id - AND stories_id = OLD.stories_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM feeds_stories_map_partitioned - WHERE feeds_id = OLD.feeds_id - AND stories_id = OLD.stories_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER feeds_stories_map_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON feeds_stories_map - FOR EACH ROW EXECUTE PROCEDURE feeds_stories_map_view_insert_update_delete(); - - -CREATE OR REPLACE FUNCTION create_missing_partitions() RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "stories_tags_map" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_partitioned" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_partitioned" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- --- Copy rows from unpartitioned table to the partitioned one --- -INSERT INTO feeds_stories_map (feeds_id, stories_id) - SELECT feeds_id, stories_id - FROM feeds_stories_map_unpartitioned; - - --- --- Drop unpartitioned table --- -DROP TABLE feeds_stories_map_unpartitioned; - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4691; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4691-4692.sql b/apps/postgresql-server/schema/migrations/mediawords-4691-4692.sql deleted file mode 100644 index 104fe3d226..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4691-4692.sql +++ /dev/null @@ -1,61 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4691 and 4692. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4691, and you would like to upgrade both the Media Cloud and the --- database to be at version 4692, import this SQL file: --- --- psql mediacloud < mediawords-4691-4692.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW feeds_stories_map - ALTER COLUMN feeds_stories_map_id - SET DEFAULT nextval(pg_get_serial_sequence('feeds_stories_map_partitioned', 'feeds_stories_map_partitioned_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('feeds_stories_map_partitioned', 'feeds_stories_map_partitioned_id')); - - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW story_sentences - ALTER COLUMN story_sentences_id - SET DEFAULT nextval(pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('story_sentences_partitioned', 'story_sentences_partitioned_id')); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4692; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4692-4693.sql b/apps/postgresql-server/schema/migrations/mediawords-4692-4693.sql deleted file mode 100644 index 9bc40549cc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4692-4693.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4692 and 4693. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4692, and you would like to upgrade both the Media Cloud and the --- database to be at version 4693, import this SQL file: --- --- psql mediacloud < mediawords-4692-4693.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -alter table topic_domains drop all_links; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4693; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4693-4694.sql b/apps/postgresql-server/schema/migrations/mediawords-4693-4694.sql deleted file mode 100644 index e30088d6dc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4693-4694.sql +++ /dev/null @@ -1,84 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4693 and 4694. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4693, and you would like to upgrade both the Media Cloud and the --- database to be at version 4694, import this SQL file: --- --- psql mediacloud < mediawords-4693-4694.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE TYPE media_sitemap_pages_change_frequency AS ENUM ( - 'always', - 'hourly', - 'daily', - 'weekly', - 'monthly', - 'yearly', - 'never' -); - - --- Pages derived from XML sitemaps (stories or not) -CREATE TABLE media_sitemap_pages ( - media_sitemap_pages_id BIGSERIAL PRIMARY KEY, - media_id INT NOT NULL REFERENCES media (media_id) ON DELETE CASCADE, - - -- -- URL of the page - url TEXT NOT NULL, - - -- -- date of last modification of the URL - last_modified TIMESTAMP WITH TIME ZONE NULL, - - -- -- how frequently the page is likely to change - change_frequency media_sitemap_pages_change_frequency NULL, - - -- -- priority of this URL relative to other URLs on your site - priority DECIMAL(2, 1) NOT NULL DEFAULT 0.5, - - -- -- title of the news article - news_title TEXT NULL, - - -- -- article publication date - news_publish_date TIMESTAMP WITH TIME ZONE NULL, - - CONSTRAINT media_sitemap_pages_priority_within_bounds - CHECK (priority IS NULL OR (priority >= 0.0 AND priority <= 1.0)) - -); - -CREATE UNIQUE INDEX media_sitemap_pages_url - ON media_sitemap_pages (url); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4694; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4694-4695.sql b/apps/postgresql-server/schema/migrations/mediawords-4694-4695.sql deleted file mode 100644 index 6783581327..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4694-4695.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4694 and 4695. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4694, and you would like to upgrade both the Media Cloud and the --- database to be at version 4695, import this SQL file: --- --- psql mediacloud < mediawords-4694-4695.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE INDEX IF NOT EXISTS media_sitemap_pages_media_id - ON media_sitemap_pages (media_id); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4695; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4695-4696.sql b/apps/postgresql-server/schema/migrations/mediawords-4695-4696.sql deleted file mode 100644 index cb597c51e3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4695-4696.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4695 and 4696. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4695, and you would like to upgrade both the Media Cloud and the --- database to be at version 4696, import this SQL file: --- --- psql mediacloud < mediawords-4695-4696.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop index topic_tweet_urls_tt; -create unique index topic_tweet_urls_tt on topic_tweet_urls ( topic_tweets_id, url ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4696; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4696-4697.sql b/apps/postgresql-server/schema/migrations/mediawords-4696-4697.sql deleted file mode 100644 index 8408584aa3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4696-4697.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4696 and 4697. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4696, and you would like to upgrade both the Media Cloud and the --- database to be at version 4697, import this SQL file: --- --- psql mediacloud < mediawords-4696-4697.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop table snap.weekly_date_counts; -drop table snap.daily_date_counts; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4697; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4697-4698.sql b/apps/postgresql-server/schema/migrations/mediawords-4697-4698.sql deleted file mode 100644 index 351526ca93..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4697-4698.sql +++ /dev/null @@ -1,58 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4697 and 4698. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4697, and you would like to upgrade both the Media Cloud and the --- database to be at version 4698, import this SQL file: --- --- psql mediacloud < mediawords-4697-4698.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DELETE FROM auth_roles -WHERE role = 'search'; - - -DELETE FROM activities -WHERE name IN ( - 'tm_remove_story_from_topic', - 'tm_media_merge', - 'tm_story_merge', - 'tm_search_tag_run', - 'tm_search_tag_change', - 'story_edit', - 'media_edit' -); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4698; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4698-4699.sql b/apps/postgresql-server/schema/migrations/mediawords-4698-4699.sql deleted file mode 100644 index 931e178fc8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4698-4699.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4698 and 4699. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4698, and you would like to upgrade both the Media Cloud and the --- database to be at version 4699, import this SQL file: --- --- psql mediacloud < mediawords-4698-4699.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE INDEX IF NOT EXISTS downloads_story_not_null - ON downloads (stories_id) - WHERE stories_id IS NOT NULL; - - --- Needed for effective migration to a partitioned table -CREATE INDEX IF NOT EXISTS downloads_type - ON downloads (type); - - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4699; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - --- --- 2 of 2. Reset the database version. --- -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4699-4700.sql b/apps/postgresql-server/schema/migrations/mediawords-4699-4700.sql deleted file mode 100644 index a16d41ff6e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4699-4700.sql +++ /dev/null @@ -1,211 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4699 and 4700. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4699, and you would like to upgrade both the Media Cloud and the --- database to be at version 4700, import this SQL file: --- --- psql mediacloud < mediawords-4699-4700.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION partition_by_stories_id_partition_name(base_table_name TEXT, stories_id INT); - -CREATE OR REPLACE FUNCTION partition_name(base_table_name TEXT, chunk_size BIGINT, object_id BIGINT) RETURNS TEXT AS $$ -DECLARE - - -- Up to 100 partitions, suffixed as "_00", "_01" ..., "_99" - -- (having more of them is not feasible) - to_char_format CONSTANT TEXT := '00'; - - -- Partition table name (e.g. "stories_tags_map_01") - table_name TEXT; - - chunk_number INT; - -BEGIN - SELECT object_id / chunk_size INTO chunk_number; - - SELECT base_table_name || '_' || TRIM(leading ' ' FROM TO_CHAR(chunk_number, to_char_format)) - INTO table_name; - - RETURN table_name; -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION partition_by_stories_id_partition_name(base_table_name TEXT, stories_id BIGINT) RETURNS TEXT AS $$ -BEGIN - - RETURN partition_name( - base_table_name := base_table_name, - chunk_size := partition_by_stories_id_chunk_size(), - object_id := stories_id - ); - -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION partition_by_stories_id_create_partitions(base_table_name TEXT) RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_stories_id INT; - partition_stories_id INT; - - -- Partition table name (e.g. "stories_tags_map_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "stories_id" chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_start BIGINT; - - -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) - stories_id_end BIGINT; -BEGIN - - SELECT partition_by_stories_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - FOR partition_stories_id IN 1..max_stories_id BY chunk_size LOOP - SELECT partition_by_stories_id_partition_name( - base_table_name := base_table_name, - stories_id := partition_stories_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (' || base_table_name || '_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE - - ) INHERITS (' || base_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION feeds_stories_map_partitioned_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "feeds_stories_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'feeds_stories_map_partitioned', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION stories_tags_map_partition_upsert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'stories_tags_map', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION story_sentences_partitioned_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'story_sentences_partitioned', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4700; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4700-4701.sql b/apps/postgresql-server/schema/migrations/mediawords-4700-4701.sql deleted file mode 100644 index 25d7e93de7..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4700-4701.sql +++ /dev/null @@ -1,429 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4700 and 4701. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4700, and you would like to upgrade both the Media Cloud and the --- database to be at version 4701, import this SQL file: --- --- psql mediacloud < mediawords-4700-4701.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -ALTER TABLE feeds_stories_map_partitioned - RENAME TO feeds_stories_map_p; - -ALTER INDEX feeds_stories_map_partitioned_pkey - RENAME TO feeds_stories_map_p_pkey; - -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'feeds_stories_map_partitioned_%' - ORDER BY tablename; - new_table_name TEXT; -BEGIN - FOR table_record IN tables LOOP - SELECT REPLACE(table_record.tablename, '_partitioned', '_p') INTO new_table_name; - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - RENAME TO ' || new_table_name || ';'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_pkey - RENAME TO ' || new_table_name || '_pkey;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_feeds_id_stories_id - RENAME TO ' || new_table_name || '_feeds_id_stories_id;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_stories_id - RENAME TO ' || new_table_name || '_stories_id;'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id - TO ' || new_table_name || '_stories_id'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_feeds_id_fkey - TO ' || new_table_name || '_feeds_id_fkey'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id_fkey - TO ' || new_table_name || '_stories_id_fkey'; - - END LOOP; -END -$$; - -ALTER TABLE feeds_stories_map_p - RENAME COLUMN feeds_stories_map_partitioned_id TO feeds_stories_map_p_id; - -ALTER SEQUENCE feeds_stories_map_partitioned_feeds_stories_map_partitioned_seq - RENAME TO feeds_stories_map_p_feeds_stories_map_p_seq; - - -ALTER TABLE story_sentences_partitioned - RENAME TO story_sentences_p; - -ALTER INDEX story_sentences_partitioned_pkey - RENAME TO story_sentences_p_pkey; - -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'story_sentences_partitioned_%' - ORDER BY tablename; - new_table_name TEXT; -BEGIN - FOR table_record IN tables LOOP - SELECT REPLACE(table_record.tablename, '_partitioned', '_p') INTO new_table_name; - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - RENAME TO ' || new_table_name || ';'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_pkey - RENAME TO ' || new_table_name || '_pkey;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_stories_id_sentence_number - RENAME TO ' || new_table_name || '_stories_id_sentence_number;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_sentence_media_week - RENAME TO ' || new_table_name || '_sentence_media_week;'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id - TO ' || new_table_name || '_stories_id'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_media_id_fkey - TO ' || new_table_name || '_media_id_fkey'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id_fkey - TO ' || new_table_name || '_stories_id_fkey'; - END LOOP; -END -$$; - -ALTER TABLE story_sentences_p - RENAME COLUMN story_sentences_partitioned_id TO story_sentences_p_id; - -ALTER SEQUENCE story_sentences_partitioned_story_sentences_partitioned_id_seq - RENAME TO story_sentences_p_story_sentences_p_id_seq; - - -DROP TRIGGER feeds_stories_map_partitioned_insert_trigger ON feeds_stories_map_p; - -DROP FUNCTION feeds_stories_map_partitioned_insert_trigger(); - -CREATE OR REPLACE FUNCTION feeds_stories_map_p_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "feeds_stories_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'feeds_stories_map_p', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER feeds_stories_map_p_insert_trigger - BEFORE INSERT ON feeds_stories_map_p - FOR EACH ROW - EXECUTE PROCEDURE feeds_stories_map_p_insert_trigger(); - - -DROP FUNCTION feeds_stories_map_create_partitions(); - -CREATE FUNCTION feeds_stories_map_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('feeds_stories_map_p')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_feeds_id_fkey - FOREIGN KEY (feeds_id) REFERENCES feeds (feeds_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_feeds_id_stories_id - ON ' || partition || ' (feeds_id, stories_id); - - CREATE INDEX ' || partition || '_stories_id - ON ' || partition || ' (stories_id); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -DROP TRIGGER IF EXISTS story_sentences_partitioned_01_insert_trigger ON story_sentences_p; -- Typo -DROP TRIGGER IF EXISTS story_sentences_partitioned_insert_trigger ON story_sentences_p; - -DROP FUNCTION story_sentences_partitioned_insert_trigger(); - -CREATE FUNCTION story_sentences_p_insert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'story_sentences_p', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER story_sentences_p_insert_trigger - BEFORE INSERT ON story_sentences_p - FOR EACH ROW - EXECUTE PROCEDURE story_sentences_p_insert_trigger(); - - -DROP FUNCTION story_sentences_create_partitions(); - -CREATE FUNCTION story_sentences_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('story_sentences_p')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_media_id_fkey - FOREIGN KEY (media_id) REFERENCES media (media_id) MATCH FULL ON DELETE CASCADE; - - CREATE UNIQUE INDEX ' || partition || '_stories_id_sentence_number - ON ' || partition || ' (stories_id, sentence_number); - - CREATE INDEX ' || partition || '_sentence_media_week - ON ' || partition || ' (half_md5(sentence), media_id, week_start_date(publish_date::date)); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -DROP FUNCTION create_missing_partitions(); - -CREATE OR REPLACE FUNCTION create_missing_partitions() RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "stories_tags_map" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_p" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_p" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - -DROP VIEW feeds_stories_map; - -CREATE VIEW feeds_stories_map AS - SELECT - feeds_stories_map_p_id AS feeds_stories_map_id, - feeds_id, - stories_id - FROM feeds_stories_map_p; - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW feeds_stories_map - ALTER COLUMN feeds_stories_map_id - SET DEFAULT nextval(pg_get_serial_sequence('feeds_stories_map_p', 'feeds_stories_map_p_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('feeds_stories_map_p', 'feeds_stories_map_p_id')); - -DROP FUNCTION feeds_stories_map_view_insert_update_delete(); - -CREATE FUNCTION feeds_stories_map_view_insert_update_delete() RETURNS trigger AS $$ - -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO feeds_stories_map_p SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE feeds_stories_map_p - SET feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id - WHERE feeds_id = OLD.feeds_id - AND stories_id = OLD.stories_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM feeds_stories_map_p - WHERE feeds_id = OLD.feeds_id - AND stories_id = OLD.stories_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER feeds_stories_map_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON feeds_stories_map - FOR EACH ROW EXECUTE PROCEDURE feeds_stories_map_view_insert_update_delete(); - - -DROP VIEW story_sentences; - -CREATE VIEW story_sentences AS - SELECT - story_sentences_p_id AS story_sentences_id, - stories_id, - sentence_number, - sentence, - media_id, - publish_date, - language, - is_dup - FROM story_sentences_p; - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW story_sentences - ALTER COLUMN story_sentences_id - SET DEFAULT nextval(pg_get_serial_sequence('story_sentences_p', 'story_sentences_p_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('story_sentences_p', 'story_sentences_p_id')); - -DROP FUNCTION story_sentences_view_insert_update_delete(); - -CREATE FUNCTION story_sentences_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO story_sentences_p SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE story_sentences_p - SET stories_id = NEW.stories_id, - sentence_number = NEW.sentence_number, - sentence = NEW.sentence, - media_id = NEW.media_id, - publish_date = NEW.publish_date, - language = NEW.language, - is_dup = NEW.is_dup - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM story_sentences_p - WHERE stories_id = OLD.stories_id - AND sentence_number = OLD.sentence_number; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER story_sentences_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON story_sentences - FOR EACH ROW EXECUTE PROCEDURE story_sentences_view_insert_update_delete(); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4701; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4701-4702.sql b/apps/postgresql-server/schema/migrations/mediawords-4701-4702.sql deleted file mode 100644 index 2a327bd986..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4701-4702.sql +++ /dev/null @@ -1,232 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4701 and 4702. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4701, and you would like to upgrade both the Media Cloud and the --- database to be at version 4702, import this SQL file: --- --- psql mediacloud < mediawords-4701-4702.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'stories_tags_map_%' - ORDER BY tablename; - new_table_name TEXT; -BEGIN - FOR table_record IN tables LOOP - SELECT REPLACE(table_record.tablename, 'stories_tags_map_', 'stories_tags_map_p_') INTO new_table_name; - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - RENAME TO ' || new_table_name || ';'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_pkey - RENAME TO ' || new_table_name || '_pkey;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_stories_id_tags_id_unique - RENAME TO ' || new_table_name || '_stories_id_tags_id_unique;'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id - TO ' || new_table_name || '_stories_id'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_stories_id_fkey - TO ' || new_table_name || '_stories_id_fkey'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_tags_id_fkey - TO ' || new_table_name || '_tags_id_fkey'; - - END LOOP; -END -$$; - - -ALTER TABLE stories_tags_map - RENAME TO stories_tags_map_p; - --- Some migrations might have led the sequence to have "1" suffix -ALTER SEQUENCE IF EXISTS stories_tags_map_stories_tags_map_id_seq1 - RENAME TO stories_tags_map_stories_tags_map_id_seq; - -ALTER SEQUENCE stories_tags_map_stories_tags_map_id_seq - RENAME TO stories_tags_map_p_stories_tags_map_p_id_seq; - -ALTER INDEX stories_tags_map_pkey - RENAME TO stories_tags_map_p_pkey; - -ALTER TABLE stories_tags_map_p - RENAME COLUMN stories_tags_map_id TO stories_tags_map_p_id; - - -CREATE OR REPLACE FUNCTION stories_tags_map_create_partitions() RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_stories_id_create_partitions('stories_tags_map_p')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - -- Add extra foreign keys / constraints to the newly created partitions - EXECUTE ' - ALTER TABLE ' || partition || ' - - -- Foreign key to tags.tags_id - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_tags_id_fkey - FOREIGN KEY (tags_id) REFERENCES tags (tags_id) MATCH FULL ON DELETE CASCADE, - - -- Unique duplets - ADD CONSTRAINT ' || REPLACE(partition, '.', '_') || '_stories_id_tags_id_unique - UNIQUE (stories_id, tags_id); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - --- Rename trigger -DROP TRIGGER stm_insert_solr_import_story ON stories_tags_map_p; - -CREATE TRIGGER stories_tags_map_p_insert_solr_import_story - BEFORE INSERT OR UPDATE OR DELETE ON stories_tags_map_p - FOR EACH ROW - EXECUTE PROCEDURE insert_solr_import_story(); - - - -DROP TRIGGER stories_tags_map_partition_upsert_trigger ON stories_tags_map_p; - -DROP FUNCTION stories_tags_map_partition_upsert_trigger(); - -CREATE OR REPLACE FUNCTION stories_tags_map_p_upsert_trigger() RETURNS TRIGGER AS $$ -DECLARE - target_table_name TEXT; -- partition table name (e.g. "stories_tags_map_01") -BEGIN - SELECT partition_by_stories_id_partition_name( - base_table_name := 'stories_tags_map_p', - stories_id := NEW.stories_id - ) INTO target_table_name; - EXECUTE ' - INSERT INTO ' || target_table_name || ' - SELECT $1.* - ON CONFLICT (stories_id, tags_id) DO NOTHING - ' USING NEW; - RETURN NULL; -END; -$$ -LANGUAGE plpgsql; - -CREATE TRIGGER stories_tags_map_p_upsert_trigger - BEFORE INSERT ON stories_tags_map_p - FOR EACH ROW - EXECUTE PROCEDURE stories_tags_map_p_upsert_trigger(); - - -CREATE OR REPLACE VIEW stories_tags_map AS - - SELECT - stories_tags_map_p_id AS stories_tags_map_id, - stories_id, - tags_id - FROM stories_tags_map_p; - - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW stories_tags_map - ALTER COLUMN stories_tags_map_id - SET DEFAULT nextval(pg_get_serial_sequence('stories_tags_map_p', 'stories_tags_map_p_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('stories_tags_map_p', 'stories_tags_map_p_id')); - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "stories_tags_map" view -CREATE OR REPLACE FUNCTION stories_tags_map_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- By INSERTing into the master table, we're letting triggers choose - -- the correct partition. - INSERT INTO stories_tags_map_p SELECT NEW.*; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - UPDATE stories_tags_map_p - SET stories_id = NEW.stories_id, - tags_id = NEW.tags_id - WHERE stories_id = OLD.stories_id - AND tags_id = OLD.tags_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - DELETE FROM stories_tags_map_p - WHERE stories_id = OLD.stories_id - AND tags_id = OLD.tags_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER stories_tags_map_view_insert_update_delete - INSTEAD OF INSERT OR UPDATE OR DELETE ON stories_tags_map - FOR EACH ROW EXECUTE PROCEDURE stories_tags_map_view_insert_update_delete(); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4702; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4702-4703.sql b/apps/postgresql-server/schema/migrations/mediawords-4702-4703.sql deleted file mode 100644 index fbd3f86c9a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4702-4703.sql +++ /dev/null @@ -1,942 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4702 and 4703. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4702, and you would like to upgrade both the Media Cloud and the --- database to be at version 4703, import this SQL file: --- --- psql mediacloud < mediawords-4702-4703.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Kill all autovacuums before proceeding with DDL changes --- -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - --- --- Drop views that use "downloads" to be recreated later --- -DROP VIEW daily_stats; -DROP VIEW downloads_media; -DROP VIEW downloads_non_media; -DROP VIEW downloads_to_be_extracted; -DROP VIEW downloads_with_error_in_past_day; -DROP VIEW downloads_in_past_day; - - --- --- Rename enums used by the non-partitioned table (and having obsolete values) --- -ALTER TYPE download_state - RENAME TO download_np_state; -ALTER TYPE download_type - RENAME TO download_np_type; - - --- --- Rename non-partitioned "downloads" table and its resources to "downloads_np" --- -ALTER TABLE downloads - RENAME TO downloads_np; - -ALTER TABLE downloads_np - RENAME COLUMN downloads_id TO downloads_np_id; - -ALTER INDEX downloads_pkey - RENAME TO downloads_np_pkey; -ALTER INDEX downloads_parent - RENAME TO downloads_np_parent; -ALTER INDEX downloads_time - RENAME TO downloads_np_time; -ALTER INDEX downloads_feed_download_time - RENAME TO downloads_np_feed_download_time; -ALTER INDEX downloads_story - RENAME TO downloads_np_story; -ALTER INDEX downloads_story_not_null - RENAME TO downloads_np_story_not_null; -ALTER INDEX downloads_type - RENAME TO downloads_np_type; -ALTER INDEX downloads_state_downloads_id_pending - RENAME TO downloads_np_state_downloads_id_pending; -ALTER INDEX downloads_extracted - RENAME TO downloads_np_extracted; -ALTER INDEX downloads_stories_to_be_extracted - RENAME TO downloads_np_stories_to_be_extracted; -ALTER INDEX downloads_extracted_stories - RENAME TO downloads_np_extracted_stories; -ALTER INDEX downloads_state_queued_or_fetching - RENAME TO downloads_np_state_queued_or_fetching; -ALTER INDEX downloads_state_fetching - RENAME TO downloads_np_state_fetching; - -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_feed_id_valid TO downloads_np_feed_id_valid; -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_path TO downloads_np_path; -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_story TO downloads_np_story; -ALTER TABLE downloads_np - RENAME CONSTRAINT valid_download_type TO downloads_np_valid_download_type; -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_feeds_id_fkey TO downloads_np_feeds_id_fkey; -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_parent_fkey TO downloads_np_parent_fkey; -ALTER TABLE downloads_np - RENAME CONSTRAINT downloads_stories_id_fkey TO downloads_np_stories_id_fkey; - - - --- --- Create the partitioned "downloads_p" table --- -CREATE TYPE download_p_state AS ENUM ( - 'error', - 'fetching', - 'pending', - 'success', - 'feed_error' -); - -CREATE TYPE download_p_type AS ENUM ( - 'content', - 'feed' -); - - --- Helper for indexing nonpartitioned "downloads.downloads_id" as BIGINT for --- faster casting -CREATE OR REPLACE FUNCTION to_bigint(p_integer INT) RETURNS BIGINT AS $$ - SELECT p_integer::bigint; -$$ LANGUAGE SQL IMMUTABLE; - - --- Convert "download_np_type" (nonpartitioned table's "type" column) --- to "download_p_type" (partitioned table's "type" column) -CREATE OR REPLACE FUNCTION download_np_type_to_download_p_type(p_type download_np_type) -RETURNS download_p_type -AS $$ - SELECT ( - CASE - -- Allow only the following types: - WHEN (p_type = 'content') THEN 'content' - WHEN (p_type = 'feed') THEN 'feed' - - -- Temporarily expose obsolete types as "content" - -- (filtering them out in WHERE wouldn't work because then - -- PostgreSQL decides to do a sequential scan) - ELSE 'content' - END - )::download_p_type; -$$ LANGUAGE SQL IMMUTABLE; - - --- Convert "download_np_state" (nonpartitioned table's "state" column) --- to "download_p_state" (partitioned table's "state" column) -CREATE OR REPLACE FUNCTION download_np_state_to_download_p_state(p_state download_np_state) -RETURNS download_p_state -AS $$ - SELECT ( - CASE - -- Rewrite obsolete states - WHEN (p_state = 'queued') THEN 'pending' - WHEN (p_state = 'extractor_error') THEN 'error' - - -- All the other states are OK - ELSE p_state::text - END - )::download_p_state; -$$ LANGUAGE SQL IMMUTABLE; - - --- Create a bunch of extra indexes on the non-partitioned table with columns --- cast to partitioned table's types for faster querying -CREATE INDEX IF NOT EXISTS downloads_np_pkey_bigint - ON downloads_np (to_bigint(downloads_np_id)); - -CREATE INDEX IF NOT EXISTS downloads_np_parent_bigint - ON downloads_np (to_bigint(parent)); - -CREATE INDEX IF NOT EXISTS downloads_np_type_p - ON downloads_np (download_np_type_to_download_p_type(type)); - -CREATE INDEX IF NOT EXISTS downloads_np_state_p_downloads_id_bigint_pending - ON downloads_np (download_np_state_to_download_p_state(state), to_bigint(downloads_np_id)) - WHERE download_np_state_to_download_p_state(state) = 'pending'; - -CREATE INDEX IF NOT EXISTS downloads_np_extracted_p - ON downloads_np (extracted, download_np_state_to_download_p_state(state), download_np_type_to_download_p_type(type)) - WHERE extracted = 'f' - AND download_np_state_to_download_p_state(state) = 'success' - AND download_np_type_to_download_p_type(type) = 'content'; - -CREATE INDEX IF NOT EXISTS downloads_np_state_p_fetching - ON downloads_np (download_np_state_to_download_p_state(state), downloads_np_id) - WHERE download_np_state_to_download_p_state(state) = 'fetching'; - - -CREATE TABLE downloads_p ( - downloads_p_id BIGSERIAL NOT NULL, - feeds_id INT NOT NULL REFERENCES feeds (feeds_id), - stories_id INT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - parent BIGINT NULL, - url TEXT NOT NULL, - host TEXT NOT NULL, - download_time TIMESTAMP NOT NULL DEFAULT NOW(), - type download_p_type NOT NULL, - state download_p_state NOT NULL, - path TEXT NULL, - error_message TEXT NULL, - priority SMALLINT NOT NULL, - sequence SMALLINT NOT NULL, - extracted BOOLEAN NOT NULL DEFAULT 'f', - - PRIMARY KEY (downloads_p_id, state, type) - -) PARTITION BY LIST (state); - --- Make partitioned table's "downloads_id" sequence start from where --- non-partitioned table's sequence left off -SELECT setval( - pg_get_serial_sequence('downloads_p', 'downloads_p_id'), - COALESCE(MAX(downloads_np_id), 1), MAX(downloads_np_id) IS NOT NULL -) FROM downloads_np; - - --- --- Create a proxy view to join partitioned and non-partitioned "downloads" --- tables --- -CREATE OR REPLACE VIEW downloads AS - - -- Non-partitioned table - SELECT - downloads_np_id::bigint AS downloads_id, - feeds_id, - stories_id, - parent::bigint, - url::text, - host::text, - download_time, - download_np_type_to_download_p_type(type) AS type, - download_np_state_to_download_p_state(state) AS state, - path::text, - error_message::text, - priority::smallint, - sequence::smallint, - extracted - FROM downloads_np - - UNION ALL - - -- Partitioned table - SELECT - downloads_p_id AS downloads_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - FROM downloads_p; - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW downloads - ALTER COLUMN downloads_id - SET DEFAULT nextval(pg_get_serial_sequence('downloads_p', 'downloads_p_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('downloads_p', 'downloads_p_id')); - - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "downloads" view -CREATE OR REPLACE FUNCTION downloads_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- New rows go into the partitioned table only - INSERT INTO downloads_p ( - downloads_p_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - ) SELECT - NEW.downloads_id, - NEW.feeds_id, - NEW.stories_id, - NEW.parent, - NEW.url, - NEW.host, - COALESCE(NEW.download_time, NOW()), - NEW.type, - NEW.state, - NEW.path, - NEW.error_message, - NEW.priority, - NEW.sequence, - COALESCE(NEW.extracted, 'f'); - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- Update both tables as one of them will have the row - UPDATE downloads_np SET - downloads_np_id = NEW.downloads_id, - feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id, - parent = NEW.parent, - url = NEW.url, - host = NEW.host, - download_time = NEW.download_time, - type = NEW.type::text::download_np_type, - state = NEW.state::text::download_np_state, - path = NEW.path, - error_message = NEW.error_message, - priority = NEW.priority, - sequence = NEW.sequence, - extracted = NEW.extracted - WHERE downloads_np_id = OLD.downloads_id; - - UPDATE downloads_p SET - downloads_p_id = NEW.downloads_id, - feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id, - parent = NEW.parent_id, - url = NEW.url, - host = NEW.host, - download_time = NEW.download_time, - type = NEW.type, - state = NEW.state, - path = NEW.path, - error_message = NEW.error_message, - priority = NEW.priority, - sequence = NEW.sequence, - extracted = NEW.extracted - WHERE downloads_p_id = OLD.downloads_id; - - -- Update record in tables that reference "downloads" with a given ID - UPDATE downloads_np - SET parent = NEW.downloads_id - WHERE parent = OLD.downloads_id; - - UPDATE downloads_p - SET parent = NEW.downloads_id - WHERE parent = OLD.downloads_id; - - UPDATE raw_downloads - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - UPDATE download_texts - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cached_extractor_results - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cache.s3_raw_downloads_cache - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- Delete from both tables as one of them will have the row - DELETE FROM downloads_np - WHERE downloads_np_id = OLD.downloads_id; - - DELETE FROM downloads_p - WHERE downloads_p_id = OLD.downloads_id; - - -- Update / delete record in tables that reference "downloads" with a - -- given ID - UPDATE downloads_np - SET parent = NULL - WHERE parent = OLD.downloads_id; - - UPDATE downloads_p - SET parent = NULL - WHERE parent = OLD.downloads_id; - - DELETE FROM raw_downloads - WHERE object_id = OLD.downloads_id; - - DELETE FROM download_texts - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cached_extractor_results - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cache.s3_raw_downloads_cache - WHERE object_id = OLD.downloads_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER downloads_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON downloads - FOR EACH ROW EXECUTE PROCEDURE downloads_view_insert_update_delete(); - - --- Imitate a foreign key by testing if a download with an INSERTed / UPDATEd --- "downloads_id" exists in "downloads" --- --- Partitioned tables don't support foreign keys being pointed to them, so this --- trigger achieves the same referential integrity for tables that point to --- "downloads". --- --- Column name from NEW (NEW.) that contains the --- INSERTed / UPDATEd "downloads_id" should be passed as an trigger argument. -CREATE OR REPLACE FUNCTION test_referenced_download_trigger() -RETURNS TRIGGER AS $$ -DECLARE - param_column_name TEXT; - param_downloads_id BIGINT; -BEGIN - - IF TG_NARGS != 1 THEN - RAISE EXCEPTION 'Trigger should be called with an column name argument.'; - END IF; - - SELECT TG_ARGV[0] INTO param_column_name; - SELECT to_json(NEW) ->> param_column_name INTO param_downloads_id; - - -- Might be NULL, e.g. downloads.parent - IF (param_downloads_id IS NOT NULL) THEN - - IF NOT EXISTS ( - SELECT 1 - FROM downloads - WHERE downloads_id = param_downloads_id - ) THEN - RAISE EXCEPTION 'Referenced download ID % from column "%" does not exist in "downloads".', param_downloads_id, param_column_name; - END IF; - - END IF; - - RETURN NEW; - -END; -$$ -LANGUAGE plpgsql; - - --- --- Create indexes and partitions of the partitioned "downloads_p" table --- -CREATE INDEX downloads_p_parent - ON downloads_p (parent); - -CREATE INDEX downloads_time_p - ON downloads_p (download_time); - -CREATE INDEX downloads_p_feed_download_time - ON downloads_p (feeds_id, download_time); - -CREATE INDEX downloads_p_story - ON downloads_p (stories_id); - - -CREATE TABLE downloads_p_error - PARTITION OF downloads_p - FOR VALUES IN ('error'); - -CREATE TRIGGER downloads_p_error_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON downloads_p_error - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('parent'); - - -CREATE TABLE downloads_p_feed_error - PARTITION OF downloads_p - FOR VALUES IN ('feed_error'); - -CREATE TRIGGER downloads_p_feed_error_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON downloads_p_feed_error - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('parent'); - - -CREATE TABLE downloads_p_fetching - PARTITION OF downloads_p - FOR VALUES IN ('fetching'); - -CREATE TRIGGER downloads_p_fetching_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON downloads_p_fetching - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('parent'); - - -CREATE TABLE downloads_p_pending - PARTITION OF downloads_p - FOR VALUES IN ('pending'); - -CREATE TRIGGER downloads_p_pending_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON downloads_p_pending - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('parent'); - - -CREATE TABLE downloads_p_success - PARTITION OF downloads_p ( - CONSTRAINT downloads_p_success_path_not_null - CHECK (path IS NOT NULL) - ) FOR VALUES IN ('success') - PARTITION BY LIST (type); - - -CREATE TABLE downloads_p_success_feed - PARTITION OF downloads_p_success ( - CONSTRAINT downloads_p_success_feed_stories_id_null - CHECK (stories_id IS NULL) - ) FOR VALUES IN ('feed') - PARTITION BY RANGE (downloads_p_id); - - -CREATE TABLE downloads_p_success_content - PARTITION OF downloads_p_success ( - CONSTRAINT downloads_p_success_content_stories_id_not_null - CHECK (stories_id IS NOT NULL) - ) FOR VALUES IN ('content') - PARTITION BY RANGE (downloads_p_id); - -CREATE INDEX downloads_p_success_content_extracted - ON downloads_p_success_content (extracted); - - --- --- Recreate views dropped previously that use "downloads" --- - -CREATE VIEW downloads_media AS - SELECT - d.*, - f.media_id AS _media_id - FROM - downloads AS d, - feeds AS f - WHERE d.feeds_id = f.feeds_id; - -CREATE VIEW downloads_non_media AS - SELECT d.* - FROM downloads AS d - WHERE d.feeds_id IS NULL; - -CREATE VIEW downloads_to_be_extracted AS - SELECT * - FROM downloads - WHERE extracted = 'f' - AND state = 'success' - AND type = 'content'; - -CREATE VIEW downloads_in_past_day AS - SELECT * - FROM downloads - WHERE download_time > NOW() - interval '1 day'; - -CREATE VIEW downloads_with_error_in_past_day AS - SELECT * - FROM downloads_in_past_day - WHERE state = 'error'; - -CREATE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - - --- --- Create partitioning tools for tables partitioned by "downloads_id" --- - --- Return partition size for every table that is partitioned by "downloads_id" -CREATE OR REPLACE FUNCTION partition_by_downloads_id_chunk_size() -RETURNS BIGINT AS $$ -BEGIN - RETURN 100 * 1000 * 1000; -- 100m downloads in each partition -END; $$ -LANGUAGE plpgsql IMMUTABLE; - - --- Return partition table name for a given base table name and "downloads_id" -CREATE OR REPLACE FUNCTION partition_by_downloads_id_partition_name( - base_table_name TEXT, - downloads_id BIGINT -) RETURNS TEXT AS $$ -BEGIN - - RETURN partition_name( - base_table_name := base_table_name, - chunk_size := partition_by_downloads_id_chunk_size(), - object_id := downloads_id - ); - -END; -$$ -LANGUAGE plpgsql IMMUTABLE; - --- Create missing partitions for tables partitioned by "downloads_id", returning --- a list of created partition tables -CREATE OR REPLACE FUNCTION partition_by_downloads_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_downloads_id BIGINT; - partition_downloads_id BIGINT; - - -- Partition table name (e.g. "downloads_success_content_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "downloads_id" chunk lower limit, inclusive (e.g. 30,000,000) - downloads_id_start BIGINT; - - -- "downloads_id" chunk upper limit, exclusive (e.g. 31,000,000) - downloads_id_end BIGINT; -BEGIN - - SELECT partition_by_downloads_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(downloads_id), 0) + chunk_size FROM downloads INTO max_downloads_id; - - FOR partition_downloads_id IN 1..max_downloads_id BY chunk_size LOOP - SELECT partition_by_downloads_id_partition_name( - base_table_name := base_table_name, - downloads_id := partition_downloads_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for download ID % already exists.', target_table_name, partition_downloads_id; - ELSE - RAISE NOTICE 'Creating partition "%" for download ID %', target_table_name, partition_downloads_id; - - SELECT (partition_downloads_id / chunk_size) * chunk_size INTO downloads_id_start; - SELECT ((partition_downloads_id / chunk_size) + 1) * chunk_size INTO downloads_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' - PARTITION OF ' || base_table_name || ' - FOR VALUES FROM (' || downloads_id_start || ') - TO (' || downloads_id_end || '); - '; - - EXECUTE ' - CREATE TRIGGER ' || target_table_name || '_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON ' || target_table_name || ' - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger(''parent''); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE ' - ALTER TABLE ' || target_table_name || ' - OWNER TO ' || target_table_owner || '; - '; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - - --- Create missing "downloads_success_content" partitions -CREATE OR REPLACE FUNCTION downloads_p_success_content_create_partitions() -RETURNS VOID AS -$$ - - SELECT partition_by_downloads_id_create_partitions('downloads_p_success_content'); - -$$ -LANGUAGE SQL; - --- Create initial "downloads_success_content" partitions for empty database -SELECT downloads_p_success_content_create_partitions(); - - --- Create missing "downloads_success_feed" partitions -CREATE OR REPLACE FUNCTION downloads_p_success_feed_create_partitions() -RETURNS VOID AS -$$ - - SELECT partition_by_downloads_id_create_partitions('downloads_p_success_feed'); - -$$ -LANGUAGE SQL; - --- Create initial "downloads_success_feed" partitions for empty database -SELECT downloads_p_success_feed_create_partitions(); - - --- --- Recreate function that creates partitions to take care of partitioned --- "downloads" too --- - --- Create missing partitions for partitioned tables -CREATE OR REPLACE FUNCTION create_missing_partitions() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "downloads_p_success_content" table...'; - PERFORM downloads_p_success_content_create_partitions(); - - RAISE NOTICE 'Creating partitions in "downloads_p_success_feed" table...'; - PERFORM downloads_p_success_feed_create_partitions(); - - RAISE NOTICE 'Creating partitions in "stories_tags_map_p" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_p" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_p" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- --- Drop foreign keys pointing to the non-partitioned table, imitate them by --- adding triggers --- - -ALTER TABLE raw_downloads - DROP CONSTRAINT raw_downloads_downloads_id_fkey; - -CREATE TRIGGER raw_downloads_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON raw_downloads - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('object_id'); - -ALTER TABLE download_texts - DROP CONSTRAINT download_texts_downloads_id_fkey; - -CREATE TRIGGER download_texts_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON download_texts - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('downloads_id'); - --- "cached_extractor_results" didn't have a foreign key reference to "downloads" - -CREATE TRIGGER cached_extractor_results_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON cached_extractor_results - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('downloads_id'); - -ALTER TABLE cache.s3_raw_downloads_cache - DROP CONSTRAINT s3_raw_downloads_cache_object_id_fkey; - -CREATE TRIGGER s3_raw_downloads_cache_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON cache.s3_raw_downloads_cache - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('object_id'); - - --- --- Move a chunk of downloads from a non-partitioned "downloads_np" to a --- partitioned "downloads_p". --- --- Expects starting and ending "downloads_id" instead of a chunk size in order --- to avoid index bloat that would happen when reading rows in sequential --- chunks. --- --- Returns number of rows that were moved. --- --- Call this repeatedly to migrate all the data to the partitioned table. -CREATE OR REPLACE FUNCTION move_chunk_of_nonpartitioned_downloads_to_partitions( - start_downloads_id INT, - end_downloads_id INT -) -RETURNS INT AS $$ - -DECLARE - moved_row_count INT; - -BEGIN - - IF NOT (start_downloads_id < end_downloads_id) THEN - RAISE EXCEPTION '"end_downloads_id" must be bigger than "start_downloads_id".'; - END IF; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - RAISE NOTICE - 'Moving downloads of downloads_id BETWEEN % AND % to the partitioned table...', - start_downloads_id, end_downloads_id; - - -- Fetch and delete downloads within bounds - WITH deleted_rows AS ( - DELETE FROM downloads_np - WHERE downloads_np_id BETWEEN start_downloads_id AND end_downloads_id - RETURNING downloads_np.* - ) - - -- Insert rows to the partitioned table - INSERT INTO downloads_p ( - downloads_p_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - ) - SELECT - downloads_np_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - download_np_type_to_download_p_type(type) AS type, - download_np_state_to_download_p_state(state) AS state, - path, - error_message, - priority, - sequence, - extracted - FROM deleted_rows - WHERE type IN ('content', 'feed'); -- Skip obsolete types like 'Calais' - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE - 'Finished moving downloads of downloads_id BETWEEN % AND % to the partitioned table, moved % rows.', - start_downloads_id, end_downloads_id, moved_row_count; - - RETURN moved_row_count; - -END; -$$ -LANGUAGE plpgsql; - - --- --- In a migration, move downloads up to downloads_id = 50,000 --- --- In case this is a development database and not a production one, we can --- afford to just move all the downloads to the partitioned table in a --- migration instead of using a separate Supervisor service to do that. --- --- On a production database, this will simply move up to 50k downloads to a --- partitioned table and should still run pretty fast. --- -SELECT move_chunk_of_nonpartitioned_downloads_to_partitions(1, 50000); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4703; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4703-4704.sql b/apps/postgresql-server/schema/migrations/mediawords-4703-4704.sql deleted file mode 100644 index b44ba53a43..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4703-4704.sql +++ /dev/null @@ -1,250 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4703 and 4704. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4703, and you would like to upgrade both the Media Cloud and the --- database to be at version 4704, import this SQL file: --- --- psql mediacloud < mediawords-4703-4704.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'cached_extractor_results'; - --- It's just some cache in it so we can just drop and recreate the table -DROP TABLE cached_extractor_results; - -CREATE UNLOGGED TABLE cache.extractor_results_cache ( - extractor_results_cache_id SERIAL PRIMARY KEY, - extracted_html TEXT NULL, - extracted_text TEXT NULL, - downloads_id BIGINT NOT NULL, - - -- Will be used to purge old cache objects; - -- don't forget to update cache.purge_object_caches() - db_row_last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() -); -CREATE UNIQUE INDEX extractor_results_cache_downloads_id - ON cache.extractor_results_cache (downloads_id); -CREATE INDEX extractor_results_cache_db_row_last_updated - ON cache.extractor_results_cache (db_row_last_updated); - -ALTER TABLE cache.extractor_results_cache - ALTER COLUMN extracted_html SET STORAGE EXTERNAL, - ALTER COLUMN extracted_text SET STORAGE EXTERNAL; - -CREATE TRIGGER extractor_results_cache_db_row_last_updated_trigger - BEFORE INSERT OR UPDATE ON cache.extractor_results_cache - FOR EACH ROW EXECUTE PROCEDURE cache.update_cache_db_row_last_updated(); - -CREATE TRIGGER extractor_results_cache_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON cache.extractor_results_cache - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('downloads_id'); - - --- Recreate helper that purges caches -CREATE OR REPLACE FUNCTION cache.purge_object_caches() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Purging "s3_raw_downloads_cache" table...'; - EXECUTE ' - DELETE FROM cache.s3_raw_downloads_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''3 days''; - '; - - RAISE NOTICE 'Purging "extractor_results_cache" table...'; - EXECUTE ' - DELETE FROM cache.extractor_results_cache - WHERE db_row_last_updated <= NOW() - INTERVAL ''3 days''; - '; - -END; -$$ -LANGUAGE plpgsql; - - --- Recreate trigger on "downloads" view -CREATE OR REPLACE FUNCTION downloads_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- New rows go into the partitioned table only - INSERT INTO downloads_p ( - downloads_p_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - ) SELECT - NEW.downloads_id, - NEW.feeds_id, - NEW.stories_id, - NEW.parent, - NEW.url, - NEW.host, - COALESCE(NEW.download_time, NOW()), - NEW.type, - NEW.state, - NEW.path, - NEW.error_message, - NEW.priority, - NEW.sequence, - COALESCE(NEW.extracted, 'f'); - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- Update both tables as one of them will have the row - UPDATE downloads_np SET - downloads_np_id = NEW.downloads_id, - feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id, - parent = NEW.parent, - url = NEW.url, - host = NEW.host, - download_time = NEW.download_time, - type = NEW.type::text::download_np_type, - state = NEW.state::text::download_np_state, - path = NEW.path, - error_message = NEW.error_message, - priority = NEW.priority, - sequence = NEW.sequence, - extracted = NEW.extracted - WHERE downloads_np_id = OLD.downloads_id; - - UPDATE downloads_p SET - downloads_p_id = NEW.downloads_id, - feeds_id = NEW.feeds_id, - stories_id = NEW.stories_id, - parent = NEW.parent, - url = NEW.url, - host = NEW.host, - download_time = NEW.download_time, - type = NEW.type, - state = NEW.state, - path = NEW.path, - error_message = NEW.error_message, - priority = NEW.priority, - sequence = NEW.sequence, - extracted = NEW.extracted - WHERE downloads_p_id = OLD.downloads_id; - - -- Update record in tables that reference "downloads" with a given ID - UPDATE downloads_np - SET parent = NEW.downloads_id - WHERE parent = OLD.downloads_id; - - UPDATE downloads_p - SET parent = NEW.downloads_id - WHERE parent = OLD.downloads_id; - - UPDATE raw_downloads - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - UPDATE download_texts - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cache.extractor_results_cache - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cache.s3_raw_downloads_cache - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- Delete from both tables as one of them will have the row - DELETE FROM downloads_np - WHERE downloads_np_id = OLD.downloads_id; - - DELETE FROM downloads_p - WHERE downloads_p_id = OLD.downloads_id; - - -- Update / delete record in tables that reference "downloads" with a - -- given ID - UPDATE downloads_np - SET parent = NULL - WHERE parent = OLD.downloads_id; - - UPDATE downloads_p - SET parent = NULL - WHERE parent = OLD.downloads_id; - - DELETE FROM raw_downloads - WHERE object_id = OLD.downloads_id; - - DELETE FROM download_texts - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cache.extractor_results_cache - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cache.s3_raw_downloads_cache - WHERE object_id = OLD.downloads_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4704; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4704-4705.sql b/apps/postgresql-server/schema/migrations/mediawords-4704-4705.sql deleted file mode 100644 index 4e2b445b68..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4704-4705.sql +++ /dev/null @@ -1,92 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4704 and 4705. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4704, and you would like to upgrade both the Media Cloud and the --- database to be at version 4705, import this SQL file: --- --- psql mediacloud < mediawords-4704-4705.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- ALTER TABLE raw_downloads ADD column might have worked too, but the table is --- empty in production or very small in development environments, so to --- preserve the column order let's just recreate everything - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'raw_downloads'; - --- Rename "raw_downloads" to "raw_downloads_int" -ALTER TABLE raw_downloads - RENAME TO raw_downloads_int; -ALTER SEQUENCE raw_downloads_raw_downloads_id_seq - RENAME TO raw_downloads_int_raw_downloads_id_seq; -ALTER INDEX raw_downloads_pkey - RENAME TO raw_downloads_int_pkey; -ALTER INDEX raw_downloads_object_id - RENAME TO raw_downloads_int_object_id; -ALTER TRIGGER raw_downloads_test_referenced_download_trigger - ON raw_downloads_int - RENAME TO raw_downloads_int_test_referenced_download_trigger; - --- Create "raw_downloads" with a BIGINT "object_id" -CREATE TABLE raw_downloads ( - raw_downloads_id BIGSERIAL PRIMARY KEY, - - -- "downloads_id" from "downloads" - object_id BIGINT NOT NULL, - - raw_data BYTEA NOT NULL -); -CREATE UNIQUE INDEX raw_downloads_object_id - ON raw_downloads (object_id); -ALTER TABLE raw_downloads - ALTER COLUMN raw_data SET STORAGE EXTERNAL; -CREATE TRIGGER raw_downloads_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON raw_downloads - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger('object_id'); - --- Copy the data -INSERT INTO raw_downloads (object_id, raw_data) - SELECT object_id::bigint, raw_data - FROM raw_downloads_int; - --- Drop old table -DROP TABLE raw_downloads_int; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4705; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4705-4706.sql b/apps/postgresql-server/schema/migrations/mediawords-4705-4706.sql deleted file mode 100644 index 3efc34d69e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4705-4706.sql +++ /dev/null @@ -1,162 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4705 and 4706. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4705, and you would like to upgrade both the Media Cloud and the --- database to be at version 4706, import this SQL file: --- --- psql mediacloud < mediawords-4705-4706.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION partition_by_downloads_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_downloads_id BIGINT; - partition_downloads_id BIGINT; - - -- Partition table name (e.g. "downloads_success_content_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "downloads_id" chunk lower limit, inclusive (e.g. 30,000,000) - downloads_id_start BIGINT; - - -- "downloads_id" chunk upper limit, exclusive (e.g. 31,000,000) - downloads_id_end BIGINT; -BEGIN - - SELECT partition_by_downloads_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(downloads_id), 0) + chunk_size FROM downloads INTO max_downloads_id; - - FOR partition_downloads_id IN 1..max_downloads_id BY chunk_size LOOP - SELECT partition_by_downloads_id_partition_name( - base_table_name := base_table_name, - downloads_id := partition_downloads_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for download ID % already exists.', target_table_name, partition_downloads_id; - ELSE - RAISE NOTICE 'Creating partition "%" for download ID %', target_table_name, partition_downloads_id; - - SELECT (partition_downloads_id / chunk_size) * chunk_size INTO downloads_id_start; - SELECT ((partition_downloads_id / chunk_size) + 1) * chunk_size INTO downloads_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' - PARTITION OF ' || base_table_name || ' - FOR VALUES FROM (' || downloads_id_start || ') - TO (' || downloads_id_end || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE ' - ALTER TABLE ' || target_table_name || ' - OWNER TO ' || target_table_owner || '; - '; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION downloads_create_subpartitions(base_table_name TEXT) -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_downloads_id_create_partitions(base_table_name)); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - CREATE TRIGGER ' || partition || '_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON ' || partition || ' - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger(''parent''); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION downloads_p_success_content_create_partitions() -RETURNS VOID AS -$$ - - SELECT downloads_create_subpartitions('downloads_p_success_content'); - -$$ -LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION downloads_p_success_feed_create_partitions() -RETURNS VOID AS -$$ - - SELECT downloads_create_subpartitions('downloads_p_success_feed'); - -$$ -LANGUAGE SQL; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4706; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4706-4707.sql b/apps/postgresql-server/schema/migrations/mediawords-4706-4707.sql deleted file mode 100644 index 6a4d20a010..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4706-4707.sql +++ /dev/null @@ -1,267 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4706 and 4707. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4706, and you would like to upgrade both the Media Cloud and the --- database to be at version 4707, import this SQL file: --- --- psql mediacloud < mediawords-4706-4707.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Kill all autovacuums before proceeding with DDL changes --- -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'download_texts'; - - --- --- Rename non-partitioned "download_texts" to "download_texts_np" --- -ALTER TABLE download_texts - RENAME TO download_texts_np; -ALTER TABLE download_texts_np - RENAME COLUMN download_texts_id TO download_texts_np_id; -ALTER TABLE download_texts_np - RENAME CONSTRAINT download_text_length_is_correct - TO download_texts_np_length_is_correct; -ALTER SEQUENCE download_texts_download_texts_id_seq - RENAME TO download_texts_np_download_texts_np_id_seq; -ALTER INDEX download_texts_pkey - RENAME TO download_texts_np_pkey; -ALTER INDEX download_texts_downloads_id_index - RENAME TO download_texts_np_downloads_id_index; -ALTER TRIGGER download_texts_test_referenced_download_trigger - ON download_texts_np - RENAME TO download_texts_np_test_referenced_download_trigger; - - --- --- Create partitioned "download_texts_p" --- -CREATE TABLE download_texts_p ( - download_texts_p_id BIGSERIAL NOT NULL, - downloads_id BIGINT NOT NULL, - download_text TEXT NOT NULL, - download_text_length INT NOT NULL, - - PRIMARY KEY (download_texts_p_id, downloads_id) - -) PARTITION BY RANGE (downloads_id); - -CREATE UNIQUE INDEX download_texts_p_downloads_id - ON download_texts_p (downloads_id); - -ALTER TABLE download_texts_p - ADD CONSTRAINT download_texts_p_length_is_correct - CHECK (length(download_text) = download_text_length); - -CREATE OR REPLACE FUNCTION download_texts_p_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_downloads_id_create_partitions('download_texts_p')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Altering created partition "%"...', partition; - - EXECUTE ' - CREATE TRIGGER ' || partition || '_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON ' || partition || ' - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger(''downloads_id''); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - --- --- Create initial "download_texts_p" partitions for empty database --- -SELECT download_texts_p_create_partitions(); - - --- Make partitioned table's "download_texts_id" sequence start from where --- non-partitioned table's sequence left off -SELECT setval( - pg_get_serial_sequence('download_texts_p', 'download_texts_p_id'), - COALESCE(MAX(download_texts_np_id), 1), MAX(download_texts_np_id) IS NOT NULL -) FROM download_texts_np; - - --- --- Create proxy view to join partitioned and non-partitioned "download_texts" --- tables --- -CREATE OR REPLACE VIEW download_texts AS - - SELECT * - FROM ( - - -- Non-partitioned table - SELECT - download_texts_np_id::bigint AS download_texts_id, - downloads_id::bigint, - download_text, - download_text_length - FROM download_texts_np - - UNION ALL - - -- Partitioned table - SELECT - download_texts_p_id AS download_texts_id, - downloads_id, - download_text, - download_text_length - FROM download_texts_p - - ) AS dt; - --- Make RETURNING work with partitioned tables --- (https://wiki.postgresql.org/wiki/INSERT_RETURNING_vs_Partitioning) -ALTER VIEW download_texts - ALTER COLUMN download_texts_id - SET DEFAULT nextval(pg_get_serial_sequence('download_texts_p', 'download_texts_p_id')); - --- Prevent the next INSERT from failing -SELECT nextval(pg_get_serial_sequence('download_texts_p', 'download_texts_p_id')); - --- Trigger that implements INSERT / UPDATE / DELETE behavior on "download_texts" view -CREATE OR REPLACE FUNCTION download_texts_view_insert_update_delete() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'INSERT') THEN - - -- New rows go into the partitioned table only - INSERT INTO download_texts_p ( - download_texts_p_id, - downloads_id, - download_text, - download_text_length - ) SELECT - NEW.download_texts_id, - NEW.downloads_id, - NEW.download_text, - NEW.download_text_length; - - RETURN NEW; - - ELSIF (TG_OP = 'UPDATE') THEN - - -- Update both tables as one of them will have the row - UPDATE download_texts_np SET - download_texts_np_id = NEW.download_texts_id, - downloads_id = NEW.downloads_id, - download_text = NEW.download_text, - download_text_length = NEW.download_text_length - WHERE download_texts_np_id = OLD.download_texts_id; - - UPDATE download_texts_p SET - download_texts_p_id = NEW.download_texts_id, - downloads_id = NEW.downloads_id, - download_text = NEW.download_text, - download_text_length = NEW.download_text_length - WHERE download_texts_p_id = OLD.download_texts_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - -- Delete from both tables as one of them will have the row - DELETE FROM download_texts_np - WHERE download_texts_np_id = OLD.download_texts_id; - - DELETE FROM download_texts_p - WHERE download_texts_p_id = OLD.download_texts_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER download_texts_view_insert_update_delete_trigger - INSTEAD OF INSERT OR UPDATE OR DELETE ON download_texts - FOR EACH ROW EXECUTE PROCEDURE download_texts_view_insert_update_delete(); - - --- --- Recreate function that creates new partitions --- -CREATE OR REPLACE FUNCTION create_missing_partitions() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "downloads_p_success_content" table...'; - PERFORM downloads_p_success_content_create_partitions(); - - RAISE NOTICE 'Creating partitions in "downloads_p_success_feed" table...'; - PERFORM downloads_p_success_feed_create_partitions(); - - RAISE NOTICE 'Creating partitions in "download_texts_p" table...'; - PERFORM download_texts_p_create_partitions(); - - RAISE NOTICE 'Creating partitions in "stories_tags_map_p" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_p" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_p" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4707; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4707-4708.sql b/apps/postgresql-server/schema/migrations/mediawords-4707-4708.sql deleted file mode 100644 index f4e139169b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4707-4708.sql +++ /dev/null @@ -1,73 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4707 and 4708. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4707, and you would like to upgrade both the Media Cloud and the --- database to be at version 4708, import this SQL file: --- --- psql mediacloud < mediawords-4707-4708.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Kill all autovacuums before proceeding with DDL changes --- -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'download_texts'; - - --- Proxy view to join partitioned and non-partitioned "download_texts" tables -CREATE OR REPLACE VIEW download_texts AS - - -- Non-partitioned table - SELECT - download_texts_np_id::bigint AS download_texts_id, - downloads_id::bigint, - download_text, - download_text_length - FROM download_texts_np - - UNION ALL - - -- Partitioned table - SELECT - download_texts_p_id AS download_texts_id, - downloads_id, - download_text, - download_text_length - FROM download_texts_p; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4708; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4708-4709.sql b/apps/postgresql-server/schema/migrations/mediawords-4708-4709.sql deleted file mode 100644 index 882b69772f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4708-4709.sql +++ /dev/null @@ -1,174 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4708 and 4709. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4708, and you would like to upgrade both the Media Cloud and the --- database to be at version 4709, import this SQL file: --- --- psql mediacloud < mediawords-4708-4709.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Due to a missing downloads.stories_id -> stories.stories_id foreign key in --- production, some downloads in the non-partitioned "downloads_np" table don't --- have a story that they point to. --- --- The partitioned table reintroduces --- downloads.stories_id -> stories.stories_id foreign key, so in order to move --- rows from a non-partitioned table to a partitioned one and not break this --- constraint, we'll move rows from "downloads_np" with no matching story to --- this table. --- -CREATE TABLE downloads_np_with_no_matching_story - AS TABLE downloads_np - WITH NO DATA; - -CREATE UNIQUE INDEX downloads_np_with_no_matching_story_downloads_np_id - ON downloads_np_with_no_matching_story (downloads_np_id); - - -CREATE OR REPLACE FUNCTION move_chunk_of_nonpartitioned_downloads_to_partitions( - start_downloads_id INT, - end_downloads_id INT -) -RETURNS INT AS $$ - -DECLARE - moved_row_count INT; - -BEGIN - - IF NOT (start_downloads_id < end_downloads_id) THEN - RAISE EXCEPTION '"end_downloads_id" must be bigger than "start_downloads_id".'; - END IF; - - - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - RAISE NOTICE 'Moving away downloads BETWEEN % AND % with no matching story...', - start_downloads_id, end_downloads_id; - - WITH deleted_rows AS ( - DELETE FROM downloads_np - WHERE downloads_np_id IN ( - SELECT downloads_np_id - FROM downloads_np - WHERE downloads_np_id BETWEEN start_downloads_id AND end_downloads_id - AND stories_id IS NOT NULL - AND NOT EXISTS ( - SELECT stories_id - FROM stories - WHERE downloads_np.stories_id = stories.stories_id - ) - ) - RETURNING * - ) - INSERT INTO downloads_np_with_no_matching_story - SELECT * - FROM deleted_rows; - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE 'Done moving away downloads BETWEEN % AND % with no matching story, moved % rows.', - start_downloads_id, end_downloads_id, moved_row_count; - - - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - RAISE NOTICE - 'Moving downloads of downloads_id BETWEEN % AND % to the partitioned table...', - start_downloads_id, end_downloads_id; - - -- Fetch and delete downloads within bounds - WITH deleted_rows AS ( - DELETE FROM downloads_np - WHERE downloads_np_id BETWEEN start_downloads_id AND end_downloads_id - RETURNING downloads_np.* - ) - - -- Insert rows to the partitioned table - INSERT INTO downloads_p ( - downloads_p_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - ) - SELECT - downloads_np_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - download_np_type_to_download_p_type(type) AS type, - download_np_state_to_download_p_state(state) AS state, - path, - error_message, - priority, - sequence, - extracted - FROM deleted_rows - WHERE type IN ('content', 'feed'); -- Skip obsolete types like 'Calais' - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE - 'Done moving downloads of downloads_id BETWEEN % AND % to the partitioned table, moved % rows.', - start_downloads_id, end_downloads_id, moved_row_count; - - RETURN moved_row_count; - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4709; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4709-4710.sql b/apps/postgresql-server/schema/migrations/mediawords-4709-4710.sql deleted file mode 100644 index e8ccbeeebd..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4709-4710.sql +++ /dev/null @@ -1,177 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4709 and 4710. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4709, and you would like to upgrade both the Media Cloud and the --- database to be at version 4710, import this SQL file: --- --- psql mediacloud < mediawords-4709-4710.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -CREATE OR REPLACE FUNCTION move_chunk_of_nonpartitioned_downloads_to_partitions( - start_downloads_id INT, - end_downloads_id INT -) -RETURNS INT AS $$ - -DECLARE - moved_row_count INT; - -BEGIN - - IF NOT (start_downloads_id < end_downloads_id) THEN - RAISE EXCEPTION '"end_downloads_id" must be bigger than "start_downloads_id".'; - END IF; - - - RAISE NOTICE 'Creating a table of downloads BETWEEN % AND %...', - start_downloads_id, end_downloads_id; - - -- For whatever reason (table stats way off? Too many tables referencing - -- "stories"? Bloated "downloads_np" primary key index so QP can't do - -- MAX(downloads_np_id)?), query planner (not the query itself!) runs for - -- the whole 6 or so minutes if we try to do - -- "FROM downloads_np LEFT JOIN stories" (to test for downloads with no - -- matching story). - -- - -- Everything seems to be snappier when we create a temporary table with - -- row IDs to be moved first. - CREATE TEMPORARY TABLE temp_downloads_np_chunk AS - SELECT downloads_np_id, stories_id - FROM downloads_np - WHERE downloads_np_id BETWEEN start_downloads_id AND end_downloads_id; - - - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - RAISE NOTICE 'Moving away downloads BETWEEN % AND % with no matching story...', - start_downloads_id, end_downloads_id; - - WITH deleted_rows AS ( - DELETE FROM downloads_np - WHERE downloads_np_id IN ( - SELECT downloads_np_id - FROM temp_downloads_np_chunk AS td - LEFT JOIN stories AS s - ON td.stories_id = s.stories_id - WHERE td.stories_id IS NOT NULL - AND s.stories_id IS NULL - ) - RETURNING * - ) - INSERT INTO downloads_np_with_no_matching_story - SELECT * - FROM deleted_rows; - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE 'Done moving away downloads BETWEEN % AND % with no matching story, moved % rows.', - start_downloads_id, end_downloads_id, moved_row_count; - - - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - RAISE NOTICE - 'Moving downloads of downloads_id BETWEEN % AND % to the partitioned table...', - start_downloads_id, end_downloads_id; - - -- Fetch and delete downloads within bounds - WITH deleted_rows AS ( - DELETE FROM downloads_np - WHERE downloads_np_id IN ( - SELECT downloads_np_id - FROM temp_downloads_np_chunk - ) - RETURNING downloads_np.* - ) - - -- Insert rows to the partitioned table - INSERT INTO downloads_p ( - downloads_p_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - type, - state, - path, - error_message, - priority, - sequence, - extracted - ) - SELECT - downloads_np_id, - feeds_id, - stories_id, - parent, - url, - host, - download_time, - download_np_type_to_download_p_type(type) AS type, - download_np_state_to_download_p_state(state) AS state, - path, - error_message, - priority, - sequence, - extracted - FROM deleted_rows - WHERE type IN ('content', 'feed'); -- Skip obsolete types like 'Calais' - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE - 'Done moving downloads of downloads_id BETWEEN % AND % to the partitioned table, moved % rows.', - start_downloads_id, end_downloads_id, moved_row_count; - - - DROP TABLE temp_downloads_np_chunk; - - - RETURN moved_row_count; - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4710; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4710-4711.sql b/apps/postgresql-server/schema/migrations/mediawords-4710-4711.sql deleted file mode 100644 index 6a9169f07e..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4710-4711.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4710 and 4711. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4710, and you would like to upgrade both the Media Cloud and the --- database to be at version 4711, import this SQL file: --- --- psql mediacloud < mediawords-4710-4711.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP TABLE similarweb_metrics; -DROP TABLE similarweb_media_metrics; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4711; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4711-4712.sql b/apps/postgresql-server/schema/migrations/mediawords-4711-4712.sql deleted file mode 100644 index d5eb66bcd0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4711-4712.sql +++ /dev/null @@ -1,97 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4711 and 4712. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4711, and you would like to upgrade both the Media Cloud and the --- database to be at version 4712, import this SQL file: --- --- psql mediacloud < mediawords-4711-4712.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- --- Domains for which we have SimilarWeb stats --- -CREATE TABLE similarweb_domains ( - similarweb_domains_id SERIAL PRIMARY KEY, - - -- Top-level (e.g. cnn.com) or second-level (e.g. edition.cnn.com) domain - domain TEXT NOT NULL - -); - -CREATE UNIQUE INDEX similarweb_domains_domain - ON similarweb_domains (domain); - - --- --- Media - SimilarWeb domain map --- -CREATE TABLE media_similarweb_domains_map ( - media_similarweb_domains_map_id SERIAL PRIMARY KEY, - - media_id INT NOT NULL REFERENCES media (media_id) ON DELETE CASCADE, - similarweb_domains_id INT NOT NULL REFERENCES similarweb_domains (similarweb_domains_id) ON DELETE CASCADE -); - --- Different media sources can point to the same domain -CREATE UNIQUE INDEX media_similarweb_domains_map_media_id_sdi - ON media_similarweb_domains_map (media_id, similarweb_domains_id); - - --- --- SimilarWeb estimated visits for domain --- (https://www.similarweb.com/corp/developer/estimated_visits_api) --- -CREATE TABLE similarweb_estimated_visits ( - similarweb_estimated_visits_id SERIAL PRIMARY KEY, - - -- Domain for which the stats were fetched - similarweb_domains_id INT NOT NULL REFERENCES similarweb_domains (similarweb_domains_id) ON DELETE CASCADE, - - -- Month, e.g. 2018-03-01 for March of 2018 - month DATE NOT NULL, - - -- Visit count is for the main domain only (value of "main_domain_only" API call argument) - main_domain_only BOOLEAN NOT NULL, - - -- Visit count - visits BIGINT NOT NULL - -); - -CREATE UNIQUE INDEX similarweb_estimated_visits_domain_month_mdo - ON similarweb_estimated_visits (similarweb_domains_id, month, main_domain_only); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4712; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4712-4713.sql b/apps/postgresql-server/schema/migrations/mediawords-4712-4713.sql deleted file mode 100644 index d2fa173cb6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4712-4713.sql +++ /dev/null @@ -1,338 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4712 and 4713. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4712, and you would like to upgrade both the Media Cloud and the --- database to be at version 4713, import this SQL file: --- --- psql mediacloud < mediawords-4712-4713.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - --- Kill all autovacuums before proceeding with DDL changes -SELECT pid -FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f -WHERE backend_type = 'autovacuum worker' - AND query ~ 'download_texts'; - - --- Temporary index to be used on JOINs with "downloads" with BIGINT primary key -CREATE UNIQUE INDEX IF NOT EXISTS download_texts_np_downloads_id_bigint_index - ON download_texts_np (to_bigint(downloads_id)); - - --- To be recreated later -DROP VIEW daily_stats; -DROP VIEW downloads_media; -DROP VIEW downloads_non_media; -DROP VIEW downloads_to_be_extracted; -DROP VIEW downloads_with_error_in_past_day; -DROP VIEW downloads_in_past_day; - - -DROP FUNCTION move_chunk_of_nonpartitioned_downloads_to_partitions(start_downloads_id INT, end_downloads_id INT); - - -DROP VIEW downloads; - - -DROP FUNCTION downloads_view_insert_update_delete(); - - -TRUNCATE downloads_np_with_no_matching_story; -DROP TABLE downloads_np_with_no_matching_story; - - -TRUNCATE downloads_np; -DROP TABLE downloads_np; - - -DROP FUNCTION download_np_type_to_download_p_type(p_type download_np_type); -DROP FUNCTION download_np_state_to_download_p_state(p_state download_np_state); - - -DROP TYPE download_np_state; -DROP TYPE download_np_type; - - -ALTER TYPE download_p_state - RENAME TO download_state; - -ALTER TYPE download_p_type - RENAME TO download_type; - - -ALTER TABLE downloads_p - RENAME TO downloads; - -ALTER TABLE downloads - RENAME COLUMN downloads_p_id TO downloads_id; - -ALTER TABLE downloads - RENAME CONSTRAINT downloads_p_feeds_id_fkey - TO downloads_feeds_id_fkey; - -ALTER TABLE downloads - RENAME CONSTRAINT downloads_p_stories_id_fkey - TO downloads_stories_id_fkey; - -ALTER INDEX downloads_p_pkey - RENAME TO downloads_pkey; - -ALTER INDEX downloads_p_feed_download_time - RENAME TO downloads_feed_download_time; - -ALTER INDEX downloads_p_parent - RENAME TO downloads_parent; - -ALTER INDEX downloads_p_story - RENAME TO downloads_story; - -ALTER INDEX downloads_time_p - RENAME TO downloads_time; - -ALTER SEQUENCE downloads_p_downloads_p_id_seq - RENAME TO downloads_downloads_id_seq; - - --- UPDATE / DELETE "downloads" trigger that enforces foreign keys on referencing tables -CREATE FUNCTION cascade_ref_downloads_trigger() RETURNS trigger AS $$ -BEGIN - - IF (TG_OP = 'UPDATE') THEN - - UPDATE downloads - SET parent = NEW.downloads_id - WHERE parent = OLD.downloads_id; - - UPDATE raw_downloads - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - UPDATE download_texts - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cache.extractor_results_cache - SET downloads_id = NEW.downloads_id - WHERE downloads_id = OLD.downloads_id; - - UPDATE cache.s3_raw_downloads_cache - SET object_id = NEW.downloads_id - WHERE object_id = OLD.downloads_id; - - RETURN NEW; - - ELSIF (TG_OP = 'DELETE') THEN - - UPDATE downloads - SET parent = NULL - WHERE parent = OLD.downloads_id; - - DELETE FROM raw_downloads - WHERE object_id = OLD.downloads_id; - - DELETE FROM download_texts - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cache.extractor_results_cache - WHERE downloads_id = OLD.downloads_id; - - DELETE FROM cache.s3_raw_downloads_cache - WHERE object_id = OLD.downloads_id; - - -- Return deleted rows - RETURN OLD; - - ELSE - RAISE EXCEPTION 'Unconfigured operation: %', TG_OP; - - END IF; - -END; -$$ LANGUAGE plpgsql; - - -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'downloads_p_%' - ORDER BY tablename; - new_table_name TEXT; -BEGIN - FOR table_record IN tables LOOP - - SELECT REPLACE(table_record.tablename, 'downloads_p_', 'downloads_') INTO new_table_name; - - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - RENAME TO ' || new_table_name || ';'; - - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_pkey - RENAME TO ' || new_table_name || '_pkey;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_download_time_idx - RENAME TO ' || new_table_name || '_download_time_idx;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_feeds_id_download_time_idx - RENAME TO ' || new_table_name || '_feeds_id_download_time_idx;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_parent_idx - RENAME TO ' || new_table_name || '_parent_idx;'; - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_stories_id_idx - RENAME TO ' || new_table_name || '_stories_id_idx;'; - - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT downloads_p_feeds_id_fkey - TO downloads_feeds_id_fkey'; - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT downloads_p_stories_id_fkey - TO downloads_stories_id_fkey'; - - IF new_table_name = 'downloads_success' THEN - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT downloads_p_success_path_not_null - TO downloads_success_path_not_null'; - END IF; - - IF new_table_name = 'downloads_success_content' THEN - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_extracted - RENAME TO ' || new_table_name || '_extracted;'; - - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT downloads_p_success_content_stories_id_not_null - TO downloads_success_content_stories_id_not_null'; - END IF; - - IF new_table_name LIKE 'downloads_success_content_%' THEN - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_extracted_idx - RENAME TO ' || new_table_name || '_extracted_idx;'; - - END IF; - - IF new_table_name = 'downloads_success_feed' THEN - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT downloads_p_success_feed_stories_id_null - TO downloads_success_feed_stories_id_null'; - END IF; - - IF new_table_name SIMILAR TO 'downloads_(error|feed_error|fetching|pending|success_content_%|success_feed_%)' THEN - - EXECUTE ' - ALTER TRIGGER ' || table_record.tablename || '_test_referenced_download_trigger - ON ' || new_table_name || ' - RENAME TO ' || new_table_name || '_test_referenced_download_trigger'; - - EXECUTE ' - CREATE TRIGGER ' || new_table_name || '_cascade_ref_downloads_trigger - AFTER UPDATE OR DELETE ON ' || new_table_name || ' - FOR EACH ROW - EXECUTE PROCEDURE cascade_ref_downloads_trigger(); - '; - - END IF; - - END LOOP; -END -$$; - - -CREATE VIEW downloads_non_media AS - SELECT d.* - FROM downloads AS d - WHERE d.feeds_id IS NULL; - -CREATE VIEW downloads_to_be_extracted AS - SELECT * - FROM downloads - WHERE extracted = 'f' - AND state = 'success' - AND type = 'content'; - -CREATE VIEW downloads_in_past_day AS - SELECT * - FROM downloads - WHERE download_time > NOW() - interval '1 day'; - -CREATE VIEW downloads_with_error_in_past_day AS - SELECT * - FROM downloads_in_past_day - WHERE state = 'error'; - - -CREATE VIEW daily_stats AS - SELECT * - FROM ( - SELECT COUNT(*) AS daily_downloads - FROM downloads_in_past_day - ) AS dd, - ( - SELECT COUNT(*) AS daily_stories - FROM stories_collected_in_past_day - ) AS ds, - ( - SELECT COUNT(*) AS downloads_to_be_extracted - FROM downloads_to_be_extracted - ) AS dex, - ( - SELECT COUNT(*) AS download_errors - FROM downloads_with_error_in_past_day - ) AS er, - ( - SELECT COALESCE( SUM( num_stories ), 0 ) AS solr_stories - FROM solr_imports WHERE import_date > now() - interval '1 day' - ) AS si; - -CREATE VIEW downloads_media AS - SELECT - d.*, - f.media_id AS _media_id - FROM - downloads AS d, - feeds AS f - WHERE d.feeds_id = f.feeds_id; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4713; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4713-4714.sql b/apps/postgresql-server/schema/migrations/mediawords-4713-4714.sql deleted file mode 100644 index 3d5f86b5fe..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4713-4714.sql +++ /dev/null @@ -1,95 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4713 and 4714. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4713, and you would like to upgrade both the Media Cloud and the --- database to be at version 4714, import this SQL file: --- --- psql mediacloud < mediawords-4713-4714.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - -SET search_path = public, pg_catalog; - - -DROP FUNCTION downloads_p_success_content_create_partitions(); -DROP FUNCTION downloads_p_success_feed_create_partitions(); -DROP FUNCTION create_missing_partitions(); - - -CREATE OR REPLACE FUNCTION downloads_success_content_create_partitions() -RETURNS VOID AS -$$ - - SELECT downloads_create_subpartitions('downloads_success_content'); - -$$ -LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION downloads_success_feed_create_partitions() -RETURNS VOID AS -$$ - - SELECT downloads_create_subpartitions('downloads_success_feed'); - -$$ -LANGUAGE SQL; - - -CREATE OR REPLACE FUNCTION create_missing_partitions() -RETURNS VOID AS -$$ -BEGIN - - RAISE NOTICE 'Creating partitions in "downloads_success_content" table...'; - PERFORM downloads_success_content_create_partitions(); - - RAISE NOTICE 'Creating partitions in "downloads_success_feed" table...'; - PERFORM downloads_success_feed_create_partitions(); - - RAISE NOTICE 'Creating partitions in "download_texts_p" table...'; - PERFORM download_texts_p_create_partitions(); - - RAISE NOTICE 'Creating partitions in "stories_tags_map_p" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_p" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_p" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4714; -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4714-4715.sql b/apps/postgresql-server/schema/migrations/mediawords-4714-4715.sql deleted file mode 100644 index 4fc330b5de..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4714-4715.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4714 and 4715. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4714, and you would like to upgrade both the Media Cloud and the --- database to be at version 4715, import this SQL file: --- --- psql mediacloud < mediawords-4714-4715.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table queued_downloads ( - queued_downloads_id bigserial primary key, - downloads_id bigint not null -); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4715; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4715-4716.sql b/apps/postgresql-server/schema/migrations/mediawords-4715-4716.sql deleted file mode 100644 index 57f84033dc..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4715-4716.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4715 and 4716. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4715, and you would like to upgrade both the Media Cloud and the --- database to be at version 4716, import this SQL file: --- --- psql mediacloud < mediawords-4715-4716.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create unique index queued_downloads_download on queued_downloads(downloads_id); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4716; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4716-4717.sql b/apps/postgresql-server/schema/migrations/mediawords-4716-4717.sql deleted file mode 100644 index 83e228387f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4716-4717.sql +++ /dev/null @@ -1,63 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4716 and 4717. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4716, and you would like to upgrade both the Media Cloud and the --- database to be at version 4717, import this SQL file: --- --- psql mediacloud < mediawords-4716-4717.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create function pop_queued_download() returns bigint as $$ - -declare - - pop_downloads_id bigint; - -begin - - select into pop_downloads_id downloads_id - from queued_downloads - order by downloads_id desc - limit 1 for - update skip locked; - - delete from queued_downloads where downloads_id = pop_downloads_id; - - return pop_downloads_id; -end; - -$$ language plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4717; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4717-4718.sql b/apps/postgresql-server/schema/migrations/mediawords-4717-4718.sql deleted file mode 100644 index 169bbed220..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4717-4718.sql +++ /dev/null @@ -1,71 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4717 and 4718. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4717, and you would like to upgrade both the Media Cloud and the --- database to be at version 4718, import this SQL file: --- --- psql mediacloud < mediawords-4717-4718.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -create or replace function get_downloads_for_queue() returns table(downloads_id bigint) as $$ -declare - pending_host record; -begin - create temporary table pending_downloads (downloads_id bigint) on commit drop; - for pending_host in - WITH RECURSIVE t AS ( - (SELECT host FROM downloads_pending ORDER BY host LIMIT 1) - UNION ALL - SELECT (SELECT host FROM downloads_pending WHERE host > t.host ORDER BY host LIMIT 1) - FROM t - WHERE t.host IS NOT NULL - ) - SELECT host FROM t WHERE host IS NOT NULL - loop - insert into pending_downloads - select dp.downloads_id - from downloads_pending dp - where host = pending_host.host - order by priority, downloads_id desc nulls last - limit 1; - end loop; - - return query select pd.downloads_id from pending_downloads pd; - end; - -$$ language plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4718; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4718-4719.sql b/apps/postgresql-server/schema/migrations/mediawords-4718-4719.sql deleted file mode 100644 index b022c5d4da..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4718-4719.sql +++ /dev/null @@ -1,75 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4718 and 4719. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4718, and you would like to upgrade both the Media Cloud and the --- database to be at version 4719, import this SQL file: --- --- psql mediacloud < mediawords-4718-4719.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- efficiently query downloads_pending for the latest downloads_id per host. postgres is not able to do this through --- its normal query planning (it just does an index scan of the whole indesx). this turns a query that --- takes ~22 seconds for a 100 million row table into one that takes ~0.25 seconds -create or replace function get_downloads_for_queue() returns table(downloads_id bigint) as $$ -declare - pending_host record; -begin - create temporary table pending_downloads (downloads_id bigint) on commit drop; - for pending_host in - WITH RECURSIVE t AS ( - (SELECT host FROM downloads_pending ORDER BY host LIMIT 1) - UNION ALL - SELECT (SELECT host FROM downloads_pending WHERE host > t.host ORDER BY host LIMIT 1) - FROM t - WHERE t.host IS NOT NULL - ) - SELECT host FROM t WHERE host IS NOT NULL - loop - insert into pending_downloads - select dp.downloads_id - from downloads_pending dp - left join queued_downloads qd on ( dp.downloads_id = qd.downloads_id ) - where - host = pending_host.host and - qd.downloads_id is null - order by priority, downloads_id desc nulls last - limit 1; - end loop; - - return query select pd.downloads_id from pending_downloads pd; - end; - -$$ language plpgsql; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4719; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4719-4720.sql b/apps/postgresql-server/schema/migrations/mediawords-4719-4720.sql deleted file mode 100644 index 550c3a2f4f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4719-4720.sql +++ /dev/null @@ -1,78 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4719 and 4720. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4719, and you would like to upgrade both the Media Cloud and the --- database to be at version 4720, import this SQL file: --- --- psql mediacloud < mediawords-4719-4720.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -DROP TRIGGER downloads_error_cascade_ref_downloads_trigger - ON downloads_error; - -DROP TRIGGER downloads_feed_error_cascade_ref_downloads_trigger - ON downloads_feed_error; - -DROP TRIGGER downloads_fetching_cascade_ref_downloads_trigger - ON downloads_fetching; - -DROP TRIGGER downloads_pending_cascade_ref_downloads_trigger - ON downloads_pending; - - -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' AND ( - tablename LIKE 'downloads_success_content_%' OR - tablename LIKE 'downloads_success_feed_%' - ) - ORDER BY tablename; -BEGIN - FOR table_record IN tables LOOP - - EXECUTE ' - DROP TRIGGER ' || table_record.tablename || '_cascade_ref_downloads_trigger - ON ' || table_record.tablename || ';'; - - END LOOP; -END -$$; - - -DROP FUNCTION cascade_ref_downloads_trigger(); - - --- --- 2 of 2. Reset the database version. --- -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4720; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4720-4721.sql b/apps/postgresql-server/schema/migrations/mediawords-4720-4721.sql deleted file mode 100644 index 6fdf167640..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4720-4721.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4720 and 4721. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4720, and you would like to upgrade both the Media Cloud and the --- database to be at version 4721, import this SQL file: --- --- psql mediacloud < mediawords-4720-4721.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -alter table auth_users add has_consented boolean not null default false; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4721; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4721-4722.sql b/apps/postgresql-server/schema/migrations/mediawords-4721-4722.sql deleted file mode 100644 index 3ad870d23c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4721-4722.sql +++ /dev/null @@ -1,123 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4721 and 4722. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4721, and you would like to upgrade both the Media Cloud and the --- database to be at version 4722, import this SQL file: --- --- psql mediacloud < mediawords-4721-4722.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view topics_with_user_permission; -drop view controversies; -drop view topic_tweet_full_urls; - -alter table topics drop twitter_topics_id; - -create type topic_platform_type AS enum ( 'web', 'twitter' ); -alter table topics add platform topic_platform_type not null default 'web'; - -create type topic_source_type AS enum ( 'mediacloud', 'crimson_hexagon', 'archive_org' ); -create table topic_seed_queries ( - topic_seed_queries_id serial primary key, - topics_id int not null references topics on delete cascade, - source topic_source_type not null, - platform topic_platform_type not null, - query text, - imported_date timestamp -); - -create index topic_seed_queries_topic on topic_seed_queries( topics_id ); - -update topics set platform = 'twitter' where ch_monitor_id is not null; -insert into topic_seed_queries (topics_id, platform, source, query) - select topics_id, 'twitter', 'crimson_hexagon', ch_monitor_id::text - from topics where ch_monitor_id is not null; - -alter table topics drop ch_monitor_id; - -alter table topic_tweet_days rename tweet_count to num_tweets; -alter table topic_tweet_days drop num_ch_tweets; - -alter table snap.tweet_stories rename tweet_count to num_tweets; -alter table snap.tweet_stories drop num_ch_tweets; - -alter table snap.story_link_counts drop normalized_tweet_count; -alter table snap.medium_link_counts drop normalized_tweet_count; - -create view controversies as select topics_id controversies_id, * from topics; -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - -create view topic_tweet_full_urls as - select distinct - t.topics_id, - tt.topic_tweets_id, tt.content, tt.publish_date, tt.twitter_user, - ttd.day, ttd.num_tweets, ttd.tweets_fetched, - ttu.url, tsu.stories_id - from - topics t - join topic_tweet_days ttd on ( t.topics_id = ttd.topics_id ) - join topic_tweets tt using ( topic_tweet_days_id ) - join topic_tweet_urls ttu using ( topic_tweets_id ) - left join topic_seed_urls tsu - on ( tsu.topics_id = t.topics_id and ttu.url = tsu.url ); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4722; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4722-4723.sql b/apps/postgresql-server/schema/migrations/mediawords-4722-4723.sql deleted file mode 100644 index 5167b01401..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4722-4723.sql +++ /dev/null @@ -1,165 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4722 and 4723. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4722, and you would like to upgrade both the Media Cloud and the --- database to be at version 4723, import this SQL file: --- --- psql mediacloud < mediawords-4722-4723.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table stories add column if not exists normalized_title_hash uuid null; -create index if not exists stories_normalized_title_hash on stories( media_id, normalized_title_hash ); - -CREATE OR REPLACE FUNCTION get_normalized_title(title text, title_media_id int) - RETURNS text - IMMUTABLE -AS $function$ -declare - title_part text; - media_title text; -begin - - -- stupid simple html stripper to avoid html messing up title_parts - select into title regexp_replace(title, '<[^\<]*>', '', 'gi'); - select into title regexp_replace(title, '\&#?[a-z0-9]*', '', 'gi'); - - select into title lower(title); - select into title regexp_replace(title,'(?:\- )|[:|]', 'SEPSEP', 'g'); - select into title regexp_replace(title, '[[:punct:]]', '', 'g'); - select into title regexp_replace(title, '\s+', ' ', 'g'); - select into title substr(title, 0, 1024); - - if title_media_id = 0 then - return title; - end if; - - select into title_part part - from ( select regexp_split_to_table(title, ' *SEPSEP *') part ) parts - order by length(part) desc limit 1; - - if title_part = title then - return title; - end if; - - if length(title_part) < 32 then - return title; - end if; - - select into media_title get_normalized_title(name, 0) from media where media_id = title_media_id; - if media_title = title_part then - return title; - end if; - - return title_part; -end -$function$ language plpgsql; - -create or replace function add_normalized_title_hash() returns trigger as $function$ -BEGIN - - if ( TG_OP = 'update' ) then - if ( OLD.title = NEW.title ) then - return new; - end if; - end if; - - select into NEW.normalized_title_hash md5( get_normalized_title( NEW.title, NEW.media_id ) )::uuid; - - return new; - -END - -$function$ language plpgsql; - -drop trigger if exists stories_add_normalized_title on stories; -create trigger stories_add_normalized_title before insert or update - on stories for each row execute procedure add_normalized_title_hash(); - --- list of all url or guid identifiers for each story -create table if not exists story_urls ( - story_urls_id bigserial primary key, - stories_id int references stories on delete cascade, - url varchar(1024) not null -); - -create unique index if not exists story_urls_url on story_urls ( url, stories_id ); -create index if not exists stories_story on story_urls ( stories_id ); - -alter table snap.live_stories add column if not exists normalized_title_hash uuid null; -create index if not exists live_stories_title_hash - on snap.live_stories ( topics_id, media_id, date_trunc('day', publish_date), normalized_title_hash ); - - -create or replace function insert_live_story() returns trigger as $insert_live_story$ - begin - - insert into snap.live_stories - ( topics_id, topic_stories_id, stories_id, media_id, url, guid, title, normalized_title_hash, description, - publish_date, collect_date, full_text_rss, language ) - select NEW.topics_id, NEW.topic_stories_id, NEW.stories_id, s.media_id, s.url, s.guid, - s.title, s.normalized_title_hash, s.description, s.publish_date, s.collect_date, s.full_text_rss, - s.language - from topic_stories cs - join stories s on ( cs.stories_id = s.stories_id ) - where - cs.stories_id = NEW.stories_id and - cs.topics_id = NEW.topics_id; - - return NEW; - END; -$insert_live_story$ LANGUAGE plpgsql; - -create or replace function update_live_story() returns trigger as $update_live_story$ - begin - - update snap.live_stories set - media_id = NEW.media_id, - url = NEW.url, - guid = NEW.guid, - title = NEW.title, - normalized_title_hash = NEW.normalized_title_hash, - description = NEW.description, - publish_date = NEW.publish_date, - collect_date = NEW.collect_date, - full_text_rss = NEW.full_text_rss, - language = NEW.language - where - stories_id = NEW.stories_id; - - return NEW; - END; -$update_live_story$ LANGUAGE plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4723; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4723-4724.sql b/apps/postgresql-server/schema/migrations/mediawords-4723-4724.sql deleted file mode 100644 index f0d0c49dea..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4723-4724.sql +++ /dev/null @@ -1,197 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4723 and 4724. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4723, and you would like to upgrade both the Media Cloud and the --- database to be at version 4724, import this SQL file: --- --- psql mediacloud < mediawords-4723-4724.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -CREATE OR REPLACE FUNCTION partition_by_downloads_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_downloads_id BIGINT; - partition_downloads_id BIGINT; - - -- Partition table name (e.g. "downloads_success_content_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "downloads_id" chunk lower limit, inclusive (e.g. 30,000,000) - downloads_id_start BIGINT; - - -- "downloads_id" chunk upper limit, exclusive (e.g. 31,000,000) - downloads_id_end BIGINT; -BEGIN - - SELECT partition_by_downloads_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(downloads_id), 0) + chunk_size FROM downloads INTO max_downloads_id; - - SELECT 1 INTO partition_downloads_id; - WHILE partition_downloads_id <= max_downloads_id LOOP - SELECT partition_by_downloads_id_partition_name( - base_table_name := base_table_name, - downloads_id := partition_downloads_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for download ID % already exists.', target_table_name, partition_downloads_id; - ELSE - RAISE NOTICE 'Creating partition "%" for download ID %', target_table_name, partition_downloads_id; - - SELECT (partition_downloads_id / chunk_size) * chunk_size INTO downloads_id_start; - SELECT ((partition_downloads_id / chunk_size) + 1) * chunk_size INTO downloads_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' - PARTITION OF ' || base_table_name || ' - FOR VALUES FROM (' || downloads_id_start || ') - TO (' || downloads_id_end || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE ' - ALTER TABLE ' || target_table_name || ' - OWNER TO ' || target_table_owner || '; - '; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - - SELECT partition_downloads_id + chunk_size INTO partition_downloads_id; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION partition_by_stories_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_stories_id INT; - partition_stories_id INT; - - -- Partition table name (e.g. "stories_tags_map_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "stories_id" chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_start BIGINT; - - -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) - stories_id_end BIGINT; -BEGIN - - SELECT partition_by_stories_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - SELECT 1 INTO partition_stories_id; - WHILE partition_stories_id <= max_stories_id LOOP - SELECT partition_by_stories_id_partition_name( - base_table_name := base_table_name, - stories_id := partition_stories_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (' || base_table_name || '_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE - - ) INHERITS (' || base_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - - SELECT partition_stories_id + chunk_size INTO partition_stories_id; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4724; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4724-4725.sql b/apps/postgresql-server/schema/migrations/mediawords-4724-4725.sql deleted file mode 100644 index 3f18944a7a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4724-4725.sql +++ /dev/null @@ -1,161 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4724 and 4725. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4724, and you would like to upgrade both the Media Cloud and the --- database to be at version 4725, import this SQL file: --- --- psql mediacloud < mediawords-4724-4725.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- Delete download texts which don't have references in "downloads" due to a missing foreign key -DELETE FROM download_texts -WHERE downloads_id IN ( - SELECT download_texts.downloads_id - FROM download_texts - LEFT JOIN downloads - ON download_texts.downloads_id = downloads.downloads_id - WHERE downloads.downloads_id IS NULL -); - - --- Delete download texts which are not successful content downloads (some --- extraction errors somehow ended up getting stored in "download_texts" as --- extracted text) -DELETE FROM download_texts_np -WHERE downloads_id IN ( - SELECT download_texts_np.downloads_id - FROM download_texts_np - INNER JOIN downloads - ON download_texts_np.downloads_id = downloads.downloads_id - WHERE downloads.state != 'success' -); - - --- Create index *only* on the base table (initially invalid) -CREATE UNIQUE INDEX downloads_success_content_downloads_id - ON ONLY downloads_success_content (downloads_id); - - --- Create partition indexes and attach them to the base table's index to make it valid -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'downloads_success_content_%' - ORDER BY tablename; -BEGIN - FOR table_record IN tables LOOP - - -- Create index on one of the partitions - EXECUTE ' - CREATE UNIQUE INDEX ' || table_record.tablename || '_downloads_id_idx - ON ' || table_record.tablename || ' (downloads_id); - '; - - -- Attach the newly created index to base table - EXECUTE ' - ALTER INDEX downloads_success_content_downloads_id - ATTACH PARTITION ' || table_record.tablename || '_downloads_id_idx; - '; - - END LOOP; -END -$$; - - --- Add foreign key constraints from "download_texts" partitions to "downloads_success_content" partitions -DO $$ -DECLARE - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'download_texts_p_%' - ORDER BY tablename; -BEGIN - FOR table_record IN tables LOOP - - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - ADD CONSTRAINT ' || table_record.tablename || '_downloads_id_fkey - FOREIGN KEY (downloads_id) - REFERENCES ' || REPLACE(table_record.tablename, 'download_texts_p', 'downloads_success_content') || ' (downloads_id) - ON DELETE CASCADE; - '; - - END LOOP; -END -$$; - - -CREATE OR REPLACE FUNCTION download_texts_p_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_downloads_id_create_partitions('download_texts_p')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Adding foreign key to created partition "%"...', partition; - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || partition || '_downloads_id_fkey - FOREIGN KEY (downloads_id) - REFERENCES ' || REPLACE(partition, 'download_texts_p', 'downloads_success_content') || ' (downloads_id) - ON DELETE CASCADE; - '; - - RAISE NOTICE 'Adding trigger to created partition "%"...', partition; - EXECUTE ' - CREATE TRIGGER ' || partition || '_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON ' || partition || ' - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger(''downloads_id''); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4725; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4725-4726.sql b/apps/postgresql-server/schema/migrations/mediawords-4725-4726.sql deleted file mode 100644 index a6d5591abe..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4725-4726.sql +++ /dev/null @@ -1,119 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4725 and 4726. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4725, and you would like to upgrade both the Media Cloud and the --- database to be at version 4726, import this SQL file: --- --- psql mediacloud < mediawords-4725-4726.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- --- Move a chunk of download texts from a non-partitioned "download_texts_np" to a --- partitioned "download_texts_p". --- --- Expects starting and ending "download_texts_id" instead of a chunk size in order --- to avoid index bloat that would happen when reading rows in sequential --- chunks. --- --- Returns number of rows that were moved. --- --- Call this repeatedly to migrate all the data to the partitioned table. -CREATE OR REPLACE FUNCTION move_chunk_of_nonpartitioned_download_texts_to_partitions( - start_download_texts_id INT, - end_download_texts_id INT -) -RETURNS INT AS $$ - -DECLARE - moved_row_count INT; - -BEGIN - - IF NOT (start_download_texts_id < end_download_texts_id) THEN - RAISE EXCEPTION '"end_download_texts_id" must be bigger than "start_download_texts_id".'; - END IF; - - - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'download_texts'; - - RAISE NOTICE - 'Moving download texts of download_texts_id BETWEEN % AND % to the partitioned table...', - start_download_texts_id, end_download_texts_id; - - -- Fetch and delete download texts within bounds - WITH deleted_rows AS ( - DELETE FROM download_texts_np - WHERE download_texts_np_id BETWEEN start_download_texts_id AND end_download_texts_id - RETURNING download_texts_np.* - ) - - -- Insert rows to the partitioned table - INSERT INTO download_texts_p ( - download_texts_p_id, - downloads_id, - download_text, - download_text_length - ) - SELECT - download_texts_np_id, - downloads_id, - download_text, - download_text_length - FROM deleted_rows; - - GET DIAGNOSTICS moved_row_count = ROW_COUNT; - - RAISE NOTICE - 'Done moving download texts of download_texts_id BETWEEN % AND % to the partitioned table, moved % rows.', - start_download_texts_id, end_download_texts_id, moved_row_count; - - RETURN moved_row_count; - -END; -$$ -LANGUAGE plpgsql; - - --- Move all of the rows in a migration --- (obviously, this wouldn't work in a production so this migration is to be --- applied automatically only in the dev environments) -SELECT move_chunk_of_nonpartitioned_download_texts_to_partitions(1, MAX(download_texts_np_id) + 1) -FROM download_texts_np; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4726; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4726-4727.sql b/apps/postgresql-server/schema/migrations/mediawords-4726-4727.sql deleted file mode 100644 index 15294480ad..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4726-4727.sql +++ /dev/null @@ -1,75 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4726 and 4727. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4726, and you would like to upgrade both the Media Cloud and the --- database to be at version 4727, import this SQL file: --- --- psql mediacloud < mediawords-4726-4727.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create or replace function get_downloads_for_queue() returns table(downloads_id bigint) as $$ -declare - pending_host record; -begin - -- quick temp copy without dead row issues for querying in loop body below - create temporary table qd on commit drop as select * from queued_downloads; - - create temporary table pending_downloads (downloads_id bigint) on commit drop; - for pending_host in - WITH RECURSIVE t AS ( - (SELECT host FROM downloads_pending ORDER BY host LIMIT 1) - UNION ALL - SELECT (SELECT host FROM downloads_pending WHERE host > t.host ORDER BY host LIMIT 1) - FROM t - WHERE t.host IS NOT NULL - ) - SELECT host FROM t WHERE host IS NOT NULL - loop - insert into pending_downloads - select dp.downloads_id - from downloads_pending dp - left join qd on ( dp.downloads_id = qd.downloads_id ) - where - host = pending_host.host and - qd.downloads_id is null - order by priority, downloads_id desc nulls last - limit 1; - end loop; - - return query select pd.downloads_id from pending_downloads pd; - end; - -$$ language plpgsql; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4727; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4727-4728.sql b/apps/postgresql-server/schema/migrations/mediawords-4727-4728.sql deleted file mode 100644 index dfe270c9d1..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4727-4728.sql +++ /dev/null @@ -1,187 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4727 and 4728. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4727, and you would like to upgrade both the Media Cloud and the --- database to be at version 4728, import this SQL file: --- --- psql mediacloud < mediawords-4727-4728.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- - --- --- 1 of 2. Import the output of 'apgdiff': --- - - -DROP VIEW download_texts; - - -TRUNCATE TABLE download_texts_np; -DROP TABLE download_texts_np; - - -DROP FUNCTION download_texts_view_insert_update_delete(); -DROP FUNCTION move_chunk_of_nonpartitioned_download_texts_to_partitions(INT, INT); - -DROP FUNCTION to_bigint(INT); - - -ALTER TABLE download_texts_p RENAME TO download_texts; -ALTER TABLE download_texts RENAME COLUMN download_texts_p_id TO download_texts_id; -ALTER TABLE download_texts RENAME CONSTRAINT download_texts_p_length_is_correct TO download_texts_length_is_correct; - -ALTER INDEX download_texts_p_pkey RENAME TO download_texts_pkey; -ALTER INDEX download_texts_p_downloads_id RENAME TO download_texts_downloads_id; - -ALTER SEQUENCE download_texts_p_download_texts_p_id_seq RENAME TO download_texts_download_texts_id_seq; - - -DO $$ -DECLARE - - new_table_name TEXT; - - tables CURSOR FOR - SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename LIKE 'download_texts_p_%' - ORDER BY tablename; - -BEGIN - FOR table_record IN tables LOOP - - SELECT REPLACE(table_record.tablename, 'download_texts_p_', 'download_texts_') INTO new_table_name; - - EXECUTE ' - ALTER TABLE ' || table_record.tablename || ' - RENAME TO ' || new_table_name || ' - '; - - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_pkey - RENAME TO ' || new_table_name || '_pkey - '; - - EXECUTE ' - ALTER INDEX ' || table_record.tablename || '_downloads_id_idx - RENAME TO ' || new_table_name || '_downloads_id_idx - '; - - EXECUTE ' - ALTER TABLE ' || new_table_name || ' - RENAME CONSTRAINT ' || table_record.tablename || '_downloads_id_fkey - TO ' || new_table_name || '_downloads_id_fkey - '; - - EXECUTE ' - ALTER TRIGGER ' || table_record.tablename || '_test_referenced_download_trigger - ON ' || new_table_name || ' - RENAME TO ' || new_table_name || '_test_referenced_download_trigger - '; - - END LOOP; -END -$$; - - --- Recreate function that creates "download_texts" partitions with a different name -DROP FUNCTION download_texts_p_create_partitions(); - -CREATE OR REPLACE FUNCTION download_texts_create_partitions() -RETURNS VOID AS -$$ -DECLARE - created_partitions TEXT[]; - partition TEXT; -BEGIN - - created_partitions := ARRAY(SELECT partition_by_downloads_id_create_partitions('download_texts')); - - FOREACH partition IN ARRAY created_partitions LOOP - - RAISE NOTICE 'Adding foreign key to created partition "%"...', partition; - EXECUTE ' - ALTER TABLE ' || partition || ' - ADD CONSTRAINT ' || partition || '_downloads_id_fkey - FOREIGN KEY (downloads_id) - REFERENCES ' || REPLACE(partition, 'download_texts', 'downloads_success_content') || ' (downloads_id) - ON DELETE CASCADE; - '; - - RAISE NOTICE 'Adding trigger to created partition "%"...', partition; - EXECUTE ' - CREATE TRIGGER ' || partition || '_test_referenced_download_trigger - BEFORE INSERT OR UPDATE ON ' || partition || ' - FOR EACH ROW - EXECUTE PROCEDURE test_referenced_download_trigger(''downloads_id''); - '; - - END LOOP; - -END; -$$ -LANGUAGE plpgsql; - - --- Update function that creates partitions of all partitioned tables to call --- the right "subsidiary" for the "download_texts" table -CREATE OR REPLACE FUNCTION create_missing_partitions() -RETURNS VOID AS -$$ -BEGIN - - -- We have to create "downloads" partitions before "download_texts" ones - -- because "download_texts" will have a foreign key reference to - -- "downloads_success_content" - - RAISE NOTICE 'Creating partitions in "downloads_success_content" table...'; - PERFORM downloads_success_content_create_partitions(); - - RAISE NOTICE 'Creating partitions in "downloads_success_feed" table...'; - PERFORM downloads_success_feed_create_partitions(); - - RAISE NOTICE 'Creating partitions in "download_texts" table...'; - PERFORM download_texts_create_partitions(); - - RAISE NOTICE 'Creating partitions in "stories_tags_map_p" table...'; - PERFORM stories_tags_map_create_partitions(); - - RAISE NOTICE 'Creating partitions in "story_sentences_p" table...'; - PERFORM story_sentences_create_partitions(); - - RAISE NOTICE 'Creating partitions in "feeds_stories_map_p" table...'; - PERFORM feeds_stories_map_create_partitions(); - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4728; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4728-4729.sql b/apps/postgresql-server/schema/migrations/mediawords-4728-4729.sql deleted file mode 100644 index f4e762000b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4728-4729.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4728 and 4729. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4728, and you would like to upgrade both the Media Cloud and the --- database to be at version 4729, import this SQL file: --- --- psql mediacloud < mediawords-4728-4729.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add respider_stories boolean not null default false; -alter table topics add respider_start_date date null; -alter table topics add respider_end_date date null; - -alter table timespans alter snapshots_id drop not null; -alter table timespans add archive_snapshots_id int null references snapshots on delete cascade; -alter table timespans add constraint topics_snapshot - check ( ( snapshots_id is null and archive_snapshots_id is not null ) or - ( snapshots_id is not null and archive_snapshots_id is null ) ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4729; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4729-4730.sql b/apps/postgresql-server/schema/migrations/mediawords-4729-4730.sql deleted file mode 100644 index 05a1419b9c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4729-4730.sql +++ /dev/null @@ -1,145 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4729 and 4730. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4729, and you would like to upgrade both the Media Cloud and the --- database to be at version 4730, import this SQL file: --- --- psql mediacloud < mediawords-4729-4730.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop view topics_with_user_permission; -drop view controversies; - -alter table topics alter platform type text; -alter table topics alter platform drop default; -alter table topic_seed_queries alter platform type text; -drop type topic_platform_type; -create type topic_platform_type AS enum ( 'web', 'twitter', 'generic_post', 'mediacloud_topic' ); -alter table topics alter platform type topic_platform_type using ( platform::topic_platform_type ); -alter table topics alter platform set default 'web'; -alter table topic_seed_queries alter platform type topic_platform_type using ( platform::topic_platform_type ); - -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' - when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' - else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - -create view controversies as select topics_id controversies_id, * from topics; - -alter table topic_seed_queries alter source type text; -drop type topic_source_type; -create type topic_source_type AS enum ( 'mediacloud', 'crimson_hexagon', 'archive_org', 'csv' ); -alter table topic_seed_queries alter source type topic_source_type using ( source::topic_source_type ); - -create type topic_mode_type AS enum ( 'web', 'url_sharing' ); - -alter table topics add mode topic_mode_type not null default 'web'; - -alter table timespans rename column tweet_count to post_count; - -alter table snap.story_link_counts rename column simple_tweet_count to post_count; - -alter table topic_tweet_days rename to topic_post_days; - -alter table topic_post_days rename column topic_tweet_days_id to topic_post_days_id; -alter table topic_post_days rename column num_tweets to num_posts; -alter table topic_post_days rename column tweets_fetched to posts_fetched; - -alter table topic_tweets rename to topic_posts; -alter table topic_posts rename topic_tweets_id to topic_posts_id; -alter table topic_posts rename topic_tweet_days_id to topic_post_days_id; -alter table topic_posts rename tweet_id to post_id; -alter table topic_posts rename twitter_user to author; -alter table topic_posts add channel varchar( 1024 ) null; -alter table topic_posts add url text null; - -create index topic_post_topic_channel on topic_posts( topic_post_days_id, channel ); - -alter table topic_tweet_urls rename to topic_post_urls; -alter table topic_post_urls rename topic_tweet_urls_id to topic_post_urls_id; -alter table topic_post_urls rename topic_tweets_id to topic_posts_id; - -drop view topic_tweet_full_urls; -create view topic_post_full_urls as - select distinct - t.topics_id, - tt.topic_posts_id, tt.content, tt.publish_date, tt.author, - ttd.day, ttd.num_posts, ttd.posts_fetched, - ttu.url, tsu.stories_id - from - topics t - join topic_post_days ttd on ( t.topics_id = ttd.topics_id ) - join topic_posts tt using ( topic_post_days_id ) - join topic_post_urls ttu using ( topic_posts_id ) - left join topic_seed_urls tsu - on ( tsu.topics_id = t.topics_id and ttu.url = tsu.url ); - -alter table snap.timespan_tweets rename to timespan_posts; -alter table snap.timespan_posts rename topic_tweets_id to topic_posts_id; - -alter table snap.tweet_stories rename to post_stories; -alter table snap.post_stories rename topic_tweets_id to topic_posts_id; -alter table snap.post_stories rename twitter_user to author; -alter table snap.post_stories rename num_tweets to num_posts; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4730; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4730-4731.sql b/apps/postgresql-server/schema/migrations/mediawords-4730-4731.sql deleted file mode 100644 index 790c1a5625..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4730-4731.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4730 and 4731. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4730, and you would like to upgrade both the Media Cloud and the --- database to be at version 4731, import this SQL file: --- --- psql mediacloud < mediawords-4730-4731.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TABLE auth_user_limits ADD PRIMARY KEY (auth_user_limits_id); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4731; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4731-4732.sql b/apps/postgresql-server/schema/migrations/mediawords-4731-4732.sql deleted file mode 100644 index 3aec6c2157..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4731-4732.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4731 and 4732. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4731, and you would like to upgrade both the Media Cloud and the --- database to be at version 4732, import this SQL file: --- --- psql mediacloud < mediawords-4731-4732.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TABLE auth_user_limits - ADD COLUMN max_topic_stories INTEGER NOT NULL DEFAULT 100000; - -UPDATE auth_user_limits -SET max_topic_stories = auth_users.max_topic_stories -FROM public.auth_users -WHERE auth_user_limits.auth_users_id = auth_users.auth_users_id; - -ALTER TABLE auth_users - DROP COLUMN max_topic_stories; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4732; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4732-4733.sql b/apps/postgresql-server/schema/migrations/mediawords-4732-4733.sql deleted file mode 100644 index 32f3891bce..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4732-4733.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4732 and 4733. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4733, and you would like to upgrade both the Media Cloud and the --- database to be at version 4733, import this SQL file: --- --- psql mediacloud < mediawords-4732-4733.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -DROP TABLE auth_users_subscribe_to_newsletter; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4733; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4733-4734.sql b/apps/postgresql-server/schema/migrations/mediawords-4733-4734.sql deleted file mode 100644 index bf3b18753d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4733-4734.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4733 and 4734. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4733, and you would like to upgrade both the Media Cloud and the --- database to be at version 4734, import this SQL file: --- --- psql mediacloud < mediawords-4733-4734.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table snapshots add seed_queries jsonb null; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4734; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4734-4735.sql b/apps/postgresql-server/schema/migrations/mediawords-4734-4735.sql deleted file mode 100644 index 7574ceaf04..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4734-4735.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4734 and 4735. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4734, and you would like to upgrade both the Media Cloud and the --- database to be at version 4735, import this SQL file: --- --- psql mediacloud < mediawords-4734-4735.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics alter column pattern drop not null; -alter table topics alter column solr_seed_query drop not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4735; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4735-4736.sql b/apps/postgresql-server/schema/migrations/mediawords-4735-4736.sql deleted file mode 100644 index d3f3e79c2f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4735-4736.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4735 and 4736. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4735, and you would like to upgrade both the Media Cloud and the --- database to be at version 4736, import this SQL file: --- --- psql mediacloud < mediawords-4735-4736.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TABLE story_statistics - ADD COLUMN facebook_reaction_count INT NULL; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4736; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4736-4737.sql b/apps/postgresql-server/schema/migrations/mediawords-4736-4737.sql deleted file mode 100644 index 61f1e61702..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4736-4737.sql +++ /dev/null @@ -1,205 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4735 and 4736. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4735, and you would like to upgrade both the Media Cloud and the --- database to be at version 4736, import this SQL file: --- --- psql mediacloud < mediawords-4735-4736.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TYPE feed_type ADD VALUE 'podcast'; - - --- --- Enclosures added to the story's feed item --- -CREATE TABLE story_enclosures ( - story_enclosures_id BIGSERIAL PRIMARY KEY, - stories_id INT NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - - -- Podcast enclosure URL - url TEXT NOT NULL, - - -- RSS spec says that enclosure's "length" and "type" are required too but - -- I guess some podcasts don't care that much about specs so both are - -- allowed to be NULL: - - -- MIME type as reported by - mime_type CITEXT NULL, - - -- Length in bytes as reported by - length BIGINT NULL -); - -CREATE UNIQUE INDEX story_enclosures_stories_id_url - ON story_enclosures (stories_id, url); - - --- --- Audio file codec; keep in sync with "_SUPPORTED_NATIVE_AUDIO_CODECS" constant --- (https://cloud.google.com/speech-to-text/docs/reference/rpc/google.cloud.speech.v1p1beta1) --- -CREATE TYPE podcast_episodes_audio_codec AS ENUM ( - 'LINEAR16', - 'FLAC', - 'MULAW', - 'OGG_OPUS', - 'MP3' -); - - --- --- Podcast story episodes (derived from enclosures) --- -CREATE TABLE podcast_episodes ( - podcast_episodes_id BIGSERIAL PRIMARY KEY, - stories_id INT NOT NULL REFERENCES stories (stories_id) ON DELETE CASCADE, - - -- Enclosure that's considered to point to a podcast episode - story_enclosures_id BIGINT NOT NULL - REFERENCES story_enclosures (story_enclosures_id) - ON DELETE CASCADE, - - -- Google Cloud Storage URI where the audio file is located at - gcs_uri TEXT NOT NULL - CONSTRAINT gcs_uri_has_gs_prefix - CHECK(gcs_uri LIKE 'gs://%'), - - -- Duration (in seconds) - duration INT NOT NULL - CONSTRAINT duration_is_positive - CHECK(duration > 0), - - -- Audio codec as determined by transcoder - codec podcast_episodes_audio_codec NOT NULL, - - -- Audio sample rate (Hz) as determined by transcoder - sample_rate INT NOT NULL - CONSTRAINT sample_rate_looks_reasonable - CHECK(sample_rate > 1000), - - -- BCP 47 language identifier - -- (https://cloud.google.com/speech-to-text/docs/languages) - bcp47_language_code CITEXT NOT NULL - CONSTRAINT bcp47_language_code_looks_reasonable - CHECK( - bcp47_language_code LIKE '%-%' - OR bcp47_language_code = 'zh' - ), - - -- Speech API operation ID to be used for retrieving transcription; if NULL, - -- transcription job hasn't been submitted yet - speech_operation_id TEXT NULL - -); - --- Only one episode per story -CREATE UNIQUE INDEX podcast_episodes_stories_id - ON podcast_episodes (stories_id); - -CREATE UNIQUE INDEX podcast_episodes_story_enclosures_id - ON podcast_episodes (story_enclosures_id); - -CREATE UNIQUE INDEX podcast_episodes_stories_id_story_enclosures_id - ON podcast_episodes (stories_id, story_enclosures_id); - - --- Result of an attempt to fetch the transcript -CREATE TYPE podcast_episode_transcript_fetch_result AS ENUM ( - - -- Operation was not yet finished yet at the time of fetching - 'in_progress', - - -- Operation was finished and transcription has succeeded - 'success', - - -- Operation was finished but the transcription has failed - 'error' - -); - - --- --- Attempts to fetch podcast episode transcript --- (we might need to try fetching the operation's results multiple times) --- -CREATE TABLE podcast_episode_transcript_fetches ( - podcast_episode_transcript_fetches_id BIGSERIAL PRIMARY KEY, - - -- Podcast that is being transcribed - podcast_episodes_id BIGINT NOT NULL - REFERENCES podcast_episodes (podcast_episodes_id) - ON DELETE CASCADE, - - -- Timestamp for when a fetch job should be added to the job broker's queue the soonest - add_to_queue_at TIMESTAMP WITH TIME ZONE NOT NULL, - - -- Timestamp for when a fetch job was added to the job broker's queue; - -- if NULL, a fetch job was never added to the queue - added_to_queue_at TIMESTAMP WITH TIME ZONE NULL, - - -- Timestamp when the operation's results were attempted to be fetched by the worker; - -- if NULL, the results weren't attempted to be fetched yet - fetched_at TIMESTAMP WITH TIME ZONE NULL, - - -- Result of the fetch attempt; - -- if NULL, the operation fetch didn't happen yet - result podcast_episode_transcript_fetch_result NULL, - - -- If result = 'error', error message that happened with the fetch attempt - error_message TEXT NULL - -); - - --- Function that returns true if results were attempted at being fetched -CREATE FUNCTION podcast_episode_transcript_was_added_to_queue(p_added_to_queue_at TIMESTAMP WITH TIME ZONE) -RETURNS BOOL AS $$ - - SELECT CASE WHEN p_added_to_queue_at::timestamp IS NULL THEN false ELSE true END; - -$$ LANGUAGE SQL IMMUTABLE; - - -CREATE INDEX podcast_episode_transcript_fetches_podcast_episodes_id - ON podcast_episode_transcript_fetches (podcast_episodes_id); - -CREATE UNIQUE INDEX podcast_episode_transcript_fetches_due - ON podcast_episode_transcript_fetches ( - add_to_queue_at, - podcast_episode_transcript_was_added_to_queue(added_to_queue_at) - ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4737; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4737-4738.sql b/apps/postgresql-server/schema/migrations/mediawords-4737-4738.sql deleted file mode 100644 index d67291f110..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4737-4738.sql +++ /dev/null @@ -1,52 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4737 and 4738. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4737, and you would like to upgrade both the Media Cloud and the --- database to be at version 4738, import this SQL file: --- --- psql mediacloud < mediawords-4737-4738.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4738; - -BEGIN - - create unique index database_variables_name on database_variables ( name ); - - drop index media_stats_medium; - create unique index media_stats_medium_date on media_stats( media_id, stat_date ); - - drop function if exists insert_story_media_stats cascade; - drop function if exists update_story_media_stats cascade; - drop function if exists delete_story_media_stats cascade; - - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4738-4739.sql b/apps/postgresql-server/schema/migrations/mediawords-4738-4739.sql deleted file mode 100644 index d5f31a91a8..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4738-4739.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4738 and 4739. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4738, and you would like to upgrade both the Media Cloud and the --- database to be at version 4739, import this SQL file: --- --- psql mediacloud < mediawords-4738-4739.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topics add only_snapshot_engaged_stories boolean default false; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4739; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4739-4740.sql b/apps/postgresql-server/schema/migrations/mediawords-4739-4740.sql deleted file mode 100644 index fa5a290e41..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4739-4740.sql +++ /dev/null @@ -1,202 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4739 and 4740. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4739, and you would like to upgrade both the Media Cloud and the --- database to be at version 4740, import this SQL file: --- --- psql mediacloud < mediawords-4739-4740.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- the mode is how we analyze the data from the platform (as web pages, social media posts, url sharing posts, etc) -create table topic_modes ( - topic_modes_id serial primary key, - name varchar(1024) not null unique, - description text not null -); - -create unique index topic_modes_name on topic_modes(name); - -insert into topic_modes ( name, description ) values - - ('web', 'analyze urls using hyperlinks as network edges'), - ('url_sharing', 'analyze urls shared on social media using co-sharing as network edges'); - --- the platform is where the analyzed data lives (web, twitter, reddit, etc) -create table topic_platforms ( - topic_platforms_id serial primary key, - name varchar(1024) not null unique, - description text not null -); - -create unique index topic_platforms_name on topic_platforms(name); - -insert into topic_platforms (name, description) values - ('web', 'pages on the open web'), - ('twitter', 'tweets from twitter.com'), - ('generic_post', 'generic social media posts'), - ('reddit', 'submissions and comments from reddit.com'); - --- the source is where we get the platforn data from (a particular database, api, csv, etc) -create table topic_sources ( - topic_sources_id serial primary key, - name varchar(1024) not null unique, - description text not null -); - -create unique index topic_sources_name on topic_sources(name); - -insert into topic_sources ( name, description ) values - ('mediacloud', 'import from the mediacloud.org archive'), - ('crimson_hexagon', 'import from the crimsonhexagon.com forsight api, only accessible to internal media cloud team'), - ('csv', 'import generic posts directly from csv'), - ('pushshift', 'import from the pushshift.io api'); - --- the pairs of platforms / sources for which the platform can fetch data -create table topic_platforms_sources_map ( - topic_platforms_id int not null references topic_platforms on delete cascade, - topic_sources_id int not null references topic_sources on delete cascade -); - -create unique index topic_platforms_sources_map_ps - on topic_platforms_sources_map ( topic_platforms_id, topic_sources_id ); - --- easily create platform source pairs -create function insert_platform_source_pair( text, text ) returns void as $$ - insert into topic_platforms_sources_map ( topic_platforms_id, topic_sources_id ) - select - tp.topic_platforms_id, - ts.topic_sources_id - from - topic_platforms tp - cross join topic_sources ts - where - tp.name = $1 and - ts.name = $2 -$$ language sql; - -select insert_platform_source_pair( 'web', 'mediacloud' ); -select insert_platform_source_pair( 'twitter', 'crimson_hexagon' ); -select insert_platform_source_pair( 'generic_post', 'csv' ); -select insert_platform_source_pair( 'reddit', 'pushshift' ); - -drop view controversies; -drop view controversy_dumps; -drop view controversy_dump_time_slices; - -drop view topics_with_user_permission; - -drop view topic_post_full_urls; - -alter table topics alter platform type text; -alter table topics add foreign key ( platform ) references topic_platforms ( name ); -alter table topics alter platform set default 'web'; - -alter table topics alter mode type text; -alter table topics add foreign key ( mode ) references topic_modes ( name ); -alter table topics alter mode set default 'web'; - -alter table topic_seed_queries alter platform type text; -alter table topic_seed_queries add foreign key ( platform ) references topic_platforms ( name ); - -alter table topic_seed_queries alter source type text; -alter table topic_seed_queries add foreign key ( source ) references topic_sources ( name ); - -alter table topic_seed_urls add topic_seed_queries_id int null references topic_seed_queries on delete cascade; - -alter table topic_post_days add topic_seed_queries_id int references topic_seed_queries on delete cascade; - -update topic_post_days tpd - set topic_seed_queries_id = tsq.topic_seed_queries_id - from topic_seed_queries tsq - where tsq.topics_id = tpd.topics_id; - -drop index topic_post_days_td; - -alter table topic_post_days drop topics_id; - -alter table topic_post_days alter topic_seed_queries_id set not null; - -create index topic_post_days_td on topic_post_days ( topic_seed_queries_id, day ); - -create view topic_post_full_urls as - select distinct - t.topics_id, - tt.topic_posts_id, tt.content, tt.publish_date, tt.author, - ttd.day, ttd.num_posts, ttd.posts_fetched, - ttu.url, tsu.stories_id - from - topics t - join topic_seed_queries tsq using ( topics_id ) - join topic_post_days ttd using ( topic_seed_queries_id ) - join topic_posts tt using ( topic_post_days_id ) - join topic_post_urls ttu using ( topic_posts_id ) - left join topic_seed_urls tsu - on ( tsu.topics_id = t.topics_id and ttu.url = tsu.url ); - -create or replace view topics_with_user_permission as - with admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin' - ), - - read_admin_users as ( - select m.auth_users_id - from auth_roles r - join auth_users_roles_map m using ( auth_roles_id ) - where - r.role = 'admin-readonly' - ) - - select - t.*, - u.auth_users_id, - case - when ( exists ( select 1 from admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'admin' when ( tp.permission is not null ) then tp.permission::text - when ( t.is_public ) then 'read' - when ( exists ( select 1 from read_admin_users a where a.auth_users_id = u.auth_users_id ) ) then 'read' else 'none' end - as user_permission - from topics t - join auth_users u on ( true ) - left join topic_permissions tp using ( topics_id, auth_users_id ); - - -drop type topic_platform_type; -drop type topic_source_type; -drop type topic_mode_type; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4740; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4740-4741.sql b/apps/postgresql-server/schema/migrations/mediawords-4740-4741.sql deleted file mode 100644 index dd87a42b44..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4740-4741.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4740 and 4741. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4740, and you would like to upgrade both the Media Cloud and the --- database to be at version 4741, import this SQL file: --- --- psql mediacloud < mediawords-4740-4741.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- -insert into topic_sources ( name, description ) values - ('google', 'import from search results on google'); - -select insert_platform_source_pair( 'web', 'google' ); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4741; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4741-4742.sql b/apps/postgresql-server/schema/migrations/mediawords-4741-4742.sql deleted file mode 100644 index 066d498f5b..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4741-4742.sql +++ /dev/null @@ -1,51 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4741 and 4742. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4741, and you would like to upgrade both the Media Cloud and the --- database to be at version 4742, import this SQL file: --- --- psql mediacloud < mediawords-4741-4742.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create table timespan_maps ( - timespan_maps_id serial primary key, - timespans_id int not null references timespans on delete cascade, - options jsonb not null, - content bytea not null, - format varchar(1024) not null -); - -create index topic_maps_timespan on timespan_maps ( timespans_id ); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4742; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4742-4743.sql b/apps/postgresql-server/schema/migrations/mediawords-4742-4743.sql deleted file mode 100644 index d7db6a427c..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4742-4743.sql +++ /dev/null @@ -1,73 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4742 and 4743. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4742, and you would like to upgrade both the Media Cloud and the --- database to be at version 4743, import this SQL file: --- --- psql mediacloud < mediawords-4742-4743.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table focal_sets alter focal_technique type text; -alter table focal_set_definitions alter focal_technique type text; - -drop type focal_technique_type; - -create type focal_technique_type as enum ( 'Boolean Query', 'URL Sharing' ); - -alter table focal_sets alter focal_technique type focal_technique_type using focal_technique::focal_technique_type; -alter table focal_set_definitions alter focal_technique type focal_technique_type - using focal_technique::focal_technique_type; - -drop view topic_post_full_urls; - -create view topic_post_stories as - select - tsq.topics_id, - tp.topic_posts_id, tp.content, tp.publish_date, tp.author, tp.channel, tp.data, - tpd.topic_seed_queries_id, - ts.stories_id, - tpu.url - from - topic_seed_queries tsq - join topic_post_days tpd using ( topic_seed_queries_id ) - join topic_posts tp using ( topic_post_days_id ) - join topic_post_urls tpu using ( topic_posts_id ) - join topic_seed_urls tsu - on ( tsu.topics_id = tsq.topics_id and tsu.url = tpu.url ) - join topic_stories ts - on ( ts.topics_id = tsq.topics_id and ts.stories_id = tsu.stories_id ); - -drop table snap.post_stories; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4743; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4743-4744.sql b/apps/postgresql-server/schema/migrations/mediawords-4743-4744.sql deleted file mode 100644 index 6b3cfc5816..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4743-4744.sql +++ /dev/null @@ -1,60 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4743 and 4744. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4743, and you would like to upgrade both the Media Cloud and the --- database to be at version 4744, import this SQL file: --- --- psql mediacloud < mediawords-4743-4744.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table snap.story_link_counts rename post_count to author_count; -alter table snap.story_link_counts add post_count int null; -alter table snap.story_link_counts add channel_count int null; - -create index story_link_counts_fb on snap.story_link_counts ( timespans_id, facebook_share_count desc nulls last ); -create index story_link_counts_post on snap.story_link_counts ( timespans_id, post_count desc nulls last); -create index story_link_counts_author on snap.story_link_counts ( timespans_id, author_count desc nulls last); -create index story_link_counts_channel on snap.story_link_counts ( timespans_id, channel_count desc nulls last); - -alter table snap.medium_link_counts rename post_count to sum_author_count; -alter table snap.medium_link_counts add sum_post_count int null; -alter table snap.medium_link_counts add sum_channel_count int null; - -create index medium_link_counts_fb on snap.medium_link_counts ( timespans_id, facebook_share_count desc nulls last); -create index medium_link_counts_sum_post on snap.medium_link_counts ( timespans_id, sum_post_count desc nulls last); -create index medium_link_counts_sum_author on snap.medium_link_counts ( timespans_id, sum_author_count desc nulls last); -create index medium_link_counts_sum_channel on snap.medium_link_counts ( timespans_id, sum_channel_count desc nulls last); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4744; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4744-4745.sql b/apps/postgresql-server/schema/migrations/mediawords-4744-4745.sql deleted file mode 100644 index 4630ebb688..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4744-4745.sql +++ /dev/null @@ -1,46 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4744 and 4745. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4744, and you would like to upgrade both the Media Cloud and the --- database to be at version 4745, import this SQL file: --- --- psql mediacloud < mediawords-4744-4745.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - alter table timespan_maps add url text null; -alter table timespan_maps alter content drop not null; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4745; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4745-4746.sql b/apps/postgresql-server/schema/migrations/mediawords-4745-4746.sql deleted file mode 100644 index adfcebc634..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4745-4746.sql +++ /dev/null @@ -1,91 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4745 and 4746. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4745, and you would like to upgrade both the Media Cloud and the --- database to be at version 4746, import this SQL file: --- --- psql mediacloud < mediawords-4745-4746.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -drop table if exists timespan_files; -create table timespan_files ( - timespan_files_id serial primary key, - timespans_id int not null references timespans on delete cascade, - name text, - url text -); - -create unique index timespan_files_timespan_name on timespan_files ( timespans_id, name ); - -drop table if exists snapshot_files; -create table snapshot_files ( - snapshot_files_id serial primary key, - snapshots_id int not null references snapshots on delete cascade, - name text, - url text -); - -create unique index snapshot_files_snapshot_name on snapshot_files ( snapshots_id, name ); - - --- table for object types used for mediawords.util.public_store -create schema public_store; - - -create table public_store.timespan_files ( - timespan_files_id bigserial primary key, - object_id bigint not null, - raw_data bytea not null -); - -create unique index timespan_files_id on public_store.timespan_files ( object_id ); - -create table public_store.snapshot_files ( - snapshot_files_id bigserial primary key, - object_id bigint not null, - raw_data bytea not null -); - -create unique index snapshot_files_id on public_store.snapshot_files ( object_id ); - -create table public_store.timespan_maps ( - timespan_maps_id bigserial primary key, - object_id bigint not null, - raw_data bytea not null -); - -create unique index timespan_maps_id on public_store.timespan_maps ( object_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4746; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4746-4747.sql b/apps/postgresql-server/schema/migrations/mediawords-4746-4747.sql deleted file mode 100644 index ac59b700ff..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4746-4747.sql +++ /dev/null @@ -1,75 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4746 and 4747. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4746, and you would like to upgrade both the Media Cloud and the --- database to be at version 4747, import this SQL file: --- --- psql mediacloud < mediawords-4746-4747.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - --- return false if there is a request for the given domain within the last domain_timeout_arg milliseconds. otherwise --- return true and insert a row into domain_web_request for the domain. this function does not lock the table and --- so may allow some parallel requests through. -create or replace function get_domain_web_requests_lock( domain_arg text, domain_timeout_arg float ) returns boolean as $$ -begin - --- we don't want this table to grow forever or to have to manage it externally, so just truncate about every --- 1 million requests. only do this if there are more than 1000 rows in the table so that unit tests will not --- randomly fail. -if ( select random() * 1000000 ) < 1 then - if exists ( select 1 from domain_web_requests offset 1000 ) then - truncate table domain_web_requests; - end if; -end if; - -if exists ( - select * - from domain_web_requests - where - domain = domain_arg and - extract( epoch from now() - request_time ) < domain_timeout_arg - ) then - - return false; -end if; - -delete from domain_web_requests where domain = domain_arg; -insert into domain_web_requests (domain) select domain_arg; - -return true; -end -$$ language plpgsql; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4747; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4747-4748.sql b/apps/postgresql-server/schema/migrations/mediawords-4747-4748.sql deleted file mode 100644 index d28aeefb3d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4747-4748.sql +++ /dev/null @@ -1,47 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4747 and 4748. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4747, and you would like to upgrade both the Media Cloud and the --- database to be at version 4748, import this SQL file: --- --- psql mediacloud < mediawords-4747-4748.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -insert into topic_sources ( name, description ) values - ('postgres', 'import generic posts from a postgres table'); - - -select insert_platform_source_pair( 'generic_post', 'postgres' ); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4748; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4748-4749.sql b/apps/postgresql-server/schema/migrations/mediawords-4748-4749.sql deleted file mode 100644 index 936ff36a16..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4748-4749.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4748 and 4749. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4748, and you would like to upgrade both the Media Cloud and the --- database to be at version 4749, import this SQL file: --- --- psql mediacloud < mediawords-4748-4749.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topic_seed_queries add ignore_pattern text; --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4749; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4749-4750.sql b/apps/postgresql-server/schema/migrations/mediawords-4749-4750.sql deleted file mode 100644 index 697ef089b3..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4749-4750.sql +++ /dev/null @@ -1,49 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4749 and 4750. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4749, and you would like to upgrade both the Media Cloud and the --- database to be at version 4750, import this SQL file: --- --- psql mediacloud < mediawords-4749-4750.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topic_post_days rename num_posts to num_posts_stored; -alter table topic_post_days add num_posts_fetched int; - -update topic_post_days set num_posts_fetched = num_posts_stored; - -alter table topic_post_days alter num_posts_fetched set not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4750; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4750-4751.sql b/apps/postgresql-server/schema/migrations/mediawords-4750-4751.sql deleted file mode 100644 index 7ec7c5758a..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4750-4751.sql +++ /dev/null @@ -1,64 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4750 and 4751. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4750, and you would like to upgrade both the Media Cloud and the --- database to be at version 4751, import this SQL file: --- --- psql mediacloud < mediawords-4750-4751.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -alter table topic_seed_urls add topic_post_urls_id int references topic_post_urls on delete cascade; - -alter table topic_seed_urls add constraint topic_seed_urls_query - foreign key ( topic_seed_queries_id ) references topic_seed_queries on delete cascade; - -create or replace view topic_post_stories as - select - tsq.topics_id, - tp.topic_posts_id, tp.content, tp.publish_date, tp.author, tp.channel, tp.data, - tpd.topic_seed_queries_id, - ts.stories_id, - tpu.url, tpu.topic_post_urls_id - from - topic_seed_queries tsq - join topic_post_days tpd using ( topic_seed_queries_id ) - join topic_posts tp using ( topic_post_days_id ) - join topic_post_urls tpu using ( topic_posts_id ) - join topic_seed_urls tsu using ( topic_post_urls_id ) - join topic_stories ts - on ( ts.topics_id = tsq.topics_id and ts.stories_id = tsu.stories_id ); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4751; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4751-4752.sql b/apps/postgresql-server/schema/migrations/mediawords-4751-4752.sql deleted file mode 100644 index e60c0ea15f..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4751-4752.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4751 and 4752. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4751, and you would like to upgrade both the Media Cloud and the --- database to be at version 4752, import this SQL file: --- --- psql mediacloud < mediawords-4751-4752.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create unique index topic_seed_urls_tpu on topic_seed_urls ( topic_post_urls_id ); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4752; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4752-4753.sql b/apps/postgresql-server/schema/migrations/mediawords-4752-4753.sql deleted file mode 100644 index 64eb5efb05..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4752-4753.sql +++ /dev/null @@ -1,59 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4752 and 4753. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4752, and you would like to upgrade both the Media Cloud and the --- database to be at version 4753, import this SQL file: --- --- psql mediacloud < mediawords-4752-4753.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -CREATE TABLE IF NOT EXISTS celery_groups ( - id INTEGER NOT NULL PRIMARY KEY, - taskset_id CHARACTER VARYING(155) NULL UNIQUE, - result BYTEA NULL, - date_done TIMESTAMP WITHOUT TIME ZONE NULL -); - -CREATE TABLE IF NOT EXISTS celery_tasks ( - id INTEGER NOT NULL PRIMARY KEY, - task_id CHARACTER VARYING(155) NULL UNIQUE, - status CHARACTER VARYING(50) NULL, - result BYTEA NULL, - date_done TIMESTAMP WITHOUT TIME ZONE NULL, - traceback TEXT NULL -); - -CREATE SEQUENCE IF NOT EXISTS task_id_sequence AS BIGINT; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4753; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4753-4754.sql b/apps/postgresql-server/schema/migrations/mediawords-4753-4754.sql deleted file mode 100644 index dcf467235d..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4753-4754.sql +++ /dev/null @@ -1,45 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4753 and 4754. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4753, and you would like to upgrade both the Media Cloud and the --- database to be at version 4754, import this SQL file: --- --- psql mediacloud < mediawords-4753-4754.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -ALTER TABLE topics - ADD COLUMN IF NOT EXISTS snapshot_periods TEXT NULL; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4754; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4754-4755.sql b/apps/postgresql-server/schema/migrations/mediawords-4754-4755.sql deleted file mode 100644 index 9d718f0c77..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4754-4755.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4754 and 4755. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4754, and you would like to upgrade both the Media Cloud and the --- database to be at version 4755, import this SQL file: --- --- psql mediacloud < mediawords-4754-4755.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -alter table stories alter publish_date drop not null; -alter table story_sentences_p alter column publish_date drop not null; -alter table snap.stories alter publish_date drop not null; -alter table snap.live_stories alter publish_date drop not null; - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4755; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4755-4756.sql b/apps/postgresql-server/schema/migrations/mediawords-4755-4756.sql deleted file mode 100644 index 9a94bccbd0..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4755-4756.sql +++ /dev/null @@ -1,67 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4755 and 4756. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4755, and you would like to upgrade both the Media Cloud and the --- database to be at version 4756, import this SQL file: --- --- psql mediacloud < mediawords-4755-4756.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - --- There's nothing worth preserving in both of these tables -DROP TABLE celery_groups; -DROP TABLE celery_tasks; - -CREATE TABLE celery_groups ( - id BIGINT NOT NULL PRIMARY KEY, - taskset_id CHARACTER VARYING(155) NULL UNIQUE, - result BYTEA NULL, - date_done TIMESTAMP WITHOUT TIME ZONE NULL -); - -CREATE TABLE celery_tasks ( - id BIGINT NOT NULL PRIMARY KEY, - task_id CHARACTER VARYING(155) NULL UNIQUE, - status CHARACTER VARYING(50) NULL, - result BYTEA NULL, - date_done TIMESTAMP WITHOUT TIME ZONE NULL, - traceback TEXT NULL -); - - --- This refers to the 4754-4755 migration, but we might want to rerun it in --- production -ALTER TABLE story_sentences_p ALTER COLUMN publish_date DROP NOT NULL; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4756; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4756-4757.sql b/apps/postgresql-server/schema/migrations/mediawords-4756-4757.sql deleted file mode 100644 index b5d0859719..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4756-4757.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4756 and 4757. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4756, and you would like to upgrade both the Media Cloud and the --- database to be at version 4757, import this SQL file: --- --- psql mediacloud < mediawords-4756-4757.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -insert into topic_sources ( name, description ) values - ('brandwatch', 'import from the brandwatch api, only accessible to internal media cloud team'); --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4757; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4757-4758.sql b/apps/postgresql-server/schema/migrations/mediawords-4757-4758.sql deleted file mode 100644 index 74de19bb80..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4757-4758.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4757 and 4758. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4757, and you would like to upgrade both the Media Cloud and the --- database to be at version 4758, import this SQL file: --- --- psql mediacloud < mediawords-4757-4758.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - -create index media_name_fts on media using gin(to_tsvector('english', name)); - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4758; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); - - diff --git a/apps/postgresql-server/schema/migrations/mediawords-4758-4759.sql b/apps/postgresql-server/schema/migrations/mediawords-4758-4759.sql deleted file mode 100644 index f67849e522..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4758-4759.sql +++ /dev/null @@ -1,212 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4758 and 4759. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4758, and you would like to upgrade both the Media Cloud and the --- database to be at version 4759, import this SQL file: --- --- psql mediacloud < mediawords-4758-4759.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -CREATE OR REPLACE FUNCTION partition_by_stories_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_stories_id INT; - partition_stories_id INT; - - -- Partition table name (e.g. "stories_tags_map_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "stories_id" chunk lower limit, inclusive (e.g. 30,000,000) - stories_id_start BIGINT; - - -- stories_id chunk upper limit, exclusive (e.g. 31,000,000) - stories_id_end BIGINT; -BEGIN - - SELECT partition_by_stories_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(stories_id), 0) + chunk_size FROM stories INTO max_stories_id; - - SELECT 1 INTO partition_stories_id; - WHILE partition_stories_id <= max_stories_id LOOP - SELECT partition_by_stories_id_partition_name( - base_table_name := base_table_name, - stories_id := partition_stories_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for story ID % already exists.', target_table_name, partition_stories_id; - ELSE - RAISE NOTICE 'Creating partition "%" for story ID %', target_table_name, partition_stories_id; - - SELECT (partition_stories_id / chunk_size) * chunk_size INTO stories_id_start; - SELECT ((partition_stories_id / chunk_size) + 1) * chunk_size INTO stories_id_end; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'stories'; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' ( - - PRIMARY KEY (' || base_table_name || '_id), - - -- Partition by stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id CHECK ( - stories_id >= ''' || stories_id_start || ''' - AND stories_id < ''' || stories_id_end || '''), - - -- Foreign key to stories.stories_id - CONSTRAINT ' || REPLACE(target_table_name, '.', '_') || '_stories_id_fkey - FOREIGN KEY (stories_id) REFERENCES stories (stories_id) MATCH FULL ON DELETE CASCADE - - ) INHERITS (' || base_table_name || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE 'ALTER TABLE ' || target_table_name || ' OWNER TO ' || target_table_owner || ';'; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - - SELECT partition_stories_id + chunk_size INTO partition_stories_id; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION partition_by_downloads_id_create_partitions(base_table_name TEXT) -RETURNS SETOF TEXT AS -$$ -DECLARE - chunk_size INT; - max_downloads_id BIGINT; - partition_downloads_id BIGINT; - - -- Partition table name (e.g. "downloads_success_content_01") - target_table_name TEXT; - - -- Partition table owner (e.g. "mediaclouduser") - target_table_owner TEXT; - - -- "downloads_id" chunk lower limit, inclusive (e.g. 30,000,000) - downloads_id_start BIGINT; - - -- "downloads_id" chunk upper limit, exclusive (e.g. 31,000,000) - downloads_id_end BIGINT; -BEGIN - - SELECT partition_by_downloads_id_chunk_size() INTO chunk_size; - - -- Create +1 partition for future insertions - SELECT COALESCE(MAX(downloads_id), 0) + chunk_size FROM downloads INTO max_downloads_id; - - SELECT 1 INTO partition_downloads_id; - WHILE partition_downloads_id <= max_downloads_id LOOP - SELECT partition_by_downloads_id_partition_name( - base_table_name := base_table_name, - downloads_id := partition_downloads_id - ) INTO target_table_name; - IF table_exists(target_table_name) THEN - RAISE NOTICE 'Partition "%" for download ID % already exists.', target_table_name, partition_downloads_id; - ELSE - RAISE NOTICE 'Creating partition "%" for download ID %', target_table_name, partition_downloads_id; - - SELECT (partition_downloads_id / chunk_size) * chunk_size INTO downloads_id_start; - SELECT ((partition_downloads_id / chunk_size) + 1) * chunk_size INTO downloads_id_end; - - -- Kill all autovacuums before proceeding with DDL changes - PERFORM pid - FROM pg_stat_activity, LATERAL pg_cancel_backend(pid) f - WHERE backend_type = 'autovacuum worker' - AND query ~ 'downloads'; - - EXECUTE ' - CREATE TABLE ' || target_table_name || ' - PARTITION OF ' || base_table_name || ' - FOR VALUES FROM (' || downloads_id_start || ') - TO (' || downloads_id_end || '); - '; - - -- Update owner - SELECT u.usename AS owner - FROM information_schema.tables AS t - JOIN pg_catalog.pg_class AS c ON t.table_name = c.relname - JOIN pg_catalog.pg_user AS u ON c.relowner = u.usesysid - WHERE t.table_name = base_table_name - AND t.table_schema = 'public' - INTO target_table_owner; - - EXECUTE ' - ALTER TABLE ' || target_table_name || ' - OWNER TO ' || target_table_owner || '; - '; - - -- Add created partition name to the list of returned partition names - RETURN NEXT target_table_name; - - END IF; - - SELECT partition_downloads_id + chunk_size INTO partition_downloads_id; - END LOOP; - - RETURN; - -END; -$$ -LANGUAGE plpgsql; - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4759; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4759-4760.sql b/apps/postgresql-server/schema/migrations/mediawords-4759-4760.sql deleted file mode 100644 index 9d8fe5e135..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4759-4760.sql +++ /dev/null @@ -1,44 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4759 and 4760. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4759, and you would like to upgrade both the Media Cloud and the --- database to be at version 4760, import this SQL file: --- --- psql mediacloud < mediawords-4759-4760.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -DROP FUNCTION IF EXISTS get_downloads_for_queue(); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4760; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/postgresql-server/schema/migrations/mediawords-4760-4761.sql b/apps/postgresql-server/schema/migrations/mediawords-4760-4761.sql deleted file mode 100644 index 9ae44c62a6..0000000000 --- a/apps/postgresql-server/schema/migrations/mediawords-4760-4761.sql +++ /dev/null @@ -1,48 +0,0 @@ --- --- This is a Media Cloud PostgreSQL schema difference file (a "diff") between schema --- versions 4760 and 4761. --- --- If you are running Media Cloud with a database that was set up with a schema version --- 4760, and you would like to upgrade both the Media Cloud and the --- database to be at version 4761, import this SQL file: --- --- psql mediacloud < mediawords-4760-4761.sql --- --- You might need to import some additional schema diff files to reach the desired version. --- --- --- 1 of 2. Import the output of 'apgdiff': --- - - -DROP TABLE podcast_episode_transcript_fetches; -DROP TABLE podcast_episodes; -DROP TYPE podcast_episodes_audio_codec; -DROP TYPE podcast_episode_transcript_fetch_result; -DROP FUNCTION podcast_episode_transcript_was_added_to_queue(TIMESTAMP WITH TIME ZONE); - - --- --- 2 of 2. Reset the database version. --- - -CREATE OR REPLACE FUNCTION set_database_schema_version() RETURNS boolean AS $$ -DECLARE - - -- Database schema version number (same as a SVN revision number) - -- Increase it by 1 if you make major database schema changes. - MEDIACLOUD_DATABASE_SCHEMA_VERSION CONSTANT INT := 4761; - -BEGIN - - -- Update / set database schema version - DELETE FROM database_variables WHERE name = 'database-schema-version'; - INSERT INTO database_variables (name, value) VALUES ('database-schema-version', MEDIACLOUD_DATABASE_SCHEMA_VERSION::int); - - return true; - -END; -$$ -LANGUAGE 'plpgsql'; - -SELECT set_database_schema_version(); diff --git a/apps/purge-object-caches/docker-compose.tests.yml b/apps/purge-object-caches/docker-compose.tests.yml index 1420a62a1c..cdc6037a60 100644 --- a/apps/purge-object-caches/docker-compose.tests.yml +++ b/apps/purge-object-caches/docker-compose.tests.yml @@ -40,8 +40,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/rescrape-media/docker-compose.tests.yml b/apps/rescrape-media/docker-compose.tests.yml index 42076ee535..5a7ce7926e 100644 --- a/apps/rescrape-media/docker-compose.tests.yml +++ b/apps/rescrape-media/docker-compose.tests.yml @@ -47,8 +47,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/tools/docker-compose.tests.yml b/apps/tools/docker-compose.tests.yml index 5c3191140b..665a29f6e3 100644 --- a/apps/tools/docker-compose.tests.yml +++ b/apps/tools/docker-compose.tests.yml @@ -42,8 +42,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-base/docker-compose.tests.yml b/apps/topics-base/docker-compose.tests.yml index 929cfda493..6dc66c83e9 100644 --- a/apps/topics-base/docker-compose.tests.yml +++ b/apps/topics-base/docker-compose.tests.yml @@ -90,8 +90,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-extract-story-links/docker-compose.tests.yml b/apps/topics-extract-story-links/docker-compose.tests.yml index 380509c5f1..94aea4a828 100644 --- a/apps/topics-extract-story-links/docker-compose.tests.yml +++ b/apps/topics-extract-story-links/docker-compose.tests.yml @@ -72,8 +72,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-fetch-link/docker-compose.tests.yml b/apps/topics-fetch-link/docker-compose.tests.yml index c4b500e35e..c9ec6f61ad 100644 --- a/apps/topics-fetch-link/docker-compose.tests.yml +++ b/apps/topics-fetch-link/docker-compose.tests.yml @@ -90,8 +90,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-fetch-twitter-urls/docker-compose.tests.yml b/apps/topics-fetch-twitter-urls/docker-compose.tests.yml index 74bd3fd650..49726d6525 100644 --- a/apps/topics-fetch-twitter-urls/docker-compose.tests.yml +++ b/apps/topics-fetch-twitter-urls/docker-compose.tests.yml @@ -94,8 +94,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-map/docker-compose.tests.yml b/apps/topics-map/docker-compose.tests.yml index d7f3d0ac2c..607c9ca4a1 100644 --- a/apps/topics-map/docker-compose.tests.yml +++ b/apps/topics-map/docker-compose.tests.yml @@ -52,8 +52,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-mine/docker-compose.tests.yml b/apps/topics-mine/docker-compose.tests.yml index 95839c3d42..011ff01846 100644 --- a/apps/topics-mine/docker-compose.tests.yml +++ b/apps/topics-mine/docker-compose.tests.yml @@ -108,8 +108,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/topics-snapshot/docker-compose.tests.yml b/apps/topics-snapshot/docker-compose.tests.yml index cd071cca44..22f1e877d0 100644 --- a/apps/topics-snapshot/docker-compose.tests.yml +++ b/apps/topics-snapshot/docker-compose.tests.yml @@ -97,8 +97,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/webapp-api/docker-compose.tests.yml b/apps/webapp-api/docker-compose.tests.yml index af67c24c42..5e2affe8ec 100644 --- a/apps/webapp-api/docker-compose.tests.yml +++ b/apps/webapp-api/docker-compose.tests.yml @@ -66,8 +66,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/webapp-httpd/docker-compose.tests.yml b/apps/webapp-httpd/docker-compose.tests.yml index a422bcb5b7..0689c6747f 100644 --- a/apps/webapp-httpd/docker-compose.tests.yml +++ b/apps/webapp-httpd/docker-compose.tests.yml @@ -95,8 +95,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml b/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml index fdc9cf0a10..2b3396af8e 100644 --- a/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml +++ b/apps/word2vec-generate-snapshot-model/docker-compose.tests.yml @@ -46,8 +46,8 @@ services: source: ./../postgresql-server/bin/ target: /opt/mediacloud/bin/ - type: bind - source: ./../postgresql-server/schema/ - target: /opt/mediacloud/schema/ + source: ./../postgresql-server/pgmigrate/ + target: /opt/postgresql-server/pgmigrate/ - type: bind source: ./../postgresql-base/conf/ target: /etc/postgresql/13/main/ diff --git a/dev/get_schema.sh b/dev/get_schema.sh new file mode 100755 index 0000000000..8286c81d53 --- /dev/null +++ b/dev/get_schema.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Generate latest schema file for reference in development +# + +set -u +set -e + +PROJECT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" + +docker run gcr.io/mcback/postgresql-server:latest cat /opt/mediawords.sql > $PROJECT_ROOT/mediawords.sql \ No newline at end of file diff --git a/doc/pycharm.markdown b/doc/pycharm.markdown index 54c73f5893..849c787e51 100644 --- a/doc/pycharm.markdown +++ b/doc/pycharm.markdown @@ -231,14 +231,9 @@ Without the SQL schema dialect and data source configuration, PyCharm will compl To configure SQL schema dialect and data source: -1. Symlink `apps/postgresql-server/schema/mediawords.sql` to `.idea/` directory: +1. To generate the latest version of the schema, navigate to the project root in your terminal and run `./dev/get_schema.sh`. - ```bash - cd apps/your_app/.idea/ - ln -s ../../postgresql-server/schema/mediawords.sql . - ``` - -2. Go to *Languages & Frameworks* -> *SQL Dialects*, set the *Project SQL Dialect:* to *PostgreSQL* and click *OK*: +2. In PyCharm, go to *Languages & Frameworks* -> *SQL Dialects*, set the *Project SQL Dialect:* to *PostgreSQL* and click *OK*: ![](https://github.com/mediacloud/backend-docs-images/raw/master/docker-pycharm/schema-02-dialect-project.png) @@ -250,7 +245,7 @@ To configure SQL schema dialect and data source: ![](https://github.com/mediacloud/backend-docs-images/raw/master/docker-pycharm/schema-04-data-source-ddl.png) -5. Leave the default value of the *Name:* field intact and under *DDL Files*, add a `mediawords.sql` file located in `.idea/` directory (you might need to click on *Show Hidden Files and Directories* button first) and click *OK*: +5. Leave the default value of the *Name:* field intact and under *DDL Files*, add the `mediawords.sql` file (now located in the project root; screenshot below outdated) and click *OK*: ![](https://github.com/mediacloud/backend-docs-images/raw/master/docker-pycharm/schema-05-data-source-mediawords.png) diff --git a/doc/schema_migrations.markdown b/doc/schema_migrations.markdown index c5ef2a6fe9..b1333a3210 100644 --- a/doc/schema_migrations.markdown +++ b/doc/schema_migrations.markdown @@ -1,20 +1,14 @@ # Schema and migrations -App `postgresql-server` provides the rest of the apps with the capability of storing and reading data from PostgreSQL. App comes pre-loaded with full schema (located in `schema/mediawords.sql`) at build time. +App `postgresql-server` provides the rest of the apps with the capability of storing and reading data from PostgreSQL. The schema migrations get applied privately, i.e. before the database instance actually goes "live" and becomes accessible to other apps, so every app is guaranteed to be connecting to the database instance with the most up-to-date schema version. Migrations are handled via [the `pgmigrate` tool](https://github.com/yandex/pgmigrate/), an open-source library from Yandex that supports both PL/pgSQL and standard SQL. -Additionally, on every start the wrapper script will test the instance for the schema version (stored in `database_variables` table) that's currently being used in the attached data volume, and if the schema version is older than the current schema version in `schema/mediawords.sql`, it will execute appropriate migration scripts (located in `schema/migrations/`) to get the data volume's schema to the newest version. +To get a dump of the current schema, run [`./dev/get_schema.sh`](https://github.com/mediacloud/backend/blob/master/dev/get_schema.sh). That will create a file called `mediawords.sql` in the app's `/schema` folder—useful as a reference in development and [for configuring PyCharm](https://github.com/mediacloud/backend/blob/master/doc/pycharm.markdown#configure-sql-schema), though excluded from version control via `.gitignore`. -The schema migrations get applied privately, i.e. before the database instance actually goes "live" and becomes accessible to other apps, so every app is guaranteed to be connecting to the database instance with the most up-to-date schema version. +To perform a new migration: -## Updating schema +1. Create a new `.sql` file in `/migrations` prefixed with `V`, the latest version number, **two** underscores, and a brief summary of the migration (e.g. `V0005__drop_foo_table.sql`). The migrations are then executed as part of the app's Docker build, which will fail if a migration can't be executed. Add comments/context (always welcome!) in your migration scripts via [`COMMENT ON` statements](https://www.postgresql.org/docs/11/sql-comment.html). +2. Rebuild `postgresql-server` app image with [the `build.py` developer script](https://github.com/mediacloud/backend/blob/master/doc/dev_scripts.markdown#buildpy---build-images), or just `git push` the changes to force the CI server to rebuild everything. +3. Pull the updated `postgresql-server` image in production, remove old container running an outdated image, and create a new container using the updated image and a data volume from the old container. +4. Start the container. The wrapper script in the container will temporarily start a private instance of PostgreSQL and apply the schema migrations before starting a public instance of the service for other apps to use. -To make changes to the schema: - -1. Edit the main schema file (located in `schema/mediawords.sql` file under the `postgresql-server` app) to make required changes; -2. In the main schema, update the `MEDIACLOUD_DATABASE_SCHEMA_VERSION` variable at the top of the file by increasing the schema version; -3. In the schema migrations directory (located in `schema/migrations/` directory under the `postgresql-server` app), add a new migration file with the name `mediawords--.sql`; in the migration file, add SQL statements that both - 1. makes the required changes in the schema itself (creates / drops tables, columns, etc.), and - 2. sets the `MEDIACLOUD_DATABASE_SCHEMA_VERSION` variable to the newest schema version; -4. Rebuild `postgresql-server` app image with `build.py` developer script, or just `git push` the changes to for the CI server to rebuild everything; -5. Pull the updated `postgresql-server` image in production, remove old container running an outdated image, and create a new container using the updated image and a data volume from the old container. -6. Start the container. The wrapper script in the container will temporarily start a private instance of PostgreSQL and apply the schema migrations before starting a public instance of the service for other apps to use. +There's some vary basic configuration for `pgmigrate` in `migrations.yml`. The tool is capable of running callbacks (stored in `/pgmigrate-callbacks`) before and after the sequence of migrations has been executed during the build, or before and after each individual migration. Currently, the only callback we use in the `beforeEach` folder (for setting `search_path`). Callbacks in each subfolder (e.g. `/pgmigrate-callbacks/beforeEach`) should be prefixed numerically to indicate the order in which they should be run. \ No newline at end of file