diff --git a/.gitignore b/.gitignore index cad2a7f..46627e0 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ !/database-data/postgres/.keep /database-data/mysql/* !/database-data/mysql/.keep +/database-data/mms-postgres/* /node_modules /yarn-error.log diff --git a/Gemfile.lock b/Gemfile.lock index 10f63b9..0e91b56 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -342,7 +342,7 @@ GEM spring (4.2.1) spring-commands-rspec (1.0.4) spring (>= 0.9.1) - stringio (3.1.1) + stringio (3.1.2) thor (1.3.2) tilt (2.4.0) timeout (0.4.1) diff --git a/app/models/capture.rb b/app/models/capture.rb new file mode 100644 index 0000000..16f984c --- /dev/null +++ b/app/models/capture.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# This class talks to the MySQL database that stores the images that are in isilon. +class Capture < ActiveRecord::Base + has_many :identifiers, as: :identifiable + + establish_connection(:ami_filestore) + self.table_name = 'captures' +end diff --git a/app/models/identifier.rb b/app/models/identifier.rb new file mode 100644 index 0000000..2d48bfd --- /dev/null +++ b/app/models/identifier.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# This class talks to the MySQL database that stores the images that are in isilon. +class Identifier < ActiveRecord::Base + belongs_to :identifiable, polymorphic: true + + establish_connection(:ami_filestore) + self.table_name = 'identifiers' +end diff --git a/app/models/link_store.rb b/app/models/link_store.rb new file mode 100644 index 0000000..9d54183 --- /dev/null +++ b/app/models/link_store.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class LinkStore < ActiveRecord::Base + establish_connection(:image_filestore) + self.table_name = 'link_store' + + def uuid + if self.url =~ /uuid:([a-f0-9\-]+)/ + uuid = $1 + #puts "Extracted UUID: #{uuid}" + else + uuid = nil + #puts "No UUID found in the URL" + end + + uuid + end + +end diff --git a/config/database.yml b/config/database.yml index 2d9f115..afe82c3 100644 --- a/config/database.yml +++ b/config/database.yml @@ -22,7 +22,7 @@ image_filestore: reconnect: true ami_filestore: - adapter: mysql2 + adapter: postgresql host: <%= ENV.fetch('AMI_FILESTORE_DATABASE_HOST') { 'localhost' } %> database: <%= ENV.fetch('AMI_FILESTORE_DATABASE_NAME') %> username: <%= ENV.fetch('AMI_FILESTORE_DATABASE_USER') %> diff --git a/db/schema.rb b/db/schema.rb index f031935..1359a3c 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2023_04_12_181309) do +ActiveRecord::Schema[7.2].define(version: 2023_04_12_181309) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -42,5 +42,4 @@ t.datetime "first_indexed", precision: nil t.index ["uuid"], name: "index_repo_solr_docs_on_uuid", unique: true end - end diff --git a/docker-compose.yml b/docker-compose.yml index 95abe5a..c299029 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ services: webapp: + #platform: linux/arm64 # uncomment for M1 macs build: context: . dockerfile: Dockerfile target: development - #platform: linux/arm64 # uncomment for M1 macs env_file: - ./.env volumes: @@ -13,38 +13,32 @@ services: - '3000:80' depends_on: - postgres - - fedora + - mms-postgres - filestore-db filestore-db: - build: - context: ../filestore_databases_docker - dockerfile: Dockerfile + #platform: linux/arm64 # uncomment for M1 macs + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: mysqlpassword volumes: - ./database-data/mysql:/var/lib/mysql ports: - '3306:3306' - worker: - image: 'fedora_ingest_rails-webapp:latest' - env_file: - - ./.env - volumes: - - '.:/home/app/fedora_ingest_rails' - entrypoint: /home/app/fedora_ingest_rails/bin/delayed_job run - depends_on: - - postgres - - fedora - - filestore-db postgres: - image: postgres:11 + #platform: linux/arm64 # uncomment for M1 macs + image: postgres:11 environment: POSTGRES_PASSWORD: mypassword volumes: - ./database-data/postgres:/var/lib/postgresql/data ports: - - '5432' - fedora: - build: - context: ../fedoracommons-3.4.2-dockerized - dockerfile: Dockerfile + - '5432:5432' + mms-postgres: + #platform: linux/arm64 # uncomment for M1 macs + image: postgres:12-bullseye + environment: + - POSTGRES_HOST_AUTH_METHOD=trust ports: - - '8080:8080' + - '5433:5432' + volumes: + - ./database-data/mms-postgres:/var/lib/postgresql/data diff --git a/lib/tasks/migrate_urls.rake b/lib/tasks/migrate_urls.rake new file mode 100644 index 0000000..3eae2f9 --- /dev/null +++ b/lib/tasks/migrate_urls.rake @@ -0,0 +1,24 @@ +namespace :urls do + task migrate: :environment do + links = 0 + start_time = Time.now + + LinkStore.where("url LIKE ?", "%/fedora/%").find_each(batch_size: 1) do |ls| + links += 1 + if (links % 1000) == 0 + elapsed_time = Time.now - start_time + hh, remainder = elapsed_time.divmod(3600) + mm, ss = remainder.divmod(60) + formatted_time = "#{hh.to_i.to_s.rjust(2, '0')}:#{mm.to_i.to_s.rjust(2, '0')}:#{ss.to_i.to_s.rjust(2, '0')}" + puts "Links: #{links} | Total time: #{formatted_time}" + end + + url = ls.url + next if url.nil? + image_id = Capture.find_by(uuid: ls.uuid)&.identifiers&.where(identifier_type: 'image_id')&.limit(1)&.pluck(:identifier_value)&.first + next if image_id.nil? + new_url = "https://iiif.nypl.org/index.php?id=#{image_id}&t=u" + ls.update(url: new_url) + end + end +end diff --git a/provisioning/docker_build/startup_scripts/01_db_migrate.sh b/provisioning/docker_build/startup_scripts/01_db_migrate.sh index 37f6335..dab9669 100755 --- a/provisioning/docker_build/startup_scripts/01_db_migrate.sh +++ b/provisioning/docker_build/startup_scripts/01_db_migrate.sh @@ -5,4 +5,4 @@ if [ "$RAILS_ENV" == "development" ]; then sleep 10 fi -cd /home/app/fedora_ingest_rails && RAILS_ENV=$RAILS_ENV bundle exec rake db:create db:migrate +#cd /home/app/fedora_ingest_rails && RAILS_ENV=$RAILS_ENV bundle exec rake db:create db:migrate