Skip to content

Commit

Permalink
rough first attempt at db:prepare:with_data
Browse files Browse the repository at this point in the history 
Lots of code copied out of rails, needs DRYing up a lot and the commits refactoring.
  • Loading branch information
@Floppy
Floppy committed Jul 1, 2023
1 parent 7bc28b8 commit 4e9cd3f
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 56 deletions.
120 changes: 109 additions & 11 deletions lib/data_migrate/database_tasks.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def dump_filename(spec_name, format = ActiveRecord::Base.schema_format)

def check_schema_file(filename)
unless File.exist?(filename)
message = +%{#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.}
message = +%(#{filename} doesn't exist yet. Run `rake data:migrate` to create it, then try again.)
Kernel.abort message
end
end
Expand All @@ -42,11 +42,11 @@ def pending_migrations
end

def sort_migrations(*migrations)
migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) }
migrations.flatten.sort { |a, b| sort_string(a) <=> sort_string(b) }
end

def sort_string migration
"#{migration[:version]}_#{migration[:kind] == :data ? 1 : 0}"
"#{migration[:version]}_#{(migration[:kind] == :data) ? 1 : 0}"
end

def data_migrations_path
Expand All @@ -55,10 +55,10 @@ def data_migrations_path

def run_migration(migration, direction)
if migration[:kind] == :data
::ActiveRecord::Migration.write("== %s %s" % ['Data', "=" * 71])
::ActiveRecord::Migration.write("== %s %s" % ["Data", "=" * 71])
::DataMigrate::DataMigrator.run(direction, data_migrations_path, migration[:version])
else
::ActiveRecord::Migration.write("== %s %s" % ['Schema', "=" * 69])
::ActiveRecord::Migration.write("== %s %s" % ["Schema", "=" * 69])
::DataMigrate::SchemaMigration.run(
direction,
::DataMigrate::SchemaMigration.migrations_paths,
Expand All @@ -80,14 +80,30 @@ def schema_dump_path(db_config, format = ActiveRecord.schema_format)
# to ensure that the sha saved in ar_internal_metadata table
# is from the original schema.rb file
def schema_sha1(file)
super(file.gsub(/data_schema.rb\z/, 'schema.rb'))
super(file.gsub(/data_schema.rb\z/, "schema.rb"))
end

def with_temporary_pool(db_config)
original_db_config = migration_class.connection_db_config
pool = migration_class.establish_connection(db_config)
yield pool
ensure
migration_class.establish_connection(original_db_config)
end

def migration_class # :nodoc:
ActiveRecord::Base
end

def migration_connection # :nodoc:
migration_class.connection
end
end

def self.forward(step = 1)
DataMigrate::DataMigrator.create_data_schema_table
migrations = pending_migrations.reverse.pop(step).reverse
migrations.each do | pending_migration |
migrations.each do |pending_migration|
if pending_migration[:kind] == :data
ActiveRecord::Migration.write("== %s %s" % ["Data", "=" * 71])
DataMigrate::DataMigrator.run(:up, data_migrations_path, pending_migration[:version])
Expand All @@ -100,8 +116,8 @@ def self.forward(step = 1)

def self.pending_data_migrations
data_migrations = DataMigrate::DataMigrator.migrations(data_migrations_path)
sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations ).
pending_migrations.map {|m| { version: m.version, name: m.name, kind: :data }})
sort_migrations(DataMigrate::DataMigrator.new(:up, data_migrations)
.pending_migrations.map { |m| {version: m.version, name: m.name, kind: :data} })
end

def self.pending_schema_migrations
Expand All @@ -111,9 +127,91 @@ def self.pending_schema_migrations
def self.past_migrations(sort = nil)
data_versions = DataMigrate::DataSchemaMigration.table_exists? ? DataMigrate::DataSchemaMigration.normalized_versions : []
schema_versions = ActiveRecord::SchemaMigration.normalized_versions
migrations = data_versions.map { |v| { version: v.to_i, kind: :data } } + schema_versions.map { |v| { version: v.to_i, kind: :schema } }
migrations = data_versions.map { |v| {version: v.to_i, kind: :data} } + schema_versions.map { |v| {version: v.to_i, kind: :schema} }

(sort&.downcase == "asc") ? sort_migrations(migrations) : sort_migrations(migrations).reverse
end

def self.migrate_with_data
DataMigrate::DataMigrator.create_data_schema_table

ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
migrations = []

if target_version.nil?
migrations = DataMigrate::DatabaseTasks.pending_migrations.map { |m| m.merge(direction: :up) }
else
current_schema_version = ActiveRecord::Migrator.current_version
schema_migrations = if target_version > current_schema_version
DataMigrate::DatabaseTasks.pending_schema_migrations.keep_if { |m| m[:version] <= target_version }.map { |m| m.merge(direction: :up) }
elsif target_version < current_schema_version
DataMigrate::DatabaseTasks.past_migrations.keep_if { |m| m[:version] > target_version }.map { |m| m.merge(direction: :down) }
else # ==
[]
end

current_data_version = DataMigrate::DataMigrator.current_version
data_migrations = if target_version > current_data_version
DataMigrate::DatabaseTasks.pending_data_migrations.keep_if { |m| m[:version] <= target_version }.map { |m| m.merge(direction: :up) }
elsif target_version < current_data_version
DataMigrate::DatabaseTasks.past_migrations.keep_if { |m| m[:version] > target_version }.map { |m| m.merge(direction: :down) }
else # ==
[]
end
migrations = if schema_migrations.empty?
data_migrations
elsif data_migrations.empty?
schema_migrations
elsif target_version > current_data_version && target_version > current_schema_version
DataMigrate::DatabaseTasks.sort_migrations data_migrations, schema_migrations
elsif target_version < current_data_version && target_version < current_schema_version
DataMigrate::DatabaseTasks.sort_migrations(data_migrations, schema_migrations).reverse
elsif target_version > current_data_version && target_version < current_schema_version
schema_migrations + data_migrations
elsif target_version < current_data_version && target_version > current_schema_version
schema_migrations + data_migrations
end
end

migrations.each do |migration|
DataMigrate::DatabaseTasks.run_migration(migration, migration[:direction])
end
end

def self.prepare_all_with_data
seed = false

each_current_configuration(env) do |db_config|
with_temporary_pool(db_config) do
begin
database_initialized = migration_connection.schema_migration.table_exists?
rescue ActiveRecord::NoDatabaseError
create(db_config)
retry
end

unless database_initialized
if File.exist?(schema_dump_path(db_config))
load_schema(db_config, ActiveRecord.schema_format, nil)
load_schema_current(
:ruby,
ENV["DATA_SCHEMA"]
)
end

seed = true
end

migrate_with_data
if ActiveRecord.dump_schema_after_migration
dump_schema(db_config)
DataMigrate::Tasks::DataMigrateTasks.dump
end
end
end

sort&.downcase == "asc" ? sort_migrations(migrations) : sort_migrations(migrations).reverse
load_seed if seed
end
end
end
63 changes: 18 additions & 45 deletions tasks/databases.rake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,55 +2,21 @@

require 'data_migrate/tasks/data_migrate_tasks'

# From https://gist.github.com/raggi/232966
def alias_task(name, old_name)
t = Rake::Task[old_name]
desc t.full_comment if t.full_comment
task name, *t.arg_names do |_, args|
args = t.arg_names.map { |a| args[a] }
t.invoke(args)
end
end

namespace :db do
namespace :migrate do
desc "Migrate the database data and schema (options: VERSION=x, VERBOSE=false)."
task :with_data => :environment do
DataMigrate::DataMigrator.create_data_schema_table

ActiveRecord::Migration.verbose = ENV["VERBOSE"] ? ENV["VERBOSE"] == "true" : true
target_version = ENV["VERSION"] ? ENV["VERSION"].to_i : nil
migrations = []

if target_version.nil?
migrations = DataMigrate::DatabaseTasks.pending_migrations.map{ |m| m.merge(:direction =>:up) }
else
current_schema_version = ActiveRecord::Migrator.current_version
schema_migrations = if target_version > current_schema_version
DataMigrate::DatabaseTasks.pending_schema_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
elsif target_version < current_schema_version
DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
else # ==
[]
end

current_data_version = DataMigrate::DataMigrator.current_version
data_migrations = if target_version > current_data_version
DataMigrate::DatabaseTasks.pending_data_migrations.keep_if{ |m| m[:version] <= target_version }.map{ |m| m.merge(:direction =>:up) }
elsif target_version < current_data_version
DataMigrate::DatabaseTasks.past_migrations.keep_if{ |m| m[:version] > target_version }.map{ |m| m.merge(:direction =>:down) }
else # ==
[]
end
migrations = if schema_migrations.empty?
data_migrations
elsif data_migrations.empty?
schema_migrations
elsif target_version > current_data_version && target_version > current_schema_version
DataMigrate::DatabaseTasks.sort_migrations data_migrations, schema_migrations
elsif target_version < current_data_version && target_version < current_schema_version
DataMigrate::DatabaseTasks.sort_migrations(data_migrations, schema_migrations).reverse
elsif target_version > current_data_version && target_version < current_schema_version
schema_migrations + data_migrations
elsif target_version < current_data_version && target_version > current_schema_version
schema_migrations + data_migrations
end
end

migrations.each do |migration|
DataMigrate::DatabaseTasks.run_migration(migration, migration[:direction])
end

DataMigrate::DatabaseTasks.migrate_with_data
Rake::Task["db:_dump"].invoke
Rake::Task["data:dump"].invoke
end
Expand Down Expand Up @@ -190,6 +156,13 @@ namespace :db do
end
end
end

namespace :prepare do
desc "Runs setup if database does not exist, or runs data and schema migrations if it does"
task with_data: :environment do
DataMigrate::DatabaseTasks.prepare_all_with_data
end
end
end

namespace :data do
Expand Down

0 comments on commit 4e9cd3f

Please sign in to comment.