From f6a1a12b601ed03b9789b2308e15e88ab4611b76 Mon Sep 17 00:00:00 2001 From: Thomas Buchinger Date: Sun, 4 Aug 2019 23:12:15 +0000 Subject: [PATCH 1/2] add skeleton for api provider --- lib/miq_flow.rb | 2 ++ lib/miq_flow/domain.rb | 26 ++++++++++++------ lib/miq_flow/pluggable/method_git.rb | 33 +++++++++++++++++++++++ lib/miq_flow/pluggable/provider_api.rb | 18 +++++++++++++ lib/miq_flow/pluggable/provider_docker.rb | 4 ++- lib/miq_flow/pluggable/provider_local.rb | 6 +++-- lib/miq_flow/pluggable/provider_noop.rb | 4 +-- 7 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 lib/miq_flow/pluggable/method_git.rb create mode 100644 lib/miq_flow/pluggable/provider_api.rb diff --git a/lib/miq_flow.rb b/lib/miq_flow.rb index 9176429..3eba14a 100644 --- a/lib/miq_flow.rb +++ b/lib/miq_flow.rb @@ -11,6 +11,8 @@ require 'miq_flow/pluggable/provider_docker' require 'miq_flow/pluggable/provider_local' require 'miq_flow/pluggable/provider_noop' +require 'miq_flow/pluggable/provider_api' +require 'miq_flow/pluggable/method_git' require 'miq_flow/pluggable/method_partial' require 'miq_flow/pluggable/method_clean' require 'miq_flow/error' diff --git a/lib/miq_flow/domain.rb b/lib/miq_flow/domain.rb index 8e45444..8cbcf40 100644 --- a/lib/miq_flow/domain.rb +++ b/lib/miq_flow/domain.rb @@ -24,10 +24,8 @@ class MiqDomain # @option opts [String] :miq_priority DOES NOTHING, since the importer does not honor it # @option opts [String] :branch_name name of the git branch. INFO only def _set_defaults(opts={}) - @miq_provider_name = opts.fetch(:miq_provider, 'noop') @export_dir = opts.fetch(:export_dir, 'automate') @export_name = opts.fetch(:export_name, @name) - @miq_import_method = opts.fetch(:miq_import_method, :partial) @miq_priority = opts.fetch(:miq_priority, 10) @branch_name = opts.fetch(:branch_name, 'No Branch') end @@ -72,10 +70,21 @@ def initialize(name, opts) @name = name _set_defaults(opts) - @miq_provider = MiqFlow::MiqProvider::Noop.new if opts[:provider_name] == 'noop' - @miq_provider = MiqFlow::MiqProvider::Appliance.new if opts[:provider_name] == 'local' - @miq_provider = MiqFlow::MiqProvider::Docker.new if opts[:provider_name] == 'docker' - @miq_provider = MiqFlow::MiqProvider::Noop.new if @miq_provider.nil? + @miq_import_method, @miq_provider = provider_from_name(opts[:provider_name]) + end + + def provider_from_name(name) + return [:partial, MiqFlow::MiqProvider::Noop.new] if name == 'noop' + + return [:git, MiqFlow::MiqProvider::Noop.new] if name == 'noop-api' + + return [:partial, MiqFlow::MiqProvider::Appliance.new] if name == 'local' + + return [:partial, MiqFlow::MiqProvider::Docker.new] if name == 'docker' + + return [:git, MiqFlow::MiqProvider::Api.new] if name == 'api' + + [:partial, MiqFlow::MiqProvider::Noop.new] end def prepare_import(domain_data, feature_data) @@ -91,7 +100,7 @@ def cleanup_import(prep_data) end def skip_deploy?(opts) - skippable_method = [:partial].include?(@miq_import_method) + skippable_method = [:partial, :git].include?(@miq_import_method) skip = skippable_method && opts[:changeset].empty?() $logger.info("Skipping Domain: #{@name}: empty") if skip skip @@ -108,7 +117,8 @@ def deploy(opts) prep_data = prepare_import(self, opts) raise MiqFlow::UnknownStrategyError, "Unknown Import method: #{@miq_import_method}" if prep_data[:error] == true - @miq_provider.import(File.join(prep_data[:import_dir], @export_dir), @export_name, @name) + prep_data.merge!(import_dir: File.join(prep_data[:import_dir], @export_dir), fs_name: @export_name) + @miq_provider.import(@name, prep_data) clean_data = cleanup_import(prep_data) raise MiqFlow::UnknownStrategyError, "Unknown cleanup method: #{@miq_import_method}" if clean_data[:error] == true end diff --git a/lib/miq_flow/pluggable/method_git.rb b/lib/miq_flow/pluggable/method_git.rb new file mode 100644 index 0000000..3884869 --- /dev/null +++ b/lib/miq_flow/pluggable/method_git.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'tmpdir' +require 'yaml' +require_relative 'method_partial' + +module MiqFlow + # This module contains everything needed to prepare an Automate domain for import + # Mostly file handling at this point + module MiqMethods + def prepare_import_git(dom, feat) + $logger.debug("Doing a git import of #{feat[:changeset].join(', ')}") + base_dir = feat[:git_workdir].to_s + import_dir = File.join($tmpdir, 'import') + + #Partial.create_fake_domain(import_dir, dom.export_dir, dom.export_name, dom.branch_name, dom.miq_priority) + #Partial.copy_to_tmp(import_dir, base_dir, Partial.file_list(feat[:changeset], base_dir)) + #Git.init_repo(import_dir) + # Git.push() + { import_dir: import_dir, git_url: 'a', ref_type: 'branch', ref_name: 'b' } + end + + def cleanup_import_git(_prep_data) + {} + end + + # Implementation for git import + module Git + def self.init_repo(import_dir) + end + end + end +end diff --git a/lib/miq_flow/pluggable/provider_api.rb b/lib/miq_flow/pluggable/provider_api.rb new file mode 100644 index 0000000..c3ad086 --- /dev/null +++ b/lib/miq_flow/pluggable/provider_api.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module MiqFlow + module MiqProvider + # This provider assumes to be running on a ManageIQ Appliance + class Api + def initialize(_opts={}) + # fix rubocop issue #6678 + true + end + + def import(miq_domain, opts) + $logger.info('Importing with Api provider') + # raise MiqFlow::ProviderError, 'Failed to Import to Appliance' unless success + end + end + end +end diff --git a/lib/miq_flow/pluggable/provider_docker.rb b/lib/miq_flow/pluggable/provider_docker.rb index 51ca849..4399353 100644 --- a/lib/miq_flow/pluggable/provider_docker.rb +++ b/lib/miq_flow/pluggable/provider_docker.rb @@ -12,7 +12,9 @@ def initialize(container_name: 'manageiq') @container_name = container_name end - def import(tmpdir, fs_domain, miq_domain) + def import(miq_domain, opts) + tmpdir = opts[:import_dir] + fs_domain = opts[:fs_domain] $logger.debug("TMPDIR=#{tmpdir}") commands = [ "docker exec #{@container_name} mkdir -p #{tmpdir}", diff --git a/lib/miq_flow/pluggable/provider_local.rb b/lib/miq_flow/pluggable/provider_local.rb index 3085f6c..1096ba2 100644 --- a/lib/miq_flow/pluggable/provider_local.rb +++ b/lib/miq_flow/pluggable/provider_local.rb @@ -11,8 +11,10 @@ def initialize(_opts={}) true end - def import(tmpdir, fs_domain, miq_domain) - commands = [ + def import(miq_domain, opts) + tmpdir = opts[:import_dir] + fs_domain = opts[:fs_domain] + commands = [ 'rake -f /var/www/miq/vmdb/Rakefile evm:automate:import'\ " DOMAIN=#{fs_domain} IMPORT_AS=#{miq_domain} IMPORT_DIR=#{tmpdir} OVERWRITE=true PREVIEW=false ENABLED=true" ] diff --git a/lib/miq_flow/pluggable/provider_noop.rb b/lib/miq_flow/pluggable/provider_noop.rb index d731a5b..45aec2a 100644 --- a/lib/miq_flow/pluggable/provider_noop.rb +++ b/lib/miq_flow/pluggable/provider_noop.rb @@ -10,10 +10,10 @@ def initialize(opts={}) @fail = opts.fetch(:flag_fail, false) end - def import(tmpdir, fs_domain, miq_domain) + def import(miq_domain, opts) raise MiqFlow::ProviderError, 'This provider cannot fail' if @fail - $logger.info("Importing with NOOP provider MIQ_DOMAIN=#{miq_domain} TMPDIR=#{tmpdir} FS_DOMAIN=#{fs_domain}") + $logger.info("Importing with NOOP provider MIQ_DOMAIN=#{miq_domain} options=#{opts}") end end end From 141b9f2793d8a5d003af21bf94ab4574cb4c49e7 Mon Sep 17 00:00:00 2001 From: Thomas Buchinger Date: Sun, 11 Aug 2019 22:45:53 +0000 Subject: [PATCH 2/2] first mostly working version --- lib/miq_flow/domain.rb | 6 +- lib/miq_flow/mixin_api.rb | 20 ++++++ lib/miq_flow/pluggable/method_git.rb | 84 ++++++++++++++++++++++-- lib/miq_flow/pluggable/method_partial.rb | 13 ++-- lib/miq_flow/pluggable/provider_api.rb | 20 ++++-- 5 files changed, 122 insertions(+), 21 deletions(-) diff --git a/lib/miq_flow/domain.rb b/lib/miq_flow/domain.rb index 8cbcf40..24c79a3 100644 --- a/lib/miq_flow/domain.rb +++ b/lib/miq_flow/domain.rb @@ -77,9 +77,9 @@ def provider_from_name(name) return [:partial, MiqFlow::MiqProvider::Noop.new] if name == 'noop' return [:git, MiqFlow::MiqProvider::Noop.new] if name == 'noop-api' - + return [:partial, MiqFlow::MiqProvider::Appliance.new] if name == 'local' - + return [:partial, MiqFlow::MiqProvider::Docker.new] if name == 'docker' return [:git, MiqFlow::MiqProvider::Api.new] if name == 'api' @@ -100,7 +100,7 @@ def cleanup_import(prep_data) end def skip_deploy?(opts) - skippable_method = [:partial, :git].include?(@miq_import_method) + skippable_method = %i[partial git].include?(@miq_import_method) skip = skippable_method && opts[:changeset].empty?() $logger.info("Skipping Domain: #{@name}: empty") if skip skip diff --git a/lib/miq_flow/mixin_api.rb b/lib/miq_flow/mixin_api.rb index bf4e53b..1da349d 100644 --- a/lib/miq_flow/mixin_api.rb +++ b/lib/miq_flow/mixin_api.rb @@ -55,5 +55,25 @@ def invoke_miq_api(path) rescue SocketError => e raise MiqFlow::ConnectionError, "Unable to connect ot ManageIQ: #{e.message}", [] end + + def miq_action_api(action_id, path, method: :post, resource: {}) + req_opts = { method: method, user: @user, password: @password, verify_ssl: false } + req_opts[:url] = @url + path + req_opts[:payload] = JSON.generate(action: action_id, resource: resource) + + $logger.debug("API #{method} Action: #{req_opts[:url]}") + $logger.debug(" Payload: #{req_opts[:payload]}") unless resource.empty? + + response = RestClient::Request.execute(req_opts) + JSON.parse(response.body) + rescue RestClient::Exceptions::Timeout => e + raise MiqFlow::ConnectionError, "Unable to connect to ManageIQ: #{e.message}", [] + rescue RestClient::Exception => e + raise MiqFlow::BadResponseError, "Invalid API call: #{e.message}", [] + rescue Errno::ECONNREFUSED => e + raise MiqFlow::ConnectionError, "ManageIQ API unavailalbe: #{e.message}", [] + rescue SocketError => e + raise MiqFlow::ConnectionError, "Unable to connect ot ManageIQ: #{e.message}", [] + end end end diff --git a/lib/miq_flow/pluggable/method_git.rb b/lib/miq_flow/pluggable/method_git.rb index 3884869..a90e3fa 100644 --- a/lib/miq_flow/pluggable/method_git.rb +++ b/lib/miq_flow/pluggable/method_git.rb @@ -13,11 +13,11 @@ def prepare_import_git(dom, feat) base_dir = feat[:git_workdir].to_s import_dir = File.join($tmpdir, 'import') - #Partial.create_fake_domain(import_dir, dom.export_dir, dom.export_name, dom.branch_name, dom.miq_priority) - #Partial.copy_to_tmp(import_dir, base_dir, Partial.file_list(feat[:changeset], base_dir)) - #Git.init_repo(import_dir) - # Git.push() - { import_dir: import_dir, git_url: 'a', ref_type: 'branch', ref_name: 'b' } + Partial.create_fake_domain(import_dir, dom.export_dir, dom.export_name, dom.branch_name, dom.miq_priority) + Partial.copy_to_tmp(import_dir, base_dir, Partial.file_list(feat[:changeset], base_dir)) + repo = Git.new(import_dir, $git_repo) + repo.headless_commit(dom.name, "Import-Branch for #{dom.name} from #{dom.branch_name}") + { import_dir: import_dir, git_url: repo.remote.url, ref_type: 'branch', ref_name: repo.branch.name, repo: repo } end def cleanup_import_git(_prep_data) @@ -25,8 +25,78 @@ def cleanup_import_git(_prep_data) end # Implementation for git import - module Git - def self.init_repo(import_dir) + class Git + attr_reader :remote, :branch + + def initialize(import_dir, global_repo) + $logger.debug("Initialize temporary repo at: #{import_dir}: remote=#{global_repo.remotes['origin'].url}") + @repo = Rugged::Repository.init_at(import_dir) + @author = { email: 'ghost@graveyard.com', name: 'Git Ghost' } + @remote = @repo.remotes.create_anonymous(global_repo.remotes['origin'].url) + user = $settings[:git][:user] + pass = $settings[:git][:password] + @cred = Rugged::Credentials::UserPassword.new(username: user, password: pass) + end + + def headless_commit(dom_name, message=nil) + message ||= "Create headless commit for #{dom_name} at #{@repo.workdir}" + index = @repo.index + + index.add_all + tree_oid = index.write_tree + commit_oid = Rugged::Commit.create( + @repo, + author: @author, + committer: @author, + message: message, + parents: [], + tree: tree_oid + ) + @branch = @repo.branches.create("tmp_miqflow_#{dom_name}", commit_oid, force: true) + @repo.checkout(@branch.name, strategy: :force) + $logger.debug("Created branch #{@branch_name} on commit #{commit_oid}") + end + + def check_connection + @remote.check_connection(:push, credentials: @cred) + true + rescue Rugged::Error + false + end + + def push(ref_spec) + unless check_connection() + $logger.error("Failed to connect to #{@remote.url}: Are the credentials valid?") + raise "Connection Error" + end + + begin + $logger.debug("Push: #{ref_spec}") + @remote.push([ref_spec], credentials: @cred) + rescue Rugged::Error => e + $logger.error(e) + raise + end + end + + def force_push(ref_spec) + old_ref = ref_spec.split(':')[0] + refs = @remote.ls(credentials: @cred) + + if !old_ref.nil? && refs.any?{ |ref_info| ref_info[:name] == old_ref } + $logger.debug("Force pushed #{ref_spec}: Deleting old Ref '#{old_ref}'") + push(":#{old_ref}") + end + + push(ref_spec) + end + + def push_to_upstream + force_push("refs/heads/#{@branch.name}") + end + + def delete_from_upstream + push(":refs/heads/#{@branch.name}") end end end diff --git a/lib/miq_flow/pluggable/method_partial.rb b/lib/miq_flow/pluggable/method_partial.rb index 280abd9..4aed335 100644 --- a/lib/miq_flow/pluggable/method_partial.rb +++ b/lib/miq_flow/pluggable/method_partial.rb @@ -43,12 +43,13 @@ def self.create_fake_domain(import_dir, automate_dir, domain_name, branch_name, end def self.generate_domain_template(domain_name, branch_name, priority, tenant) - attributes = { display_name: nil, enabled: true, source: 'user', top_level_namespace: nil } - domain = { object_type: 'domain', version: 1.0, object: { attributes: attributes } } - domain[:object][:attributes][:name] = domain_name - domain[:object][:attributes][:description] = "Development Branch for feature #{domain_name}: #{branch_name}" - domain[:object][:attributes][:priority] = priority - domain[:object][:attributes][:tenant_id] = tenant + attributes = { 'enabled' => 'true', 'source' => 'user', 'top_level_namespace' => nil } + domain = { 'object_type' => 'domain', 'version' => 1.0, 'object' => { 'attributes' => attributes } } + domain['object']['attributes']['display_name'] = domain_name + domain['object']['attributes']['name'] = domain_name + domain['object']['attributes']['description'] = "Development Branch for feature #{domain_name}: #{branch_name}" + domain['object']['attributes']['priority'] = priority + domain['object']['attributes']['tenant_id'] = tenant domain end diff --git a/lib/miq_flow/pluggable/provider_api.rb b/lib/miq_flow/pluggable/provider_api.rb index c3ad086..fa844c7 100644 --- a/lib/miq_flow/pluggable/provider_api.rb +++ b/lib/miq_flow/pluggable/provider_api.rb @@ -5,13 +5,23 @@ module MiqProvider # This provider assumes to be running on a ManageIQ Appliance class Api def initialize(_opts={}) - # fix rubocop issue #6678 - true + @api = MiqFlow::ManageIQ.new end - def import(miq_domain, opts) - $logger.info('Importing with Api provider') - # raise MiqFlow::ProviderError, 'Failed to Import to Appliance' unless success + def import(_miq_domain, opts) + $logger.info("Importing with Api provider: #{opts[:ref_name]} from #{opts[:git_url]}") + res = { git_url: opts[:git_url], ref_type: opts[:ref_type], ref_name: opts[:ref_name] } + repo = opts[:repo] + + $logger.info("Push Domain: #{opts[:ref_name]} to git") + repo.push_to_upstream + + $logger.info("Import to Automate") + re = @api.miq_action_api('create_from_git', '/automate_domains', resource: res) + $logger.debug("RestResult: #{re}") + + # $logger.info("Remove temporary branch...") + # repo.delete_from_upstream end end end