diff --git a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py index 84bdecaa..ff86f5c9 100644 --- a/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py +++ b/scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py @@ -4,9 +4,15 @@ import sys import shlex import subprocess +from itertools import repeat + import psycopg2 import requests from psycopg2.extras import execute_values +from concurrent.futures import ThreadPoolExecutor +from time import sleep + + class bcolors: @@ -23,9 +29,7 @@ class bcolors: def perform_migration(): global is_error_occurred - gate_pipeline_maps = None - try: try: drop_table_user_group_permission_3_11() @@ -270,7 +274,15 @@ def update_spinnaker_gate_url(): def get_gate_pipeline_map(): try: - platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map") + platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map ") + return platform_cursor.fetchall() + except Exception as e: + platform_conn.rollback() + raise e + +def get_service_pipeline_data(): + try: + platform_cursor.execute("SELECT service_id,pipeline_id FROM service_pipeline_map ") return platform_cursor.fetchall() except Exception as e: platform_conn.rollback() @@ -314,7 +326,7 @@ def update_pipeline_json_with_unified_url(pipelines): domain_name = host_url_comps[2] else: domain_name = host_url_comps[2] + "/" + host_url_comps[3] - + url_comps = str(gate_url).split("/") url_comps[2] = domain_name gate_url = "/" @@ -341,23 +353,33 @@ def update_pipeline_json_with_unified_url(pipelines): def update_pipeline_with_unified_url(gate_pipeline_maps, cookie): try: - for gate_pipeline_map in gate_pipeline_maps: - pipeline_id = gate_pipeline_map[0] - gate_id = gate_pipeline_map[1] - url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id) - headers = {'Cookie': cookie} - response = requests.get(url=url, headers=headers) - gate_response = json.loads(response.content) - logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n') - update_gate(gate_response, pipeline_id, gate_id, cookie) + serviceIds_pipeline_data = get_service_pipeline_data(); + serviceIds_pipeline_map = dict((y, x) for x, y in serviceIds_pipeline_data) + with ThreadPoolExecutor(max_workers = thread_pool_size) as executor: + executor.map(update_gate_information,repeat(cookie),repeat(serviceIds_pipeline_map),gate_pipeline_maps) + except Exception as e: + raise e + + +def update_gate_information(cookie,serviceIds_pipeline_map,gate_pipeline_map): + try: + pipeline_id = gate_pipeline_map[0] + gate_id = gate_pipeline_map[1] + url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id) + headers = {'Cookie': cookie} + response = requests.get(url=url, headers=headers) + gate_response = json.loads(response.content) + logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n') + update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map) except Exception as e: + logging.error("Exception occurred while updating gate information : ", exc_info=True) raise e -def update_gate(gate_response, pipeline_id, gate_id, cookie): +def update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map): try: - platform_cursor.execute("SELECT service_id FROM service_pipeline_map where pipeline_id = " + str(pipeline_id)) - service_id = platform_cursor.fetchone()[0] + if list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)] is not None: + service_id = list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)] try: if gate_response["dependsOn"] is None: gate_response["dependsOn"] = [] @@ -423,7 +445,7 @@ def update_gate(gate_response, pipeline_id, gate_id, cookie): requests.put(url=url, data=req_payload, headers=headers) logging.info("Updated the gate data : \n" + str(req_payload) + '\n') except Exception as e: - platform_conn.rollback() + logging.error("Exception occurred while updating gate information with dashboard api : ", exc_info=True) raise e @@ -716,8 +738,10 @@ def updateApprovalAuditJson(audit_events_table_id, updateJson): def drop_constraints_feature_permission(): try: - platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ") - platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ") + platform_cursor.execute( + " ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ") + platform_cursor.execute( + " ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ") except Exception as e: platform_conn.rollback() raise e @@ -741,15 +765,14 @@ def login_to_isd(): raise e - if __name__ == '__main__': n = len(sys.argv) - if n != 19: + if n != 20: print( - "Please pass valid 18 arguments " + "Please pass valid 19 arguments " " " - " ") + " ") exit(1) global is_error_occurred @@ -777,6 +800,7 @@ def login_to_isd(): isd_admin_password = sys.argv[16] db_username = sys.argv[17] db_password = sys.argv[18] + thread_pool_size = int(sys.argv[19]) # Establishing the platform db connection platform_conn = psycopg2.connect(database=platform_db, user=db_username, password=db_password, host=platform_host, @@ -820,4 +844,4 @@ def login_to_isd(): visibility_cursor = visibility_conn.cursor() - perform_migration() \ No newline at end of file + perform_migration() diff --git a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt index c39d646a..42d7ce9b 100644 --- a/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt +++ b/scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt @@ -3,7 +3,7 @@ Steps to migrate from v3.11.x to v3.12 1. Copy the file : migration_v3.11.x_to_v3.12.py file to platform pod 2. Run the below command to execute the script. - python3 + python3 - ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword + ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword 20