Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

worked on script enchancement #387

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+50 −26
Open

worked on script enchancement #387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f8a75b
worked on script enchancement
ramyaravi-opsmx Aug 19, 2022
ef2fbd7
worked on script enchancement
ramyaravi-opsmx Aug 19, 2022
10664f9
worked on script enchancement phase1
ramyaravi-opsmx Aug 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 48 additions & 24 deletions scripts/oes-data-migration-scripts/migration_v3.11.x_to_v3.12.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,15 @@
import sys
import shlex
import subprocess
from itertools import repeat

import psycopg2
import requests
from psycopg2.extras import execute_values
from concurrent.futures import ThreadPoolExecutor
from time import sleep




class bcolors:
Expand All @@ -23,9 +29,7 @@ class bcolors:

def perform_migration():
global is_error_occurred

gate_pipeline_maps = None

try:
try:
drop_table_user_group_permission_3_11()
Expand Down Expand Up @@ -270,7 +274,15 @@ def update_spinnaker_gate_url():

def get_gate_pipeline_map():
try:
platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map")
platform_cursor.execute("SELECT pipeline_id, service_gate_id FROM gate_pipeline_map ")
return platform_cursor.fetchall()
except Exception as e:
platform_conn.rollback()
raise e

def get_service_pipeline_data():
try:
platform_cursor.execute("SELECT service_id,pipeline_id FROM service_pipeline_map ")
return platform_cursor.fetchall()
except Exception as e:
platform_conn.rollback()
Expand Down Expand Up @@ -314,7 +326,7 @@ def update_pipeline_json_with_unified_url(pipelines):
domain_name = host_url_comps[2]
else:
domain_name = host_url_comps[2] + "/" + host_url_comps[3]

url_comps = str(gate_url).split("/")
url_comps[2] = domain_name
gate_url = "/"
Expand All @@ -341,23 +353,33 @@ def update_pipeline_json_with_unified_url(pipelines):

def update_pipeline_with_unified_url(gate_pipeline_maps, cookie):
try:
for gate_pipeline_map in gate_pipeline_maps:
pipeline_id = gate_pipeline_map[0]
gate_id = gate_pipeline_map[1]
url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id)
headers = {'Cookie': cookie}
response = requests.get(url=url, headers=headers)
gate_response = json.loads(response.content)
logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n')
update_gate(gate_response, pipeline_id, gate_id, cookie)
serviceIds_pipeline_data = get_service_pipeline_data();
serviceIds_pipeline_map = dict((y, x) for x, y in serviceIds_pipeline_data)
with ThreadPoolExecutor(max_workers = thread_pool_size) as executor:
executor.map(update_gate_information,repeat(cookie),repeat(serviceIds_pipeline_map),gate_pipeline_maps)
except Exception as e:
raise e


def update_gate_information(cookie,serviceIds_pipeline_map,gate_pipeline_map):
try:
pipeline_id = gate_pipeline_map[0]
gate_id = gate_pipeline_map[1]
url = host_url + "/dashboardservice/v4/pipelines/" + str(pipeline_id) + "/gates/" + str(gate_id)
headers = {'Cookie': cookie}
response = requests.get(url=url, headers=headers)
gate_response = json.loads(response.content)
logging.info("Gate data from dashboard api : \n" + str(gate_response) + '\n')
update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map)
except Exception as e:
logging.error("Exception occurred while updating gate information : ", exc_info=True)
raise e


def update_gate(gate_response, pipeline_id, gate_id, cookie):
def update_gate(gate_response, pipeline_id, gate_id, cookie,serviceIds_pipeline_map):
try:
platform_cursor.execute("SELECT service_id FROM service_pipeline_map where pipeline_id = " + str(pipeline_id))
service_id = platform_cursor.fetchone()[0]
if list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)] is not None:
service_id = list(serviceIds_pipeline_map.keys())[list(serviceIds_pipeline_map.values()).index(pipeline_id)]
try:
if gate_response["dependsOn"] is None:
gate_response["dependsOn"] = []
Expand Down Expand Up @@ -423,7 +445,7 @@ def update_gate(gate_response, pipeline_id, gate_id, cookie):
requests.put(url=url, data=req_payload, headers=headers)
logging.info("Updated the gate data : \n" + str(req_payload) + '\n')
except Exception as e:
platform_conn.rollback()
logging.error("Exception occurred while updating gate information with dashboard api : ", exc_info=True)
raise e


Expand Down Expand Up @@ -716,8 +738,10 @@ def updateApprovalAuditJson(audit_events_table_id, updateJson):

def drop_constraints_feature_permission():
try:
platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ")
platform_cursor.execute(" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ")
platform_cursor.execute(
" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fkdap3k7dwyp8yq5sn0kjf6eo42 ")
platform_cursor.execute(
" ALTER TABLE feature_permission DROP CONSTRAINT IF EXISTS fklantt6pc0wjwueula2lt5vmt8 ")
except Exception as e:
platform_conn.rollback()
raise e
Expand All @@ -741,15 +765,14 @@ def login_to_isd():
raise e



if __name__ == '__main__':
n = len(sys.argv)

if n != 19:
if n != 20:
print(
"Please pass valid 18 arguments <platform_db-name> <platform_host> <oes-db-name> <oes-db-host> "
"Please pass valid 19 arguments <platform_db-name> <platform_host> <oes-db-name> <oes-db-host> "
"<autopilot_db> <autopilot_host> <audit_db-name> <audit-db-host> <visibility_db-name> <visibility-db-host> "
"<db-port> <isd-host-url> <spinnaker-gate-url> <unified-host-url> <isd-admin-username> <isd-admin-password> <db-username> <db-password>")
"<db-port> <isd-host-url> <spinnaker-gate-url> <unified-host-url> <isd-admin-username> <isd-admin-password> <db-username> <db-password> <thread-pool-size>")
exit(1)

global is_error_occurred
Expand Down Expand Up @@ -777,6 +800,7 @@ def login_to_isd():
isd_admin_password = sys.argv[16]
db_username = sys.argv[17]
db_password = sys.argv[18]
thread_pool_size = int(sys.argv[19])

# Establishing the platform db connection
platform_conn = psycopg2.connect(database=platform_db, user=db_username, password=db_password, host=platform_host,
Expand Down Expand Up @@ -820,4 +844,4 @@ def login_to_isd():

visibility_cursor = visibility_conn.cursor()

perform_migration()
perform_migration()
4 changes: 2 additions & 2 deletions scripts/oes-data-migration-scripts/steps_to_migrate_from_v3.11.x_to_v3.12.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ Steps to migrate from v3.11.x to v3.12
1. Copy the file : migration_v3.11.x_to_v3.12.py file to platform pod
2. Run the below command to execute the script.

python3 <scriptfilename> <platform_db-name> <platform_host> <oes-db-name> <oes-db-host> <autopilot_db> <autopilot_host> <audit_db-name> <audit-db-host> <visibility_db-name> <visibility-db-host> <db-port> <isd-host-url> <spinnaker-gate-url> <unified-host-url> <isd-admin-username> <isd-admin-password> <db-username> <db-password>
python3 <scriptfilename> <platform_db-name> <platform_host> <oes-db-name> <oes-db-host> <autopilot_db> <autopilot_host> <audit_db-name> <audit-db-host> <visibility_db-name> <visibility-db-host> <db-port> <isd-host-url> <spinnaker-gate-url> <unified-host-url> <isd-admin-username> <isd-admin-password> <db-username> <db-password> <thread-pool-size>

ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword
ex: python3 migration_v3.11_to_v3.12.py platformdb oes-db oesdb oes-db opsmx oes-db auditdb oes-db visibilitydb oes-db 5432 https://isd-gate.dev.opsmx.net http://sapor-gate:8084 https://upgrade-ui.isd-dev.opsmx.net/gate isd-admin-username isd-admin-password dbusername dbpassword 20