Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AMG migrate short uid bugfix. #7935

Merged
merged 8 commits into from
Sep 9, 2024
Merged

AMG migrate short uid bugfix. #7935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
590c027
fix bug with short uid
leozhang-msft Aug 29, 2024
c9bbe56
update history
leozhang-msft Aug 30, 2024
eeb487b
Update tests and recordings
leozhang-msft Aug 30, 2024
0cea206
update history text to be more descriptive
leozhang-msft Sep 3, 2024
8395ea9
FIx up typo in annotation migrate
leozhang-msft Aug 22, 2024
7255076
check that the dashboard exists before adding in the annotation
leozhang-msft Aug 21, 2024
0306c0e
Make UID -> Uid to be consistent.
leozhang-msft Sep 3, 2024
284fc84
Merge branch 'main' of https://github.com/leozhang-msft/azure-cli-ext…
leozhang-msft Sep 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/amg/HISTORY.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,7 @@ Release History
++++++
* `az grafana private-endpoint-connection`: support private endpoint connection management
* `az grafana mpe`: support managed private endpoint management

2.3.1
++++++
* `az grafana migrate`: fix issue with remapping logic for Grafana datasources with short uids
2 changes: 1 addition & 1 deletion src/amg/azext_amg/migrate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ def _migrate_library_panels_and_dashboards(all_source_dashboards, all_library_pa

# we don't backup provisioned dashboards, so we don't need to restore them
for dashboard in all_source_dashboards:
exists_before = check_dashboard_exists(restore_url, dashboard, restore_headers)
exists_before = check_dashboard_exists(restore_url, dashboard["dashboard"]["uid"], restore_headers)
dashboard_title = dashboard['dashboard'].get('title', '')

# Skipping making/updating dashboard if the library panel it relies on is not being updated.
Expand Down
18 changes: 13 additions & 5 deletions src/amg/azext_amg/restore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ def create_dashboard(grafana_url, content, http_headers, overwrite):
return result[0] == 200


def check_dashboard_exists(grafana_url, payload, http_headers):
result = send_grafana_get(f'{grafana_url}/api/dashboards/uid/{payload["dashboard"]["uid"]}', http_headers)
def check_dashboard_exists(grafana_url, uid, http_headers):
result = send_grafana_get(f'{grafana_url}/api/dashboards/uid/{uid}', http_headers)
return result[0] == 200


Expand Down Expand Up @@ -296,21 +296,29 @@ def _load_and_create_annotation(grafana_url, file_path, http_headers):
def create_annotation(grafana_url, annotation, http_headers, overwrite):
exists_before, old_id = check_annotation_exists_and_return_id(grafana_url, annotation, http_headers)

annotation_id = annotation['id']
# check that the dashboard exists.
if not check_dashboard_exists(grafana_url, annotation['dashboardUID'], http_headers):
print_styled_text([
(Style.WARNING, f'Create annotation id {annotation_id}: '),
(Style.ERROR, 'FAILURE, dashboard does not exist')
])
return False

if exists_before:
if overwrite:
(status, content) = send_grafana_delete(f'{grafana_url}/api/annotations/{old_id}', http_headers)
logger.info("delete status: %s, msg: %s", status, content)
else:
print_styled_text([
(Style.WARNING, f'Create annotation id {old_id}: '),
(Style.WARNING, f'Create annotation id {annotation_id}: '),
(Style.ERROR, 'FAILURE'),
(Style.ERROR, ' (annotation with same time period, dashboardUID, and text already exists,'),
(Style.ERROR, ' please enable --overwrite if you want to overwrite the snapshot)')
(Style.ERROR, ' please enable --overwrite if you want to overwrite the annotation)')
])
return False

(status, content) = send_grafana_post(f'{grafana_url}/api/annotations', json.dumps(annotation), http_headers)
annotation_id = annotation['id']
print_styled_text([
(Style.WARNING, f'Create annotation id {annotation_id}: '),
(Style.SUCCESS, 'SUCCESS') if status == 200 else (Style.ERROR, 'FAILURE')
Expand Down
4,656 changes: 2,506 additions & 2,150 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_backup_restore.yaml
View file
Open in desktop

Large diffs are not rendered by default.

482 changes: 315 additions & 167 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_crud.yaml
View file
Open in desktop

Large diffs are not rendered by default.

1,225 changes: 690 additions & 535 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_e2e.yaml
View file
Open in desktop

Large diffs are not rendered by default.

2,650 changes: 1,453 additions & 1,197 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_migrate_dry_run.yaml
View file
Open in desktop

Large diffs are not rendered by default.

3,927 changes: 1,935 additions & 1,992 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_migrate_override.yaml
View file
Open in desktop

Large diffs are not rendered by default.

3,295 changes: 1,771 additions & 1,524 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_migrate_remapping.yaml
View file
Open in desktop

Large diffs are not rendered by default.

2,818 changes: 1,410 additions & 1,408 deletions src/amg/azext_amg/tests/latest/recordings/test_amg_migrate_simple_e2e.yaml
View file
Open in desktop

Large diffs are not rendered by default.

441 changes: 216 additions & 225 deletions src/amg/azext_amg/tests/latest/recordings/test_api_key_e2e.yaml
View file
Open in desktop

Large diffs are not rendered by default.

524 changes: 262 additions & 262 deletions src/amg/azext_amg/tests/latest/recordings/test_service_account_e2e.yaml
View file
Open in desktop

Large diffs are not rendered by default.

31 changes: 28 additions & 3 deletions src/amg/azext_amg/tests/latest/test_amg_migrate_scenario.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from azure.cli.testsdk import (ScenarioTest, ResourceGroupPreparer, MSGraphNameReplacer, MOCKED_USER_NAME)
from azure.cli.testsdk.scenario_tests import AllowLargeResponse

from .test_definitions import (test_data_source, test_data_source_different_uid, test_notification_channel, test_dashboard, test_dashboard_with_datasource)
from .test_definitions import (test_data_source, test_data_source_different_uid, test_notification_channel, test_dashboard, test_dashboard_with_datasource, test_dashboard_with_datasource_short_uid, test_data_source_long_uid2, test_data_source_short_uid2)
from .recording_processors import ApiKeyServiceAccountTokenReplacer

TEST_DIR = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
Expand Down Expand Up @@ -241,20 +241,40 @@ def test_amg_migrate_remapping(self, resource_group):
self.kwargs.update({
'dashboardDefinitionDatasource': test_dashboard_with_datasource,
'dashboardTitle4': dashboard_title + '4',
'dataSourceDefinitionDifferentUID': test_data_source_different_uid
'dataSourceDefinitionDifferentUid': test_data_source_different_uid,
'dashboardDefinitionDatasourceShortUid': test_dashboard_with_datasource_short_uid,
'dashboardTitle5': dashboard_title + '5',
'dataSourceDefinitionLongUid2': test_data_source_long_uid2,
'dataSourceDefinitionShortUid2': test_data_source_short_uid2,
})
self.kwargs['dashboardDefinitionDatasource']['dashboard']['uid'] = 'mg2OAlTVd' # control the uid to prevent auto generated uid with possible '-' that breaks the command
self.kwargs['dashboardDefinitionDatasourceShortUid']['dashboard']['uid'] = 'mg2OAlTVe' # control the uid to prevent auto generated uid with possible '-' that breaks the command
leozhang-msft marked this conversation as resolved.
Show resolved Hide resolved

response_create = self.cmd('grafana dashboard create -g {rg} -n {name} --definition "{dashboardDefinitionDatasource}" --title "{dashboardTitle4}"').get_output_in_json()
self.kwargs.update({
'dashboardUid4': response_create["uid"],
})

ds2 = self.cmd('grafana data-source create -g {rg} -n {name2} --definition "{dataSourceDefinitionDifferentUID}"').get_output_in_json()
ds2 = self.cmd('grafana data-source create -g {rg} -n {name2} --definition "{dataSourceDefinitionDifferentUid}"').get_output_in_json()
self.kwargs.update({
'amg2_datasource_uid': ds2['datasource']['uid']
})

# create short uid in amg 1 (to do shortuid -> longuid remapping test)
self.cmd('grafana data-source create -g {rg} -n {name} --definition "{dataSourceDefinitionShortUid2}"').get_output_in_json()

# Create the dashboard
response_create2 = self.cmd('grafana dashboard create -g {rg} -n {name} --definition "{dashboardDefinitionDatasourceShortUid}" --title "{dashboardTitle5}"').get_output_in_json()
self.kwargs.update({
'dashboardUid5': response_create2["uid"],
})

# create the long data source for amg2.
ds2 = self.cmd('grafana data-source create -g {rg} -n {name2} --definition "{dataSourceDefinitionLongUid2}"').get_output_in_json()
self.kwargs.update({
'amg2_datasource_uid_long': ds2['datasource']['uid']
})

# prepare to migrate
# get the service account token:
self.kwargs.update({
Expand All @@ -279,6 +299,11 @@ def test_amg_migrate_remapping(self, resource_group):
self.check("[dashboard.panels[0].datasource.uid]", "['{amg2_datasource_uid}']"),
self.check("[meta.folderTitle]", "['General']")])

self.cmd('grafana dashboard show -g {rg} -n {name2} --dashboard "{dashboardUid5}"', checks=[
self.check("[dashboard.title]", "['{dashboardTitle5}']"),
self.check("[dashboard.panels[0].datasource.uid]", "['{amg2_datasource_uid_long}']"),
self.check("[meta.folderTitle]", "['General']")])

# Close-out Instance
self.cmd('grafana delete -g {rg} -n {name} --yes')
self.cmd('grafana delete -g {rg} -n {name2} --yes')
Expand Down
253 changes: 253 additions & 0 deletions src/amg/azext_amg/tests/latest/test_definitions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,28 @@
"type": "grafana-azure-monitor-datasource"
}

test_data_source_long_uid2 = {
"access": "proxy",
"uid": "ea714998-ca29-4adc-851b-1bd93371cbc6",
"jsonData": {
"azureAuthType": "msi",
"subscriptionId": ""
},
"name": "Test Azure Monitor Data Source2",
"type": "grafana-azure-monitor-datasource"
}

test_data_source_short_uid2 = {
"access": "proxy",
"uid": "cdpe79jidbwu8d",
"jsonData": {
"azureAuthType": "msi",
"subscriptionId": ""
},
"name": "Test Azure Monitor Data Source2",
"type": "grafana-azure-monitor-datasource"
}

test_notification_channel = {
"name": "Test Teams Notification Channel",
"settings": {
Expand Down Expand Up @@ -269,4 +291,235 @@
"version": 1,
"weekStart": ""
}
}

test_dashboard_with_datasource_short_uid = {
"dashboard": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": True,
"hide": True,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": True,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 39,
"links": [],
"liveNow": False,
"panels": [
{
"datasource": {
"type": "grafana-azure-monitor-datasource",
"uid": "cdpe79jidbwu8d"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": False,
"axisCenteredZero": False,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": False,
"tooltip": False,
"viz": False
},
"insertNones": False,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNones": False,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": None
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": True
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": True
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "grafana-azure-monitor-datasource",
"uid": "cdpe79jidbwu8d"
},
"refId": "A",
"scenarioId": "random_walk"
}
],
"title": "Panel Title",
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-azure-monitor-datasource",
"uid": "cdpe79jidbwu8d"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": False,
"axisCenteredZero": False,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": False,
"tooltip": False,
"viz": False
},
"insertNones": False,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNones": False,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": None
},
{
"color": "red",
"value": 80
}
]
},
"unitScale": True
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": True
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "grafana-azure-monitor-datasource",
"uid": "cdpe79jidbwu8d"
},
"refId": "A",
"scenarioId": "random_walk",
"seriesCount": 1
}
],
"title": "Panel Title",
"type": "timeseries"
}
],
"refresh": "",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": False,
"timepicker": {},
"timezone": "",
"title": "Test_DATASOURCE_SHORT_UID",
"uid": "d493c0bb-f3ce-4486-8909-6d29605052bf",
"version": 1,
"weekStart": ""
}
}
Loading
Loading