From 6b270b8450a331f1c2d54c8aa1f35ea06ca7e7c7 Mon Sep 17 00:00:00 2001 From: Clare So <1740517+clarmso@users.noreply.github.com> Date: Thu, 28 Nov 2024 14:17:52 -0500 Subject: [PATCH] Add MTE-3753 TestRail backup daily (#47) * Let's do gzip file * again * Create file listing in README * Fix upload filename * Fix filename * Fix GITHUB_ENV * I meant to create a separate file for the file listing * Ignore underscore in filename in Listing.md * Fix headers * Update cfg file for a few more fields * Output result to Github Actions job page * Use year-month as sub directories * Fix subdir * Fix download path on summary page * More doc * Add Slack notification * Just output a simple message * Send just one emoji! * Fix json * Can we report backup status? * Fix json * Just a header * Try this payload * Can we have blocks? * Include all info in one mrkdwn * Put mrkdwn in one single line * Use separate json file * Fix slack.json * Use slack.json * Oh no! updated the wrong file! * Add default directory in file path * Does slackapi work? * Try again * Test slack.json * Use payload file (for real!!) * Test connection again * Fix env for URLs * Trying out mrkdwn? * Fix json * Prettify slack notif * Fix slack message * Eliminate the directory name * Add limitation * Update webhook * Try this webhook * No channel-id? * Don't worry about Slack notification until the webhooks are sorted out * Add passphrase * comment out unused step * Use apt-get (it's Ubuntu after all!) * Add passphrase to env * document how to decrypt the file * Disable gpg for now * Use sandbox channel for now * Put back Slack notifications for failed jobs * Let's trigger a failure scenario * Put password back * Put job on schedule :tada: * Nit: Add divider back to notification * Remove instructions for gpg --- .github/workflows/testrail-backup.yml | 100 +++++++++++++++++++++----- .gitignore | 2 +- backup-tools/README.md | 18 +++-- backup-tools/backup_testrail.py | 13 ++-- backup-tools/slack-fail.json | 22 ++++++ backup-tools/slack-success.json | 22 ++++++ backup-tools/testrail-import.cfg | 2 +- 7 files changed, 149 insertions(+), 30 deletions(-) create mode 100644 backup-tools/slack-fail.json create mode 100644 backup-tools/slack-success.json diff --git a/.github/workflows/testrail-backup.yml b/.github/workflows/testrail-backup.yml index 7e96990..cff8400 100644 --- a/.github/workflows/testrail-backup.yml +++ b/.github/workflows/testrail-backup.yml @@ -2,20 +2,32 @@ name: TestRail Backup on: workflow_dispatch: + schedule: + - cron: "0 15 * * *" # Run every day at 3pm + +env: + BUCKET: backups-testrail-test-suites + DEFAULT_DIR: ./backup-tools + PROJECT_IDS: 59 14 27 13 48 + # Fenix Browser: 59 + # Firefox for iOS: 14 + # Focus for iOS: 27 + # Firefox for Android: 13 + # Focus for Android:48 + STORAGE_URL_PREFIX: https://storage.googleapis.com jobs: test: name: Backup test suites - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 defaults: run: - working-directory: ./backup-tools + working-directory: ${{ env.DEFAULT_DIR }} env: TESTRAIL_HOST: ${{ secrets.TESTRAIL_HOST }} TESTRAIL_USERNAME: ${{ secrets.TESTRAIL_USERNAME }} TESTRAIL_PASSWORD: ${{ secrets.TESTRAIL_PASSWORD }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_TEST_ALERTS_SANDBOX }} steps: - name: Checkout code @@ -29,24 +41,78 @@ jobs: - name: Install dependencies run: | pip install -r requirements.txt - + sudo apt-get install gnupg + + - name: Fetch test cases + run: | + python backup_testrail.py ${{ env.PROJECT_IDS }} + + - name: Create file listing + run: | + touch Listing.md + echo "# Projects & Test Suites" >> Listing.md + echo "" >> Listing.md + echo "| Project | Test Suite | File |" >> Listing.md + echo "|---------|------------|------|" >> Listing.md + ls *.csv > files.txt + awk -F '_' '{print "| " $2 " | " $3 " | `" $0 "` |"}' < files.txt >> Listing.md + + - name: Create gzip file containing csv and listing + run: | + today=`date "+%Y-%m-%d"` + subdir=`date "+%Y-%m"` + filename=$today-TestRail-backup + + mkdir $filename + mv *.csv $filename + mv Listing.md $filename + tar -cvzf $filename.tgz $filename/* + + echo "filename=$filename" >> $GITHUB_ENV + echo "subdir=$subdir" >> $GITHUB_ENV + - name: Establish Google Cloud connection uses: google-github-actions/auth@v2 with: credentials_json: ${{ secrets.GCLOUD_AUTH }} + + - name: Upload CSV files to GCP bucket + id: upload-file + uses: google-github-actions/upload-cloud-storage@v2 + with: + path: ${{ env.DEFAULT_DIR }}/${{ env.filename }}.tgz + destination: ${{ env.BUCKET }}/${{ env.subdir }} - - name: Fetch test cases + - name: Output URL to Github Actions summary run: | - python backup_testrail.py - today=`date "+%Y-%m-%d"` - mkdir $today - cp *.csv $today - pwd - ls $today - echo "today=$today" >> $GITHUB_ENV - - - name: Upload CSV to GCP bucket - uses: google-github-actions/upload-cloud-storage@v2 + echo "[Download CSV files](${{ env.STORAGE_URL_PREFIX }}/${{ env.BUCKET }}/${{ steps.upload-file.outputs.uploaded }})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Projects Included in the Backup" >> $GITHUB_STEP_SUMMARY + cd $filename + ls *.csv > files.txt + awk -F '_' '{print "* " $2 }' < files.txt | uniq >> $GITHUB_STEP_SUMMARY + + - name: Notify Slack (Fail) + uses: slackapi/slack-github-action@v1.27.0 + if: ${{ failure() }} + with: + payload-file-path: ${{ env.DEFAULT_DIR }}/slack-fail.json + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_TEST_ALERTS_SANDBOX }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + server_url: ${{ github.server_url }} + repository: ${{ github.repository }} + run_id: ${{ github.run_id }} + + - name: Notify Slack (Success) + uses: slackapi/slack-github-action@v1.27.0 + if: ${{ success() }} with: - path: backup-tools/${{ env.today }} - destination: backups-testrail-test-suites \ No newline at end of file + payload-file-path: ${{ env.DEFAULT_DIR }}/slack-success.json + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_TEST_ALERTS_SANDBOX }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + uploaded: ${{ steps.upload-file.outputs.uploaded }} + server_url: ${{ github.server_url }} + repository: ${{ github.repository }} + run_id: ${{ github.run_id }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 82715bf..7abc892 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,6 @@ node_modules android-performance/node_modules actions-runner backup-tools/*.csv -backup-tools/*.json +backup-tools/backup*.json backup-tools/__pychache__/ *.pyc \ No newline at end of file diff --git a/backup-tools/README.md b/backup-tools/README.md index c491811..53fd07b 100644 --- a/backup-tools/README.md +++ b/backup-tools/README.md @@ -5,8 +5,9 @@ Google Cloud bucket on a regular basis. ## Recover from backup -1. Download the CSV file from the Google Cloud bucket. -1. Download the .cfg file from this directory. +1. Download the encrypted CSV file from the Google Cloud bucket. A link to the `.tgz.gpg` file is available through the + [Github Actions job summary page](https://github.com/mozilla-mobile/testops-tools/actions/workflows/testrail-backup.yml). +1. Download [testrail-import.cfg](https://github.com/mozilla-mobile/testops-tools/blob/main/backup-tools/testrail-import.cfg). 1. Navigate to TestRail project's *TestSuites & Cases* tab and open to an empty test suite. 1. Select *Import Cases* icon ➡️ *Import From CSV*. 1. From the *Import from CSV* modal: @@ -19,10 +20,15 @@ Google Cloud bucket on a regular basis. * Select *Test cases use multiple rows* * Select *Title* as the *Column to detect new test cases* * Click *Next* until the *Preview Import* screen - * (Configure the mapping for custom fields) + * Configure the mapping for custom fields if necessary 1. Click *Import* and then *Close*. -## Todo and Limitations +## Limitations -* No custom fields in .cfg file. - * [Administrator instructions](https://support.testrail.com/hc/en-us/articles/7373850291220-Configuring-custom-fields) for setting custom fields \ No newline at end of file +* Not all custom fields are captured in [testrail-import.cfg](https://github.com/mozilla-mobile/testops-tools/blob/main/backup-tools/testrail-import.cfg). + * See [Configuring custom fields](https://support.testrail.com/hc/en-us/articles/7373850291220-Configuring-custom-fields) for setting TestRail test suite custom fields for the test cases. +* An import to a non-empty test suite adds test cases to the test suite. Duplicated test cases may be added: An import does not check for duplicate titles. +* An import does not restore the test cases' original IDs. The imported cases have new IDs. +* An import does not restore the sections from test suites. +* An import does not detect duplicate test cases. +* Attachments to the test cases are not included in the backups. \ No newline at end of file diff --git a/backup-tools/backup_testrail.py b/backup-tools/backup_testrail.py index 9366afc..9941a08 100644 --- a/backup-tools/backup_testrail.py +++ b/backup-tools/backup_testrail.py @@ -1,3 +1,4 @@ +import sys import json import csv from datetime import datetime @@ -18,7 +19,7 @@ def create_csv(project_id, project_name, suite_id, suite_name): if len(cases) == 0: return - output_json_file = "data-{project}-{suite}-{year}-{month}-{day}.json".format( + output_json_file = "backup_{project}_{suite}_{year}-{month}-{day}.json".format( project=project_name, suite=suite_name, year=now.year, @@ -66,11 +67,13 @@ def create_csv(project_id, project_name, suite_id, suite_name): if __name__ == "__main__": testrail = TestRail() - - # Fenix Browser, Firefox for iOS, Focus for iOS, Firefox for Android, Focus for Android - # for project_id in [59, 14, 27, 13, 48]: - for project_id in [59, 14, 27, 13, 48]: + if len(sys.argv) == 1: + print("Usage: python backup_testrail.py ") + sys.exit(1) + + project_ids = sys.argv[1:] + for project_id in project_ids: project = testrail.project(project_id) project_name = project.get('name') suites = testrail.test_suites(project_id) diff --git a/backup-tools/slack-fail.json b/backup-tools/slack-fail.json new file mode 100644 index 0000000..575d2bb --- /dev/null +++ b/backup-tools/slack-fail.json @@ -0,0 +1,22 @@ +{ + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":x: TestRail Backup Failed", + "emoji": true + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ env.server_url }}/${{ env.repository }}/actions/runs/${{ env.run_id }}|Github Actions Job>" + } + } + ] + } \ No newline at end of file diff --git a/backup-tools/slack-success.json b/backup-tools/slack-success.json new file mode 100644 index 0000000..d6fa708 --- /dev/null +++ b/backup-tools/slack-success.json @@ -0,0 +1,22 @@ +{ + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":tada: TestRail Backup Successful", + "emoji": true + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ env.STORAGE_URL_PREFIX }}/${{ env.BUCKET }}/${{ env.uploaded }}|Download Backup (${{ env.uploaded }})> \n <${{ env.server_url }}/${{ env.repository }}/actions/runs/${{ env.run_id }}|Github Actions Job>" + } + } + ] + } \ No newline at end of file diff --git a/backup-tools/testrail-import.cfg b/backup-tools/testrail-import.cfg index 358a290..23b3c95 100644 --- a/backup-tools/testrail-import.cfg +++ b/backup-tools/testrail-import.cfg @@ -1 +1 @@ -{"import":{"format":"csv","version":1},"file":{"encoding":"UTF-8","delimiter":",","start_row":1,"has_header":true,"skip_empty":true},"layout":{"format":"multi","break":1,"template":2},"columns":["","cases:title","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","cases:custom_steps_separated.content","cases:custom_steps_separated.expected"],"values":{"1":{"remove_html":false},"34":{"remove_html":false},"35":{"remove_html":false}}} \ No newline at end of file +{"import":{"format":"csv","version":1},"file":{"encoding":"UTF-8","delimiter":",","start_row":1,"has_header":true,"skip_empty":true},"layout":{"format":"multi","break":1,"template":2},"columns":["","cases:title","cases:section_id","","cases:type_id","cases:priority_id","","cases:refs","","","","","cases:estimate","","","","","cases:assigned_to_id","cases:custom_test_case_owner","","","","cases:custom_rotation","","","cases:custom_preconds","","","","","","","","","cases:custom_steps_separated.content","cases:custom_steps_separated.expected"],"values":{"1":{"remove_html":false},"4":{"mapping":{"6":"6","7":"7","11":"11"}},"5":{"mapping":{"2":"2","3":"3"}},"7":{"remove_html":false},"17":{"mapping":[]},"18":{"mapping":{"37":"","68":"68","82":"82"}},"22":{"remove_html":false},"25":{"remove_html":false},"34":{"remove_html":false},"35":{"remove_html":false}}} \ No newline at end of file