diff --git a/__checks__/alertChannels.ts b/__checks__/alertChannels.ts index 9a5deeead..70b05b673 100644 --- a/__checks__/alertChannels.ts +++ b/__checks__/alertChannels.ts @@ -2,6 +2,6 @@ import { OpsgenieAlertChannel, SlackAlertChannel } from 'checkly/constructs' import { alertChannelIds } from './defaults' export const slackChannelOps = SlackAlertChannel.fromId(alertChannelIds.slack) -export const opsGenieChannelP1 = OpsgenieAlertChannel.fromId(alertChannelIds.opsGenieP1) +// removed temporarily for testing snapshots / visual comparisons export const opsGenieChannelP3 = OpsgenieAlertChannel.fromId(alertChannelIds.opsGenieP3) -export const alertChannels = [slackChannelOps, opsGenieChannelP3] +export const alertChannels = [slackChannelOps] diff --git a/__checks__/docs-visual.spec.ts b/__checks__/docs-visual.spec.ts new file mode 100644 index 000000000..d920ddb19 --- /dev/null +++ b/__checks__/docs-visual.spec.ts @@ -0,0 +1,8 @@ +import { test } from '@playwright/test' +import { ChecklySitePage } from './poms/ChecklySitePage' + +test('homepage visual comparison', async ({ page }) => { + const checklyPage = new ChecklySitePage(page) + await checklyPage.goto('/docs') + await checklyPage.doScreenshotCompare() +}) diff --git a/__checks__/docs-visual.spec.ts-snapshots/homepage-visual-comparison-1-chromium-linux.png b/__checks__/docs-visual.spec.ts-snapshots/homepage-visual-comparison-1-chromium-linux.png new file mode 100644 index 000000000..3d1398a19 Binary files /dev/null and b/__checks__/docs-visual.spec.ts-snapshots/homepage-visual-comparison-1-chromium-linux.png differ diff --git a/__checks__/poms/ChecklySitePage.ts b/__checks__/poms/ChecklySitePage.ts index 07573f070..88a6fd7b8 100644 --- a/__checks__/poms/ChecklySitePage.ts +++ b/__checks__/poms/ChecklySitePage.ts @@ -1,10 +1,11 @@ -import type { BrowserContext, Page } from '@playwright/test' +import type { Page } from '@playwright/test' +import { expect } from '@playwright/test' import { defaults } from '../defaults' export class ChecklySitePage { public page: Page - constructor (page) { + constructor (page: Page) { this.page = page } @@ -17,7 +18,17 @@ export class ChecklySitePage { await this.page.goto(defaults.baseURL + uri) } - async screenshot (name) { + async screenshot (name: string) { await this.page.screenshot({ path: `${defaults.screenshotPath}/${name}.jpg` }) } + + async doScreenshotCompare () { + await expect(this.page).toHaveScreenshot({ + maxDiffPixelRatio: 0.2, + mask: [ + this.page.locator('.optanon-alert-box-wrapper'), + this.page.locator('#intercom-container-body') + ] + }) + } } diff --git a/__checks__/site.check-group.ts b/__checks__/site.check-group.ts index a221db242..83676142d 100644 --- a/__checks__/site.check-group.ts +++ b/__checks__/site.check-group.ts @@ -1,4 +1,4 @@ -import { CheckGroup, RetryStrategyBuilder } from 'checkly/constructs' +import { CheckGroup, RetryStrategyBuilder, Frequency } from 'checkly/constructs' import { alertChannels } from './alertChannels' export const checklyhqComGroup = new CheckGroup('checklyhq-docs-1', { @@ -6,6 +6,7 @@ export const checklyhqComGroup = new CheckGroup('checklyhq-docs-1', { activated: true, muted: false, runtimeId: '2023.02', + frequency: Frequency.EVERY_1H, locations: [ 'us-east-1', 'us-west-1', diff --git a/package-lock.json b/package-lock.json index 2ec9b887e..3e08a7da1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "devDependencies": { "@playwright/test": "1.31.2", - "checkly": "4.0.14", + "checkly": "4.4.0-prerelease", "eslint": "7.32.0", "eslint-config-standard": "16.0.3", "eslint-plugin-html": "7.1.0", @@ -1221,12 +1221,12 @@ "integrity": "sha512-VhCw7I7qO2X49+jaKcAUwi3rR+hbxT5VcYF493+Z5kMLI0DL568b7JI4IDJaxWFH0D/xwmGJNoXisyX+w7GH/g==" }, "node_modules/@typescript-eslint/types": { - "version": "5.46.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-5.46.1.tgz", - "integrity": "sha512-Z5pvlCaZgU+93ryiYUwGwLl9AQVB/PQ1TsJ9NZ/gHzZjN7g9IAn6RSDkpCV8hqTwAiaj6fmCcKSQeBPlIpW28w==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.2.tgz", + "integrity": "sha512-flJYwMYgnUNDAN9/GAI3l8+wTmvTYdv64fcH8aoJK76Y+1FCZ08RtI5zDerM/FYT5DMkAc+19E4aLmd5KqdFyg==", "dev": true, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^16.0.0 || >=18.0.0" }, "funding": { "type": "opencollective", @@ -1234,21 +1234,21 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "5.46.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-5.46.1.tgz", - "integrity": "sha512-j9W4t67QiNp90kh5Nbr1w92wzt+toiIsaVPnEblB2Ih2U9fqBTyqV9T3pYWZBRt6QoMh/zVWP59EpuCjc4VRBg==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.2.tgz", + "integrity": "sha512-kiJKVMLkoSciGyFU0TOY0fRxnp9qq1AzVOHNeN1+B9erKFCJ4Z8WdjAkKQPP+b1pWStGFqezMLltxO+308dJTQ==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.46.1", - "@typescript-eslint/visitor-keys": "5.46.1", + "@typescript-eslint/types": "6.7.2", + "@typescript-eslint/visitor-keys": "6.7.2", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", - "semver": "^7.3.7", - "tsutils": "^3.21.0" + "semver": "^7.5.4", + "ts-api-utils": "^1.0.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^16.0.0 || >=18.0.0" }, "funding": { "type": "opencollective", @@ -1273,9 +1273,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -1294,16 +1294,16 @@ "dev": true }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "5.46.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-5.46.1.tgz", - "integrity": "sha512-jczZ9noovXwy59KjRTk1OftT78pwygdcmCuBf8yMoWt/8O8l+6x2LSEze0E4TeepXK4MezW3zGSyoDRZK7Y9cg==", + "version": "6.7.2", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.2.tgz", + "integrity": "sha512-uVw9VIMFBUTz8rIeaUT3fFe8xIUx8r4ywAdlQv1ifH+6acn/XF8Y6rwJ7XNmkNMDrTW+7+vxFFPIF40nJCVsMQ==", "dev": true, "dependencies": { - "@typescript-eslint/types": "5.46.1", - "eslint-visitor-keys": "^3.3.0" + "@typescript-eslint/types": "6.7.2", + "eslint-visitor-keys": "^3.4.1" }, "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + "node": "^16.0.0 || >=18.0.0" }, "funding": { "type": "opencollective", @@ -4129,9 +4129,9 @@ } }, "node_modules/checkly": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/checkly/-/checkly-4.0.14.tgz", - "integrity": "sha512-8ibfqa+BjaJgxMY9sl+2Vu8ZIRFoLVPHZfrDpeK6XTCXbO37Jhxr8mVq0ld/uiTMvCxuVtwBn4TfTEL+SNgXDg==", + "version": "4.4.0-prerelease", + "resolved": "https://registry.npmjs.org/checkly/-/checkly-4.4.0-prerelease.tgz", + "integrity": "sha512-o2acM0JZ5lVEHpyi6ZY2T3h3UQxUijcs8QcDDMCNSe4nq7yrO7djPtvI3oVEmgQbi9u6AV7bSTnI7oY4ddqNSQ==", "dev": true, "dependencies": { "@oclif/core": "2.8.11", @@ -4139,13 +4139,13 @@ "@oclif/plugin-not-found": "2.3.23", "@oclif/plugin-plugins": "2.3.0", "@oclif/plugin-warn-if-update-available": "2.0.24", - "@typescript-eslint/typescript-estree": "5.46.1", + "@typescript-eslint/typescript-estree": "6.7.2", "acorn": "8.8.1", "acorn-walk": "8.2.0", "async-mqtt": "2.6.3", "axios": "1.4.0", "chalk": "4.1.2", - "ci-info": "3.7.1", + "ci-info": "3.8.0", "conf": "10.2.0", "dotenv": "16.3.1", "git-repo-info": "2.1.1", @@ -4370,9 +4370,9 @@ } }, "node_modules/ci-info": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.7.1.tgz", - "integrity": "sha512-4jYS4MOAaCIStSRwiuxc4B8MYhIe676yO1sYGzARnjXkWpmzZMMYxY6zu8WYWDhSuth5zhrQ1rhNSibyyvv4/w==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.8.0.tgz", + "integrity": "sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==", "dev": true, "funding": [ { @@ -6865,12 +6865,15 @@ } }, "node_modules/eslint-visitor-keys": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", - "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "dev": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" } }, "node_modules/eslint/node_modules/@babel/code-frame": { @@ -20959,6 +20962,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/ts-api-utils": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.0.3.tgz", + "integrity": "sha512-wNMeqtMz5NtwpT/UZGY5alT+VoKdSsOOP/kqHFcUW1P/VRhH2wJ48+DN2WwUliNbQ976ETwDL0Ifd2VVvgonvg==", + "dev": true, + "engines": { + "node": ">=16.13.0" + }, + "peerDependencies": { + "typescript": ">=4.2.0" + } + }, "node_modules/ts-node": { "version": "10.9.1", "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", @@ -21031,27 +21046,6 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, - "node_modules/tsutils": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.21.0.tgz", - "integrity": "sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==", - "dev": true, - "dependencies": { - "tslib": "^1.8.1" - }, - "engines": { - "node": ">= 6" - }, - "peerDependencies": { - "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" - } - }, - "node_modules/tsutils/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", - "dev": true - }, "node_modules/tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", diff --git a/package.json b/package.json index 37b7c496a..0b31f40e7 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ }, "devDependencies": { "@playwright/test": "1.31.2", - "checkly": "4.1.0", + "checkly": "4.4.0-prerelease", "eslint": "7.32.0", "eslint-config-standard": "16.0.3", "eslint-plugin-html": "7.1.0", diff --git a/site/config.toml b/site/config.toml index 0eeaa34bd..25a5f4bf7 100644 --- a/site/config.toml +++ b/site/config.toml @@ -66,13 +66,18 @@ disqusShortname = "checkly" url = "/docs/heartbeat-checks/" weight = 35 +[[menu.resources]] + name = "Multistep API checks" + url = "/docs/multistep-checks/" + weight = 40 + [[menu.resources]] name = "Groups" url = "/docs/groups/" - weight = 40 + weight = 45 [[menu.resources]] - name = "Retries & Alerting" + name = "Alerting & retries" url = "/docs/alerting/" weight = 50 diff --git a/site/content/docs/alerting-and-retries/_index.md b/site/content/docs/alerting-and-retries/_index.md new file mode 100644 index 000000000..241dc6317 --- /dev/null +++ b/site/content/docs/alerting-and-retries/_index.md @@ -0,0 +1,18 @@ +--- +title: 'Overview' +weight: 29 +slug: / +menu: + resources: + parent: "Alerting & retries" +cli: true +aliases: + - /docs/alerting/ +--- + +Checkly will alert you when your checks [change state](/docs/alerting-and-retries/alert-states), e.g. when a passing check starts failing. To make the most out of Checkly alerts, you will want to configure them so they fit your workflow. + +You can break Checkly alerting down to three main areas: +1. [Alert settings](/docs/alerting-and-retries/alert-settings), controlling **when** and **how often** you want to be alerted when a check fails/degrades/recovers. +2. [Alert channels](/docs/alerting-and-retries/alert-channels), determining **how** alert notifications will be sent to you. +3. [Retries](/docs/alerting-and-retries/retries), which will help you avoid false positives by re-running failed checks. diff --git a/site/content/docs/retries-and-alerting/alert-channels.md b/site/content/docs/alerting-and-retries/alert-channels.md similarity index 91% rename from site/content/docs/retries-and-alerting/alert-channels.md rename to site/content/docs/alerting-and-retries/alert-channels.md index db86e64a6..e889daa51 100644 --- a/site/content/docs/retries-and-alerting/alert-channels.md +++ b/site/content/docs/alerting-and-retries/alert-channels.md @@ -3,10 +3,12 @@ title: Alert channels weight: 31 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" cli: true --- +Alert channels are the channels through which alert notifications will reach you when a check starts failing, showing degraded performance or recover. + You can add as many of the following alert channels as you want: - Email diff --git a/site/content/docs/retries-and-alerting/alert-notification-log.md b/site/content/docs/alerting-and-retries/alert-notification-log.md similarity index 98% rename from site/content/docs/retries-and-alerting/alert-notification-log.md rename to site/content/docs/alerting-and-retries/alert-notification-log.md index cbcdad18e..0ea9abd98 100644 --- a/site/content/docs/retries-and-alerting/alert-notification-log.md +++ b/site/content/docs/alerting-and-retries/alert-notification-log.md @@ -3,7 +3,7 @@ title: Alert notification log weight: 36 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" --- Checkly tries to deliver all your failure, degradation and recovery notifications as reliably as possible, but sometimes diff --git a/site/content/docs/retries-and-alerting/alert-settings.md b/site/content/docs/alerting-and-retries/alert-settings.md similarity index 87% rename from site/content/docs/retries-and-alerting/alert-settings.md rename to site/content/docs/alerting-and-retries/alert-settings.md index c94c6acb6..c3a462c79 100644 --- a/site/content/docs/retries-and-alerting/alert-settings.md +++ b/site/content/docs/alerting-and-retries/alert-settings.md @@ -3,7 +3,7 @@ title: 'Alert settings' weight: 30 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" lastmod: 01-06-2020 aliases: - /alerting/settings/ @@ -11,11 +11,7 @@ aliases: cli: true --- -Alerting in Checkly is pretty flexible and should address most common needs. You can tweak exactly when and how you should -be alerted, on what channels like email, [SMS]({{< ref "sms-delivery.md" >}}), [Slack]({{< ref "../integrations/slack.md" >}}) etc. -If you need more, you can integrate with [Pagerduty]({{< ref "../integrations/pagerduty.md" >}}), -[Opsgenie]({{< ref "../integrations/opsgenie.md" >}}) and many more! See [our integrations page]({{< ref "../integrations/" >}}) -for all options. +Alert settings allow you to control when and how often you will be notified when a check starts failing, degrades or recovers. ## Alert settings diff --git a/site/content/docs/retries-and-alerting/alert-states.md b/site/content/docs/alerting-and-retries/alert-states.md similarity index 98% rename from site/content/docs/retries-and-alerting/alert-states.md rename to site/content/docs/alerting-and-retries/alert-states.md index 2d39bd4f6..80d11d945 100644 --- a/site/content/docs/retries-and-alerting/alert-states.md +++ b/site/content/docs/alerting-and-retries/alert-states.md @@ -1,9 +1,9 @@ --- title: Alert states -weight: 32 +weight: 34 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" --- Sending out alert notifications like emails and Slack hooks depends on four factors: diff --git a/site/content/docs/retries-and-alerting/phone-calls.md b/site/content/docs/alerting-and-retries/phone-calls.md similarity index 96% rename from site/content/docs/retries-and-alerting/phone-calls.md rename to site/content/docs/alerting-and-retries/phone-calls.md index 19c6dfb91..ea3ba1a6a 100644 --- a/site/content/docs/retries-and-alerting/phone-calls.md +++ b/site/content/docs/alerting-and-retries/phone-calls.md @@ -1,9 +1,9 @@ --- title: Phone Calls -weight: 34 +weight: 35 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" --- The phone numbers used for phone call alerting need to be in [E.164 format](https://www.twilio.com/docs/glossary/what-e164). Stick to the following rules, and you'll be fine: diff --git a/site/content/docs/retries-and-alerting/_index.md b/site/content/docs/alerting-and-retries/retries.md similarity index 97% rename from site/content/docs/retries-and-alerting/_index.md rename to site/content/docs/alerting-and-retries/retries.md index 144e010c3..33551dc7b 100644 --- a/site/content/docs/retries-and-alerting/_index.md +++ b/site/content/docs/alerting-and-retries/retries.md @@ -1,13 +1,10 @@ --- title: 'Retries' -weight: 29 -slug: / +weight: 32 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" cli: true -aliases: - - /docs/alerting/ --- Sometimes the internet is flaky, or your app is just having a hiccup not worth pinging your on-call team about. Retries diff --git a/site/content/docs/retries-and-alerting/sms-delivery.md b/site/content/docs/alerting-and-retries/sms-delivery.md similarity index 97% rename from site/content/docs/retries-and-alerting/sms-delivery.md rename to site/content/docs/alerting-and-retries/sms-delivery.md index 60aef2dee..f2b981fdb 100644 --- a/site/content/docs/retries-and-alerting/sms-delivery.md +++ b/site/content/docs/alerting-and-retries/sms-delivery.md @@ -3,7 +3,7 @@ title: SMS delivery weight: 34 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" --- The phone numbers used for SMS alerting need to be in [E.164 format](https://www.twilio.com/docs/glossary/what-e164) format. Stick to the following rules and you'll be fine: diff --git a/site/content/docs/retries-and-alerting/ssl-expiration.md b/site/content/docs/alerting-and-retries/ssl-expiration.md similarity index 93% rename from site/content/docs/retries-and-alerting/ssl-expiration.md rename to site/content/docs/alerting-and-retries/ssl-expiration.md index 262a97f57..113057512 100644 --- a/site/content/docs/retries-and-alerting/ssl-expiration.md +++ b/site/content/docs/alerting-and-retries/ssl-expiration.md @@ -3,7 +3,7 @@ title: SSL certificate expiration weight: 33 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" aliases: - "/whats-new/summer-update/docs/alerting/ssl-expiration/" - "/whats-new/introducing-sms-pagerduty/docs/alerting/ssl-delivery/" @@ -13,7 +13,7 @@ An expired SSL certificate can cause havoc to sites and APIs. Checkly performs a alert you up to 30 days before your certificate expires. All alert channels (e-mail, SMS, OpsGenie, Webhook etc.) can be used for this alert. Simply create or pick an existing alert channel that your check subscribes to and enable *SSL certificate expiration* and -set the day threshold to your preference. If you don't have your alert channels set up yet, see [Alert Channels](/docs/retries-and-alerting/alert-channels/). +set the day threshold to your preference. If you don't have your alert channels set up yet, see [Alert Channels](/docs/alerting-and-retries/alert-channels/). ![Example alert channel form](/docs/images/alerting/ssl_check_example.png) diff --git a/site/content/docs/retries-and-alerting/webhooks.md b/site/content/docs/alerting-and-retries/webhooks.md similarity index 99% rename from site/content/docs/retries-and-alerting/webhooks.md rename to site/content/docs/alerting-and-retries/webhooks.md index 7c9f132f5..4128cf529 100644 --- a/site/content/docs/retries-and-alerting/webhooks.md +++ b/site/content/docs/alerting-and-retries/webhooks.md @@ -3,7 +3,7 @@ title: Webhooks weight: 35 menu: resources: - parent: "Retries & Alerting" + parent: "Alerting & retries" cli: true --- diff --git a/site/content/docs/browser-checks/visual-comparison-snapshot-testing.md b/site/content/docs/browser-checks/visual-comparison-snapshot-testing.md new file mode 100644 index 000000000..de31b34a2 --- /dev/null +++ b/site/content/docs/browser-checks/visual-comparison-snapshot-testing.md @@ -0,0 +1,262 @@ +--- +title: Visual comparison & snapshot testing +weight: 25 +menu: + resources: + parent: "Browser checks" +cli: true +beta: true +--- +Playwright Test gives you the ability to do visual comparison testing (sometimes called visual regression testing) and +snapshot testing. This is useful for testing the visual appearance of your application. The TL;DR is that you can: + +- Use the `.toHaveScreenshot()` assertion to visually compare a screenshot of your page to a golden image / reference snapshot. +- Use the `.toMatchSnapshot()` assertion to compare any `string` or `Buffer` value to a golden image / reference snapshot. + + +## Visual comparison testing + +Starting with visual comparison testing takes just three easy steps: + +1. Add a single `expect(page).toHaveScreenshot()` line of code to + your browser check script, like the example below. + + {{< tabs "Visual comparison test" >}} + {{< tab "Typescript" >}} + ```ts + import { test, expect } from '@playwright/test'; + + test('Playwright homepage', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot() + }) + ``` + {{< /tab >}} + {{< tab "Javascript" >}} + ```js + const { expect, test } = require('@playwright/test') + + test('Playwright homepage', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot() + }) + ``` + {{< /tab >}} + {{< /tabs >}} + +2. Run your browser check. The first time you run it, you will get an error indicating that no golden image / reference + snapshot exists yet. + + ``` + A snapshot doesn't exist at /tmp/19g67loplhq0j/script.spec.js-snapshots/Playwright-homepage-1-chromium-linux.png. + ``` + +3. Generate a golden image / reference snapshot by clicking the "Run script and update golden image" option in the "run script" button. + + + + This will generate the golden image, which you can inspect in the "golden files" tab in the editor. You can now save + your check and on each check run the golden image will be compared to the actual screenshot. + + + {{}} + If you are using the Checkly CLI, you can also generate a golden image / reference snapshot by running the following + command in your terminal: + ``` + npx checkly test --update-snapshots + ``` + {{}} + +Now, when your check fails due to a visual difference, you will see a diff of the golden image and the actual screenshot +in your check result. + + + + +### Configuring visual comparison testing + +To create accurate and actionable screenshot comparisons, Playwright gives you a ton of options to tweak how the `.toHaveScreenshot()` +should behave. What are acceptable differences in color, size, position, etc.? Do you want to match the full screen, or ignore +some dynamic elements that might screw up your comparison? + + Let's look at some examples, or check [the official reference docs](https://playwright.dev/docs/api/class-pageassertions#page-assertions-to-have-screenshot-1). + +### Example 1: setting pixel ratios and color thresholds + +You can control the margin of error you find acceptable between your golden image and the actual image using the following +options: + +- `maxDiffPixelRatio`: An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` +- `maxDiffPixels`: A total acceptable amount of pixels that could be different. +- `threshold`: An acceptable perceived color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between +the same pixel in compared images, between `0` (strict) and `1` (lax). + + +{{< tabs "Configuring thresholds" >}} +{{< tab "Typescript" >}} + ```ts + import { test, expect } from '@playwright/test'; + + test('Playwright homepage', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot({ maxDiffPixelRatio: 0.2 }) + await expect(page).toHaveScreenshot({ maxDiffPixels: 1000 }) + await expect(page).toHaveScreenshot({ threshold: 0.2 }) + }) + ``` +{{< /tab >}} +{{< tab "Javascript" >}} + ```js + const { test, expect } = require('@playwright/test') + + test('Playwright homepage', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot({ maxDiffPixelRatio: 0.2 }) + await expect(page).toHaveScreenshot({ maxDiffPixels: 1000 }) + await expect(page).toHaveScreenshot({ threshold: 0.2 }) + }) + ``` +{{< /tab >}} +{{< /tabs >}} + +### Example 2: ignoring specific screen elements + +A typical homepage can have dynamic elements that change on each page load, or change based on the geographical region. +Think of a "latest blog posts" section, a cookie banner or a region / language selector. Playwright allows you to ignore +these elements when doing a visual comparison using the `mask` option and using one or more `page.locator()` selectors. + +The example below hides the cookie banner and optional CTA popup from Intercom on the Checkly docs pages. + +{{< tabs "Ignoring elements" >}} +{{< tab "Typescript" >}} + ```ts + import { test, expect } from '@playwright/test'; + + test('Ignore cookie banner & cta popup', async ({ page }) => { + await page.goto('https://docs.checklyhq.com') + await expect(page).toHaveScreenshot({ + mask: [ + page.locator('.optanon-alert-box-wrapper'), + page.locator('#intercom-container-body') + ] + }) + }) + ``` +{{< /tab >}} +{{< tab "Javascript" >}} + ```js + const { test, expect } = require('@playwright/test') + + test('Playwright homepage', async ({ page }) => { + await page.goto('https://docs.checklyhq.com') + await expect(page).toHaveScreenshot({ + mask: [ + page.locator('.optanon-alert-box-wrapper'), + page.locator('#intercom-container-body') + ] + }) + }) + ``` +{{< /tab >}} +{{< /tabs >}} + +### Example 3: disabling animations + +In some cases, any ongoing animations can cause a visual difference between your golden image and the actual screenshot. +You can disable any CSS animations and transitions using the `animations` option. + +{{< tabs "Disabling animations" >}} +{{< tab "Typescript" >}} + ```ts + import { test, expect } from '@playwright/test'; + + test('Disable animations', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot({ animations: 'disabled' }) + }) + ``` +{{< /tab >}} +{{< tab "Javascript" >}} + ```js + const { test, expect } = require('@playwright/test') + + test('Disable animations', async ({ page }) => { + await page.goto('https://playwright.dev') + await expect(page).toHaveScreenshot({ animations: 'disabled' }) + }) + ``` +{{< /tab >}} +{{< /tabs >}} + +## Snapshot testing + +Snapshot testing, using the `expect().toMatchSnapshot(snapshotName)` assertion, is a great way to test the output of +any arbitrary `string` or `Buffer` value. Note that it is not optimized for visual comparison testing. + +{{< tabs "Snapshot testing" >}} +{{< tab "Typescript" >}} + ```ts + import { test, expect } from '@playwright/test' + + test('Match hero text', async ({ page }) => { + await page.goto('https://playwright.dev') + expect(await page.textContent('.hero__title')).toMatchSnapshot('hero.txt') + }) + ``` +{{< /tab >}} +{{< tab "Javascript" >}} + ```js + const { test, expect } = require('@playwright/test') + + test('Match hero text', async ({ page }) => { + await page.goto('https://playwright.dev') + expect(await page.textContent('.hero__title')).toMatchSnapshot('hero.txt') + }) + ``` +{{< /tab >}} +{{< /tabs >}} + +Creating or updating the golden image / reference snapshot works the same as with visual comparison testing. + +Check [the official reference docs](https://playwright.dev/docs/api/class-snapshotassertions#snapshot-assertions-to-match-snapshot-1) +for all options. + +## Embedding in your CI/CD workflow + +Using the [Checkly CLI](/docs/cli/) you can code and configure visual comparison and snapshot testing on your local machine +and deploy any changes either directly from your local machine or from your CI/CD pipeline of choice. + +In a typical scenario, you would follow the steps below: + +1. Create or update a browser check with visual comparison or snapshot testing on your local machine. +2. Generate the golden image / reference snapshot(s). + ```bash + npx checkly test --update-snapshots + ``` + The resulting files are stored in a `some-file-prepend.ts-snapshots` folder next to your browser check script. +3. Commit the browser check script and the golden image / reference snapshot(s) to your version control system. +4. Push your code to your CI/CD pipeline. +5. In your CI/CD pipeline, optionally run your checks again. Maybe add the `--record` flag to record the test in +Checkly. + ```bash + npx checkly test --record + ``` +6. If your tests pass, deploy your checks to production. The CLI will push your snapshot to the +Checkly cloud automatically. + ```bash + npx checkly deploy + ``` + +Learn more about setting up the Checkly CLI for your CI/CD pipeline 👇 + +{{< markdownpartial "/_shared/main-cicd-cards.md" >}} + +## Known limitations + +- Checkly currently only supports the Chromium and Chrome browsers. + +## Playwright resources + +- [The official Playwright guide on visual comparison and snapshot testing](https://playwright.dev/docs/test-snapshots) +- The `.toHaveScreenshot()` [API reference](https://playwright.dev/docs/api/class-pageassertions#page-assertions-to-have-screenshot-1) +- The `.toMatchSnapshot()` [API reference](https://playwright.dev/docs/api/class-snapshotassertions#snapshot-assertions-to-match-snapshot-1) diff --git a/site/content/docs/cli/command-line-reference.md b/site/content/docs/cli/command-line-reference.md index 3bdf44c23..983ac78d5 100644 --- a/site/content/docs/cli/command-line-reference.md +++ b/site/content/docs/cli/command-line-reference.md @@ -58,6 +58,7 @@ between environments. match any of the `--tags` filters, i.e. `--tags production,webapp --tags staging,backend` will run checks with tags (production AND webapp) OR (staging AND backend). - `--timeout`: A fallback timeout (in seconds) to wait for checks to complete. +- `--update-snapshots` or `-u`: Update the golden images / reference snapshots of your visual comparison and snapshot tests. - `--verbose` or `-v`: Always show the full logs of the checks. diff --git a/site/content/docs/cli/constructs-reference.md b/site/content/docs/cli/constructs-reference.md index 48dab7a5d..1c950daf0 100644 --- a/site/content/docs/cli/constructs-reference.md +++ b/site/content/docs/cli/constructs-reference.md @@ -62,7 +62,7 @@ dedicated docs on checkMatch and testMatch](/docs/cli/using-check-test-match/) ## `Check` -The CLI currently supports three Check types: API, Browser and Heartbeat Checks. +The CLI currently supports four Check types: API, Browser, Heartbeat and Multistep API Checks. These Check types share properties derived from the abstract class `Check`. @@ -271,6 +271,28 @@ new BrowserCheck('browser-check-1', { - `code`: an object with either an `entrypoint` property that points to `.spec.js|ts` file, or a `content` property with raw JavaScript / TypeScript as a string. +## `MultiStepCheck` + +Similar to Browser Checks, Multistep API checks uses [`@playwright/test`](https://playwright.dev/) to define the script which the check runs, but Multistep checks always need to be defined in a construct before assigning a `spec.js|ts` file. + +{{< info >}} +Multistep API checks are only supported on runtime 2023.09 or later. See [Runtimes](/docs/runtimes) for more details. +{{< /info >}} + +```ts +new MultiStepCheck('multistep-check-1', { + name: 'Multistep Check #1', + runtimeId: '2023.09', + frequency: Frequency.EVERY_10M, + locations: ['us-east-1', 'eu-west-1'], + code: { + entrypoint: path.join(__dirname, 'home.spec.ts') + }, +}) +``` + +- `code`: an object with either an `entrypoint` property that points to `.spec.js|ts` file, or a `content` property with +raw JavaScript / TypeScript as a string. ## `CheckGroup` diff --git a/site/content/docs/multistep-checks/_index.md b/site/content/docs/multistep-checks/_index.md new file mode 100644 index 000000000..5d3a8c6dc --- /dev/null +++ b/site/content/docs/multistep-checks/_index.md @@ -0,0 +1,160 @@ +--- +title: Getting started +weight: 14 +slug: / +menu: + resources: + parent: "Multistep API checks" + identifier: "multistep-checks" +aliases: + - /docs/multistep-checks/quickstart/ + - /docs/multistep-checks/getting-started/ +cli: true +beta: true +--- + +This guide gives you all the info to create your first Multistep API check with Checkly. You should have some prior +knowledge of working with JavaScript and/or Node.js. + +## What is a Multistep API check? + +A Multistep API check allows you to write Node.js scripts that can run multiple API requests in sequence, with arbitrary code between requests. This allows you to monitor entire API user flows with a single check, working with response data in between requests exactly as a user of your API would to ensure that these interactions result in the correct results. + +Examples of API sequences might be: +* Users can authenticate and access restricted data +* Users can get, set and remove data from their account +* Users can add items to a shopping cart, checkout and go through the payment flow. + +Monitoring your API user flows instead of individual endpoints gives confidence that your product as a whole works as intended and that the expected interactions between API calls are functional. + +Multistep API checks are powered by `@playwright/test`'s [API Testing](https://playwright.dev/docs/api-testing) mode. Meaning you get all of the power of our typical API tests, in the form of a programmable @playwright/test check + +{{< info >}} +Multistep API checks are only supported on runtime 2023.09 or later. See [Runtimes](/docs/runtimes) for more details. +{{< /info >}} + +The following code is a valid Multistep API check using Playwright Test. + +```ts +import { test, expect } from '@playwright/test' // 1 + +const headers = { // 2 + Authorization: `Bearer ${process.env.API_KEY}`, + 'x-checkly-account': process.env.ACCOUNT_ID, +} + +const baseUrl = process.env.API_URL ?? 'https://api.checklyhq.com/v1' + +test('create and delete a check group', async ({ request }) => { // 3 + const createdGroup = await test.step('POST /check-groups', async () => { // 4 + const response = await request.post(`${baseUrl}/check-groups`, { + data: { + locations: ['eu-west-1'], + name: 'exampleCheckGroup', + }, + headers, + }) + expect(response).toBeOK() // 5 + return response.json() // 6 + }) + + await test.step('DELETE /check-group/{id}', async () => { // 7 + const response = await request.delete(`${baseUrl}/check-groups/${createdGroup.id}`, { + headers, + }) + expect(response).toBeOK() + }) +}) +``` + +## Breaking down a Multistep check + +Let's look at the code above step by step. + +**1. Initial declarations:** To run any multistep check we first import the Playwright test framework. + +**2. Define our headers:** In many cases you will have to authenticate when requesting data by providing authorization data in your header. By using [environment variables](/docs/browser-checks/variables/) we avoid having any confidential data in our test. + +**3. Establish environment:** We create a new test using the Playwright `request` fixture, which can be used to make API requests in the test steps. + +**4. Declare our first `test.step`:** The test step uses the `request` to perform a `get` request, using the headers we defined earlier. + +{{< warning >}} +Always use `await` before `test.step`, otherwise the test will fail. +{{< /warning >}} + +**5. Define our assertion:** We use the `expect(response)` method to assert if the response was successful (The response code is in the range of 200 - 299) with `toBeOK()`. Should the request return anything outside of the 'OK' range, this will cause the check to fail and in a production scenario trigger any configured alerts. + +**6. Return the response for future usage:** We return the request response in JSON format, so we can use it in the next test step. + +**7. Declare our second `test.step`:** In order to remove the test group we just created, and avoid cluttering our system with test data, we remove it by sending a `delete` request using the group ID that was returned in our earlier test step. We use the same assertion as in the previous test step. + +If you want to build on the above example, you can add additional assertions, ensuring that the data returned is correct and contains a specific check, or add a `PUT` and `GET` test step to verify more of the `/groups` functionality. + +## Creating a Multistep check + +A valid Multistep API check is based on a valid [Playwright API test script](https://playwright.dev/docs/api-testing). You can create these scripts either in the in-app editor, or write them in your IDE and deploy them using the [Checkly CLI](https://www.checklyhq.com/docs/cli/). For production, we recommend using the CLI so you can leverage best practices such as version control and code reviews before updating your checks. + +{{< info >}} +Valid Playwright Test API scripts are the foundation of a valid Multistep API check. If the script passes, your check passes. +If the script fails, your check fails. +{{< /info >}} + +### Structuring a Multistep check + +To preserve test isolation and provide a structured report view of Multistep checks, Checkly relies on the [test.step](https://playwright.dev/docs/api/class-test#test-step) method from Playwright. Your multistep test can have several test steps. +API requests and assertions in the same test step will be presented under the same node in the reporting structure. + +To provide actionable and easy to read check run reports we recommend using this structure when writing Multistep checks. + +```ts +import { test, expect } from '@playwright/test' + +const baseUrl = 'https://api.checklyhq.com/v1' + +test('My test', async ({request}) => { + await test.step('First step', async () => { + const health = await request.get(`${baseUrl}/health`) + expect(health).toBeOK() + await expect(response).toBeOK() + }); + + await test.step('Second step', async () => { + const response = await request.post() + // Assertions for the second step + ... + }) +}) +``` + +### Building checks in the web editor + +When creating a check using the web editor, after each test run you can open up the result tree by clicking on the 'Test report' icon on the left side of the editor. Selecting an API response opens the response details. You can also view errors and any failed assertion in the report view. + +## Multistep result view + +When viewing a multistep check run result, you can select any API request in the result tree to view the full response details. If you have your API request and related assertions in the same `test.step`, related requests and failing assertions will be grouped under the same header. + +Selecting the top node in the check report shows the full job log and the check configuration for the run. + +## Pricing + +During beta, a multistep check is billed based on the number of requests done per check run. Each request in a multistep check run is billed as a single regular API check run, as they are performing the same basic operation. + +As an example, let's say you have 4 API checks, where each check doing one of the `GET`, `POST`, `PUT` and `DELETE` operations towards the same endpoint. If you replace these 4 checks with a single multistep API check that runs 4 requests towards the same endpoint, checking each method, and the check run frequency is the same as before, your cost stays the same + +A multistep check with 0 requests is billed as if it has 1 request. + +## Beta limitations + +During the beta phase, multistep API checks comes with some limitations: + - Multistep checks are not yet supported on [private locations](/docs/private-locations). + - Metrics from multistep API checks are not yet available via the [Checkly analytics API](/docs/analytics) or the [Prometheus integration](/docs/integrations/prometheus-v2/). + - Multistep checks are not included in the weekly summary email. + +## Resources + +- [Checkly's YouTube channel](https://www.youtube.com/@ChecklyHQ) where we regularly publish tutorials and tips. +- [playwright.dev](https://playwright.dev/) is the official API documentation site for the Playwright framework. +- [awesome-playwright](https://github.com/mxschmitt/awesome-playwright) is a great GitHub repo full of +Playwright-related libraries, tips and resources. \ No newline at end of file diff --git a/site/content/docs/terraform-provider/checks-groups.md b/site/content/docs/terraform-provider/checks-groups.md index 74f1ad298..c26d3ef29 100644 --- a/site/content/docs/terraform-provider/checks-groups.md +++ b/site/content/docs/terraform-provider/checks-groups.md @@ -10,7 +10,8 @@ Checks make up the most important unit of your Checkly monitoring setup. Groups ## Checks -[Browser checks](/docs/browser-checks), [API checks](/docs/api-checks) and [Heartbeat checks](/docs/heartbeat-checks) share many arguments, configuration-wise, but also have their own unique ones. The type of your check is controlled using the `type` argument. +[Browser checks](/docs/browser-checks), [API checks](/docs/api-checks), [Heartbeat checks](/docs/heartbeat-checks) and [Multistep API checks](/docs/multistep-checks) share many arguments, configuration-wise, +but also have their own unique ones. The type of your check is controlled using the `type` argument. ### Browser checks @@ -133,6 +134,35 @@ resource "checkly_check" "send-weekly-digest-v-2" { Upon applying your terraform configuration changes, you will be returned a read-only key value for the heartbeat ping token. The token is stored in your `tfstate` file. + +### Multistep API checks + +As with Browser checks, when constructing a Multistep API check it is possible to provide the script directly in-line, +but the recommended approach is to store scripts in separate files. + +{{< info >}} +Multistep API checks are only supported on runtime 2023.09 or later. See [Runtimes](/docs/runtimes) for more details. +{{< /info >}} + +For example, a Multistep check can look as follows: +```terraform + resource "checkly_check" "e2e-shopping" { + name = "Shopping Flow" + type = "MULTI_STEP" + activated = true + should_fail = false + frequency = 1 + use_global_alert_settings = true + locations = [ + "us-west-1", + "eu-central-1" + ] + + script = file("${path.module}/scripts/shopping.js") +} +``` + + ## Groups Once you start having more than just a handful of checks, it makes sense to start looking into [groups](/docs/groups) to keep things tidy: diff --git a/site/layouts/partials/docs-breadcrumb.html b/site/layouts/partials/docs-breadcrumb.html index 9320a2d03..7ed7c5294 100644 --- a/site/layouts/partials/docs-breadcrumb.html +++ b/site/layouts/partials/docs-breadcrumb.html @@ -16,7 +16,12 @@ {{ end }} {{ end }} {{ end }} - {{ if .Page.Params.cli }} -
CLI Enabled
- {{ end}} +
+ {{ if .Page.Params.beta }} +
BETA
+ {{ end}} + {{ if .Page.Params.cli }} +
CLI Enabled
+ {{ end}} +
diff --git a/site/static/docs/images/browser-checks/visual_comparison_1st_run.mp4 b/site/static/docs/images/browser-checks/visual_comparison_1st_run.mp4 new file mode 100644 index 000000000..e80bd687e Binary files /dev/null and b/site/static/docs/images/browser-checks/visual_comparison_1st_run.mp4 differ diff --git a/site/static/docs/images/browser-checks/visual_comparison_diff_modal.mp4 b/site/static/docs/images/browser-checks/visual_comparison_diff_modal.mp4 new file mode 100644 index 000000000..a4e1ba143 Binary files /dev/null and b/site/static/docs/images/browser-checks/visual_comparison_diff_modal.mp4 differ diff --git a/src/scss/_docs.scss b/src/scss/_docs.scss index 5a2c5d1d0..9023670fa 100644 --- a/src/scss/_docs.scss +++ b/src/scss/_docs.scss @@ -426,6 +426,14 @@ vertical-align: -24%; } } + + .badge-beta { + margin-left: auto; + color: $white; + background: $pink; + font-weight: 600; + text-transform: capitalize; + } } nav#TableOfContents {