Add tests to validate the content and layout of common resources

.github/workflows/run-browser-tests.yml

@@ -0,0 +1,22 @@
+name: "Scheduled jobs: Run browser tests"
+on:
+  schedule:
+    # Run every day at 2:00PM UTC.
+    - cron:  "0 14 * * *"
+  workflow_dispatch:
+jobs:
+  all:
+    env:
+      GOPATH: ${{ github.workspace }}/go
+    name: Check links
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '18.x'
+
+      - name: Run browser tests
+        run: make run-browser-tests

.gitignore

@@ -7,4 +7,6 @@ themes/default/assets
 public
 .DS_Store
 .hugo_build.lock
-.vscode
+.vscode
+cypress/videos
+cypress/screenshots

Makefile

@@ -60,3 +60,8 @@ ci_bucket_cleanup:
 check_links:
 	$(MAKE) ensure
 	./scripts/link-checker/check-links.sh "https://www.pulumi.com/registry"
+
+.PHONY: run-browser-tests
+run-browser-tests:
+	$(MAKE) ensure
+	./scripts/run-browser-tests.sh https://www.pulumi.com api-docs-spec.js

cypress/integration/api-docs-spec.js

@@ -0,0 +1,243 @@
+describe("Registry", () => {
+
+    describe("API docs", () => {
+
+        const resources = [
+            [ "aws.lambda.Function", "lambda/function" ],
+            [ "aws.s3.Bucket", "s3/bucket" ],
+            [ "aws.iam.Role", "iam/role" ],
+            [ "aws.ec2.Instance", "ec2/instance" ],
+            [ "aws.rds.Instance", "rds/instance" ],
+            [ "aws.ec2.SecurityGroup", "ec2/securitygroup" ],
+            [ "kubernetes.helm.sh.v3.Chart", "helm/v3/chart" ],
+            [ "aws.ec2.Vpc", "ec2/vpc" ],
+            [ "kubernetes.helm.sh.v3.Release", "helm/v3/release" ],
+            [ "aws.alb.LoadBalancer", "alb/loadbalancer", ],
+            [ "aws.cloudfront.Distribution", "cloudfront/distribution" ],
+            [ "aws.ecs.TaskDefinition", "ecs/taskdefinition" ],
+            [ "aws.route53.Record", "route53/record" ],
+            [ "aws.ec2.LaunchTemplate", "ec2/launchtemplate" ],
+            [ "gcp.storage.Bucket", "storage/bucket" ],
+            [ "kubernetes.apps.v1.Deployment ", "apps/v1/deployment" ],
+            [ "aws.rds.Cluster", "rds/cluster" ],
+            [ "aws.iam.Policy", "iam/policy" ],
+            [ "aws.eks.Cluster", "eks/cluster" ],
+            [ "azure-native.storage.StorageAccount", "storage/storageaccount" ],
+            [ "gcp.serviceaccount.Account", "serviceaccount/account" ],
+            [ "aws.secretsmanager.Secret", "secretsmanager/secret" ],
+            [ "aws.ec2.Subnet", "ec2/subnet" ],
+            [ "kubernetes.networking.k8s.io.v1.Ingress", "networking/v1/ingress" ],
+            [ "aws.dynamodb.Table", "dynamodb/table" ],
+            [ "kubernetes.autoscaling.v2beta2.HorizontalPodAutoscaler", "autoscaling/v2beta2/horizontalpodautoscaler" ],
+            [ "aws.ecs.Service", "ecs/service" ],
+            [ "aws.acm.Certificate", "acm/certificate" ],
+            [ "aws.apigateway.RestApi", "apigateway/restapi" ],
+            [ "gcp.container.Cluster", "container/cluster" ],
+            [ "aws.autoscaling.Group", "autoscaling/group" ],
+            [ "gcp.cloudrun.Service", "cloudrun/service" ],
+            [ "azure-native.resources.ResourceGroup", "resources/resourcegroup" ],
+            [ "aws.iam.RolePolicyAttachment ", "iam/rolepolicyattachment" ],
+            [ "azure-native.web.WebApp ", "web/webapp" ],
+            [ "aws.lb.TargetGroup", "lb/targetgroup" ],
+            [ "awsx.ec2.Vpc", "ec2/vpc" ],
+            [ "kubernetes.core.v1.Secret", "core/v1/secret" ],
+            [ "aws.sqs.Queue", "sqs/queue" ],
+            [ "gcp.compute.Instance", "compute/instance" ],
+            [ "awsx.ecs.FargateService", "ecs/fargateservice" ],
+            [ "kubernetes.yaml.ConfigFile", "yaml/configfile" ],
+            [ "aws.alb.Listener", "alb/listener" ],
+            [ "aws.eks.NodeGroup", "eks/nodegroup" ],
+            [ "aws.ecs.Cluster", "ecs/cluster" ],
+            [ "aws.iam.getPolicyDocument", "iam/getpolicydocument" ],
+            [ "aws.s3.BucketPolicy", "s3/bucketpolicy" ],
+            [ "aws.cloudwatch.MetricAlarm", "cloudwatch/metricalarm" ],
+            [ "azure-native.keyvault.Vault ", "keyvault/vault" ],
+            [ "aws.sns.Topic", "sns/topic" ],
+        ];
+
+        resources.forEach(resource => {
+            const [ token, path ] = resource;
+            const title = token;
+            const provider = token.split(".").slice(0)[0];
+            const resourceName = token.split(".").slice(-1)[0];
+
+            const page = {
+                title,
+                provider,
+                resourceName,
+                path: `/registry/packages/${provider}/api-docs/${path}/`,
+            };
+
+            describe(`${page.title} resource page (${[ Cypress.config().baseUrl, page.path ].join("")})`, () => {
+
+                beforeEach(() => {
+                    cy.visit(page.path);
+                });
+
+                it("has the correct page title", () => {
+                    cy.get("head title").should("have.text", `${page.title} | Pulumi Registry`);
+                });
+
+                describe("main content", () => {
+                    const container = "main div.docs-main-content";
+
+                    it("has the correct H1", () => {
+                        cy.get(container)
+                            .find("h1")
+                            .should("be.visible")
+                            .should("have.text", page.title);
+                    });
+
+                    describe("description", () => {
+                        it("leads with a non-empty paragraph tag", () => {
+                            cy.get(container)
+                                .find("section.docs-content > p")
+                                .first()
+                                .should("exist")
+                                .invoke("text")
+                                .should("match", /^\w+/);
+                        });
+                    });
+
+                    it("renders subsections in the correct order", () => {
+                        cy.get(container).find("section.docs-content h2").as("sections");
+
+                        cy.get("@sections").should("have.length.of", 7);
+                        cy.get("@sections").eq(0).should("have.text", "Example Usage");
+                        cy.get("@sections").eq(1).should("have.text", `Create ${page.resourceName} Resource`);
+                        cy.get("@sections").eq(2).should("have.text", `${page.resourceName} Resource Properties`);
+                        cy.get("@sections").eq(3).should("have.text", `Look up Existing ${page.resourceName} Resource`);
+                        cy.get("@sections").eq(4).should("have.text", "Supporting Types");
+                        cy.get("@sections").eq(5).should("have.text", "Import");
+                        cy.get("@sections").eq(6).should("have.text", "Package Details");
+                    });
+
+                    describe("Examples section", () => {
+                        it("contains at least one example", () => {
+                            cy.get(container)
+                                .find("h2 ~ h3")
+                                .should("have.length.of.at.least", 1);
+                        });
+
+                        describe("examples", () => {
+                            it.skip("all have language choosers", () => {
+                                // Markup structure still TBD.
+                                // https://github.com/pulumi/pulumi-aws/issues/2624
+                            });
+                        });
+                    });
+
+                    describe("Create section", () => {
+                        it.skip("renders a minimal example in all languages", () => {
+                            // Markup structure still TBD.
+                            // https://github.com/pulumi/pulumi/issues/14675
+                        });
+
+                        it.skip("renders a full example in all languages", () => {
+                            // Markup structure still TBD.
+                            // https://github.com/pulumi/pulumi/issues/14675
+                        });
+                    });
+
+                    describe("Inputs and Outputs sections", () => {
+                        const propertyLists = ".resources-properties";
+
+                        it("render deprecation notices properly", () => {
+                            cy.get(container).find(propertyLists).as("propertyLists");
+
+                            cy.get("@propertyLists").then(lists => {
+                                const deprecatedProperties = "dt.property-deprecated + dd > p.property-message";
+
+                                if (lists.find(deprecatedProperties).length > 0) {
+                                    cy.get("@propertyLists")
+                                        .find(deprecatedProperties)
+                                        .first()
+                                        .invoke("text")
+                                        .should("not.equal", "Deprecated:");
+                                }
+                            });
+                        });
+
+                        describe("type links", () => {
+                            it("all point to an on-page supporting type", () => {
+                                cy.get(container).find(propertyLists).find("dt .property-type a[href*='#']").then(links => {
+                                    links.each((i, link) => {
+                                        if (link.href.startsWith("#")) {
+                                            cy.get(`#supporting-types ~ ${link.href}`).should("exist");
+                                        }
+                                    });
+                                });
+                            });
+                        })
+                    });
+
+                    describe("Lookup section", () => {
+                        it.skip("renders an example for all languages", () => {
+                            // Markup structure still TBD.
+                            // https://github.com/pulumi/pulumi/issues/14675
+                        });
+                    });
+
+                    describe("Supporting Types section", () => {
+                        describe("type lists", () => {
+                            it("are visible for all languages", () => {
+                                const languages = [ "TypeScript", "Python", "Go", "C#", "Java", "YAML" ];
+                                const headings = "#supporting-types ~ h4";
+                                const lists = "#supporting-types + h4 ~ div pulumi-choosable div.active";
+
+                                languages.forEach(language => {
+                                    cy.get("pulumi-chooser li a").contains(language).first().click();
+
+                                    cy.get(headings).then(headings => {
+                                        cy.get(lists).then(lists => {
+                                            expect(lists.length).to.equal(headings.length);
+                                        });
+                                    });
+                                });
+                            });
+                        });
+                    });
+
+                    describe("Import section", () => {
+                        it.skip("identifies required parameters", () => {
+                            // Details here are still TBD.
+                            // https://github.com/pulumi/registry/issues/3039
+                        });
+                    });
+
+                    describe("Package Details section", () => {
+                        it.skip("exists", () => {
+                            cy.get("#package-details").should("exist");
+                        });
+                    });
+                });
+
+                describe("left column", () => {
+                    const container = "#docs-main-nav";
+
+                    it("has an API docs menu", () => {
+                        cy.get(container)
+                            .find("pulumi-api-doc-filterable-nav")
+                            .should("exist");
+                    });
+                });
+
+                describe("right column", () => {
+                    const container = ".docs-table-of-contents .table-of-contents";
+
+                    it("has a package card", () => {
+                        cy.get(container)
+                            .find("#accordion-table-of-contents")
+                            .should("exist");
+                    });
+
+                    it("has a table of contents", () => {
+                        cy.get(container)
+                            .find("#accordion-table-of-contents")
+                            .should("exist");
+                    });
+                });
+            });
+        });
+    });
+});

cypress/integration/site_spec.js → cypress/integration/site-spec.js

@@ -1,4 +1,6 @@
 describe("www.pulumi.com", () => {
+
+    // This should probably be testing the Registry home page and not the home-home page.
     describe("home page", () => {
         beforeEach(() => {
             cy.visit("/");

package.json

@@ -7,6 +7,9 @@
     "license": "Apache-2.0",
     "dependencies": {
         "@fullhuman/postcss-purgecss": "^4.0.3",
+        "@octokit/auth-action": "^1.3.2",
+        "@octokit/graphql": "^4.6.2",
+        "@octokit/rest": "^18.5.3",
         "@slack/web-api": "^5.12.0",
         "broken-link-checker": "^0.7.8",
         "concurrently": "^6.0.0",
@@ -17,13 +20,10 @@
         "js-yaml": "^4.1.0",
         "jsdom": "^22.0.0",
         "markdownlint": "^0.28.0",
-        "@octokit/auth-action": "^1.3.2",
-        "@octokit/graphql": "^4.6.2",
-        "@octokit/rest": "^18.5.3",
         "postcss": "^8.3.7",
         "postcss-cli": "^8.3.1",
         "sitemapper": "^3.2.2"
-},
+    },
     "scripts": {
         "minify-css": "node scripts/minify-css.js"
     }

scripts/run-browser-tests.sh

@@ -1,12 +1,19 @@
 #!/bin/bash
 
-MAX_RETRIES=3
-RETRY_COUNT=0
+
+CYPRESS_BASE_URL="${1:-https://www.pulumi.com}"
+SPEC_FILE="${2:-site-spec.js}"
+MAX_RETRIES="${3:-3}"
 
 # Retry on errors up to `MAX_RETRIES` with a 10 second sleep in between.
+RETRY_COUNT=0
 run_tests() {
     while true; do
-        CYPRESS_BASE_URL="$1" yarn run cypress run --headless
+        CYPRESS_BASE_URL="$CYPRESS_BASE_URL" yarn run cypress run --headless --spec "cypress/integration/${SPEC_FILE}"
+
+        # Alternatively, to run Cypress in a browser (which makes debugging much easier), you can run `cypress open`.
+        # CYPRESS_BASE_URL="$CYPRESS_BASE_URL" yarn run cypress open
+
         exit_code=$?
         if [ $exit_code -ne 0 ]; then
             RETRY_COUNT=$((RETRY_COUNT+1))