Add tests to validate the content and layout of common resources

.gitignore

@@ -7,4 +7,6 @@ themes/default/assets
 public
 .DS_Store
 .hugo_build.lock
-.vscode
+.vscode
+cypress/videos
+cypress/screenshots

cypress/integration/api-docs-spec.js

@@ -0,0 +1,211 @@
+describe("Registry", () => {
+
+    describe("API docs", () => {
+
+        const resources = [
+            // { token: "aws.lambda.Function", path: "lambda/function" },
+            // { token: "aws.s3.Bucket", path: "s3/bucket" },
+            // { token: "aws.iam.Role", path: "iam/role" },
+            // { token: "aws.ec2.Instance", path: "ec2/instance" },
+            // { token: "aws.rds.Instance", path: "rds/instance" },
+            // { token: "aws.ec2.SecurityGroup", path: "ec2/securitygroup" },
+            // { token: "kubernetes.helm.sh.v3.Chart", path: "helm/v3/chart" },
+            // { token: "aws.ec2.Vpc", path: "ec2/vpc" },
+            // { token: "kubernetes.helm.sh.v3.Release", path: "helm/v3/release" },
+            // { token: "aws.alb.LoadBalancer", path: "alb/loadbalancer", },
+            // { token: "aws.cloudfront.Distribution", path: "cloudfront/distribution" },
+            // { token: "aws.ecs.TaskDefinition", path: "ecs/taskdefinition" },
+            // { token: "aws.route53.Record", path: "route53/record" },
+            // { token: "aws.ec2.LaunchTemplate", path: "ec2/launchtemplate" },
+            // { token: "gcp.storage.Bucket", path: "storage/bucket" },
+            // { token: "kubernetes.apps.v1.Deployment ", path: "apps/v1/deployment" },
+            // { token: "aws.rds.Cluster", path: "rds/cluster" },
+            // { token: "aws.iam.Policy", path: "iam/policy" },
+            // { token: "aws.eks.Cluster", path: "eks/cluster" },
+            // { token: "azure-native.storage.StorageAccount", path: "storage/storageaccount" },
+            // { token: "gcp.serviceaccount.Account", path: "serviceaccount/account" },
+            // { token: "aws.secretsmanager.Secret", path: "secretsmanager/secret" },
+            // { token: "aws.ec2.Subnet", path: "ec2/subnet" },
+            // { token: "kubernetes.networking.k8s.io.v1.Ingress", path: "networking/v1/ingress" },
+            // { token: "aws.dynamodb.Table", path: "dynamodb/table" },
+            // { token: "kubernetes.autoscaling.v2beta2.HorizontalPodAutoscaler", path: "autoscaling/v2beta2/horizontalpodautoscaler" },
+            // { token: "aws.ecs.Service", path: "ecs/service" },
+            // { token: "aws.acm.Certificate", path: "acm/certificate" },
+            // { token: "aws.apigateway.RestApi", path: "apigateway/restapi" },
+            // { token: "gcp.container.Cluster", path: "container/cluster" },
+            // { token: "aws.autoscaling.Group", path: "autoscaling/group" },
+            // { token: "gcp.cloudrun.Service", path: "cloudrun/service" },
+            // { token: "azure-native.resources.ResourceGroup", path: "resources/resourcegroup" },
+            // { token: "aws.iam.RolePolicyAttachment ", path: "iam/rolepolicyattachment" },
+            // { token: "azure-native.web.WebApp ", path: "web/webapp" },
+            // { token: "aws.lb.TargetGroup", path: "lb/targetgroup" },
+            // { token: "awsx.ec2.Vpc", path: "ec2/vpc" },
+            // { token: "kubernetes.core.v1.Secret", path: "core/v1/secret" },
+            // { token: "aws.sqs.Queue", path: "sqs/queue" },
+            // { token: "gcp.compute.Instance", path: "compute/instance" },
+            // { token: "awsx.ecs.FargateService", path: "ecs/fargateservice" },
+            // { token: "kubernetes.yaml.ConfigFile", path: "yaml/configfile" },
+            // { token: "aws.alb.Listener", path: "alb/listener" },
+            // { token: "aws.eks.NodeGroup", path: "eks/nodegroup" },
+            // { token: "aws.ecs.Cluster", path: "ecs/cluster" },
+            // { token: "aws.iam.getPolicyDocument", path: "iam/getpolicydocument" },
+            // { token: "aws.s3.BucketPolicy", path: "s3/bucketpolicy" },
+            // { token: "aws.cloudwatch.MetricAlarm", path: "cloudwatch/metricalarm" },
+            // { token: "azure-native.keyvault.Vault ", path: "keyvault/vault" },
+            // { token: "aws.sns.Topic", path: "sns/topic" },
+
+            // https://github.com/pulumi/registry/issues/2966
+            // http://registry-production-origin-push-3b7fc727.s3-website-us-west-2.amazonaws.com/registry/packages/azure-native/api-docs/insights/component/
+            // { token: "azure-native.insights.Component", path: "insights/component" },
+        ];
+
+        resources.forEach(resource => {
+            const title = resource.token;
+            const provider = resource.token.split(".").slice(0)[0];
+            const resourceName = resource.token.split(".").slice(-1)[0];
+
+            const page = {
+                title,
+                provider,
+                resourceName,
+                path: `/registry/packages/${provider}/api-docs/${resource.path}/`,
+            };
+
+            describe(`${page.title} (${[ Cypress.config().baseUrl, page.path ].join("")})`, () => {
+
+                beforeEach(() => {
+                    cy.visit(page.path);
+                });
+
+                it("has the correct page title", () => {
+                    cy.get("head title")
+                        .should("have.text", `${page.title} | Pulumi Registry`);
+                });
+
+                describe("content pane", () => {
+                    const contentPane = "main div.docs-main-content";
+
+                    it("has the correct H1", () => {
+                        cy.get(contentPane)
+                            .find("h1")
+                            .should("be.visible")
+                            .should("have.text", page.title);
+                    });
+
+                    describe("description", () => {
+
+                        it("leads with a non-empty paragraph tag", () => {
+                            cy.get(contentPane)
+                                .find("section.docs-content > p")
+                                .first()
+                                .should("exist")
+                                .invoke("text")
+                                .should("match", /^\w+/);
+                        });
+                    });
+
+                    it("renders subsections in the correct order", () => {
+                        cy.get(contentPane)
+                            .find("section.docs-content h2")
+                            .as("sections");
+
+                        cy.get("@sections").should("have.length.of", 7);
+                        cy.get("@sections").eq(0).should("have.text", "Example Usage")
+                        cy.get("@sections").eq(1).should("have.text", `Create ${page.resourceName} Resource`)
+                        cy.get("@sections").eq(2).should("have.text", `${page.resourceName} Resource Properties`)
+                        cy.get("@sections").eq(3).should("have.text", `Look up Existing ${page.resourceName} Resource`)
+                        cy.get("@sections").eq(4).should("have.text", "Supporting Types")
+                        cy.get("@sections").eq(5).should("have.text", "Import")
+                        cy.get("@sections").eq(6).should("have.text", "Package Details");
+                    });
+
+                    describe("Examples section", () => {
+                        const examplesSection = "main div.docs-main-content";
+
+                        it("contains at least one example", () => {
+                            cy.get(examplesSection)
+                                .find("h2 ~ h3")
+                                .should("have.length.of.at.least", 1);
+                        });
+                    });
+
+                    describe("Create section", () => {
+
+                    });
+
+                    describe("Property lists", () => {
+                        const props = "main div.docs-main-content .resources-properties";
+
+                        it("render deprecation notices properly", () => {
+                            cy.get(props).as("props");
+
+                            cy.get("@props").then(lists => {
+                                const deprecatedProps = "dt.property-deprecated + dd > p.property-message";
+
+                                if (lists.find(deprecatedProps).length > 0) {
+                                    cy.get("@props")
+                                        .find(deprecatedProps)
+                                        .first()
+                                        .invoke("text")
+                                        .should("not.equal", "Deprecated:");
+                                }
+                            });
+                        });
+
+                        describe("type links", () => {
+
+                            it("link to an on-page supporting type", () => {
+
+                            });
+                        })
+                    });
+
+                    describe("Outputs section", () => {
+
+                    });
+
+                    describe("Lookup section", () => {
+
+                    });
+
+                    describe("Supporting Types section", () => {
+                        const supportingTypesSection = "main div.docs-main-content .resources-properties";
+
+                        it.only("are available in all languages", () => {
+                            const languages = [ "TypeScript", "Python", "Go", "C#", "Java", "YAML" ];
+
+                            languages.forEach(language => {
+                                cy.get("pulumi-chooser li a").contains(language).first().click();
+
+                                const headings = "#supporting-types ~ h4";
+                                const lists = "#supporting-types + h4 ~ div pulumi-choosable div.active";
+
+                                cy.get(headings).then(headings => {
+                                    cy.get(lists).then(lists => {
+                                        expect(lists.length).to.equal(headings.length);
+                                    });
+                                });
+                            })
+                        });
+                    });
+
+                    describe("Import section", () => {
+
+                    });
+
+                    describe("Package Details section", () => {
+
+                    });
+                });
+
+                describe("left nav", () => {
+
+                });
+
+                describe("right nav", () => {
+
+                });
+            });
+        });
+    });
+});

cypress/integration/site-spec.js

@@ -0,0 +1,93 @@
+describe("The Registry", () => {
+
+    // This should probably be testing the Registry home page and not the home-home page.
+    describe("home page", () => {
+        beforeEach(() => {
+            cy.visit("/");
+        });
+
+        it("loads and applies CSS", () => {
+            // Checking the computed background-color value validates that the CSS bundle
+            // was properly loaded and applied.
+            cy.get(".header-container").invoke("css", "background-color").should("equal", "rgb(255, 255, 255)");
+        });
+
+        it("loads and applies JavaScript", () => {
+            // Checking the carousel validates that the JS bundle was loaded and applied
+            // (excluding Stencil components, which are bundled separately).
+            cy.get(".header-container").should("not.have.class", "is-pinned");
+
+            cy.scrollTo(0, 250);
+
+            cy.get(".header-container").should("have.class", "is-pinned");
+        });
+    });
+
+    describe("S3 bucket resource page", () => {
+
+        beforeEach(() => {
+            cy.visit("/registry/packages/aws/api-docs/s3/bucket/");
+        });
+
+        describe("content pane", () => {
+            const contentPane = "main div.docs-main-content";
+
+            it("has an H1 at the top of the page", () => {
+                cy.get(contentPane)
+                    .find("h1")
+                    .should("be.visible")
+                    .should("have.text", "aws.s3.Bucket");
+            });
+
+            it("has a description", () => {
+                cy.get(contentPane)
+                    .find("section.docs-content p")
+                    .should("contain.text", "Provides a S3 bucket resource");
+            });
+
+            describe("Examples section", () => {
+                const examplesSection = "main div.docs-main-content";
+
+                it("has a heading", () => {
+                    cy.get(examplesSection)
+                        .find("h2#example-usage")
+                        .should("be.visible");
+                });
+
+                it("contains several examples", () => {
+                    cy.get(examplesSection)
+                        .find("h2 ~ h3")
+                        .should("have.length.of.at.least", 3);
+                });
+            });
+
+            describe("Create section", () => {
+
+            });
+
+            describe("Inputs section", () => {
+
+            });
+
+            describe("Outputs section", () => {
+
+            });
+
+            describe("Lookup section", () => {
+
+            });
+
+            describe("Supporting Types section", () => {
+
+            });
+
+            describe("Import section", () => {
+
+            });
+
+            describe("Package Details section", () => {
+
+            });
+        });
+    });
+});

package.json

@@ -7,23 +7,26 @@
     "license": "Apache-2.0",
     "dependencies": {
         "@fullhuman/postcss-purgecss": "^4.0.3",
+        "@octokit/auth-action": "^1.3.2",
+        "@octokit/graphql": "^4.6.2",
+        "@octokit/rest": "^18.5.3",
         "@slack/web-api": "^5.12.0",
         "broken-link-checker": "^0.7.8",
         "concurrently": "^6.0.0",
         "cssnano": "^5.0.8",
         "cypress": "^4.11.0",
+        "cypress-multi-reporters": "^1.6.4",
         "glob": "^7.2.0",
         "http-server": "^0.12.1",
         "js-yaml": "^4.1.0",
         "jsdom": "^22.0.0",
         "markdownlint": "^0.28.0",
-        "@octokit/auth-action": "^1.3.2",
-        "@octokit/graphql": "^4.6.2",
-        "@octokit/rest": "^18.5.3",
+        "mocha": "^10.3.0",
+        "mocha-multi-reporters": "^1.5.1",
         "postcss": "^8.3.7",
         "postcss-cli": "^8.3.1",
         "sitemapper": "^3.2.2"
-},
+    },
     "scripts": {
         "minify-css": "node scripts/minify-css.js"
     }

scripts/run-browser-tests.sh

@@ -1,12 +1,16 @@
 #!/bin/bash
 
-MAX_RETRIES=3
+
+CYPRESS_BASE_URL="${1:-https://www.pulumi.com}"
+SPEC_FILE="${2:-site-spec.js}"
 RETRY_COUNT=0
+MAX_RETRIES="${3:-3}"
 
 # Retry on errors up to `MAX_RETRIES` with a 10 second sleep in between.
 run_tests() {
     while true; do
-        CYPRESS_BASE_URL="$1" yarn run cypress run --headless
+        CYPRESS_BASE_URL="$CYPRESS_BASE_URL" yarn run cypress run --headless --spec "cypress/integration/${SPEC_FILE}"
+        # CYPRESS_BASE_URL="$CYPRESS_BASE_URL" yarn run cypress open
         exit_code=$?
         if [ $exit_code -ne 0 ]; then
             RETRY_COUNT=$((RETRY_COUNT+1))