chore: add k8s grid deployments (#26359)

packages/playwright-core/src/server/index.ts

@@ -30,4 +30,5 @@ export { createPlaywright } from './playwright';
 export type { DispatcherScope } from './dispatchers/dispatcher';
 export type { Playwright } from './playwright';
 export { openTraceInBrowser, openTraceViewerApp } from './trace/viewer/traceViewer';
-export { serverSideCallMetadata } from './instrumentation';
+export { serverSideCallMetadata } from './instrumentation';
+export { SocksProxy } from '../common/socksProxy';

packages/playwright-grid/Dockerfile

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/playwright:v1.37.0-alpha-aug-7-2023-jammy
+
+WORKDIR /app
+
+COPY package.json ./
+COPY cli.js ./
+COPY lib ./lib
+RUN npm install

packages/playwright-grid/deployment-grid.yaml

@@ -0,0 +1,42 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grid-deployment
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grid
+  template:
+    metadata:
+      labels:
+        app: grid
+    spec:
+      containers:
+      - name: grid
+        image: playwright-grid
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: DEBUG
+          value: "pw:grid*"
+        - name: PLAYWRIGHT_GRID_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: access-key-secret
+              key: access-key
+        command: ["node", "./cli.js"]
+        args: ["grid", "--port=3000"]
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grid-service
+spec:
+  selector:
+    app: grid
+  ports:
+    - protocol: TCP
+      port: 3000
+      targetPort: 3000
+  type: LoadBalancer

packages/playwright-grid/deployment-worker.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: worker-deployment
+spec:
+  replicas: 10  # or however many nodes you want
+  selector:
+    matchLabels:
+      app: worker
+  template:
+    metadata:
+      labels:
+        app: worker
+    spec:
+      containers:
+      - name: grid
+        image: playwright-grid
+        imagePullPolicy: IfNotPresent
+        env:
+        - name: DEBUG
+          value: "pw:grid*"
+        - name: PLAYWRIGHT_GRID_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: access-key-secret
+              key: access-key
+        command: ["node", "./cli.js"]
+        args: ["node", "--grid=grid-service:3000"]

packages/playwright-grid/docs/azure.md

@@ -0,0 +1,55 @@
+```sh
+# Create resource group
+az group create --name group-grid-001 --location westus3
+
+# Create ACR
+az acr create --resource-group group-grid-001 --name acrgrid001 --sku Basic
+az acr login --name acrgrid001
+az acr list --resource-group group-grid-001 --query "[].{acrLoginServer:loginServer}" --output table
+
+# Create AKS
+az aks create --resource-group group-grid-001 --name aks-grid-001 --node-count 4 --enable-addons monitoring --generate-ssh-keys
+az aks get-credentials --resource-group group-grid-001 --name aks-grid-001
+
+# Grant AKS access to ACR
+az aks show --resource-group group-grid-001 --name aks-grid-001 --query "servicePrincipalProfile.clientId" --output tsv
+# az aks show --resource-group group-grid-001 --name aks-grid-001 --query "identityProfile.kubeletidentity.clientId" -o tsv
+# az acr show --name acrgrid001 --resource-group group-grid-001 --query "id" -o tsv
+# az role assignment create --assignee <GUID> --role AcrPull --scope <SCOP PATH>
+
+# Create secrets
+kubectl create secret generic access-key-secret --from-literal=access-key=$PLAYWRIGHT_GRID_ACCESS_KEY
+
+# Create TLS
+# kubectl create secret tls grid-tls-secret --cert=../../tests/config/testserver/cert.pem --key=../../tests/config/testserver/key.pem
+# az network public-ip create --resource-group MC_group-grid-001_aks-grid-001_westus3 --name public-ip-grid-001 --sku Standard --allocation-method static
+# az network public-ip show --resource-group MC_group-grid-001_aks-grid-001_westus3 --name public-ip-grid-001 --query ipAddress --output tsv
+# # use output below
+# helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+# helm install nginx-ingress ingress-nginx/ingress-nginx \
+#     --set controller.replicaCount=1 \
+#     --set controller.nodeSelector."beta\.kubernetes\.io/os"=linux \
+#     --set defaultBackend.nodeSelector."beta\.kubernetes\.io/os"=linux \
+#     --set controller.service.loadBalancerIP="20.118.130.255"
+
+# Push Docker container
+docker build -t playwright-grid:latest -f Dockerfile .
+docker tag playwright-grid acrgrid001.azurecr.io/playwright-grid
+docker push acrgrid001.azurecr.io/playwright-grid
+
+# Delete deployment
+kubectl delete deployment grid-deployment
+kubectl delete deployment worker-deployment
+kubectl delete svc grid-service
+
+# Update deployment
+kubectl apply -f deployment-grid.yaml
+kubectl apply -f deployment-worker.yaml
+
+# Debug
+kubectl get pods -l app=grid
+kubectl logs grid-6cbbfc866c-wh8dw
+kubectl get pods -n ingress-basic
+kubectl get svc grid-service
+az aks show --resource-group group-grid-001 --name aks-grid-001 --query fqdn --output tsv
+```

packages/playwright-grid/docs/minikube.md

@@ -0,0 +1,31 @@
+```sh
+minikube config set memory 65536
+minikube config set cpus 12
+minikube start
+minikube dashboard
+
+# Point docker to minikube
+minikube -p minikube docker-env
+eval $(minikube docker-env)
+kubectl config use-context minikube
+kubectl create secret generic access-key-secret --from-literal=access-key=$PLAYWRIGHT_GRID_ACCESS_KEY
+
+# Push Docker container
+docker build -t playwright-grid:latest -f Dockerfile .
+
+# Delete deployment
+kubectl delete deployment grid-deployment
+kubectl delete deployment worker-deployment
+kubectl delete svc grid-service
+
+# Update deployment
+
+kubectl apply -f deployment-grid.yaml
+kubectl apply -f deployment-worker.yaml
+
+# Debug
+minikube ip
+kubectl get svc grid-service
+kubectl get pods -l app=grid
+kubectl logs grid-6cbbfc866c-wh8dw
+```

packages/playwright-grid/package.json

@@ -10,13 +10,13 @@
   "dependencies": {
     "commander": "^11.0.0",
     "debug": "^4.3.2",
+    "playwright-core": "1.37.0-alpha-aug-7-2023",
     "ws": "^8.1.0"
   },
   "devDependencies": {
     "@types/commander": "^2.12.2",
     "@types/debug": "^4.1.8",
-    "@types/ws": "^8.5.5",
-    "playwright-core": "1.37.0-next"
+    "@types/ws": "^8.5.5"
   },
   "repository": "github:Microsoft/playwright",
   "engines": {

packages/playwright-grid/src/cli.ts

@@ -27,7 +27,7 @@ program
     .option('--access-key <key>', 'access key to the grid')
     .action(async opts => {
       const port = opts.port || +(process.env.PLAYWRIGHT_GRID_PORT || '3333');
-      const accessKey = opts.accessKey || process.env.PLAYWRIGHT_GRID_ACCESS_KEY;
+      const accessKey = opts.accessKey || (process.env.PLAYWRIGHT_GRID_ACCESS_KEY || '');
       const { Grid } = await import('./grid/grid.js');
       const grid = new Grid(port, accessKey);
       grid.start();
@@ -40,7 +40,9 @@ program
     .option('--access-key <key>', 'access key to the grid', '')
     .action(async opts => {
       const { Node } = await import('./node/node.js');
-      new Node(opts.grid, +opts.capacity, opts.accessKey);
+      const accessKey = opts.accessKey || (process.env.PLAYWRIGHT_GRID_ACCESS_KEY || '');
+      const node = new Node(opts.grid, +opts.capacity, accessKey);
+      await node.connect();
     });
 
 program.parse(process.argv);

packages/playwright-grid/src/common/httpServer.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import debug from 'debug';
 import fs from 'fs';
 import http from 'http';
 import path from 'path';
@@ -23,11 +24,13 @@ import { Server as WebSocketServer } from 'ws';
 export type ServerRouteHandler = (request: http.IncomingMessage, response: http.ServerResponse) => boolean;
 
 export class HttpServer {
+  private _log: debug.Debugger;
   readonly server: http.Server;
   private _urlPrefix: string;
   private _routes: { prefix?: string, exact?: string, handler: ServerRouteHandler }[] = [];
 
   constructor() {
+    this._log = debug(`pw:grid:http`);
     this._urlPrefix = '';
     this.server = http.createServer(this._onRequest.bind(this));
   }
@@ -45,6 +48,7 @@ export class HttpServer {
   }
 
   async start(port?: number): Promise<string> {
+    this._log('starting server', port);
     this.server.listen(port);
     await new Promise(cb => this.server!.once('listening', cb));
     const address = this.server.address();
@@ -77,13 +81,15 @@ export class HttpServer {
   }
 
   private _onRequest(request: http.IncomingMessage, response: http.ServerResponse) {
+    this._log('web request', request.url);
     request.on('error', () => response.end());
     try {
       if (!request.url) {
         response.end();
         return;
       }
       const url = new URL('http://localhost' + request.url);
+      this._log('url pathname', url.pathname);
       for (const route of this._routes) {
         if (route.exact && url.pathname === route.exact && route.handler(request, response))
           return;

packages/playwright-grid/src/grid/grid.ts

@@ -243,14 +243,15 @@ export class Grid {
       ws.on('error', e => this._log(e));
     });
     this._server.server.on('upgrade', async (request, socket, head) => {
+      this._log('upgrade', request.url, request.headers);
+
       if (this._accessKey && request.headers['x-playwright-access-key'] !== this._accessKey) {
         socket.destroy();
         return;
       }
 
       const url = new URL('http://internal' + request.url);
       const params = url.searchParams;
-      this._log(url.pathname);
 
       if (url.pathname.startsWith('/registerNode')) {
         const nodeRequest = new WebSocketRequest(this._wsServer, request, socket, head);