test: unified JS and Go reflection API tests -- run the same test aga…

…inst both JS and Go (#617)

.github/workflows/e2e-tests.yml

@@ -0,0 +1,45 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Run e2e tests
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Run e2e tests
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: pnpm/action-setup@v3
+    - name: Set up node v20
+      uses: actions/setup-node@v4
+      with:
+        node-version: 20.x
+        cache: 'pnpm'
+    - name: Install dependencies
+      run: pnpm install
+    - name: Run build script
+      run: pnpm build
+    - name: Run pack:all script
+      run: pnpm pack:all
+    - name: Run js tests
+      run: pnpm test:js
+    - name: Run e2e tests
+      run: pnpm test:e2e
+    - name: Validate working directory is clean
+      run: .github/workflows/scripts/ensure-clean-working-tree.sh

go/ai/generate.go

@@ -53,18 +53,23 @@ type ModelMetadata struct {
 // [ModelAction] that runs it.
 func DefineModel(provider, name string, metadata *ModelMetadata, generate func(context.Context, *GenerateRequest, ModelStreamingCallback) (*GenerateResponse, error)) *Model {
 	metadataMap := map[string]any{}
-	if metadata != nil {
-		if metadata.Label != "" {
-			metadataMap["label"] = metadata.Label
+	if metadata == nil {
+		// Always make sure there's at least minimal metadata.
+		metadata = &ModelMetadata{
+			Label: name,
 		}
-		supports := map[string]bool{
-			"media":      metadata.Supports.Media,
-			"multiturn":  metadata.Supports.Multiturn,
-			"systemRole": metadata.Supports.SystemRole,
-			"tools":      metadata.Supports.Tools,
-		}
-		metadataMap["supports"] = supports
 	}
+	if metadata.Label != "" {
+		metadataMap["label"] = metadata.Label
+	}
+	supports := map[string]bool{
+		"media":      metadata.Supports.Media,
+		"multiturn":  metadata.Supports.Multiturn,
+		"systemRole": metadata.Supports.SystemRole,
+		"tools":      metadata.Supports.Tools,
+	}
+	metadataMap["supports"] = supports
+
 	return (*Model)(core.DefineStreamingAction(provider, name, atype.Model, map[string]any{
 		"model": metadataMap,
 	}, generate))

js/ai/src/model.ts

@@ -276,7 +276,7 @@ export function defineModel<
     streamingCallback?: StreamingCallback<GenerateResponseChunkData>
   ) => Promise<GenerateResponseData>
 ): ModelAction<CustomOptionsSchema> {
-  const label = options.label || `${options.name} GenAI model`;
+  const label = options.label || options.name;
   const middleware: ModelMiddleware[] = [
     ...(options.use || []),
     validateSupport(options),

package.json

@@ -22,7 +22,8 @@
     "test:genkit-tools": "pnpm test:genkit-tools-cli && pnpm test:genkit-tools-common",
     "test:genkit-tools-cli": "cd genkit-tools/cli && pnpm i && pnpm test",
     "test:genkit-tools-common": "cd genkit-tools/common && pnpm i && pnpm test",
-    "test:e2e": "pnpm build && pnpm pack:all && cd tests && pnpm test"
+    "test:e2e-local": "pnpm build && pnpm pack:all && cd tests && pnpm install && pnpm test",
+    "test:e2e": "cd tests && pnpm install && pnpm test"
   },
   "pre-commit": [
     "format:check"

tests/package.json

@@ -4,18 +4,24 @@
   "description": "",
   "main": "lib/e2e.js",
   "scripts": {
-    "test": "node --import tsx src/e2e.ts"
+    "test": "npm-run-all test:reflection_api",
+    "test:dev_ui_test": "node --import tsx src/dev_ui_test.ts",
+    "test:reflection_api": "node --import tsx src/reflection_api_test.ts"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "json-diff": "^1.0.6",
     "puppeteer": "^22.5.0",
     "puppeteer-screen-recorder": "^3.0.3",
     "semver": "^7.6.0",
-    "terminate": "^2.6.1"
+    "terminate": "^2.6.1",
+    "yaml": "^2.4.5"
   },
   "devDependencies": {
+    "@types/json-diff": "^1.0.3",
+    "npm-run-all": "^4.1.5",
     "tsx": "^4.7.1"
   }
 }