github models support (#600)

* support for github models * add gh for github * missing env * typo * add gh cli
microsoft · Aug 2, 2024 · b73b2e0 · b73b2e0
1 parent 751c9af
commit b73b2e0
Show file tree

Hide file tree

Showing 7 changed files with 146 additions and 4 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -9,7 +9,6 @@
                 "dbaeumer.vscode-eslint",
                 "bierner.markdown-mermaid",
                 "yoavbls.pretty-ts-errors",
-                "kejun.markdown-alert",
                 "astro-build.astro-vscode",
                 "unifiedjs.vscode-mdx",
                 "streetsidesoftware.code-spell-checker",
@@ -18,8 +17,9 @@
         }
     },
     "features": {
+        "ghcr.io/ghcr.io/devcontainers/features/github-cli:1.0.13": {},
         "ghcr.io/devcontainers/features/docker-in-docker:2": {},
         "ghcr.io/devcontainers/features/azure-cli:1.2.5": {},
-        "ghcr.io/devcontainers/features/python:1.6.2": {}
+        "ghcr.io/devcontainers/features/python:1.6.3": {}
     }
 }
diff --git a/.github/workflows/github-models.yml b/.github/workflows/github-models.yml
@@ -0,0 +1,43 @@
+name: github models smoke tests
+on:
+    workflow_dispatch:
+    pull_request:
+        paths:
+            - yarn.lock
+            - ".github/workflows/github-models.yml"
+            - "packages/core/**/*"
+            - "packages/cli/**/*"
+            - "packages/samples/**/*"
+    push:
+        branches:
+            - main
+        paths:
+            - yarn.lock
+            - ".github/workflows/github-models.yml"
+            - "packages/core/**/*"
+            - "packages/cli/**/*"
+            - "packages/samples/**/*"
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}-github-models
+    cancel-in-progress: true
+jobs:
+    tests:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  submodules: "recursive"
+                  fetch-depth: 0
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "20"
+                  cache: yarn
+            - run: yarn install --frozen-lockfile
+            - name: typecheck
+              run: yarn typecheck
+            - name: compile
+              run: yarn compile
+            - name: run summarize github-gpt4o
+              run: yarn test:summarize --model github:gpt-4o --out ./temp
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/docs/src/content/docs/getting-started/configuration.mdx b/docs/src/content/docs/getting-started/configuration.mdx
@@ -140,6 +140,51 @@ GENAISCRIPT_DEFAULT_MODEL=openai:gpt-4o
 
 :::
 
+## GitHub Models
+<a id="github" href=""></a>
+
+The [GitHub Models](https://github.com/marketplace/models) provider, `github`, allows running models through the GitHub Marketplace.
+This provider is useful for prototyping and subject to [rate limits](https://docs.github.com/en/github-models/prototyping-with-ai-models#rate-limits)
+depending on your subscription.
+
+<Steps>
+
+<ol>
+
+<li>
+
+Open the [GitHub Marketplace](https://github.com/marketplace/models) and find the model you want to use.
+
+</li>
+
+<li>
+
+Click **Get Started** and follow the instructions to configure the github token, or start a codespace!
+
+</li>
+
+<li>
+
+Copy the model name from the Javascript/Python samples
+
+```js "Phi-3-mini-4k-instruct"
+const modelName = "Phi-3-mini-4k-instruct";
+```
+
+to configure your script.
+
+```js
+script({
+    model: "github:Phi-3-mini-4k-instruct"
+})
+```
+
+</li>
+
+</ol>
+
+</Steps>
+
 ## Azure OpenAI
 <a id="azure" href=""></a>
 
@@ -269,7 +314,7 @@ script({
 
 </Steps>
 
-## GitHub Copilot Models
+## GitHub Copilot in Visual Studio Code
 
 If you have access to **GitHub Copilot in Visual Studio Code**, 
 GenAIScript will be able to leverage those [language models](https://code.visualstudio.com/api/extension-guides/language-model) as well.

diff --git a/packages/core/src/connection.ts b/packages/core/src/connection.ts
@@ -3,18 +3,21 @@ import {
     DEFAULT_TEMPERATURE,
     DOCS_CONFIGURATION_AICI_URL,
     DOCS_CONFIGURATION_AZURE_OPENAI_URL,
+    DOCS_CONFIGURATION_GITHUB_URL,
     DOCS_CONFIGURATION_LITELLM_URL,
     DOCS_CONFIGURATION_LLAMAFILE_URL,
     DOCS_CONFIGURATION_LOCALAI_URL,
     DOCS_CONFIGURATION_OLLAMA_URL,
     DOCS_CONFIGURATION_OPENAI_URL,
     DOT_ENV_FILENAME,
+    GITHUB_MODELS_BASE,
     LITELLM_API_BASE,
     LLAMAFILE_API_BASE,
     LOCALAI_API_BASE,
     MODEL_PROVIDER_AICI,
     MODEL_PROVIDER_AZURE,
     MODEL_PROVIDER_CLIENT,
+    MODEL_PROVIDER_GITHUB,
     MODEL_PROVIDER_LITELLM,
     MODEL_PROVIDER_LLAMAFILE,
     MODEL_PROVIDER_OLLAMA,
@@ -99,6 +102,24 @@ export async function parseTokenFromEnv(
         }
     }
 
+    if (provider === MODEL_PROVIDER_GITHUB) {
+        const token = env.GITHUB_TOKEN
+        if (!token) throw new Error("GITHUB_TOKEN must be set")
+        const type = "openai"
+        const base = GITHUB_MODELS_BASE
+        return {
+            provider,
+            model,
+            base,
+            type,
+            token,
+            source: "env: GITHUB_TOKEN",
+            curlHeaders: {
+                Authorization: `Bearer $GITHUB_TOKEN`,
+            },
+        }
+    }
+
     if (provider === MODEL_PROVIDER_AZURE) {
         const tokenVar = env.AZURE_OPENAI_API_KEY
             ? "AZURE_OPENAI_API_KEY"
@@ -298,6 +319,16 @@ OPENAI_API_TYPE="localai"
             model: `${MODEL_PROVIDER_OPENAI}:gpt-3.5-turbo`,
         }
 
+    if (provider === MODEL_PROVIDER_GITHUB)
+        return {
+            config: `
+    ## GitHub Models ${DOCS_CONFIGURATION_GITHUB_URL}
+    # use "${MODEL_PROVIDER_GITHUB}:<model>" in script({ model: ... })
+    GITHUB_TOKEN="${PLACEHOLDER_API_KEY}"
+    `,
+            model: `${MODEL_PROVIDER_GITHUB}:gpt-4o`,
+        }
+
     return {
         config: `
 ## OpenAI ${DOCS_CONFIGURATION_OPENAI_URL}

diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts
@@ -89,6 +89,7 @@ export const EMOJI_FAIL = "❌"
 export const EMOJI_UNDEFINED = "?"
 
 export const MODEL_PROVIDER_OPENAI = "openai"
+export const MODEL_PROVIDER_GITHUB = "github"
 export const MODEL_PROVIDER_AZURE = "azure"
 export const MODEL_PROVIDER_OLLAMA = "ollama"
 export const MODEL_PROVIDER_LLAMAFILE = "llamafile"
@@ -98,12 +99,16 @@ export const MODEL_PROVIDER_CLIENT = "client"
 
 export const TRACE_FILE_PREVIEW_MAX_LENGTH = 240
 
+export const GITHUB_MODELS_BASE = "https://models.inference.ai.azure.com"
+
 export const DOCS_CONFIGURATION_URL =
     "https://microsoft.github.io/genaiscript/getting-started/configuration/"
 export const DOCS_CONFIGURATION_OPENAI_URL =
     "https://microsoft.github.io/genaiscript/getting-started/configuration/#openai"
+export const DOCS_CONFIGURATION_GITHUB_URL =
+    "https://microsoft.github.io/genaiscript/getting-started/configuration/#github"
 export const DOCS_CONFIGURATION_AZURE_OPENAI_URL =
-    "https://microsoft.github.io/genaiscript/getting-started/configuration/#azure-openai"
+    "https://microsoft.github.io/genaiscript/getting-started/configuration/#azure"
 export const DOCS_CONFIGURATION_OLLAMA_URL =
     "https://microsoft.github.io/genaiscript/getting-started/configuration/#ollama"
 export const DOCS_CONFIGURATION_LLAMAFILE_URL =
@@ -121,6 +126,11 @@ export const MODEL_PROVIDERS = Object.freeze([
         detail: "OpenAI or compatible",
         url: DOCS_CONFIGURATION_OPENAI_URL,
     },
+    {
+        id: MODEL_PROVIDER_GITHUB,
+        detail: "GitHub Models",
+        url: DOCS_CONFIGURATION_GITHUB_URL,
+    },
     {
         id: MODEL_PROVIDER_AZURE,
         detail: "Azure OpenAI deployment",

diff --git a/packages/core/src/models.test.ts b/packages/core/src/models.test.ts
@@ -3,6 +3,7 @@ import { parseModelIdentifier } from "./models"
 import assert from "node:assert"
 import {
     MODEL_PROVIDER_AICI,
+    MODEL_PROVIDER_GITHUB,
     MODEL_PROVIDER_LLAMAFILE,
     MODEL_PROVIDER_OLLAMA,
     MODEL_PROVIDER_OPENAI,
@@ -38,6 +39,12 @@ describe("parseModelIdentifier", () => {
         assert(family === "*")
         assert(model === "llamafile")
     })
+    test("github:gpt4", () => {
+        const { provider, model, family } = parseModelIdentifier("github:gpt4")
+        assert(provider === MODEL_PROVIDER_GITHUB)
+        assert(model === "gpt4")
+        assert(family === "gpt4")
+    })
     test("gpt4", () => {
         const { provider, model, family } = parseModelIdentifier("gpt4")
         assert(provider === MODEL_PROVIDER_OPENAI)

diff --git a/packages/vscode/src/lmaccess.ts b/packages/vscode/src/lmaccess.ts
@@ -9,6 +9,7 @@ import {
     MODEL_PROVIDER_LITELLM,
     MODEL_PROVIDER_OPENAI,
     MODEL_PROVIDER_CLIENT,
+    MODEL_PROVIDER_GITHUB,
 } from "../../core/src/constants"
 import { APIType } from "../../core/src/host"
 import { parseModelIdentifier } from "../../core/src/models"
@@ -64,6 +65,11 @@ async function generateLanguageModelConfiguration(
             provider: MODEL_PROVIDER_AZURE,
             apiType: "azure",
         },
+        {
+            label: "GitHub Models",
+            detail: `Use a GitHub Models with a GitHub subscription.`,
+            provider: MODEL_PROVIDER_GITHUB,
+        },
         {
             label: "LocalAI",
             description: "https://localai.io/",