From 53a13aca6701dda67637aa24cb01644fa41afca9 Mon Sep 17 00:00:00 2001 From: Stefan Bokarev <38397684+maamalama@users.noreply.github.com> Date: Tue, 24 Sep 2024 17:51:33 -0700 Subject: [PATCH 1/4] some fixes (#2687) --- .../jawn/src/controllers/public/experimentController.ts | 4 ++-- .../prompts/experiments/table/ExperimentTable.tsx | 8 +++++++- web/components/templates/prompts/id/promptIdPage.tsx | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/valhalla/jawn/src/controllers/public/experimentController.ts b/valhalla/jawn/src/controllers/public/experimentController.ts index f637db722b..fdf1e3b341 100644 --- a/valhalla/jawn/src/controllers/public/experimentController.ts +++ b/valhalla/jawn/src/controllers/public/experimentController.ts @@ -147,7 +147,7 @@ export class ExperimentController extends Controller { requestBody ); - if (result.error || !result.data) { + if (result.error) { this.setStatus(500); console.error(result.error); return err(result.error); @@ -240,7 +240,7 @@ export class ExperimentController extends Controller { experiment.dataset.rows = datasetRows; experiment.hypotheses = [hypothesis]; - const runResult = run(experiment); + const runResult = await run(experiment); return runResult; } diff --git a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx index a0c789bc05..2f5e01ad41 100644 --- a/web/components/templates/prompts/experiments/table/ExperimentTable.tsx +++ b/web/components/templates/prompts/experiments/table/ExperimentTable.tsx @@ -13,6 +13,7 @@ import ProviderKeyList from "../../../enterprise/portal/id/providerKeyList"; import AddColumnHeader from "./AddColumnHeader"; import { HypothesisCellRenderer } from "./HypothesisCellRenderer"; import { HypothesisHeaderComponent } from "./HypothesisHeaderComponent"; +import { PlusIcon } from "@heroicons/react/24/outline"; interface ExperimentTableProps { promptSubversionId: string; @@ -293,7 +294,12 @@ export function ExperimentTable({ getRowId={getRowId} /> - diff --git a/web/components/templates/prompts/id/promptIdPage.tsx b/web/components/templates/prompts/id/promptIdPage.tsx index 912b6d5138..ed7e4bdb77 100644 --- a/web/components/templates/prompts/id/promptIdPage.tsx +++ b/web/components/templates/prompts/id/promptIdPage.tsx @@ -491,7 +491,7 @@ const PromptIdPage = (props: PromptIdPageProps) => { return; } router.push( - `/prompts/${id}/subversion/${prompt?.id}/experiment/${experiment.data?.data?.experimentId}` + `/prompts/${id}/subversion/${promptVersionId}/experiment/${experiment.data?.data?.experimentId}` ); }; From 16e7ff8556c7c319e590db2f49cfbab13afe4df3 Mon Sep 17 00:00:00 2001 From: Stefan Bokarev <38397684+maamalama@users.noreply.github.com> Date: Tue, 24 Sep 2024 20:09:07 -0700 Subject: [PATCH 2/4] Overview fixes (#2688) --- .../templates/prompts/id/promptIdPage.tsx | 1104 +++++++++++------ 1 file changed, 755 insertions(+), 349 deletions(-) diff --git a/web/components/templates/prompts/id/promptIdPage.tsx b/web/components/templates/prompts/id/promptIdPage.tsx index ed7e4bdb77..a0d2bc65cf 100644 --- a/web/components/templates/prompts/id/promptIdPage.tsx +++ b/web/components/templates/prompts/id/promptIdPage.tsx @@ -609,402 +609,808 @@ const PromptIdPage = (props: PromptIdPageProps) => { {prompt?.created_at && new Date(prompt?.created_at).toDateString()}

- - - - Prompt & Inputs - - {user?.email?.includes("helicone.ai") ? ( + {user?.email?.includes("helicone.ai") ? ( + + + + Prompt & Inputs + + Experiments - ) : ( - <> - )} - - Overview - - - - -
-
-
-
- { - await createSubversion(history, model); - }} - submitText="Test" - initialModel={model} - isPromptCreatedFromUi={ - prompt?.metadata?.createdFromUi as boolean | undefined - } - /> -
-
-
-
-

Versions

-
- -
- {sortedPrompts?.map((promptVersion) => { - const isProduction = - promptVersion.metadata?.isProduction === true; - const isSelected = - selectedVersion === - `${promptVersion.major_version}.${promptVersion.minor_version}`; - - return ( -
- setSelectedInputAndVersion( - `${promptVersion.major_version}.${promptVersion.minor_version}` - ) - } - > -
-
-
- {isSelected && ( -
- )} + + Overview + + + + +
+
+
+
+ { + await createSubversion(history, model); + }} + submitText="Test" + initialModel={model} + isPromptCreatedFromUi={ + prompt?.metadata?.createdFromUi as boolean | undefined + } + /> +
+
+
+
+

Versions

+
+ + +
+ {sortedPrompts?.map((promptVersion) => { + const isProduction = + promptVersion.metadata?.isProduction === true; + const isSelected = + selectedVersion === + `${promptVersion.major_version}.${promptVersion.minor_version}`; + + return ( +
+ setSelectedInputAndVersion( + `${promptVersion.major_version}.${promptVersion.minor_version}` + ) + } + > +
+
+
+ {isSelected && ( +
+ )} +
+ + V{promptVersion.major_version}. + {promptVersion.minor_version} + + + {isProduction && ( + + Prod + + )} +
- - V{promptVersion.major_version}. - {promptVersion.minor_version} - - - {isProduction && ( - - Prod - - )} - -
-
- {prompt?.metadata?.createdFromUi === - true ? ( - - - - - - {!isProduction && ( +
+ {prompt?.metadata?.createdFromUi === + true ? ( + + + + + + {!isProduction && ( + + promoteToProduction( + promptVersion.id + ) + } + > + + Promote to prod + + )} - promoteToProduction( + startExperiment( promptVersion.id ) } > - - Promote to prod + + Experiment - )} - - startExperiment(promptVersion.id) - } - > - - Experiment - - {!isProduction && ( + {!isProduction && ( + + deletePromptVersion( + promptVersion.id + ) + } + > + +

+ Delete +

+ + )} + + + ) : ( + + + + + - deletePromptVersion( + startExperiment( promptVersion.id ) } > - -

- Delete -

+ + Experiment - )} - - - ) : ( - - - - - - - startExperiment(promptVersion.id) - } - > - - Experiment - - - - )} + + + )} +
-
-
- - {getTimeAgo( - new Date(promptVersion.created_at) - )} - -
- {promptVersion.model} +
+ + {getTimeAgo( + new Date(promptVersion.created_at) + )} + +
+ {promptVersion.model} +
-
- ); - })} -
- -
-
-
-

Inputs

-
- setSearchRequestId(value)} - /> -
+ ); + })} +
+
+
+
+

Inputs

+
+ + setSearchRequestId(value) + } + /> +
+
- -
    - {inputs - ?.filter((input) => - input.source_request.includes(searchRequestId) - ) - .map((input) => ( -
  • - -
    • - ))} -
- + +
    + {inputs + ?.filter((input) => + input.source_request.includes(searchRequestId) + ) + .map((input) => ( +
  • + +
    • + ))} +
+ +
-
- - {user?.email?.includes("helicone.ai") && ( + + - )} - - -
-
-
- -
-
- - ({ - date: getTimeMap(timeIncrement)(r.time), - count: r.count, - })) ?? [] - } - index="date" - categories={["count"]} - colors={["cyan"]} - showYAxis={false} - curveType="monotone" - valueFormatter={(number: number | bigint) => { - return `${new Intl.NumberFormat("us").format( - Number(number) - )}`; + +
+
+
+ - +
+ +
+ + ({ + date: getTimeMap(timeIncrement)(r.time), + count: r.count, + })) ?? [] + } + index="date" + categories={["count"]} + colors={["cyan"]} + showYAxis={false} + curveType="monotone" + valueFormatter={(number: number | bigint) => { + return `${new Intl.NumberFormat("us").format( + Number(number) + )}`; + }} + /> + +
+
+
+

+ Experiment Logs +

+
+
+
+ { + setSelectedDatasets(value); + }} + > + {datasets.map((dataset) => ( + + {dataset.name} + + ))} + +
+
+ { + setSelectedModels(value); + }} + > + {MODEL_LIST.map((model) => ( + + {model.label} + + ))} + +
+
+ { + setSelectedDatasets([]); + setSelectedModels([]); + }} + /> +
+
+
+ {isExperimentsLoading ? ( +
+ +
+ ) : ( + ( + + {item.id} + + ), + }, + { + key: "status", + header: "Status", + render: (item) => ( + + ), + }, + { + key: "createdAt", + header: "Created At", + render: (item) => ( + {getUSDateFromString(item.createdAt)} + ), + }, + { + key: "datasetName", + header: "Dataset", + render: (item) => item.datasetName, + }, + { + key: "model", + header: "Model", + render: (item) => ( + + ), + }, + { + key: "runCount", + header: "Run Count", + render: (item) => item.runCount || 0, + }, + ]} + onSelect={(item) => { + router.push(`/prompts/${id}/experiments/${item.id}`); + }} + /> + )} + +
-
-

- Experiment Logs -

-
-
-
- { - setSelectedDatasets(value); + + + + ) : ( + + + + Prompt & Inputs + + + + Overview + + + + +
+
+
+
+ { + await createSubversion(history, model); }} - > - {datasets.map((dataset) => ( - - {dataset.name} - - ))} - + submitText="Test" + initialModel={model} + isPromptCreatedFromUi={ + prompt?.metadata?.createdFromUi as boolean | undefined + } + />
-
- { - setSelectedModels(value); - }} - > - {MODEL_LIST.map((model) => ( - - {model.label} - - ))} - +
+
+
+

Versions

+
+ + +
+ {sortedPrompts?.map((promptVersion) => { + const isProduction = + promptVersion.metadata?.isProduction === true; + const isSelected = + selectedVersion === + `${promptVersion.major_version}.${promptVersion.minor_version}`; + + return ( +
+ setSelectedInputAndVersion( + `${promptVersion.major_version}.${promptVersion.minor_version}` + ) + } + > +
+
+
+ {isSelected && ( +
+ )} +
+ + V{promptVersion.major_version}. + {promptVersion.minor_version} + + + {isProduction && ( + + Prod + + )} + +
+
+ {prompt?.metadata?.createdFromUi === + true ? ( + + + + + + {!isProduction && ( + + promoteToProduction( + promptVersion.id + ) + } + > + + Promote to prod + + )} + + startExperiment( + promptVersion.id + ) + } + > + + Experiment + + {!isProduction && ( + + deletePromptVersion( + promptVersion.id + ) + } + > + +

+ Delete +

+ + )} + + + ) : ( + + + + + + + startExperiment( + promptVersion.id + ) + } + > + + Experiment + + + + )} +
+
+
+ + {getTimeAgo( + new Date(promptVersion.created_at) + )} + +
+ {promptVersion.model} +
+
+
+ ); + })} +
+ +
+
+
+

Inputs

+
+ + setSearchRequestId(value) + } + /> +
+
+ + +
    + {inputs + ?.filter((input) => + input.source_request.includes(searchRequestId) + ) + .map((input) => ( +
  • + +
    • + ))} +
+ +
-
- { - setSelectedDatasets([]); - setSelectedModels([]); +
+
+
+ + +
+
+
+ +
+ +
+ + ({ + date: getTimeMap(timeIncrement)(r.time), + count: r.count, + })) ?? [] + } + index="date" + categories={["count"]} + colors={["cyan"]} + showYAxis={false} + curveType="monotone" + valueFormatter={(number: number | bigint) => { + return `${new Intl.NumberFormat("us").format( + Number(number) + )}`; }} /> -
+
- {isExperimentsLoading ? ( -
- +
+

+ Experiment Logs +

+
+
+
+ { + setSelectedDatasets(value); + }} + > + {datasets.map((dataset) => ( + + {dataset.name} + + ))} + +
+
+ { + setSelectedModels(value); + }} + > + {MODEL_LIST.map((model) => ( + + {model.label} + + ))} + +
+
+ { + setSelectedDatasets([]); + setSelectedModels([]); + }} + /> +
+
- ) : ( - ( - - {item.id} - - ), - }, - { - key: "status", - header: "Status", - render: (item) => ( - - ), - }, - { - key: "createdAt", - header: "Created At", - render: (item) => ( - {getUSDateFromString(item.createdAt)} - ), - }, - { - key: "datasetName", - header: "Dataset", - render: (item) => item.datasetName, - }, - { - key: "model", - header: "Model", - render: (item) => ( - - ), - }, - { - key: "runCount", - header: "Run Count", - render: (item) => item.runCount || 0, - }, - ]} - onSelect={(item) => { - router.push(`/prompts/${id}/experiments/${item.id}`); + {isExperimentsLoading ? ( +
+ +
+ ) : ( + ( + + {item.id} + + ), + }, + { + key: "status", + header: "Status", + render: (item) => ( + + ), + }, + { + key: "createdAt", + header: "Created At", + render: (item) => ( + {getUSDateFromString(item.createdAt)} + ), + }, + { + key: "datasetName", + header: "Dataset", + render: (item) => item.datasetName, + }, + { + key: "model", + header: "Model", + render: (item) => ( + + ), + }, + { + key: "runCount", + header: "Run Count", + render: (item) => item.runCount || 0, + }, + ]} + onSelect={(item) => { + router.push(`/prompts/${id}/experiments/${item.id}`); + }} + /> + )} + + - )} - - +
-
- - - + + + + )}
); }; From 3da1cd233a354f414e311e4563ddb9361992e48d Mon Sep 17 00:00:00 2001 From: colegottdank Date: Thu, 26 Sep 2024 11:10:22 -0700 Subject: [PATCH 3/4] add python sdk docs (#2690) --- docs/integrations/gemini/api/python.mdx | 55 +++++++++++++++++++++++++ docs/mint.json | 1 + 2 files changed, 56 insertions(+) create mode 100644 docs/integrations/gemini/api/python.mdx diff --git a/docs/integrations/gemini/api/python.mdx b/docs/integrations/gemini/api/python.mdx new file mode 100644 index 0000000000..5e9e2d4a70 --- /dev/null +++ b/docs/integrations/gemini/api/python.mdx @@ -0,0 +1,55 @@ +--- +title: "Gemini Python SDK Integration" +sidebarTitle: "Python" +description: "Use Gemini's Python SDK to integrate with Helicone to log your Gemini AI usage." +"twitter:title": "Gemini Python SDK Integration - Helicone OSS LLM Observability" +icon: "python" +iconType: "solid" +--- + + + + Log into [Helicone](https://www.helicone.ai) or create an account. Once you have an account, you can generate an [API key](https://helicone.ai/developer). + + + Visit the [Google Generative AI API Key](https://aistudio.google.com/app/apikey) page. Follow the instructions to create a new API key. Make sure to save the key as you will need it for the next steps. + + + ```bash + export HELICONE_API_KEY= + export GOOGLE_GENERATIVE_API_KEY= + ``` + + + Ensure you have the necessary packages installed in your Python environment: + ```bash + pip install google-generativeai + ``` + + + ```python + import google.generativeai as genai + import os + + genai.configure( + api_key=os.environ.get('GOOGLE_GENERATIVE_API_KEY'), + client_options={ + 'api_endpoint': 'gateway.helicone.ai', + }, + default_metadata=[ + ('helicone-auth', f'Bearer {os.environ.get("HELICONE_API_KEY")}'), + ('helicone-target-url', 'https://generativelanguage.googleapis.com') + ], + transport="rest" + ) + ``` + + + + ```python + model = genai.GenerativeModel('gemini-1.5-flash') + response = model.generate_content("The opposite of hot is") + print(response.result) + ``` + + diff --git a/docs/mint.json b/docs/mint.json index c15b52885b..cf466ac13a 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -130,6 +130,7 @@ "group": "API", "pages": [ "integrations/gemini/api/javascript", + "integrations/gemini/api/python", "integrations/gemini/api/curl" ], "icon": "code", From 4ee66452f017519042510772871697717b6b5bb0 Mon Sep 17 00:00:00 2001 From: colegottdank Date: Thu, 26 Sep 2024 11:33:58 -0700 Subject: [PATCH 4/4] let them add members (#2691) --- .../private/organizationController.ts | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/valhalla/jawn/src/controllers/private/organizationController.ts b/valhalla/jawn/src/controllers/private/organizationController.ts index bee841abc4..209f23a254 100644 --- a/valhalla/jawn/src/controllers/private/organizationController.ts +++ b/valhalla/jawn/src/controllers/private/organizationController.ts @@ -117,19 +117,37 @@ export class OrganizationController extends Controller { @Request() request: JawnAuthenticatedRequest ): Promise> { const organizationManager = new OrganizationManager(request.authParams); - const memberCount = await organizationManager.getMemberCount(true); - if (memberCount.error || memberCount.data == null || memberCount.data < 0) { - return err(memberCount.error ?? "Error getting member count"); + const org = await organizationManager.getOrg(); + if (org.error || !org.data) { + return err(`Error getting organization: ${org.error}`); } - const stripeManager = new StripeManager(request.authParams); + if (org.data.tier === "enterprise") { + // Enterprise tier: Proceed to add member without additional checks + } else if (org.data.tier === "pro-20240913") { + // Pro tier: Update Stripe user count before adding member + const memberCount = await organizationManager.getMemberCount(true); + if ( + memberCount.error || + memberCount.data == null || + memberCount.data < 0 + ) { + return err(memberCount.error ?? "Error getting member count"); + } - const userCount = await stripeManager.updateProUserCount( - memberCount.data + 1 - ); + const stripeManager = new StripeManager(request.authParams); + + const userCount = await stripeManager.updateProUserCount( + memberCount.data + 1 + ); - if (userCount.error) { - return err(userCount.error ?? "Error updating pro user count"); + if (userCount.error) { + return err(userCount.error ?? "Error updating pro user count"); + } + } else { + return err( + "Your current tier does not allow adding members. Please upgrade to Pro to add members." + ); } const result = await organizationManager.addMember(