Skip to content

Commit

Permalink
Displaying Scores per row (#1850)
Browse files Browse the repository at this point in the history 
* scores

* scores per row

* types fix

* pr fixes
  • Loading branch information
@maamalama
maamalama authored May 13, 2024
1 parent 8ff1400 commit 4b04b65
Show file tree
Hide file tree
Showing 6 changed files with 85 additions and 4 deletions.
8 changes: 8 additions & 0 deletions docs/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1854,6 +1854,9 @@
"rows": {
"items": {
"properties": {
"scores": {
"$ref": "#/components/schemas/Record_string.number_"
},
"inputRecord": {
"properties": {
"request": {
Expand Down Expand Up @@ -1886,6 +1889,7 @@
}
},
"required": [
"scores",
"rowId"
],
"type": "object"
Expand Down Expand Up @@ -1919,6 +1923,9 @@
"request": {
"$ref": "#/components/schemas/RequestObj"
},
"scores": {
"$ref": "#/components/schemas/Record_string.number_"
},
"response": {
"$ref": "#/components/schemas/ResponseObj"
},
Expand All @@ -1930,6 +1937,7 @@
}
},
"required": [
"scores",
"resultRequestId",
"datasetRowId"
],
Expand Down
21 changes: 19 additions & 2 deletions valhalla/jawn/src/lib/stores/experimentStore.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export interface Experiment {
response: ResponseObj;
request: RequestObj;
};
scores: Record<string, number>;
}[];
};
meta: any;
Expand All @@ -58,6 +59,7 @@ export interface Experiment {
datasetRowId: string;
resultRequestId: string;
response?: ResponseObj;
scores: Record<string, number>;
request?: RequestObj;
}[];
}[];
Expand Down Expand Up @@ -156,7 +158,15 @@ function getExperimentsQuery(
),`
: ""
}
'rowId', dsr.id
'rowId', dsr.id,
'scores', (
SELECT jsonb_object_agg(sa.score_key, sv.int_value)
FROM score_value sv
JOIN score_attribute sa ON sa.id = sv.score_attribute
JOIN prompt_input_record pir ON pir.source_request = sv.request_id
WHERE pir.id = dsr.input_record
AND sa.organization = e.organization
)
)
)
),
Expand Down Expand Up @@ -216,7 +226,14 @@ function getExperimentsQuery(
: ""
}
'datasetRowId', hr.dataset_row,
'resultRequestId', hr.result_request_id
'resultRequestId', hr.result_request_id,
'scores', (
SELECT jsonb_object_agg(sa.score_key, sv.int_value)
FROM score_value sv
JOIN score_attribute sa ON sa.id = sv.score_attribute
WHERE sv.request_id = hr.result_request_id
AND sa.organization = e.organization
)
)
)
FROM experiment_v2_hypothesis_run hr
Expand Down
4 changes: 2 additions & 2 deletions valhalla/jawn/src/tsoa-build/public/routes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -689,10 +689,10 @@ const models: TsoaRoute.Models = {
"properties": {
"id": {"dataType":"string","required":true},
"organization": {"dataType":"string","required":true},
"dataset": {"dataType":"nestedObjectLiteral","nestedProperties":{"rows":{"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"inputRecord":{"dataType":"nestedObjectLiteral","nestedProperties":{"request":{"ref":"RequestObj","required":true},"response":{"ref":"ResponseObj","required":true},"inputs":{"ref":"Record_string.string_","required":true},"requestPath":{"dataType":"string","required":true},"requestId":{"dataType":"string","required":true}}},"rowId":{"dataType":"string","required":true}}},"required":true},"name":{"dataType":"string","required":true},"id":{"dataType":"string","required":true}},"required":true},
"dataset": {"dataType":"nestedObjectLiteral","nestedProperties":{"rows":{"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"scores":{"ref":"Record_string.number_","required":true},"inputRecord":{"dataType":"nestedObjectLiteral","nestedProperties":{"request":{"ref":"RequestObj","required":true},"response":{"ref":"ResponseObj","required":true},"inputs":{"ref":"Record_string.string_","required":true},"requestPath":{"dataType":"string","required":true},"requestId":{"dataType":"string","required":true}}},"rowId":{"dataType":"string","required":true}}},"required":true},"name":{"dataType":"string","required":true},"id":{"dataType":"string","required":true}},"required":true},
"meta": {"dataType":"any","required":true},
"createdAt": {"dataType":"string","required":true},
"hypotheses": {"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"runs":{"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"request":{"ref":"RequestObj"},"response":{"ref":"ResponseObj"},"resultRequestId":{"dataType":"string","required":true},"datasetRowId":{"dataType":"string","required":true}}},"required":true},"providerKey":{"dataType":"string","required":true},"createdAt":{"dataType":"string","required":true},"status":{"dataType":"string","required":true},"model":{"dataType":"string","required":true},"parentPromptVersion":{"dataType":"nestedObjectLiteral","nestedProperties":{"template":{"dataType":"any","required":true}}},"promptVersion":{"dataType":"nestedObjectLiteral","nestedProperties":{"template":{"dataType":"any","required":true}}},"promptVersionId":{"dataType":"string","required":true},"id":{"dataType":"string","required":true}}},"required":true},
"hypotheses": {"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"runs":{"dataType":"array","array":{"dataType":"nestedObjectLiteral","nestedProperties":{"request":{"ref":"RequestObj"},"scores":{"ref":"Record_string.number_","required":true},"response":{"ref":"ResponseObj"},"resultRequestId":{"dataType":"string","required":true},"datasetRowId":{"dataType":"string","required":true}}},"required":true},"providerKey":{"dataType":"string","required":true},"createdAt":{"dataType":"string","required":true},"status":{"dataType":"string","required":true},"model":{"dataType":"string","required":true},"parentPromptVersion":{"dataType":"nestedObjectLiteral","nestedProperties":{"template":{"dataType":"any","required":true}}},"promptVersion":{"dataType":"nestedObjectLiteral","nestedProperties":{"template":{"dataType":"any","required":true}}},"promptVersionId":{"dataType":"string","required":true},"id":{"dataType":"string","required":true}}},"required":true},
"scores": {"dataType":"union","subSchemas":[{"ref":"ExperimentScores"},{"dataType":"enum","enums":[null]}],"required":true},
},
"additionalProperties": false,
Expand Down
8 changes: 8 additions & 0 deletions valhalla/jawn/src/tsoa-build/public/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1854,6 +1854,9 @@
"rows": {
"items": {
"properties": {
"scores": {
"$ref": "#/components/schemas/Record_string.number_"
},
"inputRecord": {
"properties": {
"request": {
Expand Down Expand Up @@ -1886,6 +1889,7 @@
}
},
"required": [
"scores",
"rowId"
],
"type": "object"
Expand Down Expand Up @@ -1919,6 +1923,9 @@
"request": {
"$ref": "#/components/schemas/RequestObj"
},
"scores": {
"$ref": "#/components/schemas/Record_string.number_"
},
"response": {
"$ref": "#/components/schemas/ResponseObj"
},
Expand All @@ -1930,6 +1937,7 @@
}
},
"required": [
"scores",
"resultRequestId",
"datasetRowId"
],
Expand Down
46 changes: 46 additions & 0 deletions web/components/templates/prompts/experiments/id/experimentIdPage.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,16 @@ const ExperimentIdPage = (props: PromptIdPageProps) => {
inputs: row.inputRecord?.inputs ?? {},
originResult: {
response: row.inputRecord?.response,
scores: row.scores,
},
testResult: {
response: experiment?.hypotheses?.[0]?.runs?.find(
(run) => run.datasetRowId === row.rowId
)?.response,
scores:
experiment?.hypotheses?.[0]?.runs?.find(
(run) => run.datasetRowId === row.rowId
)?.scores ?? {},
},
};
});
Expand Down Expand Up @@ -163,6 +168,9 @@ const ExperimentIdPage = (props: PromptIdPageProps) => {
<TableHeaderCell className="w-1/3 border-l border-gray-300">
<p className="text-black text-lg">Experiment Output</p>
</TableHeaderCell>
<TableHeaderCell className="w-1/3 border-l border-gray-300">
<p className="text-black text-lg">Scores</p>
</TableHeaderCell>
</TableRow>
</TableHead>

Expand Down Expand Up @@ -210,6 +218,25 @@ const ExperimentIdPage = (props: PromptIdPageProps) => {
model={run.originResult.response?.model ?? ""}
/>
</div>
<div className="w-full flex items-center gap-2">
{run.originResult.scores &&
Object.keys(run.originResult.scores).length >
0 && (
<>
Scores:{" "}
{Object.keys(run.originResult.scores).map(
(key) => (
<span
key={key}
className="bg-gray-50 text-gray-700 ring-gray-200 rounded-lg px-2 py-1 -my-1 text-xs font-medium ring-1 ring-inset"
>
{key}: {run.originResult.scores[key]}
</span>
)
)}
</>
)}
</div>
<pre className="whitespace-pre-wrap text-sm w-full h-full text-black">
{
(run.originResult.response?.body as any)
Expand Down Expand Up @@ -265,6 +292,25 @@ const ExperimentIdPage = (props: PromptIdPageProps) => {
model={run.testResult.response?.model ?? ""}
/>
</div>
<div className="w-full flex items-center gap-2">
{run.testResult.scores &&
Object.keys(run.testResult.scores).length >
0 && (
<>
Scores:{" "}
{Object.keys(run.testResult.scores).map(
(key) => (
<span
key={key}
className="bg-gray-50 text-gray-700 ring-gray-200 rounded-lg px-2 py-1 -my-1 text-xs font-medium ring-1 ring-inset"
>
{key}: {run.testResult?.scores[key]}
</span>
)
)}
</>
)}
</div>
<pre className="whitespace-pre-wrap text-sm overflow-auto h-full text-black">
{
(run.testResult.response?.body as any) // TODO: any
Expand Down
2 changes: 2 additions & 0 deletions web/lib/clients/jawnTypes/public.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ Json: JsonObject;
organization: string;
dataset: {
rows: {
scores: components["schemas"]["Record_string.number_"];
inputRecord?: {
request: components["schemas"]["RequestObj"];
response: components["schemas"]["ResponseObj"];
Expand All @@ -588,6 +589,7 @@ Json: JsonObject;
hypotheses: {
runs: {
request?: components["schemas"]["RequestObj"];
scores: components["schemas"]["Record_string.number_"];
response?: components["schemas"]["ResponseObj"];
resultRequestId: string;
datasetRowId: string;
Expand Down

0 comments on commit 4b04b65

Please sign in to comment.