Skip to content

Commit

Permalink
chore: add tracking for new testset generation (#1419)
Browse files Browse the repository at this point in the history
  • Loading branch information
jjmachan authored Oct 3, 2024
1 parent 407c2e0 commit cb62190
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 0 deletions.
1 change: 1 addition & 0 deletions src/ragas/_analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ class TestsetGenerationEvent(BaseEvent):
num_rows: int
language: str
is_experiment: bool = False
version: str = "3" # the version of testset generation pipeline


@silent
Expand Down
14 changes: 14 additions & 0 deletions src/ragas/testset/synthesizers/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import typing as t
from dataclasses import dataclass, field

from ragas._analytics import TestsetGenerationEvent, track
from ragas.callbacks import new_group
from ragas.executor import Executor
from ragas.llms import BaseRagasLLM, LangchainLLMWrapper
Expand Down Expand Up @@ -202,4 +203,17 @@ def generate(
testsets.append(TestsetSample(eval_sample=sample, **additional_info))
testset = Testset(samples=testsets)
testset_generation_rm.on_chain_end({"testset": testset})

# tracking how many samples were generated
track(
TestsetGenerationEvent(
event_type="testset_generation",
evolution_names=[
e.__class__.__name__.lower() for e, _ in query_distribution
],
evolution_percentages=[p for _, p in query_distribution],
num_rows=test_size,
language="english",
)
)
return testset

0 comments on commit cb62190

Please sign in to comment.