Skip to content

Commit

Permalink
Merge branch 'docs' of https://github.com/PromptSail/prompt_sail into…
Browse files Browse the repository at this point in the history 
… docs
  • Loading branch information
@WiolaGreen
WiolaGreen committed Dec 16, 2024
2 parents 9cc3c5d + 96e7ff8 commit 688d2d9
Show file tree
Hide file tree
Showing 15 changed files with 793 additions and 155 deletions.
265 changes: 175 additions & 90 deletions backend/src/app/web_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from collections import defaultdict
from typing import Annotated, Any

import pandas as pd
import utils
from _datetime import datetime, timezone
from app.dependencies import get_provider_pricelist, get_transaction_context
Expand All @@ -17,9 +18,7 @@
CreateProjectSchema,
GetAIProviderPriceSchema,
GetAIProviderSchema,
GetCostPerTagSchema,
GetPortfolioDetailsSchema,
GetProjectPortfolioCostPerTagSchema,
GetProjectPortfolioSchema,
GetProjectSchema,
GetProjectsUsageInTimeSchema,
Expand All @@ -44,9 +43,11 @@
from seedwork.repositories import DocumentNotFoundException
from settings.use_cases import get_organization_name
from slugify import slugify
from transactions.models import generate_uuid
from transactions.models import Transaction, generate_uuid
from transactions.schemas import (
CreateTransactionWithRawDataSchema,
GetTagStatisticsInTime,
GetTagStatisticsSchema,
GetTransactionLatencyStatisticsWithoutDateSchema,
GetTransactionPageResponseSchema,
GetTransactionSchema,
Expand All @@ -57,14 +58,14 @@
GetTransactionWithProjectSlugSchema,
GetTransactionWithRawDataSchema,
StatisticTransactionSchema,
TagStatisticTransactionSchema,
)
from transactions.use_cases import (
add_transaction,
count_transactions,
count_transactions_for_list,
delete_multiple_transactions,
get_all_filtered_and_paginated_transactions,
get_all_transactions,
get_list_of_filtered_transactions,
get_transaction,
get_transactions_for_project,
Expand Down Expand Up @@ -724,40 +725,44 @@ async def get_portfolio_usage_in_time(
)
if count == 0:
results[idx] = []
else:
transactions = ctx.call(
get_list_of_filtered_transactions,

transactions = ctx.call(
get_list_of_filtered_transactions,
project_id=idx,
date_from=date_from,
date_to=date_to,
status_codes=[200],
)
transactions = [
StatisticTransactionSchema(
project_id=idx,
date_from=date_from,
date_to=date_to,
status_codes=[200],
provider=transaction.provider,
model=transaction.model,
total_input_tokens=transaction.input_tokens or 0,
total_output_tokens=transaction.output_tokens or 0,
total_input_cost=transaction.input_cost or 0,
total_output_cost=transaction.output_cost or 0,
total_cost=transaction.total_cost or 0,
status_code=transaction.status_code,
date=transaction.response_time,
latency=(
transaction.response_time - transaction.request_time
).total_seconds(),
generation_speed=transaction.generation_speed,
total_transactions=1,
)
transactions = [
StatisticTransactionSchema(
project_id=idx,
provider=transaction.provider,
model=transaction.model,
total_input_tokens=transaction.input_tokens or 0,
total_output_tokens=transaction.output_tokens or 0,
total_input_cost=transaction.input_cost or 0,
total_output_cost=transaction.output_cost or 0,
total_cost=transaction.total_cost or 0,
status_code=transaction.status_code,
date=transaction.response_time,
latency=(
transaction.response_time - transaction.request_time
).total_seconds(),
generation_speed=transaction.generation_speed,
total_transactions=1,
)
for transaction in transactions
]
for transaction in transactions
]

if len(transactions) > 0:
stats = utils.token_counter_for_transactions(
transactions, period, date_from, date_to, False
)
results[idx] = stats

if sum([len(stat) for stat in results.items()]) == 0:
return []

aggregated_data = defaultdict(lambda: defaultdict(lambda: defaultdict(int)))

for project_id, records in results.items():
Expand Down Expand Up @@ -822,68 +827,127 @@ async def get_portfolio_usage_in_time(
dependencies=[Security(decode_and_validate_token)],
)
async def get_portfolio_costs_by_tag(
ctx: Annotated[TransactionContext, Depends(get_transaction_context)]
) -> list[GetProjectPortfolioCostPerTagSchema]:
transactions = ctx.call(get_all_transactions)
cost_by_tag, tags = {}, {"untagged-transactions": 0}
ctx: Annotated[TransactionContext, Depends(get_transaction_context)],
date_from: datetime | str | None = None,
date_to: datetime | str | None = None,
period: utils.PeriodEnum = utils.PeriodEnum.day,
) -> list[GetTagStatisticsInTime]:
date_from, date_to = utils.check_dates_for_statistics(date_from, date_to)
count = ctx.call(
count_transactions,
date_from=date_from,
date_to=date_to,
status_codes=[200],
)
if count == 0:
return []

transactions = ctx.call(
get_list_of_filtered_transactions,
date_from=date_from,
date_to=date_to,
status_codes=[200],
)

transactions_multiplied = []
for transaction in transactions:
date = transaction.response_time.date()
if date not in cost_by_tag.keys():
cost_by_tag[date] = []
if len(transaction.tags) > 0:
if len(transaction.tags) == 0:
transactions_multiplied.append(
Transaction(
id=transaction.id,
project_id=transaction.project_id,
tags=["untagged-transactions"],
provider=transaction.provider,
model=transaction.model,
type=transaction.type,
os=transaction.os,
input_tokens=transaction.input_tokens,
output_tokens=transaction.output_tokens,
library=transaction.library,
status_code=transaction.status_code,
messages=transaction.messages,
last_message=transaction.last_message,
prompt=transaction.prompt,
error_message=transaction.error_message,
generation_speed=transaction.generation_speed,
input_cost=transaction.input_cost,
output_cost=transaction.output_cost,
total_cost=transaction.total_cost,
request_time=transaction.request_time,
response_time=transaction.response_time,
hate=transaction.hate,
self_harm=transaction.self_harm,
violence=transaction.violence,
sexual=transaction.sexual,
)
)
else:
for tag in transaction.tags:
try:
tags[tag] += (
transaction.total_cost
if transaction.total_cost is not None
else 0
)
except KeyError:
tags[tag] = (
transaction.total_cost
if transaction.total_cost is not None
else 0
)

try:
idx = cost_by_tag[date].index(
[stat for stat in cost_by_tag[date] if stat["tag"] == tag][0]
)
cost_by_tag[date][idx]["cost"] += (
transaction.total_cost
if transaction.total_cost is not None
else 0
transactions_multiplied.append(
Transaction(
id=transaction.id,
project_id=transaction.project_id,
tags=[tag],
provider=transaction.provider,
model=transaction.model,
type=transaction.type,
os=transaction.os,
input_tokens=transaction.input_tokens,
output_tokens=transaction.output_tokens,
library=transaction.library,
status_code=transaction.status_code,
messages=transaction.messages,
last_message=transaction.last_message,
prompt=transaction.prompt,
error_message=transaction.error_message,
generation_speed=transaction.generation_speed,
input_cost=transaction.input_cost,
output_cost=transaction.output_cost,
total_cost=transaction.total_cost,
request_time=transaction.request_time,
response_time=transaction.response_time,
hate=transaction.hate,
self_harm=transaction.self_harm,
violence=transaction.violence,
sexual=transaction.sexual,
)
except IndexError:
cost_by_tag[date].append({"tag": tag, "cost": tags[tag]})
else:
tags["untagged-transactions"] += (
transaction.total_cost if transaction.total_cost is not None else 0
)
try:
idx = cost_by_tag[date].index(
[
stat
for stat in cost_by_tag[date]
if stat["tag"] == "untagged-transactions"
][0]
)
cost_by_tag[date][idx]["cost"] += (
transaction.total_cost if transaction.total_cost is not None else 0
)
except IndexError:
cost_by_tag[date].append(
{
"tag": "untagged-transactions",
"cost": tags["untagged-transactions"],
}
)

results = []
for key, values in cost_by_tag.items():
values = [GetCostPerTagSchema(**value) for value in values]
results.append(GetProjectPortfolioCostPerTagSchema(date=key, records=values))
return results
transactions = [
TagStatisticTransactionSchema(
tag=transaction.tags[0],
total_input_tokens=transaction.input_tokens or 0,
total_output_tokens=transaction.output_tokens or 0,
total_input_cost=transaction.input_cost or 0,
total_output_cost=transaction.output_cost or 0,
total_cost=transaction.total_cost or 0,
date=transaction.response_time,
total_transactions=1,
)
for transaction in transactions_multiplied
]

if len(transactions) > 0:
stats = utils.token_counter_for_transactions_by_tag(
transactions, period, date_from, date_to, False
)
df = pd.DataFrame([stat.model_dump() for stat in stats])
grouped = (
df.groupby("date")
.apply(lambda x: x.to_dict(orient="records"))
.reset_index(name="records")
)
output = grouped.to_dict(orient="records")

response = []
for row in output:
records = []
for record in row["records"]:
records.append(GetTagStatisticsSchema(**record))
response.append(GetTagStatisticsInTime(date=row["date"], records=records))
return response

return []


@app.get(
Expand Down Expand Up @@ -1059,18 +1123,39 @@ async def mock_transactions(
:return: A dictionary containing the status and message (code and latency).
"""
time_start = datetime.now(tz=timezone.utc)
repo = ctx["transaction_repository"]
transactions_repo = ctx["transaction_repository"]

# Delete all transactions for project-test in order to avoid duplicates transactions keys
repo.delete_cascade(project_id="project-test")
transactions_repo.delete_cascade(project_id="project-test")

# generate random transactions, with different models and providers
mocked_transactions = utils.generate_mock_transactions(count, date_from, date_to)
for transaction in mocked_transactions:
repo.add(transaction)
transactions_repo.add(transaction)
time_stop = datetime.now(tz=timezone.utc)

return {
"status_code": 200,
"message": f"{count} transactions added in {(time_stop-time_start).total_seconds()} seconds.",
}


@app.post(
"/api/only_for_purpose/remove_mocked_transactions", response_class=JSONResponse
)
async def mock_transactions(
ctx: Annotated[TransactionContext, Depends(get_transaction_context)],
) -> dict[str, Any]:
"""
API endpoint to remove mocked transactions. Warning! This endpoint is only for testing purposes and will delete all transactions for project-test.
:param ctx: The transaction context dependency.
:return: A dictionary containing the status and message.
"""
transactions_repo = ctx["transaction_repository"]
transactions_repo.delete_cascade(project_id="project-test")

return {
"status_code": 200,
"message": f"Mocked transactions removed",
}
37 changes: 37 additions & 0 deletions backend/src/transactions/schemas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,43 @@ class StatisticTransactionSchema(BaseModel):
generation_speed: int | float | None


class TagStatisticTransactionSchema(BaseModel):
tag: str
total_input_tokens: int
total_output_tokens: int
total_input_cost: int | float | None
total_output_cost: int | float | None
total_cost: int | float | None
date: datetime
total_transactions: int


class GetTagStatisticTransactionSchema(BaseModel):
tag: str
total_input_tokens: int
total_output_tokens: int
input_cumulative_total: int | float | None
output_cumulative_total: int | float | None
total_cost: int | float | None
date: datetime
total_transactions: int


class GetTagStatisticsSchema(BaseModel):
tag: str
total_input_tokens: int
total_output_tokens: int
input_cumulative_total: int | float | None
output_cumulative_total: int | float | None
total_cost: int | float | None
total_transactions: int


class GetTagStatisticsInTime(BaseModel):
date: datetime
records: list[GetTagStatisticsSchema]


class GetTransactionUsageStatisticsSchema(BaseModel):
provider: str
model: str
Expand Down
Loading

0 comments on commit 688d2d9

Please sign in to comment.