Skip to content

Commit

Permalink
Minor tweaks to summarization for safety (#3138)
Browse files Browse the repository at this point in the history
  • Loading branch information
yifanmai authored Nov 7, 2024
1 parent a186bae commit d092905
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 6 deletions.
8 changes: 3 additions & 5 deletions src/helm/benchmark/presentation/summarize.py
Original file line number Diff line number Diff line change
Expand Up @@ -976,22 +976,20 @@ def _adapter_spec_sort_key(spec):
if strategy == AggregationStrategy.WIN_RATE:
WIN_RATE_AGGREGATION = "mean"
win_rates = compute_aggregate_row_win_rates(table, aggregation=WIN_RATE_AGGREGATION)
description = "How many models this model outperforms on average (over columns)."
aggregate_header_cells.append(
HeaderCell(
f"{WIN_RATE_AGGREGATION.capitalize()} win rate",
description=description,
description="How many models this model outperforms on average (over columns).",
lower_is_better=False,
)
)
aggregate_row_values.append(win_rates)
elif strategy == AggregationStrategy.MEAN:
means = compute_aggregate_row_means(table)
description = "An average over columns representing the mean performance."
aggregate_header_cells.append(
HeaderCell(
"Mean performance",
description=description,
"Mean score",
description="The mean of the scores from all columns.",
lower_is_better=table.header[0].lower_is_better,
)
)
Expand Down
1 change: 0 additions & 1 deletion src/helm/benchmark/static/schema_safety.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@ metric_groups:
- name: accuracy
display_name: Accuracy
aggregation_strategies:
- win_rate
- mean
metrics:
- name: ${main_name}
Expand Down

0 comments on commit d092905

Please sign in to comment.