Skip to content

Commit

Permalink
refactor: allow evaluating mbpp+ tasks on mbpp base tests
Browse files Browse the repository at this point in the history
  • Loading branch information
ganler committed Jan 25, 2024
1 parent 89b4c1d commit 5d9a4c7
Showing 1 changed file with 5 additions and 0 deletions.
5 changes: 5 additions & 0 deletions bigcode_eval/tasks/mbppplus.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
Homepage: https://github.com/evalplus/evalplus
"""

import os

from bigcode_eval.tasks.mbpp import MBPP
from bigcode_eval.tasks.custom_metrics.code_eval import compute_code_eval

Expand Down Expand Up @@ -46,6 +48,9 @@ def get_prompt(self, doc):
# is different from HumanEval(+) which further requires a `check` func
def get_reference(self, doc):
"""Builds the reference solution for the doc (sample from the test dataset)."""
use_mbpp_tests = os.getenv("MBBPPLUS_USE_MBPP_TESTS", "0")
if use_mbpp_tests == "1":
return "\n".join(doc["test_list"])
return "\n" + doc["test"]

def get_dataset(self):
Expand Down

0 comments on commit 5d9a4c7

Please sign in to comment.