Skip to content

Adding the IFEval scenario #78

Adding the IFEval scenario

Adding the IFEval scenario #78

name: Scenario tests
on:
push:
branches: [ main ]
paths:
- 'src/helm/benchmark/scenarios/test_*_scenario.py'
pull_request:
paths:
- 'src/helm/benchmark/scenarios/test_*_scenario.py'
schedule:
- cron: "30 15 * * *"
jobs:
test:
name: Run scenario tests
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- name: Clear free space
run: |
sudo rm -rf /opt/ghc
df -h
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
# Installs dependencies and performs static code checks
- name: Install HELM
run: ./install-dev.sh && ./pre-commit.sh
- name: Run scenario tests
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
run: python3 -m pytest -m scenarios