configs.py

import argparse
import json

import requests
import yaml


# get configs from huggingface datasets server by doing a request
response = requests.get(
    "https://datasets-server.huggingface.co/splits?dataset=HiTZ%2FEusExams", timeout=5
)
response_json = json.loads(response.text)
CONFIGS = [split["config"] for split in response_json["splits"]]


def gen_config_yamls(output_dir: str, overwrite: bool) -> None:
    """
    Generate a yaml file for each configuage.

    :param output_dir: The directory to output the files to.
    :param overwrite: Whether to overwrite files if they already exist.
    """
    err = []
    for config in CONFIGS:
        file_name = f"eus_exams_{config}.yaml"
        try:
            with open(f"{output_dir}/{file_name}", "w" if overwrite else "x") as f:
                f.write("# Generated by utils.py\n")
                yaml.dump(
                    {
                        "include": "eus_exams_es"
                        if "eus_exams_es" in config
                        else "eus_exams_eu",
                        "dataset_name": config,
                        "task": f"eus_exams_{config}",
                    },
                    f,
                )
        except FileExistsError:
            err.append(file_name)

    if len(err) > 0:
        raise FileExistsError(
            "Files were not created because they already exist (use --overwrite flag):"
            f" {', '.join(err)}"
        )


def main() -> None:
    """Parse CLI args and generate configuage-specific yaml files."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--overwrite",
        default=False,
        action="store_true",
        help="Overwrite files if they already exist",
    )
    parser.add_argument(
        "--output-dir", default=".", help="Directory to write yaml files to"
    )
    args = parser.parse_args()

    gen_config_yamls(output_dir=args.output_dir, overwrite=args.overwrite)


if __name__ == "__main__":
    main()