Merge pull request #40 from mlfoundations/etashg/lint

Etashg/lint

.github/workflows/black.yaml

@@ -0,0 +1,10 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: psf/black@stable

create_csv_helper.py

@@ -10,6 +10,7 @@
 
 engine, SessionMaker = create_db_engine()
 
+
 @contextmanager
 def session_scope():
     """
@@ -25,54 +26,42 @@ def session_scope():
     finally:
         session.close()
 
+
 def get_model_score(name: str, model_id: uuid.UUID, annotator_model: str) -> float:
     with session_scope() as session:
         rows = session.query(EvalResult).filter_by(model_id=model_id).all()
         if not rows:
             return None
         for row in rows:
-            eval_setting = session.query(EvalSetting).filter_by(
-                id=row.eval_setting_id
-            ).first()
-            if eval_setting and name == eval_setting.name and eval_setting.parameters['annotator_model'] == annotator_model:
+            eval_setting = session.query(EvalSetting).filter_by(id=row.eval_setting_id).first()
+            if (
+                eval_setting
+                and name == eval_setting.name
+                and eval_setting.parameters["annotator_model"] == annotator_model
+            ):
                 return float(row.score)
         return None
 
+
 def get_model_name(model_id: uuid.UUID) -> str:
     with session_scope() as session:
         model = session.query(Model).filter_by(id=model_id).first()
         return model.name if model else None
 
+
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description='Generate CSV of model evaluation scores')
-    parser.add_argument(
-        '--model-ids',
-        required=True,
-        nargs='+',
-        help='List of model UUIDs to evaluate'
-    )
-    parser.add_argument(
-        '--eval-tasks',
-        required=True,
-        nargs='+',
-        help='List of evaluation task names'
-    )
-    parser.add_argument(
-        '--annotator-model',
-        required=True,
-        help='Annotator model to filter results'
-    )
-    parser.add_argument(
-        '--output',
-        default='model_scores.csv',
-        help='Output CSV filename (default: model_scores.csv)'
-    )
+    parser = argparse.ArgumentParser(description="Generate CSV of model evaluation scores")
+    parser.add_argument("--model-ids", required=True, nargs="+", help="List of model UUIDs to evaluate")
+    parser.add_argument("--eval-tasks", required=True, nargs="+", help="List of evaluation task names")
+    parser.add_argument("--annotator-model", required=True, help="Annotator model to filter results")
+    parser.add_argument("--output", default="model_scores.csv", help="Output CSV filename (default: model_scores.csv)")
     return parser.parse_args()
 
+
 def generate_eval_csv(model_ids: List[str], eval_tasks: List[str], annotator_model: str, output_file: str) -> None:
     """
     Generate CSV file with model evaluation scores.
-    
+
     Args:
         model_ids: List of model UUID strings
         eval_tasks: List of evaluation task names
@@ -87,7 +76,7 @@ def generate_eval_csv(model_ids: List[str], eval_tasks: List[str], annotator_mod
         sys.exit(1)
 
     # Prepare CSV headers
-    headers = ['model_id', 'model_name'] + eval_tasks
+    headers = ["model_id", "model_name"] + eval_tasks
 
     # Collect data for each model
     rows = []
@@ -97,21 +86,18 @@ def generate_eval_csv(model_ids: List[str], eval_tasks: List[str], annotator_mod
             print(f"Warning: Model not found for ID {model_id}", file=sys.stderr)
             continue
 
-        row = {
-            'model_id': str(model_id),
-            'model_name': model_name
-        }
-
+        row = {"model_id": str(model_id), "model_name": model_name}
+
         # Get scores for each eval task
         for task in eval_tasks:
             score = get_model_score(task, model_id, annotator_model)
-            row[task] = score if score is not None else 'N/A'
-        
+            row[task] = score if score is not None else "N/A"
+
         rows.append(row)
 
     # Write to CSV
     try:
-        with open(output_file, 'w', newline='') as csvfile:
+        with open(output_file, "w", newline="") as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=headers)
             writer.writeheader()
             writer.writerows(rows)
@@ -120,9 +106,11 @@ def generate_eval_csv(model_ids: List[str], eval_tasks: List[str], annotator_mod
         print(f"Error writing to CSV file: {e}", file=sys.stderr)
         sys.exit(1)
 
+
 def main():
     args = parse_args()
     generate_eval_csv(args.model_ids, args.eval_tasks, args.annotator_model, args.output)
 
-if __name__ == '__main__':
-    main()
+
+if __name__ == "__main__":
+    main()

eval/chat_benchmarks/MTBench/fastchat/data/clean_sharegpt.py

@@ -5,6 +5,7 @@
 Usage:
 python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json
 """
+
 import argparse
 from concurrent.futures import ProcessPoolExecutor
 import json
@@ -19,9 +20,7 @@
 
 div_pattern = re.compile("<div.*?>")
 span_pattern = re.compile("<span.*?>")
-code_lang_pattern = re.compile(
-    "```\s*" + "(.*?)" + "(?:Copy code)+" + "(.+?)" + "\s*?```", re.DOTALL
-)
+code_lang_pattern = re.compile("```\s*" + "(.*?)" + "(?:Copy code)+" + "(.+?)" + "\s*?```", re.DOTALL)
 code_lang_format = "```\g<1>\n\g<2>\n```"
 regenerate_pattern = re.compile("\d+ / \d+")
 copy_chars_pattern = re.compile("Copy\d+ chars / \d+ words")
@@ -155,9 +154,7 @@ def clean_html_all(content, begin, end):
     content = content[begin:end]
     processed = []
     with ProcessPoolExecutor() as executor:
-        for result in tqdm(
-            executor.map(clean_html_one_sample, content), total=len(content)
-        ):
+        for result in tqdm(executor.map(clean_html_one_sample, content), total=len(content)):
             processed.append(result)
 
     visited = {}

eval/chat_benchmarks/MTBench/fastchat/data/extract_gpt4_only.py

@@ -3,6 +3,7 @@
 
 Usage: python3 -m fastchat.data.extract_gpt4_only --in sharegpt.json
 """
+
 import argparse
 import json
 

eval/chat_benchmarks/MTBench/fastchat/data/extract_single_round.py

@@ -3,6 +3,7 @@
 
 Usage: python3 -m fastchat.data.extract_single_round --in sharegpt.json
 """
+
 import argparse
 import json
 

eval/chat_benchmarks/MTBench/fastchat/data/filter_wrong_format.py

@@ -5,6 +5,7 @@
 python3 -m fastchat.data.filter_wrong_format --in input.json --out output.json
 
 """
+
 import argparse
 import json
 import re

eval/chat_benchmarks/MTBench/fastchat/data/get_stats.py

@@ -26,9 +26,7 @@ def tokenize_one_sample(c):
 def tokenize_dataset(content):
     processed = []
     with ProcessPoolExecutor() as executor:
-        for result in tqdm(
-            executor.map(tokenize_one_sample, content), total=len(content)
-        ):
+        for result in tqdm(executor.map(tokenize_one_sample, content), total=len(content)):
             processed.append(result)
 
     return processed
@@ -59,9 +57,7 @@ def compute_stats(content):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--in-file", type=str)
-    parser.add_argument(
-        "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf"
-    )
+    parser.add_argument("--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf")
     args = parser.parse_args()
 
     content = json.load(open(args.in_file, "r"))

eval/chat_benchmarks/MTBench/fastchat/data/hardcoded_questions.py

@@ -1,6 +1,7 @@
 """
 Hardcoded question and answers.
 """
+
 import json
 
 

eval/chat_benchmarks/MTBench/fastchat/data/inspect_data.py

@@ -2,6 +2,7 @@
 Usage:
 python3 -m fastchat.data.inspect_data --in sharegpt_20230322_clean_lang_split.json
 """
+
 import argparse
 import json
 import random

eval/chat_benchmarks/MTBench/fastchat/data/optional_clean.py

@@ -8,6 +8,7 @@
 Requirement:
 pip3 install polyglot pyicu pycld2
 """
+
 import argparse
 import json
 import re

eval/chat_benchmarks/MTBench/fastchat/data/optional_replace.py

@@ -7,6 +7,7 @@
 Requirement:
 pip3 install transformers tqdm
 """
+
 import argparse
 import json
 import traceback
@@ -15,9 +16,7 @@
 from tqdm import tqdm
 
 
-def replace_special_tokens(
-    tokenizer: transformers.PreTrainedTokenizer, text: str
-) -> str:
+def replace_special_tokens(tokenizer: transformers.PreTrainedTokenizer, text: str) -> str:
     if not text:
         return text
 

eval/chat_benchmarks/MTBench/fastchat/data/prepare_all.py

@@ -9,20 +9,14 @@
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--prefix", type=str, default="~/datasets/sharegpt_20230521")
-    parser.add_argument(
-        "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf"
-    )
+    parser.add_argument("--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf")
     parser.add_argument("--seq-len", type=int, default=4096)
     args = parser.parse_args()
 
     in_prefix = args.prefix
     model_path = args.model_name_or_path
     seq_len = args.seq_len
-    prefix = (
-        f"{in_prefix}_{seq_len}".replace("4096", "4k")
-        .replace("8192", "8k")
-        .replace("16384", "16k")
-    )
+    prefix = f"{in_prefix}_{seq_len}".replace("4096", "4k").replace("8192", "8k").replace("16384", "16k")
 
     cmd_list = [
         f"python3 -m fastchat.data.clean_sharegpt --in {in_prefix}_html.json --out {prefix}_clean.json",

eval/chat_benchmarks/MTBench/fastchat/data/sample.py

@@ -3,6 +3,7 @@
 
 Usage: python3 -m fastchat.data.sample --in sharegpt.json --out sampled.json
 """
+
 import argparse
 import json
 

eval/chat_benchmarks/MTBench/fastchat/data/split_long_conversation.py

@@ -6,6 +6,7 @@
     --out sharegpt_split.json \
     --model-name-or-path $<model-name>
 """
+
 import argparse
 from concurrent.futures import ProcessPoolExecutor
 import json

eval/chat_benchmarks/MTBench/fastchat/data/split_train_test.py

@@ -3,6 +3,7 @@
 
 Usage: python3 -m fastchat.data.split_train_test --in sharegpt.json
 """
+
 import argparse
 import json