Milestone 1

requirements.txt

@@ -7,6 +7,7 @@ e==1.4.5
 einops==0.7.0
 filelock==3.13.1
 fire==0.5.0
+flash-attn==2.5.1.post1
 fonttools==4.47.0
 fsspec==2023.12.2
 huggingface-hub==0.20.2

src/phi/phi_utils/model_setup.py

@@ -1,5 +1,6 @@
 # TODO - import relevant model and tokenizer modules from transformers
 import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
 # helper function provided to get model info
 def get_model_info(model):
@@ -33,7 +34,11 @@ def model_and_tokenizer_setup(model_id_or_path):
 
     # End of TODO.
     ##################################################
+
+    # Adapted from sample code: https://huggingface.co/microsoft/phi-2
+    model = AutoModelForCausalLM.from_pretrained(model_id_or_path, torch_dtype=torch.float16, attn_implementation="flash_attention_2", trust_remote_code=True) 
+    tokenizer = AutoTokenizer.from_pretrained(model_id_or_path, padding="max_length", padding_side="left", pad_token="<|endoftext|>", trust_remote_code=True)
 
-    # get_model_info(model)
+    print(get_model_info(model))
 
     return model, tokenizer

src/utils/eval_utils.py

@@ -1,4 +1,6 @@
 # TODO - import relevant sklearn score modules 
+from sklearn import metrics
+
 from sklearn.metrics import accuracy_score, f1_score
 import argparse
 from utils.file_utils import load_jsonl
@@ -20,7 +22,13 @@ def evaluate_standard(gt_labels, pred_labels):
 
     # End of TODO.
     ##################################################
+
+    accuracy = metrics.accuracy_score(gt_labels, pred_labels)
+
+    # Need to tell sklearn which label is the positive one    
+    f1score = metrics.f1_score(gt_labels, pred_labels, pos_label='SUPPORTS')
 
+    print(accuracy, f1score)
     return accuracy, f1score
 
 def model_eval_report(gt_filepath, pred_filepath):