After converting pytorch to onnx my accuracy dropped a lot

[myfrndjk]

Hi,

I am new to ML. I am trying to convert my pytorch model to onnx and do inference

I followed internet samples / tutorials

it got converted however my accuracy went down a lot. Can some one help me what is the issue in my conversion and how can i fix it

import os
import torch
from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig
from torch.utils.data import Dataset, DataLoader
import time
import pandas as pd
import json
import onnx
import onnxruntime
import numpy as np
import tokenizer

device = 'cuda' if torch.cuda.is_available() else 'cpu'
torch.multiprocessing.set_sharing_strategy('file_system')
batch_size=2
num_workers = 2
model_name = "./repo_name/ms-marco-MiniLM-L-4-v2"
load_path = "./checkpoint/"
max_length = 150

## model load


def load_checkpoint(model, path, fn='checkpoint.pth', strict=True):
    model = model.module if hasattr(model, 'module') else model
    checkpoint_fn = os.path.join(path, fn)
    checkpoint = torch.load(checkpoint_fn, map_location=("cuda"))
    model_to_load = checkpoint['model']
    model.load_state_dict(model_to_load, strict=strict)
    
model = AutoModelForSequenceClassification.from_pretrained(model_name, local_files_only=True).to(device)
tokenizer = AutoTokenizer.from_pretrained(model_name, local_files_only=True)
config = AutoConfig.from_pretrained(model_name, local_files_only=True)
load_checkpoint(model, load_path, strict=False)
torch.multiprocessing.set_sharing_strategy('file_system')

class RelevanceDataset(Dataset):
    def __init__(self, data, tokenizer, label_column, max_length=150, do_lower_case=True, transform=None):
        self.data = data
        self.transform = transform
        self.data_columns = list(self.data.columns)
        self.tokenizer = tokenizer
        self.max_length = max_length
        self.label_column = label_column

    def __len__(self):
        return len(self.data)

    def __getitem__(self, sample_idx):
        if torch.is_tensor(sample_idx):
            sample_idx = sample_idx.tolist()
                # Ensure 'jk_title' is in the list of columns
        if 'jk_title' not in self.data_columns:
            raise ValueError("'jk_title' not found in the list of columns.")

        sample = self.data.iloc[sample_idx].values 
        jk_text_feature = str(sample[self.data_columns.index('jk_title')])
        query_text_features = str(sample[self.data_columns.index('query')])

        text_inputs = self.tokenizer(query_text_features,
                                      jk_text_feature,
                                      padding='max_length',
                                      max_length=self.max_length,
                                      truncation='longest_first',
                                      return_tensors="pt")

        labels = torch.squeeze(torch.FloatTensor([int(sample[self.data_columns.index(self.label_column)])]))

        sample = {k: torch.squeeze(v) for k,v in text_inputs.items()}
        sample["labels"] = labels

        if self.transform:
            sample = self.transform(sample, self.data_columns, sample_idx)

        return sample


def run_eval_epoch(dataloader, model, device):
    s = time.time()
    emb_scores_list = []
    labels_list = []
    model.eval()
    with torch.no_grad():
        for i, batch_data in enumerate(dataloader):

            batch_data = {key: value.to(device) for key, value in batch_data.items()}
            outputs = model(**batch_data, return_dict=True)
            batch_softmax_logits = torch.sigmoid(outputs.logits)
            emb_scores_list.extend(batch_softmax_logits[:, -1].detach().cpu().numpy().tolist())
            labels_list.extend(batch_data["labels"].detach().cpu().numpy().tolist())

            del batch_data, outputs
            torch.cuda.empty_cache()  # Free up GPU memory explicitly
            #print(" Batch Loading Time after delcache = {}".format(time.time()-s))
    return emb_scores_list, labels_list

def invoke(df_test):
    df_test['standardRelevancy'] = 1
    test_dataset = RelevanceDataset(df_test, 
                                    tokenizer=tokenizer,
                                    label_column="standardRelevancy",
                                    max_length=100, 
                                    do_lower_case=True)

    test_dataloader = DataLoader(test_dataset,
                                  batch_size=batch_size,
                                  shuffle=False,
                                  num_workers=num_workers,
                                  pin_memory=True) 

    # Export the PyTorch model to ONNX
    print("exporting model")
    example_input = torch.zeros(batch_size, max_length).to(device).to(torch.long)  # Cast to torch.long
    onnx_model_path = 'your_model.onnx'
    

    torch.onnx.export(
        model,
        (example_input,),
        onnx_model_path,
        verbose=True,
        input_names=['input_ids'],
        output_names=['output'],
        dynamic_axes={'input_ids': {0: 'batch_size', 1: 'sequence_length'}},
        opset_version=13
    )

    # Load ONNX model
    onnx_model_path = 'your_model.onnx'
    onnx_model = onnx.load(onnx_model_path)
    onnx_session = onnxruntime.InferenceSession(onnx_model_path)


    # Run inference with PyTorch model
    emb_scores_list_pytorch, labels_list = run_eval_epoch(test_dataloader, model, device)

    # # Convert PyTorch scores to NumPy array and apply threshold
    # emb_scores_array_pytorch = np.array(emb_scores_list_pytorch)
    # binary_predictions_pytorch = (emb_scores_array_pytorch >= threshold).astype(int)

    # ...

    # Run inference with ONNX model
    emb_scores_list_onnx = []
    for i, batch_data in enumerate(test_dataloader):
        batch_data = {key: value.numpy() for key, value in batch_data.items()}
        input_data = batch_data['input_ids']

        # Run inference with ONNX model
        onnx_inputs = {onnx_session.get_inputs()[0].name: input_data}
        onnx_outputs = onnx_session.run(None, onnx_inputs)
        onnx_scores = torch.sigmoid(torch.tensor(onnx_outputs[0]))

        emb_scores_list_onnx.extend(onnx_scores[:, -1].detach().cpu().numpy().tolist())
        del batch_data, input_data, onnx_outputs
        torch.cuda.empty_cache()



    # Compare the results
    emb_scores_list_pytorch = np.array(emb_scores_list_pytorch)
    emb_scores_list_onnx = np.array(emb_scores_list_onnx)

    print("PyTorch Scores:")
    print(emb_scores_list_pytorch)

    print("ONNX Scores:")
    print(emb_scores_list_onnx)
    


    df_test["score"] = emb_scores_list_pytorch
    x = df_test[['query', 'jk_title', 'score']].to_json(orient='records')

    return x

# Example usage:
df_test = pd.read_csv('jj_one_line_input.csv', sep='\t')  # Replace with your actual test data
result = invoke(df_test)
print(result)

This is the result i got

exporting model
PyTorch Scores:
[0.99658054 0.9994173 ]
ONNX Scores:
[0.42117319 0.43306887]

input.csv

jk_title query

someTile1 somequery1
someTile2 somequery2