in progress

turbo_alignment/common/tf/loaders/model/model.py

@@ -7,7 +7,6 @@
     PeftConfigRegistry,
     TransformersAutoModelRegistry,
 )
-from turbo_alignment.modeling.liger_kernels import apply_liger_kernel_to_gemma2
 from turbo_alignment.settings.model import (
     ModelForPeftSettings,
     ModelType,
@@ -46,13 +45,6 @@ def load_model(
     model_settings: PreTrainedModelSettings,
     tokenizer: PreTrainedTokenizerBase,
 ) -> PreTrainedModel:
-    if model_settings.liger_kernels_settings is not None:
-        apply_liger_kernel_to_gemma2(
-            rope=model_settings.liger_kernels_settings.use_rope,
-            cross_entropy=model_settings.liger_kernels_settings.use_cross_entropy,
-            geglu=model_settings.liger_kernels_settings.use_geglu,
-        )
-
     model = TransformersAutoModelRegistry.by_name(model_settings.model_type).from_pretrained(
         model_settings.model_path,
         **model_settings.transformers_settings.dict(exclude_none=True),

turbo_alignment/dataset/multimodal/collators.py

@@ -8,8 +8,8 @@ def torch_call(self, features):
         label_name = 'label' if 'label' in features[0].keys() else 'labels'
         labels = [feature[label_name] for feature in features] if label_name in features[0].keys() else None
         if 'modality_inputs' in features[0].keys():
-            print([feature['modality_inputs'] for feature in features])
-            modality_inputs = torch.stack([torch.stack(feature['modality_inputs']) for feature in features])
+            # print([feature['modality_inputs'] for feature in features])
+            modality_inputs = torch.stack([torch.stack(feature['modality_inputs']) for feature in features]).contiguous()
         else:
             modality_inputs = [None for _ in features]
 
@@ -54,7 +54,7 @@ def torch_call(self, features):
                 )
                 for feature in features
             ]
-        )
+        ).contiguous()
 
         if labels is None:
             return batch
@@ -63,6 +63,6 @@ def torch_call(self, features):
             label.tolist() + [self.label_pad_token_id] * (sequence_length - len(label)) for label in labels
         ]
 
-        batch[label_name] = torch.tensor(batch[label_name], dtype=torch.int64)
+        batch[label_name] = torch.tensor(batch[label_name], dtype=torch.int64).contiguous()
 
         return batch