Skip loading model if its allready fitted

Signed-off-by: Vibhu Jawa <[email protected]>
rapidsai · Aug 5, 2024 · de6fb38 · de6fb38
1 parent a6f4ac2
commit de6fb38
Showing 1 changed file with 0 additions and 5 deletions.
diff --git a/crossfit/backend/torch/loader.py b/crossfit/backend/torch/loader.py
@@ -75,7 +75,6 @@ def __next__(self):
 
         batch = {key: val[self.current_idx : end] for key, val in self.tensor_dict.items()}
         if self.max_seq_len is not None:
-            # TODO: Check this
             if self.padding_side == "right":
                 batch = {key: val[:, : self.max_seq_len] for key, val in batch.items()}
             else:
@@ -162,10 +161,6 @@ def __next__(self):
                     for key, val in self.tensor_dict.items()
                     if key not in self.to_ignore
                 }
-                # TODO: Fix max_length
-                if self.max_seq_len is None:
-                    self.max_seq_len = self.model.max_seq_length()
-
                 batch = clip_tokens(
                     token_o=batch,
                     max_length=self.max_seq_len,