From 623648b243bc951671f0a7f4fd7ceac9d357971f Mon Sep 17 00:00:00 2001 From: Joshua Lochner Date: Tue, 3 Sep 2024 10:45:02 +0000 Subject: [PATCH 1/2] Ignore kobert from unit tests (now requires `trust_remote_code=True` --- tests/generate_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/generate_tests.py b/tests/generate_tests.py index d529160dd..b4ffe9137 100644 --- a/tests/generate_tests.py +++ b/tests/generate_tests.py @@ -72,6 +72,9 @@ # TODO: remove when https://github.com/huggingface/transformers/issues/28096 is addressed 'RajuKandasamy/tamillama_tiny_30m', + + # Requires `trust_remote_code` + 'monologg/kobert', ] MAX_TESTS = { From dfaf8dceb6e93c45262836e4d58c9ce8b40837d0 Mon Sep 17 00:00:00 2001 From: Joshua Lochner Date: Tue, 3 Sep 2024 11:25:36 +0000 Subject: [PATCH 2/2] Remove tokenizers without chat template --- tests/generate_tests.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/generate_tests.py b/tests/generate_tests.py index b4ffe9137..3f103778b 100644 --- a/tests/generate_tests.py +++ b/tests/generate_tests.py @@ -231,10 +231,6 @@ TOKENIZERS_WITH_CHAT_TEMPLATES = { # https://huggingface.co/docs/transformers/main/en/chat_templating - 'Xenova/blenderbot-400M-distill': [ - 'basic', - ], - 'Xenova/mistral-tokenizer-v1': [ 'basic', ], @@ -243,16 +239,6 @@ 'system', ], - 'Xenova/llama-tokenizer': [ - 'basic', - 'system', - 'system + assistant', - ], - 'Xenova/llama2-tokenizer': [ - 'basic', - 'system', - 'system + assistant', - ], 'Xenova/llama2-chat-tokenizer': [ 'basic', 'system',