diff --git a/tests/format/data/structured/demo-dataset.jsonl b/tests/format/data/structured/demo-dataset.jsonl index 590c029f5..77a0a1d88 100644 --- a/tests/format/data/structured/demo-dataset.jsonl +++ b/tests/format/data/structured/demo-dataset.jsonl @@ -1,6 +1,6 @@ -{"text": "Today is Sunday and it's a happy day!", "meta": {"src": "Arxiv"}} -{"text": "Do you need a cup of coffee?", "meta": {"src": "code"}} -{"text": "你好,请问你是谁", "meta": {"src": "customized"}} -{"text": "Sur la plateforme MT4, plusieurs manières d'accéder à ces fonctionnalités sont conçues simultanément.", "meta": {"src": "Oscar"}} -{"text": "欢迎来到阿里巴巴!", "meta": {"src": "customized"}} -{"text": "This paper proposed a novel method on LLM pretraining.", "meta": {"src": "customized"}} \ No newline at end of file +{"text": "Today is Sunday and it's a happy day!", "meta": {"src": "Arxiv", "date": "2023-04-27", "version": "1.0"}} +{"text": "Do you need a cup of coffee?", "meta": {"src": "code", "author": "xxx"}} +{"text": "Today is Sunday and it's a happy day!", "meta": {"src": "Arxiv", "date": "2023-04-27", "version": "1.0"}} +{"text": "Do you need a cup of coffee?", "meta": {"src": "code", "author": "xxx"}} +{"text": "Today is Sunday and it's a happy day!", "meta": {"src": "Arxiv", "date": "2023-04-27", "version": "1.0"}} +{"text": "Do you need a cup of coffee?", "meta": {"src": "code", "author": "xxx"}} \ No newline at end of file diff --git a/tests/format/test_unify_format.py b/tests/format/test_unify_format.py index 2f64d0dcf..c9b41d19d 100644 --- a/tests/format/test_unify_format.py +++ b/tests/format/test_unify_format.py @@ -366,8 +366,7 @@ def test_hetero_meta(self): 'author': 'xxx' } }] - unified_sample_list = ds.to_list() - self.assertEqual(unified_sample_list, sample) + # test nested and missing field for the following cases: # 1. first row, then column unified_sample_first = ds[0]