From 9f03ddf19fedfe593dc4515837754bb92caa04d6 Mon Sep 17 00:00:00 2001 From: humu789 <88702197+humu789@users.noreply.github.com> Date: Tue, 29 Aug 2023 18:46:57 +0800 Subject: [PATCH] [Feature & Bug] Refactor apis & Update examples & Fix re-import (#53) * refactor apis & update examples & fix re-import * update process_hf_dataset * add example readme * fix conflict * adapt new dataset * passed examples and train.py * Update examples/README.md Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * Update examples/README.md Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * Update examples/README.md Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * Update examples/README.md Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * del slurm_train.sh * add 5 datasets apis * Update xtuner/apis/model.py Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * Update xtuner/apis/model.py Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> * Update xtuner/apis/model.py Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> --------- Co-authored-by: LZHgrla <36994684+LZHgrla@users.noreply.github.com> --- examples/README.md | 96 +++--------- examples/data_utils.py | 14 -- examples/deepspeed_config.json | 31 ---- ...rain_alpaca-qlora_with_mmchat_deepspeed.sh | 5 - .../run_train_alpaca-qlora_with_mmchat_hf.sh | 22 --- ...rain_alpaca-qlora_with_mmchat_deepspeed.py | 145 ------------------ examples/train_alpaca-qlora_with_mmchat_hf.py | 103 ------------- examples/train_hf.py | 5 +- examples/train_lora_hf.py | 37 +++++ examples/train_qlora_hf.py | 37 +++++ xtuner/apis/__init__.py | 2 +- xtuner/apis/datasets/__init__.py | 62 ++++---- xtuner/apis/datasets/alpaca.py | 98 +++++++----- xtuner/apis/datasets/alpaca_enzh.py | 60 -------- xtuner/apis/datasets/alpaca_zh.py | 62 -------- xtuner/apis/datasets/arxiv.py | 57 ++----- xtuner/apis/datasets/code_alpaca.py | 32 ++++ xtuner/apis/datasets/colorist.py | 33 ++++ xtuner/apis/datasets/lawyer.py | 91 +++++++++++ xtuner/apis/datasets/medical.py | 58 ++----- xtuner/apis/datasets/moss_003_sft.py | 71 +++++++++ xtuner/apis/datasets/moss_003_sft_all.py | 57 ------- .../apis/datasets/moss_003_sft_no_plugins.py | 54 ------- xtuner/apis/datasets/moss_003_sft_plugins.py | 54 ------- xtuner/apis/datasets/oasst1.py | 58 ++----- xtuner/apis/datasets/open_orca.py | 58 ++----- xtuner/apis/datasets/sql.py | 32 ++++ xtuner/apis/datasets/tiny_codes.py | 32 ++++ xtuner/apis/model.py | 82 ++++++++++ xtuner/apis/models/__init__.py | 12 -- xtuner/apis/models/baichuan.py | 74 --------- xtuner/apis/models/base.py | 36 ----- xtuner/apis/models/internlm.py | 51 ------ xtuner/apis/models/llama.py | 74 --------- xtuner/apis/models/qwen.py | 51 ------ .../default.py => training_args.py} | 14 +- xtuner/apis/training_args/__init__.py | 3 - .../baichuan_13b_base_qlora_alpaca_e3.py | 8 +- .../baichuan_13b_base_qlora_arxiv_e3.py | 8 +- ...baichuan_13b_base_qlora_moss_sft_all_e1.py | 6 +- ...uan_13b_base_qlora_moss_sft_all_e2_gpu8.py | 6 +- ...huan_13b_base_qlora_moss_sft_plugins_e1.py | 6 +- .../baichuan_13b_base_qlora_oasst1_e3.py | 8 +- .../baichuan_13b_base_qlora_openorca_e1.py | 8 +- .../baichuan_13b_chat_qlora_alpaca_e3.py | 8 +- .../baichuan_13b_chat_qlora_arxiv_e3.py | 8 +- ...baichuan_13b_chat_qlora_moss_sft_all_e1.py | 6 +- ...uan_13b_chat_qlora_moss_sft_all_e2_gpu8.py | 6 +- ...huan_13b_chat_qlora_moss_sft_plugins_e1.py | 6 +- .../baichuan_13b_chat_qlora_oasst1_e3.py | 8 +- .../baichuan_13b_chat_qlora_openorca_e1.py | 8 +- .../baichuan_7b_qlora_alpaca_e3.py | 8 +- .../baichuan_7b/baichuan_7b_qlora_arxiv_e3.py | 8 +- .../baichuan_7b_qlora_moss_sft_all_e1.py | 6 +- .../baichuan_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../baichuan_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../baichuan_7b_qlora_oasst1_e3.py | 8 +- .../baichuan_7b_qlora_openorca_e1.py | 8 +- xtuner/configs/deepspeed/deepspeed_zero3.json | 1 + .../internlm_7b_qlora_alpaca_e3.py | 8 +- .../internlm_7b/internlm_7b_qlora_arxiv_e3.py | 8 +- .../internlm_7b_qlora_code_alpaca_e3.py | 8 +- .../internlm_7b_qlora_moss_sft_all_e1.py | 6 +- .../internlm_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../internlm_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../internlm_7b_qlora_oasst1_e3.py | 8 +- .../internlm_7b_qlora_open_platypus_e3.py | 8 +- .../internlm_7b_qlora_openorca_e1.py | 8 +- .../internlm_7b/internlm_7b_qlora_sql_e3.py | 8 +- .../internlm_7b_qlora_tiny_codes_e1.py | 8 +- .../internlm_chat_7b_qlora_alpaca_e3.py | 8 +- .../internlm_chat_7b_qlora_arxiv_e3.py | 8 +- .../internlm_chat_7b_qlora_moss_sft_all_e1.py | 6 +- ...rnlm_chat_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- ...ernlm_chat_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../internlm_chat_7b_qlora_oasst1_e3.py | 8 +- .../internlm_chat_7b_qlora_openorca_e1.py | 8 +- .../llama2_70b_int8_lora_open_platypus_e1.py | 8 +- ...lama2_70b_int8_lora_open_platypus_e1_hf.py | 4 +- .../llama2_70b_qlora_open_platypus_e1.py | 8 +- .../llama2_70b_qlora_open_platypus_e1_hf.py | 4 +- .../llama2_7b/llama2_7b_qlora_alpaca_e3.py | 8 +- .../llama2_7b/llama2_7b_qlora_arxiv_e3.py | 8 +- .../llama2_7b_qlora_moss_sft_all_e1.py | 6 +- .../llama2_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../llama2_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../llama2_7b/llama2_7b_qlora_oasst1_e3.py | 8 +- .../llama2_7b/llama2_7b_qlora_openorca_e1.py | 8 +- .../llama2_7b_chat_qlora_alpaca_e3.py | 8 +- .../llama2_7b_chat_qlora_arxiv_e3.py | 8 +- .../llama2_7b_chat_qlora_moss_sft_all_e1.py | 6 +- ...ama2_7b_chat_qlora_moss_sft_all_e2_gpu8.py | 6 +- ...lama2_7b_chat_qlora_moss_sft_plugins_e1.py | 6 +- .../llama2_7b_chat_qlora_oasst1_e3.py | 8 +- .../llama2_7b_chat_qlora_openorca_e1.py | 8 +- .../llama_7b/llama_7b_qlora_alpaca_e3.py | 10 +- .../llama/llama_7b/llama_7b_qlora_arxiv_e3.py | 8 +- .../llama_7b_qlora_moss_sft_all_e1.py | 6 +- .../llama_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../llama_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../llama_7b/llama_7b_qlora_oasst1_e3.py | 8 +- .../llama_7b/llama_7b_qlora_openorca_e1.py | 8 +- .../qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py | 8 +- .../qwen/qwen_7b/qwen_7b_qlora_arxiv_e3.py | 8 +- .../qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py | 6 +- .../qwen_7b_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../qwen_7b_qlora_moss_sft_plugins_e1.py | 6 +- .../qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py | 8 +- .../qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py | 8 +- .../qwen_7b_chat_qlora_alpaca_e3.py | 8 +- .../qwen_7b_chat_qlora_arxiv_e3.py | 8 +- .../qwen_7b_chat_qlora_moss_sft_all_e1.py | 6 +- ...qwen_7b_chat_qlora_moss_sft_all_e2_gpu8.py | 6 +- .../qwen_7b_chat_qlora_moss_sft_plugins_e1.py | 6 +- .../qwen_7b_chat_qlora_oasst1_e3.py | 8 +- .../qwen_7b_chat_qlora_openorca_e1.py | 8 +- xtuner/{datasets => dataset}/__init__.py | 0 .../collate_fns/__init__.py | 0 .../collate_fns/defalut_collate_fn.py | 0 .../collate_fns/mmlu_collate_fn.py | 0 .../{datasets => dataset}/concat_dataset.py | 0 xtuner/{datasets => dataset}/huggingface.py | 6 +- .../{datasets => dataset}/map_fns/__init__.py | 0 .../map_fns/dataset_map_fns/__init__.py | 0 .../map_fns/dataset_map_fns/alpaca_map_fn.py | 0 .../dataset_map_fns/alpaca_zh_map_fn.py | 0 .../map_fns/dataset_map_fns/arxiv_map_fn.py | 0 .../dataset_map_fns/code_alpaca_map_fn.py | 0 .../map_fns/dataset_map_fns/colors_map_fn.py | 0 .../crime_kg_assitant_map_fn.py | 0 .../dataset_map_fns/law_reference_map_fn.py | 0 .../map_fns/dataset_map_fns/medical_map_fn.py | 0 .../map_fns/dataset_map_fns/oasst1_map_fn.py | 0 .../map_fns/dataset_map_fns/openai_map_fn.py | 0 .../dataset_map_fns/openorca_map_fn.py | 0 .../map_fns/dataset_map_fns/sql_map_fn.py | 0 .../dataset_map_fns/tiny_codes_map_fn.py | 0 .../map_fns/template_map_fn.py | 0 xtuner/{datasets => dataset}/moss_sft.py | 0 xtuner/{datasets => dataset}/utils.py | 0 xtuner/{models => model}/__init__.py | 0 .../fast_forward/__init__.py | 0 .../fast_forward/internlm_attn.py | 0 .../fast_forward/llama_attn.py | 0 xtuner/{models => model}/sft.py | 0 xtuner/{models => model}/utils.py | 0 xtuner/tools/train.py | 7 +- 147 files changed, 911 insertions(+), 1524 deletions(-) delete mode 100644 examples/data_utils.py delete mode 100644 examples/deepspeed_config.json delete mode 100644 examples/run_train_alpaca-qlora_with_mmchat_deepspeed.sh delete mode 100644 examples/run_train_alpaca-qlora_with_mmchat_hf.sh delete mode 100644 examples/train_alpaca-qlora_with_mmchat_deepspeed.py delete mode 100644 examples/train_alpaca-qlora_with_mmchat_hf.py create mode 100644 examples/train_lora_hf.py create mode 100644 examples/train_qlora_hf.py delete mode 100644 xtuner/apis/datasets/alpaca_enzh.py delete mode 100644 xtuner/apis/datasets/alpaca_zh.py create mode 100644 xtuner/apis/datasets/code_alpaca.py create mode 100644 xtuner/apis/datasets/colorist.py create mode 100644 xtuner/apis/datasets/lawyer.py create mode 100644 xtuner/apis/datasets/moss_003_sft.py delete mode 100644 xtuner/apis/datasets/moss_003_sft_all.py delete mode 100644 xtuner/apis/datasets/moss_003_sft_no_plugins.py delete mode 100644 xtuner/apis/datasets/moss_003_sft_plugins.py create mode 100644 xtuner/apis/datasets/sql.py create mode 100644 xtuner/apis/datasets/tiny_codes.py create mode 100644 xtuner/apis/model.py delete mode 100644 xtuner/apis/models/__init__.py delete mode 100644 xtuner/apis/models/baichuan.py delete mode 100644 xtuner/apis/models/base.py delete mode 100644 xtuner/apis/models/internlm.py delete mode 100644 xtuner/apis/models/llama.py delete mode 100644 xtuner/apis/models/qwen.py rename xtuner/apis/{training_args/default.py => training_args.py} (83%) delete mode 100644 xtuner/apis/training_args/__init__.py rename xtuner/{datasets => dataset}/__init__.py (100%) rename xtuner/{datasets => dataset}/collate_fns/__init__.py (100%) rename xtuner/{datasets => dataset}/collate_fns/defalut_collate_fn.py (100%) rename xtuner/{datasets => dataset}/collate_fns/mmlu_collate_fn.py (100%) rename xtuner/{datasets => dataset}/concat_dataset.py (100%) rename xtuner/{datasets => dataset}/huggingface.py (94%) rename xtuner/{datasets => dataset}/map_fns/__init__.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/__init__.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/alpaca_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/alpaca_zh_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/arxiv_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/code_alpaca_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/colors_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/law_reference_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/medical_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/oasst1_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/openai_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/openorca_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/sql_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/dataset_map_fns/tiny_codes_map_fn.py (100%) rename xtuner/{datasets => dataset}/map_fns/template_map_fn.py (100%) rename xtuner/{datasets => dataset}/moss_sft.py (100%) rename xtuner/{datasets => dataset}/utils.py (100%) rename xtuner/{models => model}/__init__.py (100%) rename xtuner/{models => model}/fast_forward/__init__.py (100%) rename xtuner/{models => model}/fast_forward/internlm_attn.py (100%) rename xtuner/{models => model}/fast_forward/llama_attn.py (100%) rename xtuner/{models => model}/sft.py (100%) rename xtuner/{models => model}/utils.py (100%) diff --git a/examples/README.md b/examples/README.md index 43341097d..c827c99c3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,4 +1,4 @@ -# How to use XTuner in HuggingFace & DeepSpeed training pipelines +# How to use XTuner in HuggingFace training pipeline ## Quick run @@ -11,86 +11,30 @@ 2. run training scripts ```shell - # HuggingFace training pipeline - bash run_train_alpaca-qlora_with_xtuner_hf.sh - - # DeepSpeed training pipeline - bash run_train_alpaca-qlora_with_xtuner_deepspeed.sh - ``` - -3. (optional) whether to use `qlora` / `lora` or not - - you can choose whether to use `qlora` / `lora` in training just by change the setting in scripts. - + # qlora-training internlm-7b with alpaca dataset + python train_qlora_hf.py --model_name_or_path internlm/internlm-7b --dataset_name_or_path tatsu-lab/alpaca ``` - # case 1. use qlora - --use_qlora True - --use_lora False - - # case 2. use lora - --use_qlora False - --use_lora True - - # case 3. neither - --use_qlora False - --use_lora False - ``` - -## Training pipeline -If you want to use xtuner for efficient finetuning in your original training pipelines, you just need to change the implement of building `model` and `dataloader`, reserve other parts. Thus you can quickly fine-tune various models with various datasets by changing the relevant configs. + `--model_name_or_path`: specify the model name or path to train. -## How to build model with XTuner for QLoRA + `--dataset_name_or_path`: specify the dataset name or path to use. -```python -from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig -from xtuner.models import SupervisedFinetune +## How to customize your experiment -model_name_or_path = 'internlm/internlm-7b' - -tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, - trust_remote_code=True) -# if use lora, `quantization_config = None` -quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - load_in_8bit=False, - llm_int8_threshold=6.0, - llm_int8_has_fp16_weight=False, - bnb_4bit_compute_dtype=torch.float16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type='nf4') - -# if neither use QLoRA or LoRA, `lora_config = None` -lora_config = LoraConfig( - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM') - -# build base llm model -llm = AutoModelForCausalLM.from_pretrained( - model_name_or_path, - quantization_config=quantization_config, - trust_remote_code=True -) - -# build model with XTuner -model = SupervisedFinetune(llm=llm, lora=lora_config, tokenizer=tokenizer) +XTuner APIs are compatible with the usage of HuggingFace's transformers. +If you want to customize your experiment, you just need to pass in your hyperparameters like HuggingFace. ``` - -## How to build train_dataloader - -```python -from .data_utils import get_train_dataloader - -dataset_cfg_path = '../configs/_base_/datasets/alpaca.py' - -# need to pass in `tokenizer` previously prepared -train_dataloader = get_train_dataloader(dataset_cfg_path, tokenizer) - -# (optional) you can get also dataset or collate_fn by train_dataloader -train_dataset = train_dataloader.dataset -data_collator = train_dataloader.collate_fn +# training example +python train_qlora_hf.py \ + # custom training args + --model_name_or_path internlm/internlm-7b \ + --dataset_name_or_path tatsu-lab/alpaca \ + # HuggingFace's default training args + --do_train = True + --per_device_train_batch_size = 1 + --learning_rate = 2e-5 + --save_strategy = 'epoch' + --lr_scheduler_type = 'cosine' + --logging_steps = 1 ``` diff --git a/examples/data_utils.py b/examples/data_utils.py deleted file mode 100644 index 7c8dd1583..000000000 --- a/examples/data_utils.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -from mmengine.config import Config -from mmengine.runner import Runner - - -def get_train_dataloader(dataset_cfg_path, tokenizer): - dataset_cfg = Config.fromfile(dataset_cfg_path) - if dataset_cfg.train_dataloader.dataset.type.__name__ == 'ConcatDataset': - dataset_cfg.train_dataloader.dataset.datasets_kwargs.tokenizer = \ - tokenizer - else: - dataset_cfg.train_dataloader.dataset.tokenizer = tokenizer - train_dataloader = Runner.build_dataloader(dataset_cfg.train_dataloader) - return train_dataloader diff --git a/examples/deepspeed_config.json b/examples/deepspeed_config.json deleted file mode 100644 index 0d22f8c15..000000000 --- a/examples/deepspeed_config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "gradient_accumulation_steps": 1, - "train_micro_batch_size_per_gpu": 1, - "prescale_gradients": false, - "zero_allow_untested_optimizer": true, - "optimizer": { - "type": "AdamW", - "params": { - "lr": 1e-8, - "eps": 1.0e-8, - "betas": [ - 0.9, - 0.95 - ], - "weight_decay": 0.1 - } - }, - "zero_optimization": { - "stage": 2, - "contiguous_gradients": false, - "allgather_bucket_size": 1e8, - "reduce_bucket_size": 1e8, - "overlap_comm": true, - "reduce_scatter": true - }, - "gradient_clipping": 1.0, - "wall_clock_breakdown": true, - "bf16": { - "enabled": true - } - } diff --git a/examples/run_train_alpaca-qlora_with_mmchat_deepspeed.sh b/examples/run_train_alpaca-qlora_with_mmchat_deepspeed.sh deleted file mode 100644 index 4e554bb21..000000000 --- a/examples/run_train_alpaca-qlora_with_mmchat_deepspeed.sh +++ /dev/null @@ -1,5 +0,0 @@ -CUDA_VISIBLE_DEVICES=1,2,3,4 deepspeed train_alpaca-qlora_with_xtuner_deepspeed.py \ - --model_name_or_path internlm/internlm-7b \ - --use_qlora True \ - --use_lora False \ - --output_dir work_dirs/xtuner_ds_internlm-7b_qlora diff --git a/examples/run_train_alpaca-qlora_with_mmchat_hf.sh b/examples/run_train_alpaca-qlora_with_mmchat_hf.sh deleted file mode 100644 index 40ecbf937..000000000 --- a/examples/run_train_alpaca-qlora_with_mmchat_hf.sh +++ /dev/null @@ -1,22 +0,0 @@ -CUDA_VISIBLE_DEVICES=1,2,3,4 torchrun --nproc_per_node=4 --master_port=10000 train_alpaca-qlora_with_xtuner_hf.py \ - --model_name_or_path internlm/internlm-7b \ - --use_qlora True \ - --use_lora False \ - --bf16 True \ - --output_dir work_dirs/xtuner_hf_internlm-7b_qlora \ - --num_train_epochs 3 \ - --per_device_train_batch_size 1 \ - --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 1 \ - --evaluation_strategy "no" \ - --save_strategy "steps" \ - --save_steps 1000 \ - --save_total_limit 1 \ - --learning_rate 2e-5 \ - --weight_decay 0. \ - --warmup_ratio 0.03 \ - --lr_scheduler_type "cosine" \ - --logging_steps 1 \ - --tf32 True \ - --remove_unused_columns False \ - --ddp_find_unused_parameters False diff --git a/examples/train_alpaca-qlora_with_mmchat_deepspeed.py b/examples/train_alpaca-qlora_with_mmchat_deepspeed.py deleted file mode 100644 index 7190d6481..000000000 --- a/examples/train_alpaca-qlora_with_mmchat_deepspeed.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -import argparse -import os - -import deepspeed -import torch -import transformers -from data_utils import get_train_dataloader -from peft import LoraConfig -from tqdm import tqdm -from transformers import AutoTokenizer, BitsAndBytesConfig - -from xtuner.models import SupervisedFinetune - - -def get_argument_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--model_name_or_path', type=str, default='internlm/internlm-7b') - parser.add_argument('--use_qlora', type=bool, default=True) - parser.add_argument('--use_lora', type=bool, default=False) - parser.add_argument( - '--dataset_cfg_path', - type=str, - default='../configs/_base_/datasets/alpaca.py', - help='Path to xtuner dataset config') - parser.add_argument( - '--deepspeed_config', - type=str, - default='./deepspeed_config.json', - help='Path to deepspeed config') - parser.add_argument( - '--num_train_epochs', - type=int, - default=3, - help='Number of epochs to train') - parser.add_argument( - '--output_dir', - type=str, - default='work_dirs', - help='Dir to store checkpoint files') - parser.add_argument( - '--local_rank', - type=int, - default=-1, - help='Reserved for deepspeed framework') - return parser - - -arg_parser = get_argument_parser() -args = arg_parser.parse_args() -if args.local_rank == -1: - device = torch.device('cuda') -else: - torch.cuda.set_device(args.local_rank) - device = torch.device('cuda', args.local_rank) - deepspeed.init_distributed() - - -def to_device(batch, device): - output = {} - for k, v in batch.items(): - try: - output[k] = v.to(device) - except Exception: - output[k] = v - return output - - -def print_rank_0(msg, rank=0): - if rank <= 0: - print(msg) - - -def train(): - # build model - if args.use_qlora: - quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - load_in_8bit=False, - llm_int8_threshold=6.0, - llm_int8_has_fp16_weight=False, - bnb_4bit_compute_dtype=torch.float16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type='nf4') - lora_config = LoraConfig( - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM') - elif not args.use_qlora and args.use_lora: - quantization_config = None - lora_config = LoraConfig( - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM') - else: - quantization_config = None - lora_config = None - - llm = transformers.AutoModelForCausalLM.from_pretrained( - args.model_name_or_path, - quantization_config=quantization_config, - trust_remote_code=True) - tokenizer = AutoTokenizer.from_pretrained( - args.model_name_or_path, trust_remote_code=True) - - model = SupervisedFinetune(llm=llm, lora=lora_config, tokenizer=tokenizer) - - # build deepspeed engine - model_parameters = filter(lambda p: p.requires_grad, model.parameters()) - model_engine, _, _, _ = deepspeed.initialize( - args=args, - model=model, - optimizer=None, - model_parameters=model_parameters) - - # build dataloader - train_dataloader = get_train_dataloader(args.dataset_cfg_path, tokenizer) - - # training - model_engine.train() - for epoch in range(args.num_train_epochs): - for i, inputs in enumerate(tqdm(train_dataloader)): - inputs = to_device(inputs, device) - loss = model_engine(**inputs)['loss'] - model_engine.backward(loss) - model_engine.step() - print_rank_0( - f'Epoch [{epoch+1}/{args.num_train_epochs}], ' - f'Step [{i}/{len(train_dataloader)}], Loss: {loss.item()}') - - save_dir = os.path.join(args.output_dir, f'epoch_{epoch+1}') - os.makedirs(save_dir, exist_ok=True) - model_to_save = model_engine.module.llm if hasattr( - model_engine, 'module') else model_engine.llm - model_to_save.save_pretrained( - save_dir, state_dict=model_to_save.state_dict()) - - -if __name__ == '__main__': - train() diff --git a/examples/train_alpaca-qlora_with_mmchat_hf.py b/examples/train_alpaca-qlora_with_mmchat_hf.py deleted file mode 100644 index edc99fe85..000000000 --- a/examples/train_alpaca-qlora_with_mmchat_hf.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -from dataclasses import dataclass, field -from typing import Optional - -import torch -import transformers -from data_utils import get_train_dataloader -from peft import LoraConfig -from transformers import AutoTokenizer, BitsAndBytesConfig, Trainer - -from xtuner.models import SupervisedFinetune - - -@dataclass -class ModelArguments: - model_name_or_path: Optional[str] = field(default='internlm/internlm-7b') - use_qlora: bool = field(default=True) - use_lora: bool = field(default=False) - - -@dataclass -class DataArguments: - dataset_cfg_path: str = field( - default='../configs/_base_/datasets/alpaca.py') - - -@dataclass -class TrainingArguments(transformers.TrainingArguments): - cache_dir: Optional[str] = field(default=None) - optim: str = field(default='adamw_torch') - model_max_length: int = field( - default=512, - metadata={ - 'help': - 'Maximum sequence length. Sequences will be right padded ' - '(and possibly truncated).' - }, - ) - - -def train(): - parser = transformers.HfArgumentParser( - (ModelArguments, DataArguments, TrainingArguments)) - model_args, data_args, training_args = parser.parse_args_into_dataclasses() - - # build model - if model_args.use_qlora: - quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - load_in_8bit=False, - llm_int8_threshold=6.0, - llm_int8_has_fp16_weight=False, - bnb_4bit_compute_dtype=torch.float16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type='nf4') - lora_config = LoraConfig( - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM') - elif not model_args.use_qlora and model_args.use_lora: - quantization_config = None - lora_config = LoraConfig( - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM') - else: - quantization_config = None - lora_config = None - - llm = transformers.AutoModelForCausalLM.from_pretrained( - model_args.model_name_or_path, - cache_dir=training_args.cache_dir, - quantization_config=quantization_config, - trust_remote_code=True) - tokenizer = AutoTokenizer.from_pretrained( - model_args.model_name_or_path, trust_remote_code=True) - - model = SupervisedFinetune(llm=llm, lora=lora_config, tokenizer=tokenizer) - - # build trainer_hf - train_dataloader = get_train_dataloader(data_args.dataset_cfg_path, - tokenizer) - train_dataset = train_dataloader.dataset - data_collator = train_dataloader.collate_fn - data_module = dict( - train_dataset=train_dataset, - eval_dataset=None, - data_collator=data_collator) - trainer = Trainer(model=model, args=training_args, **data_module) - - # training - trainer.train() - - trainer.save_state() - trainer.save_model(output_dir=training_args.output_dir) - - -if __name__ == '__main__': - train() diff --git a/examples/train_hf.py b/examples/train_hf.py index 930dcea24..7a7af2eb7 100644 --- a/examples/train_hf.py +++ b/examples/train_hf.py @@ -2,9 +2,8 @@ import transformers from transformers import Trainer +from xtuner.apis import DefaultTrainingArguments, build_model from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset -from xtuner.apis.models import internlm_7b_qlora -from xtuner.apis.training_args import DefaultTrainingArguments def train(): @@ -13,7 +12,7 @@ def train(): training_args = parser.parse_args_into_dataclasses()[0] # init model and dataset - model, tokenizer = internlm_7b_qlora( + model, tokenizer = build_model( model_name_or_path=training_args.model_name_or_path, return_tokenizer=True) train_dataset = alpaca_dataset( diff --git a/examples/train_lora_hf.py b/examples/train_lora_hf.py new file mode 100644 index 000000000..60dd949c5 --- /dev/null +++ b/examples/train_lora_hf.py @@ -0,0 +1,37 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import transformers +from transformers import Trainer + +from xtuner.apis import DefaultTrainingArguments, build_lora_model +from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset + + +def train(): + # get DefaultTrainingArguments and to be updated with passed args + parser = transformers.HfArgumentParser(DefaultTrainingArguments) + training_args = parser.parse_args_into_dataclasses()[0] + + # init model and dataset + model, tokenizer = build_lora_model( + model_name_or_path=training_args.model_name_or_path, + return_tokenizer=True) + train_dataset = alpaca_dataset( + tokenizer=tokenizer, path=training_args.dataset_name_or_path) + data_collator = alpaca_data_collator(return_hf_format=True) + + # build trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + data_collator=data_collator) + + # training + trainer.train() + + trainer.save_state() + trainer.save_model(output_dir=training_args.output_dir) + + +if __name__ == '__main__': + train() diff --git a/examples/train_qlora_hf.py b/examples/train_qlora_hf.py new file mode 100644 index 000000000..2e8dbab71 --- /dev/null +++ b/examples/train_qlora_hf.py @@ -0,0 +1,37 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import transformers +from transformers import Trainer + +from xtuner.apis import DefaultTrainingArguments, build_qlora_model +from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset + + +def train(): + # get DefaultTrainingArguments and to be updated with passed args + parser = transformers.HfArgumentParser(DefaultTrainingArguments) + training_args = parser.parse_args_into_dataclasses()[0] + + # init model and dataset + model, tokenizer = build_qlora_model( + model_name_or_path=training_args.model_name_or_path, + return_tokenizer=True) + train_dataset = alpaca_dataset( + tokenizer=tokenizer, path=training_args.dataset_name_or_path) + data_collator = alpaca_data_collator(return_hf_format=True) + + # build trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + data_collator=data_collator) + + # training + trainer.train() + + trainer.save_state() + trainer.save_model(output_dir=training_args.output_dir) + + +if __name__ == '__main__': + train() diff --git a/xtuner/apis/__init__.py b/xtuner/apis/__init__.py index 8ee9456d2..27157e743 100644 --- a/xtuner/apis/__init__.py +++ b/xtuner/apis/__init__.py @@ -1,3 +1,3 @@ from .datasets import * # noqa: F401, F403 -from .models import * # noqa: F401, F403 +from .model import * # noqa: F401, F403 from .training_args import * # noqa: F401, F403 diff --git a/xtuner/apis/datasets/__init__.py b/xtuner/apis/datasets/__init__.py index 473406c88..4eaa91d1b 100644 --- a/xtuner/apis/datasets/__init__.py +++ b/xtuner/apis/datasets/__init__.py @@ -1,33 +1,35 @@ -from .alpaca import alpaca_data_collator, alpaca_dataloader, alpaca_dataset -from .alpaca_enzh import (alpaca_enzh_data_collator, alpaca_enzh_dataloader, - alpaca_enzh_dataset) -from .alpaca_zh import (alpaca_zh_data_collator, alpaca_zh_dataloader, - alpaca_zh_dataset) -from .arxiv import arxiv_data_collator, arxiv_dataloader, arxiv_dataset -from .medical import medical_data_collator, medical_dataloader, medical_dataset -from .moss_003_sft_all import (moss_003_sft_data_collator, - moss_003_sft_dataloader, moss_003_sft_dataset) -from .moss_003_sft_no_plugins import (moss_003_sft_no_plugins_data_collator, - moss_003_sft_no_plugins_dataloader, - moss_003_sft_no_plugins_dataset) -from .moss_003_sft_plugins import (moss_003_sft_plugins_data_collator, - moss_003_sft_plugins_dataloader, - moss_003_sft_plugins_dataset) -from .oasst1 import oasst1_data_collator, oasst1_dataloader, oasst1_dataset -from .open_orca import (openorca_data_collator, openorca_dataloader, - openorca_dataset) +from .alpaca import (alpaca_data_collator, alpaca_dataset, + alpaca_enzh_data_collator, alpaca_enzh_dataset, + alpaca_zh_data_collator, alpaca_zh_dataset) +from .arxiv import arxiv_data_collator, arxiv_dataset +from .code_alpaca import code_alpaca_data_collator, code_alpaca_dataset +from .colorist import colorist_data_collator, colorist_dataset +from .lawyer import (lawyer_crime_data_collator, lawyer_crime_dataset, + lawyer_data_collator, lawyer_dataset, + lawyer_reference_data_collator, lawyer_reference_dataset) +from .medical import medical_data_collator, medical_dataset +from .moss_003_sft import (moss_003_sft_data_collator, moss_003_sft_dataset, + moss_003_sft_no_plugins_data_collator, + moss_003_sft_no_plugins_dataset, + moss_003_sft_plugins_data_collator, + moss_003_sft_plugins_dataset) +from .oasst1 import oasst1_data_collator, oasst1_dataset +from .open_orca import openorca_data_collator, openorca_dataset +from .sql import sql_data_collator, sql_dataset +from .tiny_codes import tiny_codes_data_collator, tiny_codes_dataset __all__ = [ - 'alpaca_data_collator', 'alpaca_dataloader', 'alpaca_dataset', - 'alpaca_enzh_data_collator', 'alpaca_enzh_dataloader', - 'alpaca_enzh_dataset', 'alpaca_zh_data_collator', 'alpaca_zh_dataloader', - 'alpaca_zh_dataset', 'arxiv_data_collator', 'arxiv_dataloader', - 'arxiv_dataset', 'medical_data_collator', 'medical_dataloader', - 'medical_dataset', 'moss_003_sft_data_collator', 'moss_003_sft_dataloader', - 'moss_003_sft_dataset', 'moss_003_sft_no_plugins_data_collator', - 'moss_003_sft_no_plugins_dataloader', 'moss_003_sft_no_plugins_dataset', - 'moss_003_sft_plugins_data_collator', 'moss_003_sft_plugins_dataloader', - 'moss_003_sft_plugins_dataset', 'oasst1_data_collator', - 'oasst1_dataloader', 'oasst1_dataset', 'openorca_data_collator', - 'openorca_dataloader', 'openorca_dataset' + 'alpaca_data_collator', 'alpaca_dataset', 'alpaca_enzh_data_collator', + 'alpaca_enzh_dataset', 'alpaca_zh_data_collator', 'alpaca_zh_dataset', + 'arxiv_data_collator', 'arxiv_dataset', 'medical_data_collator', + 'medical_dataset', 'moss_003_sft_data_collator', 'moss_003_sft_dataset', + 'moss_003_sft_no_plugins_data_collator', 'moss_003_sft_no_plugins_dataset', + 'moss_003_sft_plugins_data_collator', 'moss_003_sft_plugins_dataset', + 'oasst1_data_collator', 'oasst1_dataset', 'openorca_data_collator', + 'openorca_dataset', 'lawyer_crime_dataset', 'lawyer_crime_data_collator', + 'lawyer_reference_dataset', 'lawyer_reference_data_collator', + 'lawyer_dataset', 'lawyer_data_collator', 'colorist_dataset', + 'colorist_data_collator', 'sql_dataset', 'sql_data_collator', + 'code_alpaca_dataset', 'code_alpaca_data_collator', 'tiny_codes_dataset', + 'tiny_codes_data_collator' ] diff --git a/xtuner/apis/datasets/alpaca.py b/xtuner/apis/datasets/alpaca.py index 695e04de0..50aefe9b1 100644 --- a/xtuner/apis/datasets/alpaca.py +++ b/xtuner/apis/datasets/alpaca.py @@ -1,59 +1,85 @@ from functools import partial from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner +from torch.utils.data import ConcatDataset -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn -from xtuner.registry import BUILDER +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import (alpaca_map_fn, alpaca_zh_map_fn, + template_map_fn_factory) +from xtuner.utils import PROMPT_TEMPLATE -def alpaca_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path=None, +def alpaca_enzh_dataset(tokenizer, + path_en='tatsu-lab/alpaca', + path_zh='silk-road/alpaca-data-gpt4-chinese', + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + alpaca = alpaca_dataset( + tokenizer, + path=path_en, + max_length=max_length, + shuffle_before_pack=True, + remove_unused_columns=remove_unused_columns, + pack_to_max_length=pack_to_max_length) + alpaca_zh = alpaca_zh_dataset( + tokenizer, + path=path_zh, + max_length=max_length, + shuffle_before_pack=True, + remove_unused_columns=remove_unused_columns, + pack_to_max_length=pack_to_max_length) + dataset = ConcatDataset([alpaca, alpaca_zh]) + return dataset + + +def alpaca_enzh_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) + + +def alpaca_zh_dataset(tokenizer, + path='silk-road/alpaca-data-gpt4-chinese', max_length=2048, + remove_unused_columns=True, pack_to_max_length=True): - if path is None: - path = 'tatsu-lab/alpaca' - ds = alpaca_dataset( - tokenizer, - path=path, + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.alpaca) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, max_length=max_length, + dataset_map_fn=alpaca_zh_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl + + return dataset + + +def alpaca_zh_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) def alpaca_dataset(tokenizer, - path=None, + path='tatsu-lab/alpaca', max_length=2048, + remove_unused_columns=True, pack_to_max_length=True): - if path is None: - path = 'tatsu-lab/alpaca' - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict(type=load_dataset, path=path), + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.alpaca) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, tokenizer=tokenizer, max_length=max_length, - map_fn=alpaca_map_fn, - remove_columns=['instruction', 'text'], + dataset_map_fn=alpaca_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds + + return dataset def alpaca_data_collator(return_hf_format=False): diff --git a/xtuner/apis/datasets/alpaca_enzh.py b/xtuner/apis/datasets/alpaca_enzh.py deleted file mode 100644 index 5d13235c9..000000000 --- a/xtuner/apis/datasets/alpaca_enzh.py +++ /dev/null @@ -1,60 +0,0 @@ -from functools import partial - -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from torch.utils.data import ConcatDataset - -from xtuner.datasets.collate_fns import default_collate_fn -from .alpaca import alpaca_dataset -from .alpaca_zh import alpaca_zh_dataset - - -def alpaca_enzh_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path_en='tatsu-lab/alpaca', - path_zh='silk-road/alpaca-data-gpt4-chinese', - max_length=2048, - pack_to_max_length=True): - ds = alpaca_enzh_dataset( - tokenizer, - path_en=path_en, - path_zh=path_zh, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl - - -def alpaca_enzh_dataset(tokenizer, - path_en='tatsu-lab/alpaca', - path_zh='silk-road/alpaca-data-gpt4-chinese', - max_length=2048, - pack_to_max_length=True): - alpaca = alpaca_dataset( - tokenizer, - path=path_en, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - alpaca_zh = alpaca_zh_dataset( - tokenizer, - path=path_zh, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - ds = ConcatDataset([alpaca, alpaca_zh]) - return ds - - -def alpaca_enzh_data_collator(return_hf_format=False): - return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/alpaca_zh.py b/xtuner/apis/datasets/alpaca_zh.py deleted file mode 100644 index efdda5176..000000000 --- a/xtuner/apis/datasets/alpaca_zh.py +++ /dev/null @@ -1,62 +0,0 @@ -from functools import partial - -from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner - -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_zh_map_fn -from xtuner.registry import BUILDER - - -def alpaca_zh_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path=None, - max_length=2048, - pack_to_max_length=True): - if path is None: - path = 'silk-road/alpaca-data-gpt4-chinese' - ds = alpaca_zh_dataset( - tokenizer, - path=path, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl - - -def alpaca_zh_dataset(tokenizer, - path=None, - max_length=2048, - pack_to_max_length=True): - if path is None: - path = 'silk-road/alpaca-data-gpt4-chinese' - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict(type=load_dataset, path=path), - tokenizer=tokenizer, - max_length=max_length, - map_fn=alpaca_zh_map_fn, - remove_columns=[ - 'instruction', 'instruction_zh', 'input_zh', 'output_zh' - ], - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds - - -def alpaca_zh_data_collator(return_hf_format=False): - return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/arxiv.py b/xtuner/apis/datasets/arxiv.py index df010e6d3..2efea321b 100644 --- a/xtuner/apis/datasets/arxiv.py +++ b/xtuner/apis/datasets/arxiv.py @@ -1,66 +1,35 @@ from functools import partial from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn -from xtuner.registry import BUILDER - - -def arxiv_dataloader(tokenizer, - batch_size=1, - num_workers=0, - data_file=None, - max_length=2048, - pack_to_max_length=True): - ds = arxiv_dataset( - tokenizer, - data_file=data_file, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE def arxiv_dataset(tokenizer, data_file=None, max_length=2048, + remove_unused_columns=True, pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.title) # 1. Download data from https://kaggle.com/datasets/Cornell-University/arxiv # noqa: E501 # 2. Process data with `./tools/data_preprocess/arxiv.py` if data_file is None: data_file = './data/arxiv_postprocess_csAIcsCLcsCV_20200101.json' - - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict( - type=load_dataset, path='json', data_files=dict(train=data_file)), + dataset_org = load_dataset(path='json', data_files=dict(train=data_file)) + dataset = process_hf_dataset( + dataset=dataset_org, tokenizer=tokenizer, max_length=max_length, - map_fn=arxiv_map_fn, - remove_columns=[ - 'id', 'submitter', 'authors', 'title', 'comments', 'journal-ref', - 'doi', 'report-no', 'categories', 'license', 'abstract', - 'versions', 'update_date', 'authors_parsed' - ], + dataset_map_fn=arxiv_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds + return dataset def arxiv_data_collator(return_hf_format=False): diff --git a/xtuner/apis/datasets/code_alpaca.py b/xtuner/apis/datasets/code_alpaca.py new file mode 100644 index 000000000..e03a87fda --- /dev/null +++ b/xtuner/apis/datasets/code_alpaca.py @@ -0,0 +1,32 @@ +from functools import partial + +from datasets import load_dataset + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import code_alpaca_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE + + +def code_alpaca_dataset(tokenizer, + path='HuggingFaceH4/CodeAlpaca_20K', + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.coder) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=code_alpaca_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def code_alpaca_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/colorist.py b/xtuner/apis/datasets/colorist.py new file mode 100644 index 000000000..4050ec9d1 --- /dev/null +++ b/xtuner/apis/datasets/colorist.py @@ -0,0 +1,33 @@ +from functools import partial + +from datasets import load_dataset + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import colors_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE + + +def colorist_dataset(tokenizer, + path='burkelibbey/colors', + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory( + template=PROMPT_TEMPLATE.colorist) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=colors_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def colorist_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/lawyer.py b/xtuner/apis/datasets/lawyer.py new file mode 100644 index 000000000..dea7e8832 --- /dev/null +++ b/xtuner/apis/datasets/lawyer.py @@ -0,0 +1,91 @@ +from functools import partial + +from datasets import load_dataset +from torch.utils.data import ConcatDataset + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import (crime_kg_assitant_map_fn, + law_reference_map_fn, + template_map_fn_factory) +from xtuner.utils import PROMPT_TEMPLATE + + +def lawyer_dataset(tokenizer, + crime_data_file=None, + reference_data_file=None, + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + crime_dataset = lawyer_crime_dataset( + tokenizer, + data_file=crime_data_file, + max_length=max_length, + remove_unused_columns=remove_unused_columns, + pack_to_max_length=pack_to_max_length) + reference_dataset = lawyer_reference_dataset( + tokenizer, + data_file=reference_data_file, + max_length=max_length, + remove_unused_columns=remove_unused_columns, + pack_to_max_length=pack_to_max_length) + dataset = ConcatDataset([crime_dataset, reference_dataset]) + return dataset + + +def lawyer_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) + + +def lawyer_crime_dataset(tokenizer, + data_file=None, + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.lawyer) + # Download data from https://github.com/LiuHC0428/LAW-GPT # noqa: E501 + if data_file is None: + data_file = './data/law/CrimeKgAssitant清洗后_52k.json' + dataset_org = load_dataset(path='json', data_files=dict(train=data_file)) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=crime_kg_assitant_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def lawyer_crime_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) + + +def lawyer_reference_dataset(tokenizer, + data_file=None, + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.lawyer) + # Download data from https://github.com/LiuHC0428/LAW-GPT # noqa: E501 + if data_file is None: + data_file = './data/law/训练数据_带法律依据_92k.json' + dataset_org = load_dataset(path='json', data_files=dict(train=data_file)) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=law_reference_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def lawyer_reference_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/medical.py b/xtuner/apis/datasets/medical.py index 87d6aead8..3ce93a20e 100644 --- a/xtuner/apis/datasets/medical.py +++ b/xtuner/apis/datasets/medical.py @@ -1,61 +1,31 @@ from functools import partial from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import medical_map_fn -from xtuner.registry import BUILDER - - -def medical_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path=None, - data_config_name='finetune', - max_length=2048, - pack_to_max_length=True): - if path is None: - path = 'shibing624/medical' - ds = medical_dataset( - tokenizer, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import medical_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE def medical_dataset(tokenizer, - path=None, - data_config_name='finetune', + path='shibing624/medical', max_length=2048, + remove_unused_columns=False, pack_to_max_length=True): - if path is None: - path = 'shibing624/medical' - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict(type=load_dataset, path=path, name=data_config_name), + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.medical) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, tokenizer=tokenizer, max_length=max_length, - map_fn=medical_map_fn, + dataset_map_fn=medical_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds + return dataset def medical_data_collator(return_hf_format=False): diff --git a/xtuner/apis/datasets/moss_003_sft.py b/xtuner/apis/datasets/moss_003_sft.py new file mode 100644 index 000000000..999210965 --- /dev/null +++ b/xtuner/apis/datasets/moss_003_sft.py @@ -0,0 +1,71 @@ +from functools import partial + +from torch.utils.data import ConcatDataset + +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn + + +def moss_003_sft_dataset(tokenizer, + plugins_data_file=None, + no_plugins_data_file=None, + bot_name=None, + max_length=2048): + plugins = moss_003_sft_plugins_dataset( + tokenizer, + data_file=plugins_data_file, + bot_name=bot_name, + max_length=max_length) + no_plugins = moss_003_sft_no_plugins_dataset( + tokenizer, + data_file=no_plugins_data_file, + bot_name=bot_name, + max_length=max_length) + dataset = ConcatDataset([plugins, no_plugins]) + return dataset + + +def moss_003_sft_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) + + +def moss_003_sft_no_plugins_dataset(tokenizer, + data_file=None, + bot_name=None, + max_length=2048): + + # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data + if data_file is None: + data_file = './data/moss-003-sft-no-tools.jsonl' + dataset = MOSSSFTDataset( + data_file=data_file, + bot_name=bot_name, + tokenizer=tokenizer, + max_length=max_length) + + return dataset + + +def moss_003_sft_no_plugins_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) + + +def moss_003_sft_plugins_dataset(tokenizer, + data_file=None, + bot_name=None, + max_length=2048): + + # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data + if data_file is None: + data_file = './data/conversations_with_tools_with_inner_instruction_no_text2image_train_all_random_meta0.5_0.1_0.01_moss_0709.jsonl' # noqa: E501 + dataset = MOSSSFTDataset( + data_file=data_file, + bot_name=bot_name, + tokenizer=tokenizer, + max_length=max_length) + + return dataset + + +def moss_003_sft_plugins_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/moss_003_sft_all.py b/xtuner/apis/datasets/moss_003_sft_all.py deleted file mode 100644 index b69fff3d0..000000000 --- a/xtuner/apis/datasets/moss_003_sft_all.py +++ /dev/null @@ -1,57 +0,0 @@ -from functools import partial - -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from torch.utils.data import ConcatDataset - -from xtuner.datasets.collate_fns import default_collate_fn -from .moss_003_sft_no_plugins import moss_003_sft_no_plugins_dataset -from .moss_003_sft_plugins import moss_003_sft_plugins_dataset - - -def moss_003_sft_dataloader(tokenizer, - batch_size=1, - num_workers=0, - plugins_data_file=None, - no_plugins_data_file=None, - bot_name=None, - max_length=2048): - ds = moss_003_sft_dataset( - tokenizer, - plugins_data_file=plugins_data_file, - no_plugins_data_file=no_plugins_data_file, - bot_name=bot_name, - max_length=max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl - - -def moss_003_sft_dataset(tokenizer, - plugins_data_file=None, - no_plugins_data_file=None, - bot_name=None, - max_length=2048): - plugins = moss_003_sft_plugins_dataset( - tokenizer, - data_file=plugins_data_file, - bot_name=bot_name, - max_length=max_length) - no_plugins = moss_003_sft_no_plugins_dataset( - tokenizer, - data_file=no_plugins_data_file, - bot_name=bot_name, - max_length=max_length) - ds = ConcatDataset([plugins, no_plugins]) - return ds - - -def moss_003_sft_data_collator(return_hf_format=False): - return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/moss_003_sft_no_plugins.py b/xtuner/apis/datasets/moss_003_sft_no_plugins.py deleted file mode 100644 index 7957a0848..000000000 --- a/xtuner/apis/datasets/moss_003_sft_no_plugins.py +++ /dev/null @@ -1,54 +0,0 @@ -from functools import partial - -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner - -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.registry import BUILDER - - -def moss_003_sft_no_plugins_dataloader(tokenizer, - batch_size=1, - num_workers=0, - data_file=None, - bot_name=None, - max_length=2048): - ds = moss_003_sft_no_plugins_dataset( - tokenizer, - data_file=data_file, - bot_name=bot_name, - max_length=max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl - - -def moss_003_sft_no_plugins_dataset(tokenizer, - data_file=None, - bot_name=None, - max_length=2048): - - # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data - if data_file is None: - data_file = './data/moss-003-sft-no-tools.jsonl' - ds_cfg = dict( - type=MOSSSFTDataset, - data_file=data_file, - bot_name=bot_name, - tokenizer=tokenizer, - max_length=max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds - - -def moss_003_sft_no_plugins_data_collator(return_hf_format=False): - return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/moss_003_sft_plugins.py b/xtuner/apis/datasets/moss_003_sft_plugins.py deleted file mode 100644 index 07bb03d71..000000000 --- a/xtuner/apis/datasets/moss_003_sft_plugins.py +++ /dev/null @@ -1,54 +0,0 @@ -from functools import partial - -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner - -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.registry import BUILDER - - -def moss_003_sft_plugins_dataloader(tokenizer, - batch_size=1, - num_workers=0, - data_file=None, - bot_name=None, - max_length=2048): - ds = moss_003_sft_plugins_dataset( - tokenizer, - data_file=data_file, - bot_name=bot_name, - max_length=max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl - - -def moss_003_sft_plugins_dataset(tokenizer, - data_file=None, - bot_name=None, - max_length=2048): - - # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data - if data_file is None: - data_file = './data/conversations_with_tools_with_inner_instruction_no_text2image_train_all_random_meta0.5_0.1_0.01_moss_0709.jsonl' # noqa: E501 - ds_cfg = dict( - type=MOSSSFTDataset, - data_file=data_file, - bot_name=bot_name, - tokenizer=tokenizer, - max_length=max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds - - -def moss_003_sft_plugins_data_collator(return_hf_format=False): - return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/oasst1.py b/xtuner/apis/datasets/oasst1.py index 897645b8a..f29b02a8d 100644 --- a/xtuner/apis/datasets/oasst1.py +++ b/xtuner/apis/datasets/oasst1.py @@ -1,60 +1,32 @@ from functools import partial from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn -from xtuner.registry import BUILDER - - -def oasst1_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path=None, - max_length=2048, - pack_to_max_length=True): - if path is None: - path = 'timdettmers/openassistant-guanaco' - ds = oasst1_dataset( - tokenizer, - path=path, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE def oasst1_dataset(tokenizer, - path=None, + path='timdettmers/openassistant-guanaco', max_length=2048, + remove_unused_columns=False, pack_to_max_length=True): - if path is None: - path = 'timdettmers/openassistant-guanaco' - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict(type=load_dataset, path=path), + template_map_fn = template_map_fn_factory( + template=PROMPT_TEMPLATE.openassistant) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, tokenizer=tokenizer, max_length=max_length, - map_fn=oasst1_map_fn, + dataset_map_fn=oasst1_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds + return dataset def oasst1_data_collator(return_hf_format=False): diff --git a/xtuner/apis/datasets/open_orca.py b/xtuner/apis/datasets/open_orca.py index 25b9acdaf..f840bc980 100644 --- a/xtuner/apis/datasets/open_orca.py +++ b/xtuner/apis/datasets/open_orca.py @@ -1,61 +1,31 @@ from functools import partial from datasets import load_dataset -from mmengine.config import Config -from mmengine.dataset import DefaultSampler -from mmengine.runner import Runner -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn -from xtuner.registry import BUILDER - - -def openorca_dataloader(tokenizer, - batch_size=1, - num_workers=0, - path=None, - max_length=2048, - pack_to_max_length=True): - if path is None: - path = 'Open-Orca/OpenOrca' - ds = openorca_dataset( - tokenizer, - path=path, - max_length=max_length, - shuffle_before_pack=True, - pack_to_max_length=pack_to_max_length) - dl_cfg = dict( - batch_size=batch_size, - num_workers=num_workers, - dataset=ds, - sampler=dict(type=DefaultSampler, shuffle=True), - collate_fn=dict(type=default_collate_fn)) - - dl_cfg = Config(dl_cfg) - dl = Runner.build_dataloader(dl_cfg) - return dl +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE def openorca_dataset(tokenizer, - path=None, + path='Open-Orca/OpenOrca', max_length=2048, + remove_unused_columns=True, pack_to_max_length=True): - if path is None: - path = 'Open-Orca/OpenOrca' - ds_cfg = dict( - type=process_hf_dataset, - dataset=dict(type=load_dataset, path=path), + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.alpaca) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, tokenizer=tokenizer, max_length=max_length, - map_fn=openorca_map_fn, - remove_columns=['id', 'system_prompt', 'question', 'response'], + dataset_map_fn=openorca_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, shuffle_before_pack=True, pack_to_max_length=pack_to_max_length) - ds_cfg = Config(ds_cfg) - ds = BUILDER.build(ds_cfg) - return ds + return dataset def openorca_data_collator(return_hf_format=False): diff --git a/xtuner/apis/datasets/sql.py b/xtuner/apis/datasets/sql.py new file mode 100644 index 000000000..6d50e5117 --- /dev/null +++ b/xtuner/apis/datasets/sql.py @@ -0,0 +1,32 @@ +from functools import partial + +from datasets import load_dataset + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import sql_map_fn, template_map_fn_factory +from xtuner.utils import PROMPT_TEMPLATE + + +def sql_dataset(tokenizer, + path='b-mc2/sql-create-context', + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.sql) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=sql_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def sql_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/datasets/tiny_codes.py b/xtuner/apis/datasets/tiny_codes.py new file mode 100644 index 000000000..d07f59809 --- /dev/null +++ b/xtuner/apis/datasets/tiny_codes.py @@ -0,0 +1,32 @@ +from functools import partial + +from datasets import load_dataset + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import template_map_fn_factory, tiny_codes_map_fn +from xtuner.utils import PROMPT_TEMPLATE + + +def tiny_codes_dataset(tokenizer, + path='nampdn-ai/tiny-codes', + max_length=2048, + remove_unused_columns=True, + pack_to_max_length=True): + template_map_fn = template_map_fn_factory(template=PROMPT_TEMPLATE.coder) + dataset_org = load_dataset(path) + dataset = process_hf_dataset( + dataset=dataset_org, + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=tiny_codes_map_fn, + template_map_fn=template_map_fn, + remove_unused_columns=remove_unused_columns, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length) + + return dataset + + +def tiny_codes_data_collator(return_hf_format=False): + return partial(default_collate_fn, return_hf_format=return_hf_format) diff --git a/xtuner/apis/model.py b/xtuner/apis/model.py new file mode 100644 index 000000000..e998f851e --- /dev/null +++ b/xtuner/apis/model.py @@ -0,0 +1,82 @@ +import torch +from peft import LoraConfig +from transformers import (AutoModelForCausalLM, AutoTokenizer, + BitsAndBytesConfig) + +from xtuner.model import SupervisedFinetune + +__all__ = ['build_model', 'build_lora_model', 'build_qlora_model'] + + +def build_qlora_model(model_name_or_path, + quantization_config=None, + lora_config=None, + return_tokenizer=True): + + if quantization_config is None: + quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + load_in_8bit=False, + llm_int8_threshold=6.0, + llm_int8_has_fp16_weight=False, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4') + if lora_config is None: + lora_config = LoraConfig( + r=64, + lora_alpha=16, + lora_dropout=0.1, + bias='none', + task_type='CAUSAL_LM') + + llm = AutoModelForCausalLM.from_pretrained( + model_name_or_path, + torch_dtype=torch.float16, + trust_remote_code=True, + quantization_config=quantization_config) + + model = SupervisedFinetune(llm, lora=lora_config) + + if return_tokenizer: + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, trust_remote_code=True) + return model.llm, tokenizer + else: + return model.llm + + +def build_lora_model(model_name_or_path, + lora_config=None, + return_tokenizer=True): + if lora_config is None: + lora_config = LoraConfig( + r=64, + lora_alpha=16, + lora_dropout=0.1, + bias='none', + task_type='CAUSAL_LM') + + llm = AutoModelForCausalLM.from_pretrained( + model_name_or_path, torch_dtype=torch.float16, trust_remote_code=True) + + model = SupervisedFinetune(llm, lora=lora_config) + + if return_tokenizer: + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, trust_remote_code=True) + return model.llm, tokenizer + else: + return model.llm + + +def build_model(model_name_or_path, return_tokenizer=True): + model = AutoModelForCausalLM.from_pretrained( + model_name_or_path, torch_dtype=torch.float16, trust_remote_code=True) + + if return_tokenizer: + tokenizer = AutoTokenizer.from_pretrained( + model_name_or_path, trust_remote_code=True) + return model, tokenizer + else: + return model diff --git a/xtuner/apis/models/__init__.py b/xtuner/apis/models/__init__.py deleted file mode 100644 index 300fd29e6..000000000 --- a/xtuner/apis/models/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .baichuan import (baichuan_7b_qlora, baichuan_13b_base_qlora, - baichuan_13b_chat_qlora) -from .internlm import internlm_7b_qlora, internlm_chat_7b_qlora -from .llama import llama2_7b_chat_qlora, llama2_7b_qlora, llama_7b_qlora -from .qwen import qwen_7b_chat_qlora, qwen_7b_qlora - -__all__ = [ - 'llama_7b_qlora', 'llama2_7b_qlora', 'llama2_7b_chat_qlora', - 'internlm_7b_qlora', 'internlm_chat_7b_qlora', 'baichuan_7b_qlora', - 'baichuan_13b_base_qlora', 'baichuan_13b_chat_qlora', 'qwen_7b_qlora', - 'qwen_7b_chat_qlora' -] diff --git a/xtuner/apis/models/baichuan.py b/xtuner/apis/models/baichuan.py deleted file mode 100644 index f07dbf077..000000000 --- a/xtuner/apis/models/baichuan.py +++ /dev/null @@ -1,74 +0,0 @@ -from mmengine.config import Config - -from xtuner.registry import BUILDER -from .base import model_qlora as model_qlora_cfg_dict -from .base import tokenizer as tokenizer_cfg_dict - - -def baichuan_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'baichuan-inc/Baichuan-7B' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def baichuan_13b_base_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'baichuan-inc/Baichuan-13B-Base' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def baichuan_13b_chat_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'baichuan-inc/Baichuan-13B-Chat' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm diff --git a/xtuner/apis/models/base.py b/xtuner/apis/models/base.py deleted file mode 100644 index d701c9b42..000000000 --- a/xtuner/apis/models/base.py +++ /dev/null @@ -1,36 +0,0 @@ -import torch -from peft import LoraConfig -from transformers import (AutoModelForCausalLM, AutoTokenizer, - BitsAndBytesConfig) - -from xtuner.models import SupervisedFinetune - -tokenizer = dict( - type=AutoTokenizer.from_pretrained, - pretrained_model_name_or_path=None, - padding_side='right', - trust_remote_code=True) - -model_qlora = dict( - type=SupervisedFinetune, - llm=dict( - type=AutoModelForCausalLM.from_pretrained, - pretrained_model_name_or_path=None, - torch_dtype=torch.float16, - trust_remote_code=True, - quantization_config=dict( - type=BitsAndBytesConfig, - load_in_4bit=True, - load_in_8bit=False, - llm_int8_threshold=6.0, - llm_int8_has_fp16_weight=False, - bnb_4bit_compute_dtype=torch.float16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type='nf4')), - lora=dict( - type=LoraConfig, - r=64, - lora_alpha=16, - lora_dropout=0.1, - bias='none', - task_type='CAUSAL_LM')) diff --git a/xtuner/apis/models/internlm.py b/xtuner/apis/models/internlm.py deleted file mode 100644 index 7eef92916..000000000 --- a/xtuner/apis/models/internlm.py +++ /dev/null @@ -1,51 +0,0 @@ -from mmengine.config import Config - -from xtuner.registry import BUILDER -from .base import model_qlora as model_qlora_cfg_dict -from .base import tokenizer as tokenizer_cfg_dict - - -def internlm_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'internlm/internlm-7b' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def internlm_chat_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'internlm/internlm-chat-7b' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm diff --git a/xtuner/apis/models/llama.py b/xtuner/apis/models/llama.py deleted file mode 100644 index a3de77fd3..000000000 --- a/xtuner/apis/models/llama.py +++ /dev/null @@ -1,74 +0,0 @@ -from mmengine.config import Config - -from xtuner.registry import BUILDER -from .base import model_qlora as model_qlora_cfg_dict -from .base import tokenizer as tokenizer_cfg_dict - - -def llama_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'huggyllama/llama-7b' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def llama2_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'meta-llama/Llama-2-7b-hf' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def llama2_7b_chat_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'meta-llama/Llama-2-7b-chat-hf' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm diff --git a/xtuner/apis/models/qwen.py b/xtuner/apis/models/qwen.py deleted file mode 100644 index 9a55b4091..000000000 --- a/xtuner/apis/models/qwen.py +++ /dev/null @@ -1,51 +0,0 @@ -from mmengine.config import Config - -from xtuner.registry import BUILDER -from .base import model_qlora as model_qlora_cfg_dict -from .base import tokenizer as tokenizer_cfg_dict - - -def qwen_7b_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'Qwen/Qwen-7B' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm - - -def qwen_7b_chat_qlora(model_name_or_path=None, - quantization_config=None, - lora_config=None, - return_tokenizer=True): - if model_name_or_path is None: - model_name_or_path = 'Qwen/Qwen-7B-Chat' - model_cfg = Config(model_qlora_cfg_dict) - model_cfg.llm.pretrained_model_name_or_path = model_name_or_path - if quantization_config: - model_cfg.llm.quantization_config = quantization_config - if lora_config: - model_cfg.lora = lora_config - - model = BUILDER.build(model_cfg) - if return_tokenizer: - tokenizer_cfg = Config(tokenizer_cfg_dict) - tokenizer_cfg.pretrained_model_name_or_path = model_name_or_path - tokenizer = BUILDER.build(tokenizer_cfg) - return model.llm, tokenizer - else: - return model.llm diff --git a/xtuner/apis/training_args/default.py b/xtuner/apis/training_args.py similarity index 83% rename from xtuner/apis/training_args/default.py rename to xtuner/apis/training_args.py index 24081b848..c28e46741 100644 --- a/xtuner/apis/training_args/default.py +++ b/xtuner/apis/training_args.py @@ -4,6 +4,8 @@ from transformers import TrainingArguments from transformers.trainer_utils import IntervalStrategy, SchedulerType +__all__ = ['DefaultTrainingArguments'] + @dataclass class DefaultTrainingArguments(TrainingArguments): @@ -19,13 +21,12 @@ class DefaultTrainingArguments(TrainingArguments): # huggingface default_output_dir = './work_dirs' - default_bf16 = True default_do_train = True default_per_device_train_batch_size = 1 default_learning_rate = 2e-5 default_save_strategy = 'epoch' default_lr_scheduler_type = 'cosine' - default_logging_steps = 1 + default_logging_steps = 5 output_dir: str = field( default=default_output_dir, @@ -33,15 +34,6 @@ class DefaultTrainingArguments(TrainingArguments): 'help': ('The output directory where the model predictions and ' 'checkpoints will be written.') }) - bf16: bool = field( - default=default_bf16, - metadata={ - 'help': - ('Whether to use bf16 (mixed) precision instead of 32-bit. ' - 'Requires Ampere or higher NVIDIA architecture or using ' - 'CPU (no_cuda). This is an experimental API and it may ' - 'change.') - }) do_train: bool = field( default=default_do_train, metadata={'help': 'Whether to run training.'}) diff --git a/xtuner/apis/training_args/__init__.py b/xtuner/apis/training_args/__init__.py deleted file mode 100644 index b9e974b55..000000000 --- a/xtuner/apis/training_args/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .default import DefaultTrainingArguments - -__all__ = ['DefaultTrainingArguments'] diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_alpaca_e3.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_alpaca_e3.py index ec446a3f4..4f63b2bbc 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_alpaca_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_arxiv_e3.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_arxiv_e3.py index b374f98bc..27278ac79 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_arxiv_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e1.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e1.py index 487f24173..57c6d9668 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py index 3230f467a..d8f36f124 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_plugins_e1.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_plugins_e1.py index 35771c8e6..8ac302788 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_oasst1_e3.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_oasst1_e3.py index 2739e6c91..0eccc44a6 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_oasst1_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_openorca_e1.py b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_openorca_e1.py index 90d91489a..7841c73f0 100644 --- a/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_openorca_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_base/baichuan_13b_base_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_alpaca_e3.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_alpaca_e3.py index c60614e93..8b7d85dd6 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_alpaca_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_arxiv_e3.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_arxiv_e3.py index c3f892f55..e17dcd806 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_arxiv_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e1.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e1.py index 354fb217c..5910c9922 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e2_gpu8.py index 214f93c3e..eaabbc105 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_plugins_e1.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_plugins_e1.py index 5bfa868a5..39eb0d7a0 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_oasst1_e3.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_oasst1_e3.py index 9aa051e8c..b6e8661f1 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_oasst1_e3.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_openorca_e1.py b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_openorca_e1.py index 126151e55..e164f3894 100644 --- a/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_openorca_e1.py +++ b/xtuner/configs/baichuan/baichuan_13b_chat/baichuan_13b_chat_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_alpaca_e3.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_alpaca_e3.py index 6e7665beb..7208fa3cb 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_arxiv_e3.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_arxiv_e3.py index c77a90cdf..c9403e69e 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e1.py index 7db314f94..bf13b6241 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e2_gpu8.py index c6d442d46..5c5d00887 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_plugins_e1.py index a4b63b88d..a101eed51 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_oasst1_e3.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_oasst1_e3.py index 5a9b6f10e..5d63ce6c2 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_openorca_e1.py b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_openorca_e1.py index 07ad8521d..4e0a0c329 100644 --- a/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_openorca_e1.py +++ b/xtuner/configs/baichuan/baichuan_7b/baichuan_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/deepspeed/deepspeed_zero3.json b/xtuner/configs/deepspeed/deepspeed_zero3.json index 07d127c25..c7a0e802b 100644 --- a/xtuner/configs/deepspeed/deepspeed_zero3.json +++ b/xtuner/configs/deepspeed/deepspeed_zero3.json @@ -13,6 +13,7 @@ "reduce_scatter": true, "stage3_gather_16bit_weights_on_model_save": true }, + "low_cpu_mem_usage": false, "fp16": { "enabled": true, "initial_scale_power": 16 diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_alpaca_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_alpaca_e3.py index 2a9a7196c..2fb0556ec 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_arxiv_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_arxiv_e3.py index 4084ba774..bcf04073f 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_code_alpaca_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_code_alpaca_e3.py index 82c697ce6..fe7040247 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_code_alpaca_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_code_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import code_alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import code_alpaca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e1.py index 42ad908b7..885a2a0c8 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e2_gpu8.py index efa717a0c..5d7a40a9e 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_plugins_e1.py index 54d85b1ca..81047cfa7 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_oasst1_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_oasst1_e3.py index 80ac9b8cc..127c72a43 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_open_platypus_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_open_platypus_e3.py index 63bf7cbc6..a43724b01 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_open_platypus_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_open_platypus_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_openorca_e1.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_openorca_e1.py index 897e58120..097308036 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_openorca_e1.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_sql_e3.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_sql_e3.py index cc540f574..6f387ca8b 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_sql_e3.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_sql_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import sql_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import sql_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_tiny_codes_e1.py b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_tiny_codes_e1.py index 8613b4e7a..8ebd032a6 100644 --- a/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_tiny_codes_e1.py +++ b/xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_tiny_codes_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import tiny_codes_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import tiny_codes_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_alpaca_e3.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_alpaca_e3.py index 4e722b5b5..d195cd41e 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_arxiv_e3.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_arxiv_e3.py index bb291bfd3..9418a0449 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e1.py index 33b623140..02347ef73 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e2_gpu8.py index 2ceca583b..3504eb4c7 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_plugins_e1.py index c360f3bd7..45cec54a0 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_oasst1_e3.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_oasst1_e3.py index 9dd8559fb..0d9a6fd68 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_openorca_e1.py b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_openorca_e1.py index 4f4d03503..d4ea4d7b8 100644 --- a/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_openorca_e1.py +++ b/xtuner/configs/internlm/internlm_chat_7b/internlm_chat_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1.py b/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1.py index a196eb044..337228c3b 100644 --- a/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1.py +++ b/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1.py @@ -9,11 +9,11 @@ from peft import LoraConfig from transformers import AutoModelForCausalLM, AutoTokenizer -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1_hf.py b/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1_hf.py index a3fc44bd0..c0f48cb3d 100644 --- a/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1_hf.py +++ b/xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1_hf.py @@ -4,8 +4,8 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.map_fns import alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.map_fns import alpaca_map_fn framework = 'huggingface' pretrained_model_name_or_path = 'meta-llama/Llama-2-70b-hf' diff --git a/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1.py b/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1.py index 8a572f763..8b24fdf8b 100644 --- a/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1.py +++ b/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1.py @@ -10,11 +10,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1_hf.py b/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1_hf.py index 8ac807bbb..ae047ac37 100644 --- a/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1_hf.py +++ b/xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1_hf.py @@ -4,8 +4,8 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, Trainer, TrainingArguments) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.map_fns import alpaca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.map_fns import alpaca_map_fn framework = 'huggingface' pretrained_model_name_or_path = 'meta-llama/Llama-2-70b-hf' diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_alpaca_e3.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_alpaca_e3.py index 4357848fc..e1f76268c 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_arxiv_e3.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_arxiv_e3.py index 935db4a4a..e8a1869a2 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e1.py index b5dcd2fb3..d20fc0783 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e2_gpu8.py index 45ba3ddb4..3c02e97fd 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_plugins_e1.py index 953f37733..ae328f8c5 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_oasst1_e3.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_oasst1_e3.py index 3876e965e..e56f4657d 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_openorca_e1.py b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_openorca_e1.py index 1049f722e..ec6476d52 100644 --- a/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_openorca_e1.py +++ b/xtuner/configs/llama/llama2_7b/llama2_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_alpaca_e3.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_alpaca_e3.py index 016f1a394..42e177714 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_alpaca_e3.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_arxiv_e3.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_arxiv_e3.py index 19e401390..94d27ab91 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_arxiv_e3.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e1.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e1.py index cd2dd75f0..edd447062 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e2_gpu8.py index 1614b5a63..121e3bb64 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_plugins_e1.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_plugins_e1.py index 5b8a1ff0a..6ab11dcff 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_oasst1_e3.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_oasst1_e3.py index 5ee6aeb69..1b5179261 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_oasst1_e3.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_openorca_e1.py b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_openorca_e1.py index 9d1b20b9a..80c99f864 100644 --- a/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_openorca_e1.py +++ b/xtuner/configs/llama/llama2_7b_chat/llama2_7b_chat_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_alpaca_e3.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_alpaca_e3.py index 738791d13..5a7883c66 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_alpaca_e3.py @@ -9,12 +9,12 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import (alpaca_map_fn, alpaca_zh_map_fn, - template_map_fn_factory) +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import (alpaca_map_fn, alpaca_zh_map_fn, + template_map_fn_factory) from xtuner.engine import DatasetInfoHook, EvaluateChatHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_arxiv_e3.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_arxiv_e3.py index 47ff133f6..5ec9f4334 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e1.py index d9f70c44b..62157f560 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e2_gpu8.py index ac847e4db..4ad05c2e1 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_plugins_e1.py index 9b842acc9..ffbe5002d 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_oasst1_e3.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_oasst1_e3.py index 639171e69..25d88a3da 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn, template_map_fn_factory +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn, template_map_fn_factory from xtuner.engine import DatasetInfoHook, EvaluateChatHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/llama/llama_7b/llama_7b_qlora_openorca_e1.py b/xtuner/configs/llama/llama_7b/llama_7b_qlora_openorca_e1.py index d7e895d82..70134cead 100644 --- a/xtuner/configs/llama/llama_7b/llama_7b_qlora_openorca_e1.py +++ b/xtuner/configs/llama/llama_7b/llama_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py index 659dd005c..1408a36e7 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_e3.py index e21e6cee1..5513237b0 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_e3.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py index 8849c8de7..3d77a939f 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py index df33bc28b..467e75488 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py index c89d11e51..98d448061 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py index 75048de75..d60574cc1 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py index c99b1e9e9..277917ed5 100644 --- a/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py +++ b/xtuner/configs/qwen/qwen_7b/qwen_7b_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_alpaca_e3.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_alpaca_e3.py index 3d3e561ff..f6128debd 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_alpaca_e3.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_alpaca_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import alpaca_map_fn, alpaca_zh_map_fn +from xtuner.dataset import ConcatDataset, process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, alpaca_zh_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_arxiv_e3.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_arxiv_e3.py index cf73b350d..410585904 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_arxiv_e3.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_arxiv_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import arxiv_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import arxiv_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e1.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e1.py index df898a5bd..fed197a92 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e1.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e2_gpu8.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e2_gpu8.py index 420d91712..5f25fd2da 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e2_gpu8.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_all_e2_gpu8.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import ConcatDataset, MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import ConcatDataset, MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_plugins_e1.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_plugins_e1.py index c0d1b45e3..8727c06c9 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_plugins_e1.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_moss_sft_plugins_e1.py @@ -8,10 +8,10 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import MOSSSFTDataset -from xtuner.datasets.collate_fns import default_collate_fn +from xtuner.dataset import MOSSSFTDataset +from xtuner.dataset.collate_fns import default_collate_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_oasst1_e3.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_oasst1_e3.py index 7a4804d94..81e65dcad 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_oasst1_e3.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_oasst1_e3.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import oasst1_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import oasst1_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_openorca_e1.py b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_openorca_e1.py index a5e093db5..41b955ba8 100644 --- a/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_openorca_e1.py +++ b/xtuner/configs/qwen/qwen_7b_chat/qwen_7b_chat_qlora_openorca_e1.py @@ -9,11 +9,11 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig) -from xtuner.datasets import process_hf_dataset -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.datasets.map_fns import openorca_map_fn +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import openorca_map_fn from xtuner.engine import LogSampleHook, SampleGenerateHook -from xtuner.models import SupervisedFinetune +from xtuner.model import SupervisedFinetune from xtuner.utils import PROMPT_TEMPLATE ####################################################################### diff --git a/xtuner/datasets/__init__.py b/xtuner/dataset/__init__.py similarity index 100% rename from xtuner/datasets/__init__.py rename to xtuner/dataset/__init__.py diff --git a/xtuner/datasets/collate_fns/__init__.py b/xtuner/dataset/collate_fns/__init__.py similarity index 100% rename from xtuner/datasets/collate_fns/__init__.py rename to xtuner/dataset/collate_fns/__init__.py diff --git a/xtuner/datasets/collate_fns/defalut_collate_fn.py b/xtuner/dataset/collate_fns/defalut_collate_fn.py similarity index 100% rename from xtuner/datasets/collate_fns/defalut_collate_fn.py rename to xtuner/dataset/collate_fns/defalut_collate_fn.py diff --git a/xtuner/datasets/collate_fns/mmlu_collate_fn.py b/xtuner/dataset/collate_fns/mmlu_collate_fn.py similarity index 100% rename from xtuner/datasets/collate_fns/mmlu_collate_fn.py rename to xtuner/dataset/collate_fns/mmlu_collate_fn.py diff --git a/xtuner/datasets/concat_dataset.py b/xtuner/dataset/concat_dataset.py similarity index 100% rename from xtuner/datasets/concat_dataset.py rename to xtuner/dataset/concat_dataset.py diff --git a/xtuner/datasets/huggingface.py b/xtuner/dataset/huggingface.py similarity index 94% rename from xtuner/datasets/huggingface.py rename to xtuner/dataset/huggingface.py index bf03bfc6f..132d762ab 100644 --- a/xtuner/datasets/huggingface.py +++ b/xtuner/dataset/huggingface.py @@ -53,9 +53,13 @@ def process_hf_dataset(dataset, it to True during training and False during testing. """ - dataset = BUILDER.build(dataset) if isinstance(dataset, DatasetDict): dataset = dataset[split] + elif isinstance(dataset, dict) or isinstance( + dataset, Config) or isinstance(dataset, ConfigDict): + dataset = BUILDER.build(dataset) + if isinstance(dataset, DatasetDict): + dataset = dataset[split] # sample `max_dataset_length` items from the original dataset to # save time consumed by map function diff --git a/xtuner/datasets/map_fns/__init__.py b/xtuner/dataset/map_fns/__init__.py similarity index 100% rename from xtuner/datasets/map_fns/__init__.py rename to xtuner/dataset/map_fns/__init__.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/__init__.py b/xtuner/dataset/map_fns/dataset_map_fns/__init__.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/__init__.py rename to xtuner/dataset/map_fns/dataset_map_fns/__init__.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/alpaca_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/alpaca_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/alpaca_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/alpaca_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/alpaca_zh_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/alpaca_zh_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/alpaca_zh_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/alpaca_zh_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/arxiv_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/arxiv_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/arxiv_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/arxiv_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/code_alpaca_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/code_alpaca_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/code_alpaca_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/code_alpaca_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/colors_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/colors_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/colors_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/colors_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/law_reference_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/law_reference_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/law_reference_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/law_reference_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/medical_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/medical_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/medical_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/medical_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/oasst1_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/oasst1_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/oasst1_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/oasst1_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/openai_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/openai_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/openorca_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/openorca_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/openorca_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/openorca_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/sql_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/sql_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/sql_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/sql_map_fn.py diff --git a/xtuner/datasets/map_fns/dataset_map_fns/tiny_codes_map_fn.py b/xtuner/dataset/map_fns/dataset_map_fns/tiny_codes_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/dataset_map_fns/tiny_codes_map_fn.py rename to xtuner/dataset/map_fns/dataset_map_fns/tiny_codes_map_fn.py diff --git a/xtuner/datasets/map_fns/template_map_fn.py b/xtuner/dataset/map_fns/template_map_fn.py similarity index 100% rename from xtuner/datasets/map_fns/template_map_fn.py rename to xtuner/dataset/map_fns/template_map_fn.py diff --git a/xtuner/datasets/moss_sft.py b/xtuner/dataset/moss_sft.py similarity index 100% rename from xtuner/datasets/moss_sft.py rename to xtuner/dataset/moss_sft.py diff --git a/xtuner/datasets/utils.py b/xtuner/dataset/utils.py similarity index 100% rename from xtuner/datasets/utils.py rename to xtuner/dataset/utils.py diff --git a/xtuner/models/__init__.py b/xtuner/model/__init__.py similarity index 100% rename from xtuner/models/__init__.py rename to xtuner/model/__init__.py diff --git a/xtuner/models/fast_forward/__init__.py b/xtuner/model/fast_forward/__init__.py similarity index 100% rename from xtuner/models/fast_forward/__init__.py rename to xtuner/model/fast_forward/__init__.py diff --git a/xtuner/models/fast_forward/internlm_attn.py b/xtuner/model/fast_forward/internlm_attn.py similarity index 100% rename from xtuner/models/fast_forward/internlm_attn.py rename to xtuner/model/fast_forward/internlm_attn.py diff --git a/xtuner/models/fast_forward/llama_attn.py b/xtuner/model/fast_forward/llama_attn.py similarity index 100% rename from xtuner/models/fast_forward/llama_attn.py rename to xtuner/model/fast_forward/llama_attn.py diff --git a/xtuner/models/sft.py b/xtuner/model/sft.py similarity index 100% rename from xtuner/models/sft.py rename to xtuner/model/sft.py diff --git a/xtuner/models/utils.py b/xtuner/model/utils.py similarity index 100% rename from xtuner/models/utils.py rename to xtuner/model/utils.py diff --git a/xtuner/tools/train.py b/xtuner/tools/train.py index df01057a7..88f51affc 100644 --- a/xtuner/tools/train.py +++ b/xtuner/tools/train.py @@ -14,10 +14,9 @@ from transformers import TrainingArguments from xtuner.configs import cfgs_name_path -from xtuner.datasets.collate_fns import default_collate_fn -from xtuner.models.fast_forward import dispatch_fast_forward -from xtuner.models.utils import (LoadWoInit, find_all_linear_names, - traverse_dict) +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.model.fast_forward import dispatch_fast_forward +from xtuner.model.utils import LoadWoInit, find_all_linear_names, traverse_dict from xtuner.registry import BUILDER