-
Notifications
You must be signed in to change notification settings - Fork 22
/
dpo.json
executable file
·142 lines (142 loc) · 4.33 KB
/
dpo.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
{
"train_dataset_settings": {
"sources": [
{
"name": "train_preferences",
"records_path": "train_preferences.jsonl",
"num_samples": 10
}
],
"chat_settings":{
"prompt_template": {
"role_tag_mapping": {
"bot": "assistant",
"user": "user",
"system": "system"
},
"prefix_template": "<|start_header_id|>{role}<|end_header_id|>\n\n",
"suffix_template": "<|eot_id|>"
},
"max_tokens_count": 2000
},
"add_labels": true,
"dataset_type": "pair_preferences"
},
"val_dataset_settings": {
"sources": [
{
"name": "val_preferences",
"records_path": "val_preferences.jsonl",
"num_samples": 10
}
],
"chat_settings":{
"prompt_template": {
"role_tag_mapping": {
"bot": "assistant",
"user": "user",
"system": "system"
},
"prefix_template": "<|start_header_id|>{role}<|end_header_id|>\n\n",
"suffix_template": "<|eot_id|>"
},
"max_tokens_count": 2000
},
"add_labels": true,
"dataset_type": "pair_preferences"
},
"cherry_pick_settings": {
"generator_transformers_settings": {
"num_beams": 1,
"do_sample": true,
"repetition_penalty": 1.02,
"stop_strings": "<|eot_id|>",
"max_new_tokens": 8
},
"custom_generation_settings": {
"skip_special_tokens": true
},
"dataset_settings": {
"sources": [
{
"name": "chat_dataset",
"records_path": "val_chat.jsonl",
"num_samples": 3
}
],
"prompt_template": {
"role_tag_mapping": {
"bot": "assistant",
"user": "user",
"system": "system"
},
"prefix_template": "<|start_header_id|>{role}<|end_header_id|>\n\n",
"suffix_template": "<|eot_id|>"
},
"dataset_type": "chat",
"max_tokens_count": 2000,
"only_answer_loss": true
},
"metric_settings": [
{
"type": "kl",
"parameters": {
"need_average": [true],
"ref_logits_type": "reference"
}
}
]
},
"model_settings": {
"model_path": "yujiepan/llama-3-tiny-random",
"model_type": "causal",
"transformers_settings": {}
},
"tokenizer_settings": {},
"special_tokens_settings": {
"bos_token": "<|begin_of_text|>",
"eos_token": "<|end_of_text|>"
},
"trainer_settings": {
"evaluation_strategy": "steps",
"per_device_train_batch_size": 1,
"per_device_eval_batch_size": 1,
"gradient_accumulation_steps": 2,
"load_best_model_at_end": false,
"eval_steps": 32,
"save_steps": 32,
"logging_steps": 1,
"learning_rate": 1e-6,
"num_train_epochs": 2,
"lr_scheduler_type": "cosine",
"warmup_ratio": 0.3,
"fp16": false,
"bf16": false,
"adam_beta1": 0.9,
"adam_beta2": 0.95,
"adam_epsilon": 1e-12,
"max_grad_norm": 2.0,
"weight_decay": 0.0,
"optim": "adamw_torch",
"save_total_limit": 16,
"dataloader_num_workers": 18,
"gradient_checkpointing": false,
"save_only_model": true,
"loss_settings": {
"loss_type": "sigmoid",
"beta": 0.01
},
"sync_ref_settings": {
"sync_ref_model": false
},
"use_sft_model": false,
"use_ref_model": true
},
"logging_settings": {
"project_name": "alignment",
"run_name": "dpo",
"entity": "turbo-alignment"
},
"seed": 0,
"log_path": "train_output"
}