-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsave_hf_model_locally.py
39 lines (31 loc) · 1.99 KB
/
save_hf_model_locally.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#from transformers import AutoModelForSequenceClassification, BertTokenizer
#
#sourcename = "Rostlab/prot_t5_xxl_bfd"
#modelname = "prot_t5_xxl_bfd"
#outdir = "/scratch/gpfs/cmcwhite/hfmodels/" + modelname
#print(sourcename, modelname, outdir)
#
#tokenizer = BertTokenizer.from_pretrained(sourcename)
#tokenizer.save_pretrained(outdir)
#model = AutoModelForSequenceClassification.from_pretrained(sourcename)
#model.save_pretrained(outdir)
from transformers import T5Tokenizer, T5Model
tokenizer = T5Tokenizer.from_pretrained("Rostlab/prot_t5_xl_uniref50")
#model = T5Model.from_pretrained("Rostlab/prot_t5_xl_uniref50", cache_dir="/scratch/gpfs/cmcwhite/cache/")
tokenizer = T5Tokenizer.from_pretrained("Rostlab/prot_t5_xl_uniref50", cache_dir="/scratch/gpfs/cmcwhite/cache/")
tokenizer.save_pretrained('/scratch/gpfs/cmcwhite/prot_t5_xl_uniref50')
#model.save_pretrained('/scratch/gpfs/cmcwhite/prot_t5_xl_uniref50')
#from transformers import AutoTokenizer, AutoModelForTokenClassification
#tokenizer = AutoTokenizer.from_pretrained("Rostlab/prot_bert_bfd_ss3")
#tokenizer.save_pretrained('/scratch/gpfs/cmcwhite/hfmodels/prot_bert_bfd_ss3')
#model = AutoModelForTokenClassification.from_pretrained("Rostlab/prot_bert_bfd_ss3")
#model.save_pretrained("/scratch/gpfs/cmcwhite/hfmodels/prot_bert_bfd_ss3")
#tokenizer = AlbertTokenizer.from_pretrained("Rostlab/prot_t5_xxl_bfd")
#tokenizer.save_pretrained('/scratch/gpfs/cmcwhite/hfmodels/prot_t5_xxl_bfd')
#model = AutoModel.from_pretrained("Rostlab/prot_t5_xxl_bfd")
#model.save_pretrained('/scratch/gpfs/cmcwhite/hfmodels/prot_t5_xxl_bfd')
##from transformers import AutoModelForSequenceClassification, BertTokenizer
#tokenizer = BertTokenizer.from_pretrained('Rostlab/prot_t5_xxl_bfd')
#tokenizer.save_pretrained('/scratch/gpfs/cmcwhite/hfmodels/prot_t5_xxl_bfd')
#model = AutoModelForSequenceClassification.from_pretrained('Rostlab/prot_t5_xxl_bfd')
##model.save_pretrained('/scratch/gpfs/cmcwhite/hfmodels/prot_t5_xxl_bfd')