peft_lora_wrapper_for_seq2seq_transformer_finetuning.py

1from transformers import AutoModelForSeq2SeqLM
2from peft import get_peft_config, get_peft_model, LoraConfig, TaskType
3
4model_name_or_path = "bigscience/mt0-large"
5tokenizer_name_or_path = "bigscience/mt0-large"
6
7peft_config = LoraConfig(
8    task_type=TaskType.SEQ_2_SEQ_LM, 
9    inference_mode=False, 
10    r=8, 
11    lora_alpha=32, 
12    lora_dropout=0.1
13)
14
15model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
16model = get_peft_model(model, peft_config)
17model.print_trainable_parameters()
18
19# output: trainable params: 2,359,296 || all params: 1,231,940,608 || trainable%: 0.19151053100118282