import os
import tempfile
from modelscope.msdatasets import MsDataset
from modelscope.metainfo import Trainers
from modelscope.trainers import build_trainer

通过data_files参数传入list来批量加载

my_csv_1 = 'results/1.csv'
my_csv_2 = 'results/sam.csv'

训练数据的输入出均为文本，需要将数据集预处理为输入为 src_txt，输出为 tgt_txt 的格式：

train_dataset = MsDataset.load('csv', data_files=[my_csv_1]).remap_columns(
{'input_text': 'src_txt', 'category': 'tgt_txt'})
eval_dataset = MsDataset.load('csv', data_files=[my_csv_2]).remap_columns(
{'input_text': 'src_txt', 'category': 'tgt_txt'})

num_warmup_steps = 500

def noam_lambda(current_step: int):
current_step += 1
return min(current_step (-0.5), current_step * num_warmup_steps (-1.5))

可以在代码修改 configuration 的配置

def cfg_modify_fn(cfg):
cfg.train.lr_scheduler = {
'type': 'LambdaLR',
'lr_lambda': noam_lambda,
'options': {
'by_epoch': False
}
}
cfg.train.optimizer = {
"type": "AdamW",
"lr": 1e-3,
"options": {}
}
cfg.train.max_epochs = 1
cfg.train.dataloader = {
"batch_size_per_gpu": 1, # 减小 batch_size
"workers_per_gpu": 1
}
cfg.fp16 = {
'enabled': True # 启用混合精度训练
}
return cfg

if name == 'main':

# 指定模型导出路径
export_dir = 'F:/study/graduationProject/openingReport/exported_model'

kwargs = dict(
    model='F:/study/graduationProject/openingReport/nlp_mt5_zero-shot-augment_chinese-base',
    model_revision="master",
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    work_dir=tempfile.TemporaryDirectory().name,
    cfg_modify_fn=cfg_modify_fn
)
trainer = build_trainer(
    name=Trainers.text_generation_trainer, default_args=kwargs)
trainer.train()

# 导出模型到指定位置
if not os.path.exists(export_dir):
    os.makedirs(export_dir)

# 确保 trainer.model 和 trainer.tokenizer 是正确的对象
if hasattr(trainer, 'model') and hasattr(trainer, 'tokenizer'):
    trainer.model.save_pretrained(export_dir)
    trainer.tokenizer.save_pretrained(export_dir)
    print(f"模型已导出到 {export_dir}")
else:
    print("无法找到模型或分词器对象进行保存")ModelScope这个模型怎么自定义路径？每次都保存在c盘