ModelScope代码：import tempfile

from modelscope.msdatasets import MsDataset
from modelscope.metainfo import Trainers
from modelscope.trainers import build_trainer

my_csv = 'results/1.csv'

train_ds = MsDataset.load('csv', data_files=[my_csv])

print(next(iter(train_ds)))

通过data_files参数传入list来批量加载

my_csv_1 = 'results/1.csv'
my_csv_2 = 'results/sam.csv'

ds = MsDataset.load('csv', data_files={'train': [my_csv_1], 'test': [my_csv_2]})

以此类推，其它类型（txt/json/jsonl）数据也可采用此方式加载

训练数据的输入出均为文本，需要将数据集预处理为输入为 src_txt，输出为 tgt_txt 的格式：

train_dataset = MsDataset.load('csv', data_files=[my_csv_1]).remap_columns({'input_text': 'src_txt', 'category': 'tgt_txt'})
eval_dataset = MsDataset.load('csv', data_files=[my_csv_2]).remap_columns({'input_text': 'src_txt', 'category': 'tgt_txt'})

num_warmup_steps = 500

def noam_lambda(current_step: int):
current_step += 1
return min(current_step(-0.5),
current_step * num_warmup_steps(-1.5))

可以在代码修改 configuration 的配置

def cfg_modify_fn(cfg):
cfg.train.lr_scheduler = {
'type': 'LambdaLR',
'lr_lambda': noam_lambda,
'options': {
'by_epoch': False
}
}
cfg.train.optimizer = {
"type": "AdamW",
"lr": 1e-3,
"options": {}
}
cfg.train.max_epochs = 3
cfg.train.dataloader = {
"batch_size_per_gpu": 8,
"workers_per_gpu": 1
}
return cfg

if name == 'main':
kwargs = dict(
model='F:/study/graduationProject/openingReport/nlp_mt5_zero-shot-augment_chinese-base',
model_revision="master",
train_dataset=train_dataset,
eval_dataset=eval_dataset,
work_dir=tempfile.TemporaryDirectory().name,
cfg_modify_fn=cfg_modify_fn)
trainer = build_trainer(
name=Trainers.text_generation_trainer, default_args=kwargs)
trainer.train()
报错：RuntimeError: expected scalar type Long but found Int这个怎么处理呀？