代码:
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
from swift.llm import DatasetName, ModelType, SftArguments, sft_main
sft_args = SftArguments(
model_type=ModelType.qwen1half_4b_chat,
dataset=[DatasetName.ms_bench_mini],
train_dataset_sample=1000,
logging_steps=5,
max_length=2048,
learning_rate=5e-5,
warmup_ratio=0.4,
output_dir='output',
lora_target_modules=['ALL'],
self_cognition_sample=500,
model_name=['小黄', 'Xiao Huang'],
model_author=['魔搭', 'ModelScope'])
output = sft_main(sft_args)
best_model_checkpoint = output['best_model_checkpoint']
print(f'best_model_checkpoint: {best_model_checkpoint}')
报错:
Traceback (most recent call last):
File "r:\LLM\Project\qwen\Train.py", line 21, in
output = sft_main(sft_args)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\utils\run_utils.py", line 31, in x_main
result = llm_x(args, kwargs)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\llm\sft.py", line 134, in llm_sft
train_dataset, val_dataset = get_dataset(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\llm\utils\dataset.py", line 1510, in get_dataset
dataset = get_function(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\llm\utils\dataset.py", line 352, in get_dataset_from_repo
dataset = load_ms_dataset(dataset_id, subset_split_list)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\llm\utils\dataset.py", line 236, in load_ms_dataset
dataset = MsDataset.load(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\swift\llm\utils\utils.py", line 102, in _msdataset_ddp_load
dataset = _old_msdataset_load(*args, kwargs)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\ms_dataset.py", line 315, in load
dataset_inst = remote_dataloader_manager.load_dataset(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\data_loader\data_loader_manager.py", line 132, in load_dataset
oss_downloader.process()
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\data_loader\data_loader.py", line 83, in process
self._prepare_and_download()
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\data_loader\data_loader.py", line 147, in _prepare_and_download
self.dataset = self.data_files_manager.fetch_data_files(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\data_files\data_files_manager.py", line 116, in fetch_data_files
return builder.as_dataset()
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\download\dataset_builder.py", line 249, in as_dataset
return DatasetDict({
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\download\dataset_builder.py", line 250, in
k: self._convert_csv_to_dataset(k, v)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\modelscope\msdatasets\download\dataset_builder.py", line 218, in _convert_csv_to_dataset
df = pd.read_csv(
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\datasets\streaming.py", line 75, in wrapper
return function(args, download_config=download_config, kwargs)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\datasets\download\streaming_download_manager.py", line 784, in xpandas_read_csv
return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), kwargs)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\pandas\io\parsers\readers.py", line 948, in read_csv
return _read(filepath_or_buffer, kwds)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\pandas\io\parsers\readers.py", line 611, in _read
parser = TextFileReader(filepath_or_buffer, kwds)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\pandas\io\parsers\readers.py", line 1448, in init
self._engine = self._make_engine(f, self.engine)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\pandas\io\parsers\readers.py", line 1723, in _make_engine
return mapping[engine](f, self.options)
File "R:\Python\Anaconda3\envs\LLM\lib\site-packages\pandas\io\parsers\c_parser_wrapper.py", line 93, in init
self._reader = parsers.TextReader(src, *kwds)
File "parsers.pyx", line 579, in pandas._libs.parsers.TextReader.cinit
File "parsers.pyx", line 668, in pandas._libs.parsers.TextReader._get_header
File "parsers.pyx", line 879, in pandas._libs.parsers.TextReader._tokenize_rows
File "parsers.pyx", line 890, in pandas._libs.parsers.TextReader._check_tokenize_status
File "parsers.pyx", line 2050, in pandas._libs.parsers.raise_parser_error
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb5 in position 128: invalid start byte
在读取CSV文件时,指定正确的文件编码。例如,如果文件是使用latin1或ISO-8859-1编码的,你应该在read_csv函数中添加encoding='latin1'参数。
ModelScope旨在打造下一代开源的模型即服务共享平台,为泛AI开发者提供灵活、易用、低成本的一站式模型服务产品,让模型应用更简单!欢迎加入技术交流群:微信公众号:魔搭ModelScope社区,钉钉群号:44837352