modelscope-funasr这个微调uniasr闽南语版本问题怎么解决？
modelscope-funasr这个微调uniasr闽南语版本问题怎么解决？ File "finetune.py", line 60, in
modelscope_finetune(params)
File "finetune.py", line 48, in modelscope_finetune
trainer.train()
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/modelscope/trainers/audio/asr_trainer.py", line 168, in train
self.trainer.run()
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/tasks/abs_task.py", line 1147, in run
cls.main_worker(args)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/tasks/abs_task.py", line 1445, in main_worker
cls.trainer.run(
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/train/trainer.py", line 287, in run
all_steps_are_invalid, max_update_stop = cls.train_one_epoch(
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/train/trainer.py", line 612, in train_one_epoch
retval = model(batch)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, kwargs)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/models/e2e_uni_asr.py", line 364, in forward
encoder_out, encoder_out_lens = self.encode2(encoder_out, encoder_out_lens, speech_raw, speech_lengths, ind=ind)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/models/e2e_uni_asr.py", line 562, in encode2
speech, speech_lengths = self.stride_conv(encoder_out, encoder_out_lens)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(args, *kwargs)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/modules/subsampling.py", line 361, in forward
x = F.relu(self.conv(x))
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/torch/nn/functional.py", line 1457, in relu
result = torch.relu(input)
(Triggered internally at /opt/conda/conda-bld/pytorch_1682343998658/work/torch/csrc/autograd/python_anomaly_mode.cpp:114.)
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
Traceback (most recent call last):
File "finetune.py", line 60, in
modelscope_finetune(params)
File "finetune.py", line 48, in modelscope_finetune
trainer.train()
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/modelscope/trainers/audio/asr_trainer.py", line 168, in train
self.trainer.run()
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/tasks/abs_task.py", line 1147, in run
cls.main_worker(args)
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/tasks/abs_task.py", line 1445, in main_worker
cls.trainer.run(
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/train/trainer.py", line 287, in run
all_steps_are_invalid, max_update_stop = cls.train_one_epoch(
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/funasr/train/trainer.py", line 677, in train_one_epoch
loss.backward()
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/torch/_tensor.py", line 487, in backward
torch.autograd.backward(
File "/home/xueziqiang/miniconda3/envs/ass/lib/python3.8/site-packages/torch/autograd/init.py", line 200, in backward
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [14, 880, 72]], which is output 0 of ReluBackward0, is at version 1; expected version 0 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!
