三、模型训练
1.transforms定义
# 定义训练和验证时的transforms # API说明：https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/apis/transforms/transforms.md import paddlex as pdx from paddlex import transforms as T train_transforms = T.Compose([ T.MixupImage(mixup_epoch=250), T.RandomDistort(), T.RandomExpand(im_padding_value=[123.675, 116.28, 103.53]), T.RandomCrop(), T.RandomDistort(brightness_range=0.5, brightness_prob=0.5, contrast_range=0.5, contrast_prob=0.5, saturation_range=0.5, saturation_prob=0.5, hue_range=18, hue_prob=0.5, random_apply=True, count=4, shuffle_channel=False), T.RandomHorizontalFlip(), T.RandomVerticalFlip(), T.BatchRandomResize( target_sizes=[320, 352, 384, 416, 448, 480, 512, 544, 576, 608], interp='RANDOM'), T.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) eval_transforms = T.Compose([ T.Resize( 608, interp='CUBIC'), T.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ])
[08-21 14:22:29 MainThread @utils.py:79] WRN paddlepaddle version: 2.3.1. The dynamic graph version of PARL is under development, not fully tested and supported /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/parl/remote/communication.py:38: DeprecationWarning: 'pyarrow.default_serialization_context' is deprecated as of 2.0.0 and will be removed in a future version. Use pickle or the pyarrow IPC functionality instead. context = pyarrow.default_serialization_context() /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/matplotlib/__init__.py:107: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working from collections import MutableMapping /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/matplotlib/rcsetup.py:20: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working from collections import Iterable, Mapping /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/matplotlib/colors.py:53: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working from collections import Sized 2022-08-21 14:22:29,871-WARNING: type object 'QuantizationTransformPass' has no attribute '_supported_quantizable_op_type' 2022-08-21 14:22:29,873-WARNING: If you want to use training-aware and post-training quantization, please use Paddle >= 1.8.4 or develop version
2.数据集定义
# 定义训练和验证所用的数据集 # API说明：https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/apis/datasets.md train_dataset = pdx.datasets.VOCDetection( data_dir='data/round1/train/', file_list='data/round1/train/train_list.txt', label_list='data/round1/train/labels.txt', transforms=train_transforms, shuffle=True) eval_dataset = pdx.datasets.VOCDetection( data_dir='data/round1/train', file_list='data/round1/train/val_list.txt', label_list='data/round1/train/labels.txt', transforms=eval_transforms, shuffle=False)
2022-08-21 14:22:31 [INFO] Starting to read file list from dataset... 2022-08-21 14:22:35 [INFO] 3212 samples in file data/round1/train/train_list.txt, including 3212 positive samples and 0 negative samples. creating index... index created! 2022-08-21 14:22:35 [INFO] Starting to read file list from dataset... 2022-08-21 14:22:36 [INFO] 802 samples in file data/round1/train/val_list.txt, including 802 positive samples and 0 negative samples. creating index... index created!
3.模型定义
# 初始化模型，并进行训练 # 可使用VisualDL查看训练指标，参考https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/visualdl.md num_classes = len(train_dataset.labels) model = pdx.det.YOLOv3(num_classes=num_classes, backbone='DarkNet53')
W0821 14:22:36.573086 10280 gpu_resources.cc:61] Please NOTE: device: 0, GPU Compute Capability: 7.0, Driver API Version: 11.2, Runtime API Version: 10.1 W0821 14:22:36.577816 10280 gpu_resources.cc:91] device: 0, cuDNN Version: 7.6.
4.模型定义
主要是batch size 选择：
- batch size: 8， 对应显存 13421
- batch size:19.37， 对应显存 32000 选择16即可。
# API说明：https://github.com/PaddlePaddle/PaddleX/blob/develop//docs/apis/models/detection.md # 各参数介绍与调整说明：https://github.com/PaddlePaddle/PaddleX/blob/develop//docs/parameters.md model.train( num_epochs=270, train_dataset=train_dataset, train_batch_size=16, eval_dataset=eval_dataset, learning_rate=0.001 / 8, warmup_steps=1000, warmup_start_lr=0.0, save_interval_epochs=10, lr_decay_epochs=[216, 243], save_dir='output/yolov3_darknet53', use_vdl=True)
2022-08-21 14:22:36 [INFO] Downloading DarkNet53_pretrained.pdparams from https://paddledet.bj.bcebos.com/models/pretrained/DarkNet53_pretrained.pdparams 0%| | 291/158704 [00:00<00:56, 2800.01KB/s]100%|██████████| 158704/158704 [00:03<00:00, 50714.52KB/s] 2022-08-21 14:22:39 [INFO] Loading pretrained model from output/yolov3_darknet53/pretrain/DarkNet53_pretrained.pdparams 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv0.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv0.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv0.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv0.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv0.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv1.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv1.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv1.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv1.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv1.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv2.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv2.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv2.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv2.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv2.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv3.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv3.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv3.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv3.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.conv3.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.route.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.route.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.route.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.route.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.conv_module.route.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.tip.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.tip.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.tip.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.tip.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.0.tip.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.0.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.0.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.0.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.0.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.0.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv0.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv0.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv0.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv0.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv0.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv1.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv1.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv1.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv1.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv1.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv2.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv2.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv2.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv2.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv2.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv3.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv3.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv3.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv3.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.conv3.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.route.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.route.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.route.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.route.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.conv_module.route.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.tip.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.tip.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.tip.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.tip.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.1.tip.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.1.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.1.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.1.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.1.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_transition.1.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv0.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv0.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv0.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv0.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv0.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv1.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv1.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv1.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv1.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv1.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv2.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv2.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv2.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv2.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv2.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv3.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv3.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv3.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv3.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.conv3.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.route.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.route.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.route.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.route.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.conv_module.route.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.tip.conv.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.tip.batch_norm.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.tip.batch_norm.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.tip.batch_norm._mean is not in pretrained model 2022-08-21 14:22:40 [WARNING] neck.yolo_block.2.tip.batch_norm._variance is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.0.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.0.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.1.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.1.bias is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.2.weight is not in pretrained model 2022-08-21 14:22:40 [WARNING] yolo_head.yolo_output.2.bias is not in pretrained model 2022-08-21 14:22:40 [INFO] There are 260/366 variables loaded into YOLOv3. 2022-08-21 14:23:05 [INFO] [TRAIN] Epoch=1/270, Step=10/200, loss_xy=7.773049, loss_wh=8.631678, loss_obj=8424.009766, loss_cls=14.103580, loss=8454.517578, lr=0.000001, time_each_step=2.48s, eta=37:16:9 2022-08-21 14:23:23 [INFO] [TRAIN] Epoch=1/270, Step=20/200, loss_xy=6.570035, loss_wh=6.603613, loss_obj=578.750854, loss_cls=12.434158, loss=604.358643, lr=0.000002, time_each_step=1.81s, eta=27:12:17 2022-08-21 14:23:44 [INFO] [TRAIN] Epoch=1/270, Step=30/200, loss_xy=6.975683, loss_wh=7.730688, loss_obj=47.920227, loss_cls=14.573313, loss=77.199913, lr=0.000004, time_each_step=2.02s, eta=30:17:17 2022-08-21 14:24:06 [INFO] [TRAIN] Epoch=1/270, Step=40/200, loss_xy=6.988766, loss_wh=6.911714, loss_obj=39.952698, loss_cls=13.511222, loss=67.364403, lr=0.000005, time_each_step=2.24s, eta=33:32:9 2022-08-21 14:24:31 [INFO] [TRAIN] Epoch=1/270, Step=50/200, loss_xy=6.662724, loss_wh=7.118166, loss_obj=31.094318, loss_cls=13.199011, loss=58.074219, lr=0.000006, time_each_step=2.55s, eta=38:15:22 2022-08-21 14:24:49 [INFO] [TRAIN] Epoch=1/270, Step=60/200, loss_xy=5.612492, loss_wh=6.151673, loss_obj=23.123552, loss_cls=12.082365, loss=46.970085, lr=0.000007, time_each_step=1.76s, eta=26:26:55 2022-08-21 14:25:15 [INFO] [TRAIN] Epoch=1/270, Step=70/200, loss_xy=6.188484, loss_wh=7.226328, loss_obj=22.821095, loss_cls=11.426645, loss=47.662552, lr=0.000009, time_each_step=2.6s, eta=38:56:18 2022-08-21 14:25:39 [INFO] [TRAIN] Epoch=1/270, Step=80/200, loss_xy=7.388262, loss_wh=8.146477, loss_obj=25.573256, loss_cls=14.415861, loss=55.523857, lr=0.000010, time_each_step=2.42s, eta=36:12:28 2022-08-21 14:26:02 [INFO] [TRAIN] Epoch=1/270, Step=90/200, loss_xy=6.698905, loss_wh=6.328094, loss_obj=25.467766, loss_cls=13.299914, loss=51.794682, lr=0.000011, time_each_step=2.24s, eta=33:33:32 Exception in thread Thread-7: Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py", line 620, in _get_data data = self._data_queue.get(timeout=self._timeout) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/multiprocessing/queues.py", line 105, in get raise Empty _queue.Empty During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/threading.py", line 926, in _bootstrap_inner self.run() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/threading.py", line 870, in run self._target(*self._args, **self._kwargs) File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py", line 534, in _thread_loop batch = self._get_data() File "/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py", line 636, in _get_data "pids: {}".format(len(failed_workers), pids)) RuntimeError: DataLoader 2 workers exit unexpectedly, pids: 10395, 10396 --------------------------------------------------------------------------- SystemError Traceback (most recent call last) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py in __next__(self) 745 if _in_legacy_dygraph(): --> 746 data = self._reader.read_next_var_list() 747 data = _restore_batch(data, self._structure_infos.pop(0)) SystemError: (Fatal) Blocking queue is killed because the data reader raises an exception. [Hint: Expected killed_ != true, but received killed_:1 == true:1.] (at /paddle/paddle/fluid/operators/reader/blocking_queue.h:166)
During handling of the above exception, another exception occurred:
KeyboardInterrupt Traceback (most recent call last) /tmp/ipykernel_10280/328266389.py in <module> 12 lr_decay_epochs=[216, 243], 13 save_dir='output/yolov3_darknet53', ---> 14 use_vdl=True) /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/detector.py in train(self, num_epochs, train_dataset, train_batch_size, eval_dataset, optimizer, save_interval_epochs, log_interval_steps, save_dir, pretrain_weights, learning_rate, warmup_steps, warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric, use_ema, early_stop, early_stop_patience, use_vdl, resume_checkpoint) 332 early_stop=early_stop, 333 early_stop_patience=early_stop_patience, --> 334 use_vdl=use_vdl) 335 336 def quant_aware_train(self, /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddlex/cv/models/base.py in train_loop(self, num_epochs, train_dataset, train_batch_size, eval_dataset, save_interval_epochs, log_interval_steps, save_dir, ema, early_stop, early_stop_patience, use_vdl) 331 step_time_tic = time.time() 332 --> 333 for step, data in enumerate(self.train_data_loader()): 334 if nranks > 1: 335 outputs = self.run(ddp_net, data, mode='train') /opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/dataloader/dataloader_iter.py in __next__(self) 744 else: 745 if _in_legacy_dygraph(): --> 746 data = self._reader.read_next_var_list() 747 data = _restore_batch(data, self._structure_infos.pop(0)) 748 else: KeyboardInterrupt:
