# 先把paddleclas安装上再说
# 安装paddleclas以及相关三方包(好像studio自带的已经够用了，无需安装了)
!git clone https://gitee.com/paddlepaddle/PaddleClas.git -b release/2.2
# 我这里安装相关包时，花了30几分钟还有错误提示，不管他即可
#!pip install --upgrade -r PaddleClas/requirements.txt -i https://mirror.baidu.com/pypi/simple

Cloning into 'PaddleClas'...
remote: Enumerating objects: 538, done.[K
remote: Counting objects: 100% (538/538), done.[K
remote: Compressing objects: 100% (323/323), done.[K
remote: Total 15290 (delta 344), reused 349 (delta 210), pack-reused 14752[K
Receiving objects: 100% (15290/15290), 113.56 MiB | 8.20 MiB/s, done.
Resolving deltas: 100% (10236/10236), done.
Checking connectivity... done.

# 先导入库 （这个项目不一定都用得到，我习惯了先把这一堆搬过来）
from sklearn.utils import shuffle
import os
import pandas as pd
import numpy as np
from PIL import Image
import paddle
import paddle.nn as nn
from paddle.io import Dataset
import random

/opt/conda/envs/python35-paddle120-env/lib/python3.7/site-packages/paddle/fluid/layers/utils.py:26: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.
Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations
  def convert_to_list(value, n, name, dtype=np.int):

# 忽略（垃圾）警告信息
# 在python中运行代码经常会遇到的情况是——代码可以正常运行但是会提示警告，有时特别讨厌。
# 那么如何来控制警告输出呢？其实很简单，python通过调用warnings模块中定义的warn()函数来发出警告。我们可以通过警告过滤器进行控制是否发出警告消息。
import warnings
warnings.filterwarnings("ignore")

# 更改文件夹名字，规范一些
!mv 初赛_测试集 testdata
!mv 训练集 traindata

# -*- coding: utf-8 -*-
# 根据官方paddleclas的提示，我们需要把图像变为两个txt文件
# train_list.txt（训练集）
# val_list.txt（验证集）
# 先把路径搞定 比如：训练集/0/77398.jpg ,读取到并写入txt 
dirpath = "traindata"
# 先得到总的txt后续再进行划分，因为要划分出验证集，所以要先打乱，因为原本是有序的
def get_all_txt():
    all_list = []
    i = 0
    for root,dirs,files in os.walk(dirpath): # 分别代表根目录、文件夹、文件
        for file in files:
            i = i + 1 
            # 文件中每行格式： 图像相对路径      图像的label_id（注意：中间有空格）。              
            #                traindata/0/77398.jpg        0
            #                traindata/101/6259.jpg     101
            #all_list.append(os.path.join(root,file)+" "+dirs+"\n")
            imgpath = os.path.join(root,file)
            category_id = os.path.join(root,file).split("/")
            all_list.append(imgpath+" "+category_id[1]+"\n")
    allstr = ''.join(all_list)
    f = open('all_list.txt','w',encoding='utf-8')
    f.write(allstr)
    return all_list , i
all_list,all_lenth = get_all_txt()
print(all_lenth)

51273

# 先把数据打乱，然后按照比例划分数据集 数据有5万多，所以9:1即可
random.shuffle(all_list)
train_size = int(all_lenth * 0.85)
train_list = all_list[:train_size]
val_list = all_list[train_size:]
print(len(train_list))
print(len(val_list))

43582
7691

# 运行cell，生成txt 
train_txt = ''.join(train_list)
f_train = open('train_list.txt','w',encoding='utf-8')
f_train.write(train_txt)
f_train.close()
print("train_list.txt 生成成功！")
# 运行cell，生成txt
val_txt = ''.join(val_list)
f_val = open('val_list.txt','w',encoding='utf-8')
f_val.write(val_txt)
f_val.close()
print("val_list.txt 生成成功！")

train_list.txt 生成成功！
val_list.txt 生成成功！

# 将图片移动到paddleclas下面的数据集里面
# 至于为什么现在移动，也是我的一点小技巧，防止之前移动的话，生成的txt的路径是全路径，反而需要去掉路径的一部分
!mv traindata/ PaddleClas/dataset/
!mv testdata/ PaddleClas/dataset/

!mv all_list.txt PaddleClas/dataset/traindata
!mv train_list.txt PaddleClas/dataset/traindata
!mv val_list.txt PaddleClas/dataset/traindata

/home/aistudio/PaddleClas
dataset  hubconf.py   MANIFEST.in    README_ch.md  requirements.txt
deploy   __init__.py  paddleclas.py  README_en.md  setup.py
docs   LICENSE      ppcls      README.md     tools

# global configs
Global:
  checkpoints: null
  pretrained_model: null
  output_dir: ./output/
  device: gpu
  save_interval: 10
  eval_during_train: True
  eval_interval: 3
  epochs: 150
  print_batch_step: 10
  use_visualdl: True
  # used for static mode and model export
  image_shape: [3, 224, 224]
  save_inference_dir: ./inference
  # training model under @to_static
  to_static: False
# model architecture
Arch:
  name: ResNet50
  class_num: 137
# loss function config for traing/eval process
Loss:
  Train:
    - CELoss:
        weight: 1.0
  Eval:
    - CELoss:
        weight: 1.0
Optimizer:
  name: Momentum
  momentum: 0.9
  lr:
    name: Piecewise
    learning_rate: 0.015
    decay_epochs: [30, 60, 90]
    values: [0.1, 0.01, 0.001, 0.0001]
  regularizer:
    name: 'L2'
    coeff: 0.0005
# data loader for train and eval
DataLoader:
  Train:
    dataset:
      name: ImageNetDataset
      image_root: ./dataset/
      cls_label_path: ./dataset/traindata/train_list.txt
      transform_ops:
        - DecodeImage:
            to_rgb: True
            channel_first: False
        - ResizeImage:
            resize_short: 256
        - CropImage:
            size: 224
        - RandFlipImage:
            flip_code: 1
        - NormalizeImage:
            scale: 1.0/255.0
            mean: [0.485, 0.456, 0.406]
            std: [0.229, 0.224, 0.225]
            order: ''
    sampler:
      name: DistributedBatchSampler
      batch_size: 128
      drop_last: False
      shuffle: True
    loader:
      num_workers: 0
      use_shared_memory: True
  Eval:
    dataset: 
      name: ImageNetDataset
      image_root: ./dataset/
      cls_label_path: ./dataset/traindata/val_list.txt
      transform_ops:
        - DecodeImage:
            to_rgb: True
            channel_first: False
        - ResizeImage:
            resize_short: 256
        - CropImage:
            size: 224
        - NormalizeImage:
            scale: 1.0/255.0
            mean: [0.485, 0.456, 0.406]
            std: [0.229, 0.224, 0.225]
            order: ''
    sampler:
      name: DistributedBatchSampler
      batch_size: 128
      drop_last: False
      shuffle: True
    loader:
      num_workers: 0
      use_shared_memory: True
Infer:
  infer_imgs: dataset/testdata/a237.jpg
  batch_size: 10
  transforms:
    - DecodeImage:
        to_rgb: True
        channel_first: False
    - ResizeImage:
        resize_short: 256
    - CropImage:
        size: 224
    - NormalizeImage:
        scale: 1.0/255.0
        mean: [0.485, 0.456, 0.406]
        std: [0.229, 0.224, 0.225]
        order: ''
    - ToCHWImage:
  PostProcess:
    name: Topk
    topk: 5
    class_id_map_file: dataset/label_list.txt
Metric:
  Train:
    - TopkAcc:
        topk: [1, 5]
  Eval:
    - TopkAcc:
        topk: [1, 5]

[Errno 2] No such file or directory: 'PaddleClas'
/home/aistudio/PaddleClas
dataset  hubconf.py   MANIFEST.in    ppcls     README.md       tools
deploy   __init__.py  output       README_ch.md  requirements.txt
docs   LICENSE      paddleclas.py  README_en.md  setup.py

# 模型评估
!python tools/eval.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50.yaml \
    -o Global.pretrained_model=./output/ResNet50/epoch_200

!python3 tools/infer.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50.yaml \
    -o Infer.infer_imgs=dataset/traindata/0/77398.jpg \
    -o Global.pretrained_model=output/ResNet50/best_model

!python3 tools/infer.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50.yaml \
    -o Global.pretrained_model=output/ResNet50/best_model \
    -o Infer.infer_imgs=dataset/testdata

None
6409
['dataset/testdata/a2411.jpg\n']

#找到问题图片之后将其删除即可
!rm dataset/testdata/a2411.jpg

# 再次预测，得到全部结果，除了那个坏图
# 这里，运行之前我先把yaml的预测结果topk改成了1，防止生成太多数据，毕竟只有第一个数据才是我们需要的值
!python3 tools/infer.py \
    -c ./ppcls/configs/ImageNet/ResNet/ResNet50.yaml \
    -o Global.pretrained_model=output/ResNet50/best_model \
    -o Infer.infer_imgs=dataset/testdata

# 将其复制一分出来，方便处理，看着也舒服
!cp output/ResNet50/infer.log output/ResNet50/infer.txt

import json
# 手动先把txt里面的如下之前全删了（包含这一行）再说
# [2021/07/30 16:15:03] root INFO: train with paddle 2.0.2 and device CUDAPlace(0)这行删了
# 这样下面处理就是统一的样式了
result = []
save_path = "output/ResNet50/submit_sample.csv"
f = open("output/ResNet50/infer.txt","r") 
lines = f.readlines()      #读取全部内容 ，并以列表方式返回
for line in lines:
    # 先把前面的时间都干掉,然后将单引号换成双引号，就变成了json格式的文本了
    txt = line.split(" root INFO: ")[1]
    print(txt)
    jsontxt = txt.replace("\'","\"")
    # print(jsontxt)
    list_txt = json.loads(jsontxt)
    for i in range(len(list_txt)):
        name, ids = list_txt[i]['file_name'],list_txt[i]['class_ids']
        onefilepath = name.split("/")[2]
        result.append([onefilepath,ids[0]])
df = pd.DataFrame(result, columns=['image_id', 'category_id'])
df.sort_values(by="image_id",inplace=True,ascending=True)  #千万记得排序！
df.to_csv(save_path, index=None)