1. 简单介绍
本文的应用场景是二分类问题，采用的数据集为猫狗分类数据集，为了减少训练时间，训练集图片有2123张，验证集有909 张图片，测试的图片有1000张，分为猫和狗两个类别，图片已经放置在dc_2000文件夹下面。
2 加载图片
2.1 导入相关包并加载图片
# 导入相关包 from google.colab import drive drive.mount('/content/gdrive') import os os.chdir("/content/gdrive/My Drive/Colab Notebooks/tensorflow") import tensorflow as tf print('Tensorflow version: {}'.format(tf.__version__)) from tensorflow import keras import matplotlib.pyplot as plt %matplotlib inline import numpy as np import pathlib #配置数据集路径 path_root = os.path.join(os.path.realpath("."),"DS","dc_2000") data_dir = pathlib.Path(path_root) #数量构成 train_image_count = len(list(data_dir.glob('train/*/*.jpg'))) test_image_count = len(list(data_dir.glob('test/*/*.jpg'))) CLASS_NAMES = np.array([item.name for item in data_dir.glob('train/*')]) print("训练集的数量:{}\n验证集的数量:{}\n数据集类别:{}".format(train_image_count,test_image_count,CLASS_NAMES)) #保存文件的路径,随机打乱 import random train_all_image_path = list(data_dir.glob("train/*/*")) test_all_image_path = list(data_dir.glob("test/*/*")) train_all_image_path = [str(path) for path in train_all_image_path] test_all_image_path = [str(path) for path in test_all_image_path] random.shuffle(train_all_image_path) random.shuffle(test_all_image_path)
训练集的数量:3032 验证集的数量:1000 数据集类别:['cat' 'dog']
2.2 读取图片
#确定每个图像的标签 lable_names = sorted(item.name for item in data_dir.glob("train/*/")) #为每个标签分配索引,构建字典 lable_to_index = dict((name,index) for index,name in enumerate(lable_names)) print(lable_to_index) #创建一个列表，包含每个文件的标签索引 train_image_label = [lable_to_index[pathlib.Path(path).parent.name] for path in train_all_image_path] test_image_label = [lable_to_index[pathlib.Path(path).parent.name] for path in test_all_image_path] #包装为函数，以备后用 def preprocess_image(image): image = tf.image.decode_jpeg(image, channels=3) image = tf.image.resize(image, [200, 200]) image /= 255.0 # normalize to [0,1] range return image #加载图片 def load_and_preprocess_image(path): image = tf.io.read_file(path) return preprocess_image(image) image_path = test_all_image_path[11] label = test_image_label[11] plt.imshow(load_and_preprocess_image(image_path)) plt.grid(False) ##plt.xlabel(caption_image(image_path)) plt.title(lable_names[label].title()) plt.axis("off") print()
{'cat': 0, 'dog': 1}
3. 图片预处理
在这一部分我们采用from_tensor_slices的方法对图片数据集进行构建，对比tf1.x版本采用队列形式读取数据，这一种方法比较简单切易于理解。并构建（图片，标签）对数据集
#构建一个tf.data.Dataset #一个图片数据集构建 tf.data.Dataset 最简单的方法就是使用 from_tensor_slices 方法。 #将字符串数组切片，得到一个字符串数据集： train_path_ds = tf.data.Dataset.from_tensor_slices(train_all_image_path) print(train_path_ds) test_path_ds = tf.data.Dataset.from_tensor_slices(test_all_image_path) #现在创建一个新的数据集，通过在路径数据集上映射 preprocess_image来动态加载和格式化图片。 AUTOTUNE = tf.data.experimental.AUTOTUNE train_image_ds = train_path_ds.map(load_and_preprocess_image,num_parallel_calls=AUTOTUNE) test_image_ds = test_path_ds.map(load_and_preprocess_image,num_parallel_calls=AUTOTUNE) train_lable_ds = tf.data.Dataset.from_tensor_slices(tf.cast(train_image_label,tf.int64)) test_lable_ds = tf.data.Dataset.from_tensor_slices(tf.cast(test_image_label,tf.int64)) for label in train_lable_ds.take(5): print(lable_names[label.numpy()]) #%%构建一个（图片，标签）对数据集 #因为这些数据集顺序相同，可以将他们打包起来 image_label_ds = tf.data.Dataset.zip((train_image_ds,train_lable_ds)) test_data = tf.data.Dataset.zip((test_image_ds,test_lable_ds)) print(test_data) #注意：当你拥有形似 all_image_labels 和 all_image_paths 的数组，tf.data.dataset.Dataset.zip 的替代方法是将这对数组切片 # =================================im============================================ # ds = tf.data.Dataset.from_tensor_slices((all_image_path,all_image_label)) # def load_and_preprocess_from_path_label(path, label): # return load_and_preprocess_image(path),label # image_label_ds = ds.map(load_and_preprocess_from_path_label) # =============================================================================
<TensorSliceDataset shapes: (), types: tf.string> cat dog cat dog cat <ZipDataset shapes: ((200, 200, 3), ()), types: (tf.float32, tf.int64)>
4. 训练阶段
4.1 设置验证集与数据集
#%%设置训练数据和验证集数据的大小 test_count = int(train_image_count*0.3) train_count = train_image_count - test_count print(test_count,train_count) #跳过test_count个 train_dataset = image_label_ds.skip(test_count) test_dataset = image_label_ds.take(test_count) batch_size = 32 # 设置一个和数据集大小一致的 shuffle buffer size（随机缓冲区大小）以保证数据被充分打乱。 train_ds = train_dataset.shuffle(buffer_size=train_count).repeat().batch(batch_size) test_ds = test_dataset.batch(batch_size)
909 2123 • 1
