1. 简单介绍
本文的应用场景是二分类问题，采用的数据集为猫狗分类数据集，为了减少训练时间，训练集图片有2123张，验证集有909 张图片，测试的图片有1000张，分为猫和狗两个类别，图片已经放置在dc_2000文件夹下面。
2 加载图片
2.1 导入相关包并加载图片
# 导入相关包
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir("/content/gdrive/My Drive/Colab Notebooks/tensorflow")
import tensorflow as tf
print('Tensorflow version: {}'.format(tf.__version__))
from tensorflow import keras
import matplotlib.pyplot as plt
%matplotlib inline
import numpy as np
import pathlib
#配置数据集路径
path_root = os.path.join(os.path.realpath("."),"DS","dc_2000")
data_dir = pathlib.Path(path_root)
#数量构成
train_image_count = len(list(data_dir.glob('train/*/*.jpg')))
test_image_count = len(list(data_dir.glob('test/*/*.jpg')))
CLASS_NAMES = np.array([item.name for item in data_dir.glob('train/*')])
print("训练集的数量:{}\n验证集的数量:{}\n数据集类别:{}".format(train_image_count,test_image_count,CLASS_NAMES))
#保存文件的路径,随机打乱
import random
train_all_image_path = list(data_dir.glob("train/*/*"))
test_all_image_path = list(data_dir.glob("test/*/*"))
train_all_image_path = [str(path) for path in train_all_image_path]
test_all_image_path = [str(path) for path in test_all_image_path]
random.shuffle(train_all_image_path)
random.shuffle(test_all_image_path)训练集的数量:3032
验证集的数量:1000
数据集类别:['cat' 'dog']2.2 读取图片
#确定每个图像的标签
lable_names = sorted(item.name for item in data_dir.glob("train/*/"))
#为每个标签分配索引,构建字典
lable_to_index = dict((name,index) for index,name in enumerate(lable_names))
print(lable_to_index)
#创建一个列表，包含每个文件的标签索引
train_image_label = [lable_to_index[pathlib.Path(path).parent.name] for path in train_all_image_path]
test_image_label = [lable_to_index[pathlib.Path(path).parent.name] for path in test_all_image_path]
#包装为函数，以备后用
def preprocess_image(image):
image = tf.image.decode_jpeg(image, channels=3)
image = tf.image.resize(image, [200, 200])
image /= 255.0 # normalize to [0,1] range
return image
#加载图片
def load_and_preprocess_image(path):
image = tf.io.read_file(path)
return preprocess_image(image)
image_path = test_all_image_path[11]
label = test_image_label[11]
plt.imshow(load_and_preprocess_image(image_path))
plt.grid(False)
##plt.xlabel(caption_image(image_path))
plt.title(lable_names[label].title())
plt.axis("off")
print(){'cat': 0, 'dog': 1}
3. 图片预处理
在这一部分我们采用from_tensor_slices的方法对图片数据集进行构建，对比tf1.x版本采用队列形式读取数据，这一种方法比较简单切易于理解。并构建（图片，标签）对数据集。
#构建一个tf.data.Dataset
#一个图片数据集构建 tf.data.Dataset 最简单的方法就是使用 from_tensor_slices 方法。
#将字符串数组切片，得到一个字符串数据集：
train_path_ds = tf.data.Dataset.from_tensor_slices(train_all_image_path)
print(train_path_ds)
test_path_ds = tf.data.Dataset.from_tensor_slices(test_all_image_path)
#现在创建一个新的数据集，通过在路径数据集上映射 preprocess_image来动态加载和格式化图片。
AUTOTUNE = tf.data.experimental.AUTOTUNE
train_image_ds = train_path_ds.map(load_and_preprocess_image,num_parallel_calls=AUTOTUNE)
test_image_ds = test_path_ds.map(load_and_preprocess_image,num_parallel_calls=AUTOTUNE)
train_lable_ds = tf.data.Dataset.from_tensor_slices(tf.cast(train_image_label,tf.int64))
test_lable_ds = tf.data.Dataset.from_tensor_slices(tf.cast(test_image_label,tf.int64))
for label in train_lable_ds.take(5):
print(lable_names[label.numpy()])
#%%构建一个（图片，标签）对数据集
#因为这些数据集顺序相同，可以将他们打包起来
image_label_ds = tf.data.Dataset.zip((train_image_ds,train_lable_ds))
test_data = tf.data.Dataset.zip((test_image_ds,test_lable_ds))
print(test_data)
#注意：当你拥有形似 all_image_labels 和 all_image_paths 的数组，tf.data.dataset.Dataset.zip 的替代方法是将这对数组切片
# =================================im============================================
# ds = tf.data.Dataset.from_tensor_slices((all_image_path,all_image_label))
# def load_and_preprocess_from_path_label(path, label):
# return load_and_preprocess_image(path),label
# image_label_ds = ds.map(load_and_preprocess_from_path_label)
# =============================================================================<TensorSliceDataset shapes: (), types: tf.string>
cat
dog
cat
dog
cat
<ZipDataset shapes: ((200, 200, 3), ()), types: (tf.float32, tf.int64)>4. 训练阶段
4.1 设置验证集与数据集
#%%设置训练数据和验证集数据的大小
test_count = int(train_image_count*0.3)
train_count = train_image_count - test_count
print(test_count,train_count)
#跳过test_count个
train_dataset = image_label_ds.skip(test_count)
test_dataset = image_label_ds.take(test_count)
batch_size = 32
# 设置一个和数据集大小一致的 shuffle buffer size（随机缓冲区大小）以保证数据被充分打乱。
train_ds = train_dataset.shuffle(buffer_size=train_count).repeat().batch(batch_size)
test_ds = test_dataset.batch(batch_size)909 21234.2 构建模型并训练
model = tf.keras.Sequential() #顺序模型
model.add(tf.keras.layers.Conv2D(64, (3, 3), input_shape=(200, 200, 3), activation='relu'))
model.add(tf.keras.layers.Conv2D(64, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D())
model.add(tf.keras.layers.Conv2D(128, (3, 3), activation='relu'))
model.add(tf.keras.layers.Conv2D(128, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D())
model.add(tf.keras.layers.Conv2D(256, (3, 3), activation='relu'))
model.add(tf.keras.layers.Conv2D(256, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D())
model.add(tf.keras.layers.Conv2D(512, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D())
model.add(tf.keras.layers.Conv2D(512, (3, 3), activation='relu'))
model.add(tf.keras.layers.MaxPooling2D())
model.add(tf.keras.layers.Conv2D(1024, (3, 3), activation='relu'))
model.add(tf.keras.layers.GlobalAveragePooling2D())
model.add(tf.keras.layers.Dense(1024, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
#%%
model.summary()
model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.0001),
loss='binary_crossentropy',
metrics=['acc'])Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d (Conv2D) (None, 198, 198, 64) 1792
_________________________________________________________________
conv2d_1 (Conv2D) (None, 196, 196, 64) 36928
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 98, 98, 64) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 96, 96, 128) 73856
_________________________________________________________________
conv2d_3 (Conv2D) (None, 94, 94, 128) 147584
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 47, 47, 128) 0
_________________________________________________________________
conv2d_4 (Conv2D) (None, 45, 45, 256) 295168
_________________________________________________________________
conv2d_5 (Conv2D) (None, 43, 43, 256) 590080
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 21, 21, 256) 0
_________________________________________________________________
conv2d_6 (Conv2D) (None, 19, 19, 512) 1180160
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 9, 9, 512) 0
_________________________________________________________________
conv2d_7 (Conv2D) (None, 7, 7, 512) 2359808
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 3, 3, 512) 0
_________________________________________________________________
conv2d_8 (Conv2D) (None, 1, 1, 1024) 4719616
_________________________________________________________________
global_average_pooling2d (Gl (None, 1024) 0
_________________________________________________________________
dense (Dense) (None, 1024) 1049600
_________________________________________________________________
dense_1 (Dense) (None, 256) 262400
_________________________________________________________________
dense_2 (Dense) (None, 1) 257
=================================================================
Total params: 10,717,249
Trainable params: 10,717,249
Non-trainable params: 0
_________________________________________________________________steps_per_eooch = train_count//batch_size
validation_steps = test_count//batch_size
history = model.fit(train_ds,epochs=100,steps_per_epoch=steps_per_eooch,validation_data=test_ds,validation_steps=validation_steps)Epoch 1/100
66/66 [==============================] - 11s 171ms/step - loss: 0.6936 - acc: 0.5208 - val_loss: 0.6931 - val_acc: 0.4833
Epoch 2/100
66/66 [==============================] - 11s 169ms/step - loss: 0.6902 - acc: 0.5312 - val_loss: 0.6919 - val_acc: 0.5502
Epoch 3/100
66/66 [==============================] - 11s 168ms/step - loss: 0.6851 - acc: 0.5611 - val_loss: 0.6799 - val_acc: 0.6105
Epoch 4/100
66/66 [==============================] - 17s 254ms/step - loss: 0.6650 - acc: 0.6080 - val_loss: 0.6568 - val_acc: 0.6228
Epoch 5/100
66/66 [==============================] - 17s 257ms/step - loss: 0.6341 - acc: 0.6501 - val_loss: 0.6466 - val_acc: 0.6395
.......
Epoch 98/100
66/66 [==============================] - 17s 256ms/step - loss: 4.7811e-08 - acc: 1.0000 - val_loss: 3.8404 - val_acc: 0.7422
Epoch 99/100
66/66 [==============================] - 17s 259ms/step - loss: 5.1801e-08 - acc: 1.0000 - val_loss: 3.8481 - val_acc: 0.7433
Epoch 100/100
66/66 [==============================] - 17s 256ms/step - loss: 4.4815e-08 - acc: 1.0000 - val_loss: 3.8532 - val_acc: 0.74444.3 分析评估
我们对比一下在训练集和验证集的准确度和损失的变化曲线，我们可以发现，本网络有些过拟合。
history.history.keys()
plt.plot(history.epoch, history.history.get('acc'), label='acc')
plt.plot(history.epoch, history.history.get('val_acc'), label='val_acc')
plt.legend()
plt.plot(history.epoch, history.history.get('loss'), label='loss')
plt.plot(history.epoch, history.history.get('val_loss'), label='val_loss')
plt.legend()
最后评估一下在测试集上的误差
#test_ds = test_data.batch(batch_size)
loss,acc = model.evaluate(test_data.batch(batch_size))32/32 [==============================] - 4s 111ms/step - loss: 3.8820 - acc: 0.7300总结一下：本测试的结果有些过拟合，考虑到图片较少，不能提取更为高级的特征，所以在测试集的效果只有73%,后期我们可以通过增加数据集，数据增强，并引入正则化和丢弃法等方法来抑制过拟合并提高精度。
