一、神经网络搭建
1. 神经网络组成
tensor 张量：
表示数据 多维数组*（列表）*
计算图：
搭建神经网络*（只描述计算过程，不执行计算结果）*
import tensorflow as tf
x=tf.constant([[1.0,2.0]]) #输入
w=tf.constant([[3.0],[4.0]]) #w 作为权重
y=tf.matmul(x,w) #计算图的计算过程 y =x1*w1+x2*w2
session 会话：
执行计算图
with session as sess:
print sess.run(y)2. 神经网络实现过程
- 准备数据集，提取特征，作为输入喂给神经网络(NN)
- 搭建NN结构，计算图+会话（NN的前向传播算法 -> 输出）
- 大量特征数据喂给NN，迭代优化参数（NN反向传播算法->优化参数训练模型）
- 使用训练好的模型预测和分类
3. 神经网络搭建栗子
假设输入是商品的体积V和质量M
假设如果V + M<1 标签Y为1
神经网络图
搭建神经网络代码
1 #coding: utf-8
2 #导入模块，生成数据集（自制数据集）
3 import tensorflow as tf
4 import numpy as np #python 的科学计算模块
5 BATCH_SIZE = 8 #每次喂给神经网络的数据大小
6 seed = 23455 #随机种子，为了使每次生成数据一致
7
8 #利用seed生成随机数
9 rng = np.random.RandomState(seed)
10 #返回 32*2 矩阵 代表32组，体积和质量的数据
11 X = rng.rand(32,2)
12 #如果V+M<1 Y=1否则 Y=0
13 #Y 作为数据集的标签
14 Y = [[int(x0 + x1 < 1)] for (x0, x1) in X]
15 print "X:\n",X
16 print "Y:\n",Y
17
18 #1.定义输入，参数和输出 /前向传播过程
19 x = tf.placeholder(tf.float32, shape = (None,2))
20 y_= tf.placeholder(tf.float32, shape = (None,1))
21
22 w1 = tf.Variable(tf.random_normal([2,3], stddev=1, seed=1))
23 w2 = tf.Variable(tf.random_normal([3,1], stddev=1, seed=1))
24
25 a = tf.matmul(x,w1)
26 y = tf.matmul(a,w2)
27
28 #2.定义损失函数和反向传播方法
29 loss = tf.reduce_mean(tf.square(y-y_))
30 train_step = tf.train.GradientDescentOptimizer(0.001).minimize(loss)
31 #train_step=tf.train.MomentumOptimizer(0.001,0.9).minimize(loss)
32 #train_step=tf.train.AdamOptimizer(0.001).minimize(loss)
33
34 #3.生成会话 训练STEPS 次
35 with tf.Session() as sess:
36 init_op = tf.initialize_all_variables()
37 sess.run(init_op)
38 #打印训练前的参数
39 print "w1:\n", sess.run(w1)
40 print "w2:\n", sess.run(w2)
41
42 #4.训练模型
43 STEPS = 3000
44 for i in range(STEPS):
45 start = (i*BATCH_SIZE) % 32
46 end = start +BATCH_SIZE
47 sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})
48 if i % 500 == 0:
49 total_loss = sess.run(loss, feed_dict={x: X, y_: Y})
50 print("After %d training steps(s), loss on all data is %g" % (i,total_loss))
51
52 #输出训练后的参数
53 print "\n"
54 print "w1:\n", sess.run(w1)
55 print "w2:\n", sess.run(w2)函数：
函数 | 功能 |
tf.placeholder(type,shape=None, name=None) | 类似形参，搭建神经网络时占位用 |
np.random.RandomState(seed) | seed可有可无，有seed为了使每次生成数据一致 |
tf.Variable(initializer,name) | 初始化参数 |
tf.train.GradientDescentOptimizer(0.001).minimize(loss) | 这三个都是梯度下降训练损失函数 |
tf.train.MomentumOptimizer(0.001,0.9).minimize(loss) | – |
tf.train.AdamOptimizer(0.001).minimize(loss) | |
run(fetches, feed_dict=None, options=None, run_metadata=None) | tensorflow并没有计算整个图，只是计算了与想要fetch 的值相关的部分 |
二、神经网络优化
1. 损失函数
1、均方误差
loss = tf.reduce_mean(tf.square(y-y_))
2、自定义损失函数
3、交叉熵
2. 学习率
学习率是参数优化过程中，参数每次变化的幅度。
更新后参数 = 更新前参数 - 学习率 × 损失函数的梯度（导数）
通过改变下列代码中，第四行的0.1，可以感受到学习率对参数优化的影响，如果学习率过大，可能会难以达到正确的优化结果，但如果参数过小，达到最优结果速度会比较慢。
1 import tensorflow as tf
2 w = tf.Variable(tf.constant(5, dtype = tf.float32))
3 loss = tf.square(w+1)
4 train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
5 with tf.Session() as sess:
6 init_op =tf.initialize_all_variables()
7 sess.run(init_op)
8 for i in range(80):
9 sess.run(train_step)
10 w_val = sess.run(w)
11 loss_val = sess.run(loss)
12 print "After %s steps w is %f loss is %f" %( i, w_val, loss_val)所以可以采用指数下降学习率，刚开始学习率较大，提高学习速度，后来学习率变小，提高训练精度。
指数下降学习率
学习率 = 初始学习率 × 学习率衰减率^(训练轮数/BATCH_SIZE)
#BATCH_SIZE 是一次喂给神经网络的数据的组数
LEARNING_RATE = LEARNING_RATE_BASE * LEARNING_RATE_DECAY^(global_step/BATCH_SIZE)所以学习率呈指数下降，在刚开始时候rate比较大，学习速度较快；到后来rate减小，使结果更精确。
函数： y = 0.1 * (0.99) ^x (x= [0,3000]) 图像：
指数下降学习率代码：
1 import tensorflow as tf
2
3 LEARNING_RATE_BASE = 0.1 #初始学习率
4 LEARNING_RATE_DECAY = 0.99 #学习率衰减率
5 LEARNING_RATE_STEP = 1 #学习率更新频率，通常是总数/BATCH_SIZE （一次喂给神经网络的数据大小
6
7 #运行几组 BATCH_SIZE的计数器 初值是 0, 设为不训练
8 global_step = tf.Variable(0, trainable=False)
9 #定义指数下降学习率
10 learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,LEARNING_RATE_STEP, LEARNING_RATE_DECAY, staircase=True)
11 #定义待优化的参数 初值为5
12 w = tf.Variable(tf.constant(5, dtype=tf.float32))
13 #定义损失函数
14 loss = tf.square(w+1)
15 #定义反向传播方法
16 train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_ step)
17 #会话
18 with tf.Session() as sess:
19 init_op = tf.initialize_all_variables() #python2 用这个代替 global_variables_initializer()
20 sess.run(init_op)
21 for i in range(180):
22 sess.run(train_step)
23 learning_rate_val = sess.run(learning_rate)
24 global_step_val = sess.run(global_step)
25 w_val = sess.run(w)
26 loss_val = sess.run(loss)
27 print "After %s steps: global_step is %f, learning rate is %f, w is %f, loss is %f" % (i, global_step_val, learning_rate_val, w_val, loss_val)3. 滑动平均
4. 正则化
三、mnist手写数字体识别
mnist_forward.py
import tensorflow as tf
INPUT_NODE = 784
OUTPUT_NODE = 10
LAYER1_NODE = 500
def get_weight(shape, regularizer):
w = tf.Variable(tf.truncated_normal(shape,stddev=0.1))
if regularizer !=None: tf.add_to_collection('losses',tf.contrib.layers.l2_regularizer(regularizer)(w))
return w
def get_bias(shape):
b = tf.Variable(tf.zeros(shape))
return b
def forward(x, regularizer):
w1 = get_weight([INPUT_NODE, LAYER1_NODE],regularizer)
b1 = get_bias([LAYER1_NODE])
y1 = tf.nn.relu(tf.matmul(x,w1) + b1)
w2 = get_weight([LAYER1_NODE,OUTPUT_NODE],regularizer)
b2 = get_bias([OUTPUT_NODE])
y = tf.matmul(y1,w2) + b2
return ymnist_backward.py
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import mnist_forward
import os
BATCH_SIZE = 200
LEARNING_RATE_BASE = 0.1
LEARNING_RATE_DECAY = 0.99
REGULARIZER = 0.0001
STEPS = 50000
MOVING_AVERAGE_DECAY = 0.99
MODEL_SAVE_PATH = "./model/"
MODEL_NAME = "mnist_model"
def backward(mnist):
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, REGULARIZER)
global_step = tf.Variable(0, trainable = False)
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_,1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name = 'train')
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.initialize_all_variables()
sess.run(init_op)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
_, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})
if i % 1000 == 0:
print("After %d training step(s), loss on training batch is %g." % (step, loss_value))
saver.save(sess,os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def main():
mnist = input_data.read_data_sets("./data/", one_hot=True)
backward(mnist)
if __name__ == '__main__':
main()mnist_test.py
import time
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
import mnist_forward
import mnist_backward
TEST_INTERVAL_SECS = 5
def test(mnist):
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, None)
ema = tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/'[-1].split('-')[-1]
accuracy_score =sess.run(accuracy, feed_dict={x: mnist.test.images, y_:mnist.test.labels})
print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score))
else:
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def main():
mnist = input_data.read_data_sets("./data/", one_hot=True)
test(mnist)
if __name__ == '__main__':
main()
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
