dropout函数python python中dropout

• 背景介绍
  Neural Network之模型复杂度主要取决于优化参数个数与参数变化范围. 优化参数个数可手动调节, 参数变化范围可通过正则化技术加以限制. 本文从优化参数个数出发, 以dropout技术为例, 简要演示dropout参数丢弃比例对Neural Network模型复杂度的影响.
• 算法特征
  ①. 训练阶段以概率丢弃数据点; ②. 测试阶段保留所有数据点
• 算法推导 以概率\(p\)对数据点\(x\)进行如下变换,

\[\begin{equation*}
x' = \left\{\begin{split}
&0 &\quad\text{with probability $p$,} \\
&\frac{x}{1-p} &\quad\text{otherwise,}
\end{split}\right.
\end{equation*}
\]

• 即数据点\(x\)以概率\(p\)置零, 以概率\(1-p\)放大\(1/(1-p)\)倍. 此时有,

\[\begin{equation*}
\mathbf{E}[x'] = p\mathbf{E}[0] + (1-p)\mathbf{E}[\frac{x}{1-p}] = \mathbf{E}[x],
\end{equation*}
\]

• 
  此变换不改变数据点均值, 为无偏变换.
  若数据点\(x\)作为某线性变换之输入, 将其置零, 则对此线性变换无贡献, 等效于无效化该数据点及相关权重参数, 减少了优化参数个数, 降低了模型复杂度.
• 数据、模型与损失函数 数据生成策略如下,

\[\begin{equation*}
\left\{\begin{aligned}
x &= r + 2g + 3b \\
y &= r^2 + 2g^2 + 3b^2 \\
lv &= -3r - 4g - 5b
\end{aligned}\right.
\end{equation*}
\]

• Neural Network网络模型如下, 其中, 输入层为$(r, g, b)$, 隐藏层取激活函数$\tanh$, 输出层为$(x, y, lv)$且不取激活函数. 损失函数如下,

$$
\begin{equation*}
L = \sum_i\frac{1}{2}(\bar{x}^{(i)}-x^{(i)})^2+\frac{1}{2}(\bar{y}^{(i)}-y^{(i)})^2+\frac{1}{2}(\bar{lv}^{(i)}-lv^{(i)})^2
\end{equation*}
$$

• 其中, $i$为data序号, $(\bar{x}, \bar{y}, \bar{lv})$为相应观测值.
• 代码实现
  本文拟将中间隐藏层节点数设置为300, 使模型具备较高复杂度. 后逐步提升置零概率\(p\), 使模型复杂度降低, 以此观察泛化误差的变化. 具体实现如下, code

import numpy
import torch
from torch import nn
from torch import optim
from torch.utils import data
from matplotlib import pyplot as plt


# 获取数据与封装数据
def xFunc(r, g, b):
    x = r + 2 * g + 3 * b
    return x


def yFunc(r, g, b):
    y = r ** 2 + 2 * g ** 2 + 3 * b ** 2
    return y


def lvFunc(r, g, b):
    lv = -3 * r - 4 * g - 5 * b
    return lv


class GeneDataset(data.Dataset):
    
    def __init__(self, rRange=[-1, 1], gRange=[-1, 1], bRange=[-1, 1], num=100, transform=None,\
                 target_transform=None):
        self.__rRange = rRange
        self.__gRange = gRange
        self.__bRange = bRange
        self.__num = num
        self.__transform = transform
        self.__target_transform = transform
        
        self.__X = self.__build_X()
        self.__Y_ = self.__build_Y_()
        
    
    def __build_X(self):
        rArr = numpy.random.uniform(*self.__rRange, (self.__num, 1))
        gArr = numpy.random.uniform(*self.__gRange, (self.__num, 1))
        bArr = numpy.random.uniform(*self.__bRange, (self.__num, 1))
        X = numpy.hstack((rArr, gArr, bArr))
        return X
    
    
    def __build_Y_(self):
        rArr = self.__X[:, 0:1]
        gArr = self.__X[:, 1:2]
        bArr = self.__X[:, 2:3]
        xArr = xFunc(rArr, gArr, bArr)
        yArr = yFunc(rArr, gArr, bArr)
        lvArr = lvFunc(rArr, gArr, bArr)
        Y_ = numpy.hstack((xArr, yArr, lvArr))
        return Y_
    
    
    def __len__(self):
        return self.__num
    
    
    def __getitem__(self, idx):
        x = self.__X[idx]
        y_ = self.__Y_[idx]
        if self.__transform:
            x = self.__transform(x)
        if self.__target_transform:
            y_ = self.__target_transform(y_)
        return x, y_


# 构建模型
class Linear(nn.Module):
    
    def __init__(self, dim_in, dim_out):
        super(Linear, self).__init__()
        
        self.__dim_in = dim_in
        self.__dim_out = dim_out
        self.weight = nn.Parameter(torch.randn((dim_in, dim_out)))
        self.bias = nn.Parameter(torch.randn((dim_out,)))
        
        
    def forward(self, X):
        X = torch.matmul(X, self.weight) + self.bias
        return X
    
    
class Tanh(nn.Module):
    
    def __init__(self):
        super(Tanh, self).__init__()
        
        
    def forward(self, X):
        X = torch.tanh(X)
        return X


class Dropout(nn.Module):
    
    def __init__(self, p):
        super(Dropout, self).__init__()
        
        assert 0 <= p <= 1
        self.__p = p     # 置零概率
        
        
    def forward(self, X):
        if self.__p == 0:
            return X
        if self.__p == 1:
            return torch.zeros_like(X)
        mark = (torch.rand(X.shape) > self.__p).type(torch.float)
        X = X * mark / (1 - self.__p)
        return X
    

class MLP(nn.Module):
    
    def __init__(self, dim_hidden=50, p=0, is_training=True):
        super(MLP, self).__init__()
        
        self.__dim_hidden = dim_hidden
        self.__p = p
        self.training = True
        self.__dim_in = 3
        self.__dim_out = 3
        
        self.lin1 = Linear(self.__dim_in, self.__dim_hidden)
        self.tanh = Tanh()
        self.drop = Dropout(self.__p)
        self.lin2 = Linear(self.__dim_hidden, self.__dim_out)

        
    def forward(self, X):
        X = self.tanh(self.lin1(X))
        if self.training:
            X = self.drop(X)
        X = self.lin2(X)
        return X


# 构建损失函数
class MSE(nn.Module):
        
    def __init__(self):
        super(MSE, self).__init__()
        
        
    def forward(self, Y, Y_):
        loss = torch.sum((Y - Y_) ** 2) / 2
        return loss


# 训练单元与测试单元
def train_epoch(trainLoader, model, loss_fn, optimizer):
    model.train()
    loss = 0
    
    with torch.enable_grad():
        for X, Y_ in trainLoader:
            optimizer.zero_grad()
            Y = model(X)
            loss_tmp = loss_fn(Y, Y_)
            loss_tmp.backward()
            optimizer.step()
            
            loss += loss_tmp.item()
    return loss
            
        
def test_epoch(testLoader, model, loss_fn):
    model.eval（)
    loss = 0
    
    with torch.no_grad():
        for X, Y_ in testLoader:
            Y = model(X)
            loss_tmp = loss_fn(Y, Y_)
            loss += loss_tmp.item()
            
    return loss


# 进行训练与测试
def train(trainLoader, testLoader, model, loss_fn, optimizer, epochs):
    minLoss = numpy.inf
    for epoch in range(epochs):
        trainLoss = train_epoch(trainLoader, model, loss_fn, optimizer) / len(trainLoader.dataset)
        testLoss = test_epoch(testLoader, model, loss_fn) / len(testLoader.dataset)
        if testLoss < minLoss:
            minLoss = testLoss
            torch.save(model.state_dict(), "./mlp.params")
#         if epoch % 100 == 0:
#             print(f"epoch = {epoch:8}, trainLoss = {trainLoss:15.9f}, testLoss = {testLoss:15.9f}")
    return minLoss


numpy.random.seed(0)
torch.random.manual_seed(0)

def search_dropout():
    trainData = GeneDataset(num=50, transform=torch.Tensor, target_transform=torch.Tensor)
    trainLoader = data.DataLoader(trainData, batch_size=50, shuffle=True)
    testData = GeneDataset(num=1000, transform=torch.Tensor, target_transform=torch.Tensor)
    testLoader = data.DataLoader(testData, batch_size=1000, shuffle=False)

    dim_hidden1 = 300
    p = 0.005
    model = MLP(dim_hidden1, p)
    loss_fn = MSE()
    optimizer = optim.Adam(model.parameters(), lr=0.003)
    train(trainLoader, testLoader, model, loss_fn, optimizer, 100000)

    pRange = numpy.linspace(0, 1, 101)
    lossList = list()
    for idx, p in enumerate(pRange):
        model = MLP(dim_hidden1, p)
        loss_fn = MSE()
        optimizer = optim.Adam(model.parameters(), lr=0.003)
        model.load_state_dict(torch.load("./mlp.params"))
        loss = train(trainLoader, testLoader, model, loss_fn, optimizer, 100000)
        lossList.append(loss)
        print(f"p = {p:10f}, loss = {loss:15.9f}")

    minIdx = numpy.argmin(lossList)
    pBest = pRange[minIdx]
    lossBest = lossList[minIdx]

    fig = plt.figure(figsize=(5, 4))
    ax1 = fig.add_subplot(1, 1, 1)
    ax1.plot(pRange, lossList, ".--", lw=1, markersize=5, label="testing error", zorder=1)
    ax1.scatter(pBest, lossBest, marker="*", s=30, c="red", label="optimal", zorder=2)
    ax1.set(xlabel="$p$", ylabel="error", title="optimal dropout probability = {:.5f}".format(pBest))
    ax1.legend()
    fig.tight_layout()
    fig.savefig("search_p.png", dpi=100)
    # plt.show()



if __name__ == "__main__":
    search_dropout()

• 结果展示 可以看到, 泛化误差在提升置零概率后先下降后上升, 大致对应降低模型复杂度使模型表现从过拟合至欠拟合.
• 使用建议
  ①. dropout为使整个节点失效, 通常作用在节点的最终输出上(即激活函数后);
  ②. dropout适用于神经网络全连接层.
• 参考文档
  ①. 动手学深度学习 - 李牧