关于反向传播算法中几个公式的推导

参考自资料，戳此处

J (W, b; x, y) = \frac{1}{2} | | h_{W, b} (x) - y | |^{2}

符号说明：

\begin{aligned} l & = 神 经 网 络 的 某 一 层 \\ S_{l} & = 第 l 层 神 经 元 的 数 目 \\ L & = 神 经 网 络 的 总 层 数 \end{aligned}

如下图中：

S_{1} = 2; S_{L} = S_{3} = 2

推导之前先列出几个用到的等式：

\begin{aligned} z_{j}^{l} & = \sum_{k = 1}^{S_{l - 1}} f (z_{k}^{l - 1}) \cdot W_{j k}^{l - 1} \\ = f (z_{1}^{l - 1}) \cdot W_{j 1}^{l - 1} + f (z_{2}^{l - 1}) \cdot W_{j 2}^{l - 1} + \dots + f (z_{S_{l - 1}}^{l - 1}) \cdot W_{j S_{l - 1}}^{l - 1} \dots \dots \dots (1) \\ = a_{1}^{l - 1} \cdot W_{j 1}^{l - 1} + a_{2}^{l - 1} \cdot W_{j 2}^{l - 1} + \dots + a_{S_{l - 1}}^{l - 1} \cdot W_{j S_{l - 1}}^{l - 1} \dots \dots \dots \dots \dots (2) \end{aligned}

例如：

\begin{aligned} z_{1}^{2} = \sum_{k = 1}^{3} f (z_{k}^{1}) \cdot W_{1 k}^{1} = a_{1}^{1} \cdot W_{11}^{1} + a_{2}^{1} \cdot W_{12}^{1} + a_{3}^{1} \cdot W_{13}^{1} \end{aligned}

\begin{aligned} h (x_{i}) = a_{i}^{L} = f (z_{i}^{L}); z_{i}^{l} = \sum_{j = 1}^{S_{(l - 1)}} W_{i j}^{l - 1} \cdot a_{j}^{l - 1} \dots \dots \dots \dots (3) \end{aligned}

定义：

δ_{i}^{l} = \frac{\partial J}{\partial z_{i}^{l}}

；这只是定义的一个称之为“残差”的中间变量

推导：

公式一： $\frac{\partial J}{\partial W_{i j}^{l}} = δ_{i}^{l + 1} \cdot a_{j}^{l}$

\begin{aligned} \frac{\partial J}{\partial W_{i j}^{l}} & = \sum_{k = 1}^{S_{l + 1}} \frac{\partial J}{\partial z_{k}^{l + 1}} \cdot \frac{\partial z_{k}^{l + 1}}{\partial W_{i j}^{l}} \\ = \frac{\partial J}{\partial z_{i}^{l + 1}} \cdot \frac{\partial z_{i}^{l + 1}}{\partial W_{i j}^{l}} \dots \dots 其 中 只 有 当 k 等 于 i 时 ， \frac{\partial z_{i}^{l + 1}}{\partial W_{i j}^{l}} 才 不 为 零 \\ = δ_{i}^{l + 1} \cdot \frac{\partial z_{i}^{l + 1}}{\partial W_{i j}^{l}} \\ = δ_{i}^{l + 1} \cdot \frac{\partial}{\partial W_{i j}^{l}} [a_{1}^{l} W_{i 1}^{l} + a_{2}^{l} W_{i 2}^{l} + \dots + a_{1}^{l} W_{i S_{l}}^{l}] \dots \dots 红 色 部 分 由 (2) 可 知 \\ = δ_{i}^{l + 1} \cdot a_{j}^{l} \dots \dots 观 察 可 知 ， 当 a_{？}^{l} W_{i j}^{l} 等 于 上 面 红 色 部 分 中 的 某 一 项 时 ， 问 号 处 刚 好 为 j, 注 意 观 察 角 标 \end{aligned}

\begin{aligned} 矢 量 化 形 式 为 ： \frac{\partial J}{\partial W^{l}} = δ^{l + 1} \cdot (a^{l})^{T} 怎 么 来 的 ？ 看 下 面 例 子 \end{aligned}

由上面的网络图可知：

\begin{aligned} \frac{\partial J}{W_{11}^{2}} & = δ_{1}^{3} a_{1}^{2}; \frac{\partial J}{W_{12}^{2}} = δ_{1}^{3} a_{2}^{2}; \frac{\partial J}{W_{21}^{2}} = δ_{2}^{3} a_{1}^{2}; \frac{\partial J}{W_{22}^{2}} = δ_{2}^{3} a_{2}^{2}; \\ ⟹ \frac{\partial J}{W^{2}} = [\begin{matrix} δ_{1}^{3} \\ δ_{2}^{3} \end{matrix}] \cdot {[\begin{matrix} a_{1}^{2} \\ a_{2}^{2} \end{matrix}]}^{T} = [\begin{matrix} δ_{1}^{3} \\ δ_{2}^{3} \end{matrix}] \cdot [\begin{matrix} a_{1}^{2} & a_{2}^{2} \end{matrix}] = [\begin{matrix} δ_{1}^{3} a_{1}^{2} & δ_{1}^{3} a_{2}^{2} \\ δ_{2}^{3} a_{1}^{2} & δ_{2}^{3} a_{2}^{2} \end{matrix}] = δ^{l + 1} \cdot (a^{l})^{T} \end{aligned}

公式二： $δ_{i}^{L} = - [y_{i} - a_{i}^{L}] \cdot f^{'} (z_{i}^{L})$

\begin{aligned} δ_{i}^{L} & = \frac{\partial J}{\partial z_{i}^{L}} = \frac{\partial}{\partial z_{i}^{L}} \cdot [\frac{1}{2} \cdot \sum_{k = 1}^{S_{L}} (y_{k} - h (x)_{k})^{2}] \\ = \frac{\partial}{\partial z_{i}^{L}} \cdot [\frac{1}{2} \cdot \sum_{k = 1}^{S_{L}} (y_{k} - f (z_{k}^{L}))^{2}] \dots \dots 红 色 部 分 由 (3) 可 知 \\ = \frac{\partial}{\partial z_{i}^{L}} \cdot \frac{1}{2} \cdot {[y_{1} - f (z_{1}^{L})]^{2} + [y_{2} - f (z_{2}^{L})]^{2} + \dots + [y_{i} - f (z_{i}^{L})]^{2} + \dots + [y_{S_{L}} - f (z_{S_{L}}^{L})]^{2}} \\ = - [y_{i} - f (z_{i}^{L})] \cdot \frac{\partial f (z_{i}^{L})}{\partial z_{i}^{L}} \dots \dots \dots 因 为 上 面 式 子 中 只 有 红 色 项 含 有 z_{i}^{L} 这 一 因 子 ， 所 以 其 他 项 对 z_{i}^{L} 求 导 均 为 0 \\ = - [y_{i} - f (z_{i}^{L})] \cdot f^{'} (z_{i}^{L}) \\ = - [y_{i} - a_{i}^{L}] \cdot f^{'} (z_{i}^{L}) \dots \dots 红 色 部 分 由 (3) 可 知 \end{aligned}

\begin{aligned} 矢 量 化 形 式 ： δ^{L} = - [y - a^{L}] \cdot f^{'} (z^{L}) \end{aligned}

这个表达式是根据一个特定的代价函数推导出来的，更为一般的形式如下：(另，本文所有的公式推导均不依赖任何具体的代价函数)

\begin{aligned} δ_{i}^{L} & = \frac{\partial J}{\partial z_{i}^{L}} = \frac{\partial J}{\partial a_{i}^{L}} \cdot \frac{\partial a_{i}^{L}}{\partial z_{i}^{L}} = \frac{\partial J}{\partial a_{i}^{L}} \cdot \frac{\partial f (z_{i}^{L})}{\partial z_{i}^{L}} = \frac{\partial J}{\partial a_{i}^{L}} \cdot f^{'} (z_{i}^{L}) \end{aligned}

例如对于代价函数

\begin{aligned} J (Θ) = - \frac{1}{m} \sum_{i = 1}^{m} \sum_{k = 1}^{K} [y_{k}^{(i)} \log ((h_{Θ} (x^{(i)}))_{k}) + (1 - y_{k}^{(i)}) \log (1 - (h_{Θ} (x^{(i)}))_{k})] + \frac{λ}{2 m} \sum_{l = 1}^{L - 1} \sum_{i = 1}^{s_{l}} \sum_{j = 1}^{s_{l + 1}} (Θ_{j, i}^{(l)})^{2} \end{aligned}

求导时只考虑一个训练数据即可即m=1，且regular term对 $a_{i}^{L}$ 求导均为0

\begin{aligned} δ_{i}^{L} & = \frac{\partial J}{\partial a_{i}^{L}} \cdot f^{'} (z_{i}^{L}) \\ = \frac{\partial}{\partial a_{i}^{L}} - \sum_{k = 1}^{S_{L}} [y_{k} \cdot l o g (h (x)_{k}) + (1 - y_{k}) \cdot l o g (1 - h (x)_{k})] \cdot f^{'} (z_{i}^{L}) \\ = - \sum_{k = 1}^{S_{L}} \frac{\partial}{\partial a_{i}^{L}} [y_{k} \cdot l o g (a_{k}^{L}) + (1 - y_{k}) \cdot l o g (1 - a_{k}^{L})] \cdot f^{'} (z_{i}^{L}) \\ = - \sum_{k = 1}^{S_{L}} [y_{k} \cdot \frac{1}{a_{k}^{L}} \cdot \frac{\partial a_{k}^{L}}{\partial a_{i}^{L}} + (1 - y_{k}) \cdot \frac{- 1}{1 - a_{k}^{L}} \cdot \frac{\partial a_{k}^{L}}{\partial a_{i}^{L}}] \cdot f^{'} (z_{i}^{L}) \dots \dots 此 处 的 l o g 就 是 l n, 即 l o g e = l n e = 1 \\ = - [y_{i} \cdot \frac{1}{a_{i}^{L}} \cdot 1 + (1 - y_{i}) \cdot \frac{- 1}{1 - a_{i}^{L}} \cdot 1] \cdot f^{'} (z_{i}^{L}) \dots \dots 当 k 等 于 i 时 ， \frac{\partial a_{k}^{L}}{\partial a_{i}^{L}} = 1 ， 其 他 情 况 为 0 \\ = [\frac{- y_{i}}{a_{i}^{L}} + \frac{1 - y_{i}}{1 - a_{i}^{L}}] \cdot [a_{i}^{L} \cdot (1 - a_{i}^{L})] \\ = a_{i}^{L} - y_{i} \dots \dots 这 就 是 c o u r s e r a 中 A n g d r e w N g 直 接 给 出 来 的 公 式 \end{aligned}

公式三： $δ^{l} = (W^{l})^{T} δ^{l + 1} . * f^{'} (z^{l})$

\begin{aligned} δ_{i}^{L - 1} & = \frac{\partial J}{\partial z_{i}^{L - 1}} \\ = \frac{\partial J}{\partial z_{1}^{L}} \cdot \frac{\partial z_{1}^{L}}{\partial z_{i}^{L - 1}} + \frac{\partial J}{\partial z_{2}^{L}} \cdot \frac{\partial z_{2}^{L}}{\partial z_{i}^{L - 1}} + \dots + \frac{\partial J}{\partial z_{S_{L}}^{L}} \cdot \frac{\partial z_{S_{L}}^{L}}{\partial z_{i}^{L - 1}} \\ = \sum_{k = 1}^{S_{L}} \frac{\partial J}{\partial z_{k}^{L}} \cdot \frac{\partial z_{k}^{L}}{\partial z_{i}^{L - 1}} \\ = \sum_{k = 1}^{S_{L}} δ_{k}^{L} \cdot \frac{\partial}{\partial z_{i}^{L - 1}} \sum_{j = 1}^{S_{(L - 1)}} f (z_{j}^{L - 1}) W_{k j}^{L - 1} \dots \dots \dots 红 色 部 分 是 由 z_{k}^{L} 展 开 而 来 \\ = \sum_{k = 1}^{S_{L}} δ_{k}^{L} \cdot W_{k i}^{L - 1} \cdot f^{'} (z_{i}^{L - 1}) \dots \dots 注 意 观 察 角 标 ， 之 后 当 j 等 于 i 的 时 候 ， f^{'} (z_{j}^{L - 1}) 才 不 为 0 ， 所 以 其 他 项 也 就 都 没 了 \end{aligned}

\begin{aligned} 把 L - 1 替 换 为 l, L 替 换 为 l + 1 即 ： \\ δ_{i}^{l} & = \sum_{k = 1}^{S_{l + 1}} δ_{k}^{l + 1} \cdot W_{k i}^{l} \cdot f^{'} (z_{i}^{l}) \dots 从 表 达 式 可 知 ， 要 求 第 l 层 残 差 ， 必 须 先 求 l + 1 层 残 差 ， 所 以 称 为 反 向 传 播 \end{aligned}

\begin{aligned} 矢 量 化 形 式 为 ： δ^{l} & = (W^{l})^{T} δ^{l + 1} . * f^{'} (z^{l}) \dots \dots 怎 么 来 的 ？ 像 公 式 一 中 一 样 ， 举 个 例 子 就 明 白 了 \end{aligned}

公式四： $\frac{\partial J}{\partial b_{i}^{l}} = δ_{i}^{l + 1}$

注意：敲黑板！！！

在Michael Nielsen 的笔记NeuralNetworkandDeepLearning中， $\frac{\partial J}{\partial b_{i}^{l}} = δ_{i}^{l}$ ，到底哪个对呢？当然是都对，只是在定义参数时候用的角标指代不同。在Michael Nielsen 笔记中 $W_{i j}^{l}$ 指的是第 $l - 1$ 层的第 $j$ 个神经元指向第 $l$ 层的第 $i$ 个神经元之间的权重，当然 $b_{i}^{l}$ 也就指的是第 $l - l$ 到第l层之间的偏置，所以它定义了 $z^{l} = w^{l} a^{l - 1} + b^{l}$

而

在其它一些地方包括此处， $W_{i j}^{l}$ 指的是第 $l$ 层第 $j$ 个神经元指向第 $l + 1$ 层的第 $i$ 个神经元之间的权重， $b_{i}^{l}$ 同理，所以定义了 $z^{l + 1} = w^{l} a^{l} + b^{l}$

先证明第一种：

\begin{aligned} \frac{\partial J}{\partial b_{i}^{l}} & = \frac{\partial J}{\partial z_{1}^{l}} \cdot \frac{\partial z_{1}^{l}}{\partial b_{i}^{l}} + \frac{\partial J}{\partial z_{2}^{l}} \cdot \frac{\partial z_{2}^{l}}{\partial b_{i}^{l}} + \dots + \frac{\partial J}{\partial z_{S_{l}}^{l}} \cdot \frac{\partial z_{S_{l}}^{l}}{\partial b_{i}^{l}} \\ = δ_{1}^{l} \cdot \frac{\partial z_{1}^{l}}{\partial b_{i}^{l}} + δ_{2}^{l} \cdot \frac{\partial z_{2}^{l}}{\partial b_{i}^{l}} + \dots + δ_{i}^{l} \cdot \frac{\partial z_{i}^{l}}{\partial b_{i}^{l}} + \dots + δ_{S_{l}}^{l} \cdot \frac{\partial z_{S_{l}}^{l}}{\partial b_{i}^{l}} \\ = δ_{1}^{l} \cdot 0 + δ_{2}^{l} \cdot 0 + \dots + δ_{i}^{l} \cdot 1 + \dots + δ_{S_{l}}^{l} \cdot 0 \dots \dots 由 z^{l} = W^{l} a^{(l - 1)} + b^{l} 可 知 \\ = δ_{i}^{l} \end{aligned}

第二种

\begin{aligned} \frac{\partial J}{\partial b_{i}^{l}} & = \frac{\partial J}{\partial z_{1}^{l + 1}} \cdot \frac{\partial z_{1}^{l + 1}}{\partial b_{i}^{l}} + \frac{\partial J}{\partial z_{2}^{l + 1}} \cdot \frac{\partial z_{2}^{l + 1}}{\partial b_{i}^{l}} + \dots + \frac{\partial J}{\partial z_{S_{l}}^{l + 1}} \cdot \frac{\partial z_{S_{l}}^{l + 1}}{\partial b_{i}^{l}} \\ = δ_{1}^{l + 1} \cdot \frac{\partial z_{1}^{l + 1}}{\partial b_{i}^{l}} + δ_{2}^{l + 1} \cdot \frac{\partial z_{2}^{l + 1}}{\partial b_{i}^{l}} + \dots + δ_{i}^{l + 1} \cdot \frac{\partial z_{i}^{l + 1}}{\partial b_{i}^{l}} + \dots + δ_{S_{l}}^{l + 1} \cdot \frac{\partial z_{S_{l}}^{l + 1}}{\partial b_{i}^{l}} \\ = δ_{1}^{l + 1} \cdot 0 + δ_{2}^{l + 1} \cdot 0 + \dots + δ_{i}^{l + 1} \cdot 1 + \dots + δ_{S_{l}}^{l + 1} \cdot 0 \dots \dots 由 z^{l + 1} = W^{l} a^{l} + b^{l} 可 知 \\ = δ_{i}^{l + 1} \end{aligned}

公式五： $f^{'} (z) = f (z) \cdot (1 - f (z))$

\begin{aligned} f (z) & = \frac{1}{1 + e^{- z}} ⟹ f (z) \cdot e^{- z} = 1 - f (z) \\ ⟹ & f^{'} (z) = \frac{e^{- z}}{(1 + e^{- z})^{2}} = f^{2} (z) \cdot e^{- z} \\ = f (z) \cdot (1 - f (z)) \end{aligned}

关于反向传播算法中几个公式的推导

相关推荐