Variational Inference

相比MCMC，在大数据量采样下，VI要快。 1

\begin{eqnarray*} \ln{P(X)}&=&\ln{(P(X,Z))}-\ln{P(Z|X)}\\ &=&\ln{(\frac{P(X,Z)}{q(Z)})}-\ln{(\frac{P(Z|X)}{q(Z)})}\\ &=&\ln{P(X,Z)}-\ln{q(Z)}-\ln{(\frac{P(Z|X)}{q(Z)})} \end{eqnarray*}

\begin{eqnarray*} \int_{Z}q(Z)\ln{P(X)}\,dZ&=&\int_{Z}q(Z)\ln{P(X,Z)}\,dZ-\int_{Z}q(Z)\ln{q(Z)}\,dZ-\int_{Z}q(Z)\ln{(\frac{P(Z|X)}{q(Z)})}\,dZ\\ \ln{P(X)}&=&\underbrace{\int_{Z}q(Z)\ln{P(X,Z)}\,dZ-\int_{Z}q(Z)\ln{q(Z)}\,dZ}_{\text{L(q):ELOB}}\underbrace{-\int_{Z}q(Z)\ln{(\frac{P(Z|X)}{q(Z)})}\,dZ}_{\text{KL(q(Z)$||$P(Z|X))}} \end{eqnarray*}

3 真实的后验概率 $P(Z|X)$ 往往是十分复杂的，我们用 $q(Z)$ 近似 $P(Z)$ ，并且选择 $q(Z)=q_1(Z_1)q_2(Z_2)⋯q_M(Z_M) = \prod_{i=1}^{M}q_{i}(Z_{i})$ ，将每个q分解。这样便于计算积分等。这叫做平均场理论(mean field theory)，主要基于基于系统中个体的局部相互作用可以产生宏观层面较为稳定的行为这个物理思想。

\begin{eqnarray*} L(q)&=&\int_{Z}q(Z)\ln{P(X,Z)}\,dZ-\int_{Z}q(Z)\ln{q(Z)}\,dZ\\ &=&\underbrace{\int_{Z}\prod_{i=1}^{M}q_{i}(Z_{i})\ln{P(X,Z)}\,dZ}_{\text{$L_{1}$}} - \underbrace{\int_{Z}\prod_{i=1}^{M}q_{i}(Z_{i})\ln{\prod_{i=1}^{M}q_{i}(Z_{i})}\,dZ}_{\text{$L_{2}$}} \end{eqnarray*}

l1部分，只关心j部分

\begin{eqnarray*} L_{1}&=&\int_{Z}\prod_{i=1}^{M}q_{i}(Z_{i})\ln{P(X,Z)}\,dZ\\ &=&\idotsint\limits_{Z_{1},Z_{2}\cdots Z_{M}} \prod_{i=1}^{M}q_{i}(Z_{i})\ln{P(X,Z)}\,dZ_{1}dZ_{2}\ldots dZ_{M}\\ &=&\int_{Z_{j}} q_{j}(Z_{j})(\idotsint\limits_{Z_{i\neq j}} q_{i}(Z_{i})\prod_{i\neq j}^{M} q_{i}(Z_{i})\ln{P(X,Z)}\prod_{i\neq j}^{M}dZ_i)dZ_{j}\\ &=&\int_{Z_{j}}q_{j}(Z_{j})(E_{i\neq j}(\ln{P(X,Z)}))\,dZ_{j} \end{eqnarray*}

l2部分，只关心第j部分 \begin{eqnarray*} L_{2}&=&\int\prod_{i=1}^{M}q_{i}(Z_{i})\sum_{i=1}^{M}\ln{q_{i}(Z_{i})}\,dZ\\ &=&\sum_{i=1}^{M}(\int\limits_{Z_{i}}q_{i}(Z_{i})\ln{q_{i}(Z_{i})}\,dZ_{i}\\ &=&\int\limits_{Z_{j}}q_{j}(Z_{j})\ln{q_{j}(Z_{j})}\,dZ_{j}+const \end{eqnarray*}

4 所以最终 $L(q)$ 可以简化成：

\begin{eqnarray*} L(q)&=&L_{1}-L_{2}\\ &=&\int\limits_{Z_{j}}q_{j}(Z_{j})(E_{i\neq j}(\ln{P(X,Z)}))\,dZ_{j}-\int\limits_{Z_{j}}q_{j}(Z_{j})\ln{q_{j}(Z_{j})}\,dZ_{j}\\ &=&\int\limits_{Z_{j}}q_{j}(Z_{j})\frac{E_{i\neq j}(\ln{P(X,Z)})}{\ln{q_{j}(Z_{j})}}\,dZ_{j} \end{eqnarray*}

再简化：

\begin{equation} \ln{\tilde{P}(X,Z_{j})}=E_{i\neq j}(\ln{P(X,Z)}) \end{equation}

\begin{equation} L(q)=\int\limits_{Z_{j}}q_{j}(Z_{j})\ln{\frac{\tilde{P}(X,Z_{j})}{q_{j}(Z_{j})}}\,dZ_{j}+const=-D_{KL}(q_{j}(Z_{j})||\tilde{P}(X,Z_{j}))+const \end{equation}