最大似然估计的方法以及与其他表达式的关联，贝叶斯统计方法概述及与最大似然的不同；MAP方法
ML
\[\theta_{ML} = \arg_{\theta} \max p(\mathbb{X} ; \theta )\]ML log
\[\begin{split} \theta_{ML} &= \arg_{\theta} \max \Pi_{i=1}^{m}p(x^{(i)} ; \theta ) \\ &= \arg_{\theta} \max \sum_{i=1}^m \log p(x^{(i)} ; \theta) \end{split}\]ML 期望
\[\begin{split} \theta_{ML} &= \arg_{\theta} \max \frac{1}{m}\sum_{i=1}^m \log p(x^{(i)} ; \theta) \\ &= \arg_{\theta} \max \sum_{i=1}^m \underbrace{ \frac{1}{m}}_{变量经验分布的概率} \underbrace{\log p(x^{(i)} ; \theta)}_{变量} \\ &= \arg_{\theta} \max \mathbb{E}_{x ~ \sim ~ \tilde p } [ ~\log p(x ; \theta) ~] \end{split}\]KL散度
\[\begin{split} D_{KL}(\tilde p(x) ~ ||~ p(x;\theta)) &= \sum_{i=1}^m \tilde p(x^{(i)}) \log \frac{\tilde p(x^{(i)})}{p(x^{(i)} ;\theta )} \\ &= \sum_{i=1}^m \tilde p(x^{(i)}) \log \tilde p(x^{(i)}) - \tilde p(x^{(i)}) \log p(x^{(i)} ;\theta) \\ &= \sum_{i=1}^m \tilde p(x^{(i)}) \log \tilde p(x^{(i)}) - \sum_{i=1}^m\tilde p(x^{(i)}) \log p(x^{(i)};\theta) \\ &= \mathbb{E}_{x \sim \tilde p} \log \tilde p(x) - \mathbb{E}_{x \sim \tilde p} \log p(x;\theta) \end{split}\]最小化KL散度
\[\begin{split} &=> minimize ~~ D_{KL}(\tilde p(x) ~ ||~ p(x;\theta)) \\ &=> minimize ~~ \underbrace{\mathbb{E}_{x \sim \tilde p} \log \tilde p(x)}_{常量} - \mathbb{E}_{x \sim \tilde p} \log p(x;\theta) \\ &=> maximize ~~ \mathbb{E}_{x \sim \tilde p} \log p(x;\theta) \end{split}\]KL与似然的等价
\[\begin{split} minimize ~~ D_{KL}(\tilde p(x) ~ ||~ p(x;\theta)) &\Longleftrightarrow maximize ~~\log p(x;\theta) \\ &\Longleftrightarrow maximize ~~ p(x;\theta) \end{split}\]与最小化熵等价
\[\begin{split} minimize ~~ H(x , p(x)) &\Longleftrightarrow maximize ~~ \log p(x;\theta ) \\ &\Longleftrightarrow maximize ~~ p(x;\theta) \end{split}\]全部等价
\[\begin{split} maximize ~~ p(x;\theta) &\Longleftrightarrow maximize ~~ \log p(x;\theta ) \\ &\Longleftrightarrow minimize ~~ H(x , p(x)) \\ &\Longleftrightarrow minimize ~~ D_{KL}(\tilde p(x) ~ ||~ p(x;\theta)) \end{split}\]最大似然等价 经验分布
条件对数似然
\[\begin{split} \theta_{ML} &= \arg_{\theta} \max p(Y | X ; \theta) \\ &= \arg_{\theta} \max \Pi_{i=1}^m p(y^{(i)}| x^{(i)} ; \theta) ~~(~~&i.i.d ~ 条件~~)\\ &= \arg_{\theta} \max \sum_{i=1}^{m} \log p(y^{(i)}| x^{(i)} ; \theta) \end{split}\]对数似然 推导 MSE
\[\begin{split} \theta_{ML} &= \arg_{\theta} \max \Pi_{i=1}^{m} \frac{1}{\sqrt{2 \pi } \sigma} \exp({ - \frac{ (\hat y^{(i)} - y^{(i)})^2 }{2\sigma^2} }) \\ &= \arg_{\theta} \max \sum_{i=1}^{m} \log \frac{1}{\sqrt{2 \pi } \sigma} \exp({ - \frac{ (\hat y^{(i)} - y^{(i)})^2 }{2\sigma^2} }) \\ &= \arg_{\theta} \max \sum_{i=1}^{m} [ - \frac{1}{2\sigma^2} (\hat y^{(i)} - y^{(i)})^2 + \log \frac{1}{\sqrt{ 2 \pi } \sigma}]\\ &= \arg_{\theta} \max \sum_{i=1}^{m} - \frac{1}{2\sigma^2} ||\hat y^{(i)} - y^{(i)}||^2 \\ &= \arg_{\theta} \min \sum_{i=1}^{m} ||\hat y^{(i)} - y^{(i)}||^2 \\ \end{split}\]贝叶斯估计
\[\begin{split} p(x^{(m+1)} | \mathbb{X}) = \int \underbrace{p(x^{(m+1)} | \theta )}_{似然项} ~ \underbrace{p(\theta | \mathbb{X})}_{\theta 取该值时的后验概率} ~~d\theta \end{split}\]MAP
\[\begin{split} \theta_{MAP} &= \arg_{\theta} \max ~~ \frac{p(\mathbb{X}| \theta) p(\theta)}{p(\mathbb{X})} \\ &= \arg_{\theta} \max~~ \log \frac{p(\mathbb{X}| \theta) p(\theta)}{p(\mathbb{X})} \\ &= \arg_{\theta} \max ~~\log p(\mathbb{X} | \theta) + \log p(\theta) - \log p(\mathbb{X}) \\ &= \arg_{\theta} \max ~~\log p(\mathbb{X} | \theta) + \log p(\theta) \end{split}\]