\begin{align} p_1(x)&=\mathcal{N}(x\mid-4,1) \tag{11.6}\\ p_2(x)&=\mathcal{N}(x\mid 0,0.2) \tag{11.7}\\ p_3(x)&=\mathcal{N}(x\mid 8,3) \tag{11.8} \end{align}

p(\mathcal{X}\mid\boldsymbol{\theta})=\prod_{n=1}^N p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right),\quad p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)=\sum_{k=1}^K\pi_k\,\mathcal{N}\left(\boldsymbol{x}_n\mid\boldsymbol{\mu}_k,\boldsymbol{\Sigma}_k\right), \tag{11.9}

\log p(\mathcal{X}\mid\boldsymbol{\theta})=\sum_{n=1}^{N}\log p\left(\boldsymbol{x}_{n}\mid\boldsymbol{\theta}\right)=\underbrace{\sum_{n=1}^{N}\log\sum_{k=1}^{K}\pi_{k}\,\mathcal{N}\left(\boldsymbol{x}_{n}\mid\boldsymbol{\mu}_{k},\boldsymbol{\Sigma}_{k}\right)}_{=:\mathcal{L}}. \tag{11.10}

\log\mathcal{N}(\boldsymbol{x}\mid\boldsymbol{\mu},\boldsymbol{\Sigma})=-\frac{D}{2}\log(2\pi)-\frac{1}{2}\log\operatorname{det}(\boldsymbol{\Sigma})-\frac{1}{2}(\boldsymbol{x}-\boldsymbol{\mu})^{\top}\boldsymbol{\Sigma}^{-1}(\boldsymbol{x}-\boldsymbol{\mu}).

\begin{align*} \frac{\partial\mathcal{L}}{\partial\boldsymbol{\mu}_k}=\boldsymbol{0}^\top & \Longleftrightarrow\sum_{n=1}^N\frac{\partial\log p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}{\partial\boldsymbol{\mu}_k}=\boldsymbol{0}^\top,\tag{11.12}\\ \frac{\partial\mathcal{L}}{\partial\boldsymbol{\Sigma}_k}=\boldsymbol{0} & \Longleftrightarrow\sum_{n=1}^N\frac{\partial\log p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}{\partial\boldsymbol{\Sigma}_k}=\boldsymbol{0},\tag{11.13}\\ \frac{\partial\mathcal{L}}{\partial\pi_k}=0 & \Longleftrightarrow\sum_{n=1}^N\frac{\partial\log p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}{\partial\pi_k}=0.\tag{11.14} \end{align*}

\frac{\partial\log p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}{\partial\boldsymbol{\theta}}= {\color{orange} \frac{1}{p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)} } {\color{blue} \frac{\partial p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}{\partial\boldsymbol{\theta}} } , \tag{11.15}

{\color{orange} \frac{1}{p\left(\boldsymbol{x}_n\mid\boldsymbol{\theta}\right)}=\frac{1}{\sum_{j=1}^K\pi_j\mathcal{N}\left(\boldsymbol{x}_n\mid\boldsymbol{\mu}_j,\boldsymbol{\Sigma}_j\right)} } . \tag{11.16}

r_{n, k}:=\frac{\pi_k\mathcal{N}\left(\boldsymbol{x}_n\mid\boldsymbol{\mu}_k,\boldsymbol{\Sigma}_k\right)}{\sum_{j=1}^K\pi_j\mathcal{N}\left(\boldsymbol{x}_n\mid\boldsymbol{\mu}_j,\boldsymbol{\Sigma}_j\right)} \tag{11.17}

p(\boldsymbol{x}_{n}|\pi_{k}, \boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) = \pi\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) \tag{11.18}

\left[\begin{array}{cccc} 1.0& 0.0& 0.0\\ 1.0& 0.0& 0.0\\ 0.057& 0.943& 0.0\\ 0.001& 0.999& 0.0\\ 0.0& 0.066& 0.934\\ 0.0& 0.0& 1.0\\ 0.0& 0.0& 1.0\end{array}\right]\in \mathbb{R}^{N\times K}. \tag{11.19}

\boldsymbol{\mu}_{k}^{\text{new}}=\frac{\sum_{n=1}^{N} r_{n, k} \boldsymbol{x}_{n}}{\sum_{n=1}^{N} r_{n, k}}, \tag{11.20}

\begin{align} {\color{blue} \frac{\partial p\left(\boldsymbol{x}_{n}\mid\theta\right)}{\partial\boldsymbol{\mu}_{k}} } &= \sum\limits_{j=1}^{K} \pi_{j} \displaystyle \frac{ \partial \mathcal{N}(\boldsymbol{x}_{n}| \boldsymbol{\mu}_{j}, \boldsymbol{\Sigma}_{j}) }{ \partial \boldsymbol{\mu}_{k} } = \pi_{k} \displaystyle \frac{ \partial \mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) }{ \partial \boldsymbol{\mu}_{k} } \tag{11.21a}\\ &={\color{blue} \pi_{k}\left(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k}\right)^{\top}\boldsymbol{\Sigma}_{k}^{-1}\mathcal{N}\left(\boldsymbol{x}_{n}\mid\boldsymbol{\mu}_{k},\boldsymbol{\Sigma}_{k}\right) } , \tag{11.21b} \end{align}

\begin{align} \displaystyle \frac{ \partial \mathcal{L} }{ \partial \boldsymbol{\mu}_{k} } &= \sum\limits_{n=1}^{N} \displaystyle \frac{ \partial \log p(\boldsymbol{x}_{n}|\boldsymbol{\theta}) }{ \partial \boldsymbol{\mu}_{k} } = \sum\limits_{n=1}^{N} {\color{orange} \frac{1}{p(\boldsymbol{x}_{n}|\boldsymbol{\theta})} } {\color{blue} \displaystyle \frac{ \partial p(\boldsymbol{x}_{n}|\boldsymbol{\theta}) }{ \partial \boldsymbol{\mu}_{k} } } \tag{11.22a} \\&= \sum\limits_{n=1}^{N} {\color{blue} (\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1} } \underbrace{ \boxed{\frac{{\color{blue} \pi_{k}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) }}{{\color{orange} \sum_{j=1}^{K} \pi_{j}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{j}, \boldsymbol{\Sigma}_{j}) } } } }_{ =r_{n,k} }\tag{11.22b} \\ &=\sum_{n=1}^{N} r_{n, k}\left(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k}\right)^{\top}\boldsymbol{\Sigma}_{k}^{-1}. \tag{11.22c} \end{align}

\begin{align} \sum\limits_{n=1}^{N} r_{n, k}\boldsymbol{x}_{n} &= \sum\limits_{n=1}^{N} r_{n, k} \boldsymbol{\mu}_{k}^{\text{new}} \\ &\Updownarrow \\ \boldsymbol{\mu}_{k}^{\text{new}} = \frac{\displaystyle \sum_{n=1}^{N}r_{n, k}\boldsymbol{x}_{n}}{\boxed{\sum_{n=1}^{N} r_{n, k}}} &= \frac{1}{\boxed{N_{k}}}\sum\limits_{n=1}^{N} r_{n, k}\boldsymbol{x}_{n}, \end{align} \tag{11.23}

N_k:=\sum_{n=1}^N r_{n, k} \tag{11.24}

r_{k}:=\left[r_{1, k},\ldots, r_{n, k}\right]^{\top}/ N_{k}, \tag{11.25}

\boldsymbol{\mu}_{k}\leftarrow \mathbb{E}_{r_{k}}[\mathcal{X}]. \tag{11.26}

\begin{align} \boldsymbol{\mu}_{1} :& -4 \rightarrow 2.7 \tag{11.27}\\ \boldsymbol{\mu}_{2} :& 0 \rightarrow -0.4\tag{11.28}\\ \boldsymbol{\mu}_{3} :& 8 \rightarrow 3.7\tag{11.29} \end{align}

\boldsymbol{\Sigma}_k^{\text{new}}=\frac{1}{N_k}\sum_{n=1}^N r_{n, k}\left(\boldsymbol{x}_n-\boldsymbol{\mu}_k\right)\left(\boldsymbol{x}_n-\boldsymbol{\mu}_k\right)^{\top}, \tag{11.30}

\frac{\partial\mathcal{L}}{\partial\boldsymbol{\Sigma}_k}=\sum_{n=1}^N\frac{\partial\log p\left(\boldsymbol{x}_n\mid\theta\right)}{\partial\boldsymbol{\Sigma}_k}=\sum_{n=1}^N {\color{orange} \frac{1}{p\left(\boldsymbol{x}_n\mid\theta\right)} } {\color{blue} \frac{\partial p\left(\boldsymbol{x}_n\mid\theta\right)}{\partial\boldsymbol{\Sigma}_k} } . \tag{11.31}

\begin{align*} &\,{\color{blue} \frac{\partial p(\boldsymbol{x}_n\mid\theta)}{\partial\boldsymbol{\Sigma}_k} } \tag{11.32a}\\ =&\,\frac{\partial}{\partial\boldsymbol{\Sigma}_{k}}\left(\pi_{k}(2\pi)^{-\frac{D}{2}}\det(\boldsymbol{\Sigma}_{k})^{-\frac{1}{2}}\exp\left(-\frac{1}{2}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})\right)\right) \tag{11.32b}\\ =&\,\pi_k(2\pi)^{-\frac D2}\left[{\color{red} \frac\partial{\partial\boldsymbol{\Sigma}_k}\det(\boldsymbol{\Sigma}_k)^{-\frac12} } \exp\left(-\frac12(\boldsymbol{x}_n-\boldsymbol{\mu}_k)^\top\boldsymbol{\Sigma}_k^{-1}(\boldsymbol{x}_n-\boldsymbol{\mu}_k)\right)\right] \\ &\,+\det(\boldsymbol{\Sigma}_{k})^{-\frac{1}{2}}\frac{\partial}{\partial\boldsymbol{\Sigma}_{k}}\exp\left(-\frac{1}{2}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})\right)\bigg] .\tag{11.32c} \end{align*}

\begin{align} {\color{red} \displaystyle \frac{ \partial }{ \partial \boldsymbol{\Sigma}_{k} } \det(\boldsymbol{\Sigma}_{k})^{-1/2} } \xlongequal{\text{(5.101)}}& {\color{red} -\frac{1}{2}\det(\boldsymbol{\Sigma}_{k})^{-1/2}\boldsymbol{\Sigma}_{k}^{-1} } , \tag{11.33}\\ \displaystyle \frac{ \partial }{ \partial \boldsymbol{\sigma}_{k} } (\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1}(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k}) \xlongequal{\text{(5.103)}}& -\boldsymbol{\Sigma}_{k}^{-1} (\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1} \tag{11.34} \end{align}

\begin{align} {\color{blue} \displaystyle \frac{ \partial p(\boldsymbol{x}_{n}|\boldsymbol{\theta}) }{ \partial \boldsymbol{\Sigma}_{k} } } &= {\color{blue} \pi_{k}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) \cdot \left[ -\frac{1}{2} (\boldsymbol{\Sigma}_{k}^{-1} - \boldsymbol{\Sigma}_{k}^{-1}(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1}) \right] } .\tag{11.35} \end{align}

\begin{align} \frac{\partial\mathcal{L}}{\partial\boldsymbol{\Sigma}_{k}} & =\sum_{n=1}^N\frac{\partial\log p(\boldsymbol{x}_n\mid\theta)}{\partial\boldsymbol{\Sigma}_k}=\sum_{n=1}^N {\color{orange} \frac{1}{p(\boldsymbol{x}_n\mid\theta)} } {\color{blue} \frac{\partial p(\boldsymbol{x}_n\mid\theta)}{\partial\boldsymbol{\Sigma}_k} } \tag{11.36a} \\ & =\sum_{n=1}^N\frac{\color{blue} \pi_k\mathcal{N}(\boldsymbol{x}_n\mid\boldsymbol{\mu}_k,\boldsymbol{\Sigma}_k)}{\underbrace{\color{orange} \sum_{j=1}^K\pi_j\mathcal{N}(\boldsymbol{x}_n\mid\boldsymbol{\mu}_j,\boldsymbol{\Sigma}_j)}_{=r_{nk}}}\cdot{\color{blue} \left[-\frac{1}{2}(\boldsymbol{\Sigma}_k^{-1}-\boldsymbol{\Sigma}_k^{-1}(\boldsymbol{x}_n-\boldsymbol{\mu}_k)(\boldsymbol{x}_n-\boldsymbol{\mu}_k)^\top\boldsymbol{\Sigma}_k^{-1})\right]} \tag{11.36b} \\ & =-\frac{1}{2}\sum_{n=1}^{N}r_{nk}(\boldsymbol{\Sigma}_{k}^{-1}-\boldsymbol{\Sigma}_{k}^{-1}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})^{\top}\boldsymbol{\Sigma}_{k}^{-1}) \tag{11.36c} \\ & =-\frac{1}{2}\boldsymbol{\Sigma}_{k}^{-1}\underbrace{\sum_{n=1}^{N}r_{nk}}_{=N_{k}}+\frac{1}{2}\boldsymbol{\Sigma}_{k}^{-1}\left(\sum_{n=1}^{N}r_{nk}(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})(\boldsymbol{x}_{n}-\boldsymbol{\mu}_{k})^{\top}\right)\boldsymbol{\Sigma}_{k}^{-1}. \tag{11.36d} \end{align}

\begin{align} & N_{k}\boldsymbol{\Sigma}_{k}^{-1} = \boldsymbol{\Sigma}_{k}^{-1}\left[ \sum\limits_{n=1}^{N} r_{n, k}(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top} \right] \boldsymbol{\Sigma}_{k}^{-1} \tag{11.37a}\\ \iff & N_{k}\boldsymbol{I} = \left[ \sum\limits_{n=1}^{N} r_{n, k} (\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top} \right] \boldsymbol{\Sigma}_{k}^{-1}. \tag{11.37b} \end{align}

\boldsymbol{\Sigma}_{k}^{\text{new}} = \frac{1}{N_{k}} \sum\limits_{n=1}^{N} r_{n, k} (\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})(\boldsymbol{x}_{n} - \boldsymbol{\mu}_{k})^{\top}, \tag{11.38}

\begin{align} \boldsymbol{\mu}_{1} :& 1 \rightarrow 0.14 \tag{11.39}\\ \boldsymbol{\mu}_{2} :& 0.2 \rightarrow 0.44\tag{11.40}\\ \boldsymbol{\mu}_{3} :& 3 \rightarrow 1.53\tag{11.41} \end{align}

\pi_{k}^{\text{new}}=\frac{N_{k}}{N},\quad k=1,\ldots, K, \tag{11.42}

\begin{align} \mathfrak{L}&=\mathcal{L}+\lambda\left(\sum_{k=1}^{K}\pi_{k}-1\right) \tag{11.43a} \\ &= \sum\limits_{n=1}^{N} \log\left[ \sum\limits_{k=1}^{K} \pi_{k}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k}) \right] + \lambda\left(\sum_{k=1}^{K}\pi_{k}-1\right) \tag{11.43b} \end{align}

\begin{align} \displaystyle \frac{ \partial \mathfrak{L} }{ \partial \pi_{k} } &= \sum\limits_{n=1}^{N} \frac{\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k})}{\sum_{j=1}^{K} \pi_{j}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{j}, \boldsymbol{\Sigma}_{j})} + \lambda \tag{11.44a}\\ &= \frac{1}{\pi_{k}} \sum\limits_{n=1}^{N} \frac{\pi_{k}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{k}, \boldsymbol{\Sigma}_{k})}{\sum_{j=1}^{K} \pi_{j}\,\mathcal{N}(\boldsymbol{x}_{n}|\boldsymbol{\mu}_{j}, \boldsymbol{\Sigma}_{j})} + \lambda = \frac{N_{k}}{\pi_{k}} + \lambda \tag{11.44b} \end{align}

\displaystyle \frac{ \partial \mathfrak{L} }{ \partial \lambda } = \sum\limits_{k=1}^{K} \pi_{k} - 1. \tag{11.45}

\begin{align} \pi_{k} &= -\frac{N_{k}}{\lambda}, \tag{11.46}\\ 1 &= \sum\limits_{k=1}^{K} \pi_{k}. \tag{11.47} \end{align}

\sum\limits_{k=1}^{K} \pi_{k} = 1 \iff -\sum\limits_{k=1}^{K} \frac{N_{k}}{\lambda} = 1 \iff -\frac{N}{\lambda} = 1 \iff \lambda = -N. \tag{11.48}

\pi_{k}^{\text{new}} = \frac{N_{k}}{N}, \tag{11.49}

\begin{align} \pi_{1} :& \frac{1}{2} \rightarrow 0.29 \tag{11.50}\\ \pi_{2} :& \frac{1}{3} \rightarrow 0.29 \tag{11.51}\\ \pi_{3} :& \frac{1}{3} \rightarrow 0.42 \tag{11.52} \end{align}