PRML Part 1.lyx~

#LyX 2.3 created this file. For more info see http://www.lyx.org/
\lyxformat 544
\begin_document
\begin_header
\save_transient_properties true
\origin unavailable
\textclass book
\begin_preamble
\usepackage{ifxetex}
\usepackage{ifluatex}\usepackage{fixltx2e}% provides \textsubscript
\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
  \else % if luatex or xelatex
  \ifxetex
    \usepackage{mathspec}
\else
    \fi
  \defaultfontfeatures{Ligatures=TeX,Scale=MatchLowercase}
\fi
% use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
% use microtype if available
\IfFileExists{microtype.sty}{%
\usepackage[]{microtype}
\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\PassOptionsToPackage{hyphens}{url} % url is loaded by hyperref

\urlstyle{same}  % don't use monospace font for urls
\usepackage{grffile}
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}

% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
\IfFileExists{parskip.sty}{%
\usepackage{parskip}
}{% else
\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt plus 2pt minus 1pt}
}
\setlength{\emergencystretch}{3em}  % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}

% Redefines (sub)paragraphs to behave more like sections
\ifx\paragraph\undefined\else
\let\oldparagraph\paragraph
\renewcommand{\paragraph}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
\let\oldsubparagraph\subparagraph
\renewcommand{\subparagraph}[1]{\oldsubparagraph{#1}\mbox{}}
\fi

% set default figure placement to htbp

\def\fps@figure{htbp}
\end_preamble
\use_default_options true
\begin_modules
theorems-ams
eqs-within-sections
figs-within-sections
\end_modules
\maintain_unincluded_children false
\language english
\language_package none
\inputencoding utf8
\fontencoding T1
\font_roman "lmodern" "default"
\font_sans "default" "default"
\font_typewriter "default" "default"
\font_math "auto" "auto"
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100 100
\font_tt_scale 100 100
\use_microtype false
\use_dash_ligatures true
\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command makeindex
\paperfontsize default
\spacing single
\use_hyperref true
\pdf_bookmarks true
\pdf_bookmarksnumbered false
\pdf_bookmarksopen true
\pdf_bookmarksopenlevel 1
\pdf_breaklinks true
\pdf_pdfborder true
\pdf_colorlinks false
\pdf_backref section
\pdf_pdfusetitle false
\papersize a4paper
\use_geometry true
\use_package amsmath 2
\use_package amssymb 2
\use_package cancel 2
\use_package esint 1
\use_package mathdots 2
\use_package mathtools 2
\use_package mhchem 2
\use_package stackrel 2
\use_package stmaryrd 2
\use_package undertilde 1
\cite_engine basic
\cite_engine_type default
\biblio_style plain
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date true
\justification true
\use_refstyle 0
\use_minted 0
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 2
\tocdepth 2
\paragraph_separation indent
\paragraph_indentation default
\is_math_indent 0
\math_numbering_side default
\quotes_style english
\dynamic_quotes 0
\papercolumns 1
\papersides 2
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Chapter*
2.
 Probability Distribution
\end_layout

\begin_layout Standard

\emph on
Density estimation
\end_layout

\begin_layout Standard
Data points are independent and identically distributed.
 There are infinitely many probability distributions that could have given
 rise to the observed finite data set.
\end_layout

\begin_layout Standard

\series bold
Parametric
\series default
 and 
\series bold
non-parametric
\series default
 approaches.
\end_layout

\begin_layout Section*
2.1 Binary Variables 
\end_layout

\begin_layout Standard
Given 
\begin_inset Formula $x\in\left\{ 0,1\right\} $
\end_inset

 and 
\begin_inset Formula $0\leq\mu\leq1$
\end_inset

, the one-coin-tossing distribution, i.e.
 the Bernoulli distribution
\begin_inset Formula 
\begin{equation}
\text{\text{Bern}\left(x|\mu\right) \ensuremath{=\mu^{x}\left(1-\mu\right){}^{1-x}}}
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
whose 
\begin_inset Formula $\mu_{ML}=\dfrac{1}{N}\sum\limits _{n=1}^{N}x_{n}=\dfrac{m}{N}$
\end_inset

, 
\begin_inset Formula $m$
\end_inset

 is the number of observations.
 
\end_layout

\begin_layout Standard
Suppose we toss this coin 
\begin_inset Formula $N$
\end_inset

 times, the number of heads is 
\begin_inset Formula $m$
\end_inset

, the distribution of 
\begin_inset Formula $m$
\end_inset

 will be
\begin_inset Formula 
\begin{equation}
\text{Bin}\left(m|N,\mu\right)=\dbinom{N}{m}\mu^{m}\left(1-\mu\right){}^{N-m}
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
If we obtain 
\begin_inset Formula $m=N$
\end_inset

 in an experiment, the estimation will be unreasonable for the both distribution
s.
\end_layout

\begin_layout Subsection*
2.1.1 The beta distribution 
\end_layout

\begin_layout Standard
To solve the problem of the unreasonble estimation (overfitting) when 
\begin_inset Formula $m=N$
\end_inset

, we introduce a prior to be proportional to powers of 
\begin_inset Formula $\mu$
\end_inset

 and 
\begin_inset Formula $1-\mu$
\end_inset

, which results the same functional form of the posterior as the prior (
\shape italic
conjugacy
\shape default
), this prior we choose is 
\shape italic
beta
\shape default
 distribution:
\begin_inset Formula 
\begin{equation}
\text{Beta}\left(\mu|a,b\right)=\frac{\Gamma\left(a+b\right)}{\Gamma\left(a\right)\Gamma\left(b\right)}\mu^{a-1}\left(1-\mu\right){}^{b-1}
\end{equation}

\end_inset

where the gamma function 
\begin_inset Formula 
\begin{equation}
\Gamma\left(z\right)=\int_{0}^{\infty}x^{z-1}e^{-x}dx
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
The parameters 
\begin_inset Formula $a$
\end_inset

 and 
\begin_inset Formula $b$
\end_inset

 control the distribution of the parameter 
\begin_inset Formula $\mu$
\end_inset

, thus called 
\shape italic
hyperparameters.
\end_layout

\begin_layout Standard
Multiplying (0.1) by (0.2) yields the posterior 
\begin_inset Formula 
\[
p\left(\mu|m,l,a,b\right)\propto\dbinom{l+m}{m}\mu^{m+a-1}\left(1-\mu\right){}^{l+b-1}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $l=N-m$
\end_inset

, taking the same form as the prior.
 The parameter estimation now depends both its own (prior) distribution
 and the data observed, and this gives rise to sequential learning, by updating
 
\begin_inset Formula $\mu$
\end_inset

 repeatedly with 
\begin_inset Formula $l$
\end_inset

 and 
\begin_inset Formula $m$
\end_inset

.
\end_layout

\begin_layout Standard
In Bayesian learning, as we observe more and more data, the uncertainty
 represented by the posterior distribution will steadily decrese.
\end_layout

\begin_layout Standard
Given a parameter 
\begin_inset Formula $\theta$
\end_inset

 for which we have observed a data set 
\begin_inset Formula $D$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
E_{\theta}(\theta) & =E_{D}\underbrace{\left[E_{\theta}\left[\theta|D\right]\right]}_{\text{posterior mean}}\\
\text{var}_{\theta}[\theta] & =E_{D}\left[\text{var}_{\theta}\left[\theta|D\right]\right]+\text{var}_{D}\left[E_{\theta}\left[\theta|D\right]\right]
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
On average, the expectation of the prior expectation becomes more stable
 and the posterior variance of 
\begin_inset Formula $\theta$
\end_inset

 is smaller than the prior (less uncertainty).
\end_layout

\begin_layout Section*
2.2 Multinomial Variables
\end_layout

\begin_layout Standard
The probability of 
\begin_inset Formula $x_{k}=1$
\end_inset

 is denoted by the parameter 
\begin_inset Formula $\mu_{k}$
\end_inset

, then the distribution
\begin_inset Formula 
\begin{equation}
p\left(x|\mu\right)=\prod_{k=1}^{K}\mu_{k}^{x_{k}}
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mathrm{x}=\left(\dots,x_{k},\dots\right){}^{T}$
\end_inset

and 
\begin_inset Formula $\mu=\left(\mu_{1},\dots,\mu_{K}\right){}^{T}$
\end_inset

 and 
\begin_inset Formula $\sum\limits _{k}\mu_{_{k}}=1,\sum\limits _{k}x_{k}=1,\mathbb{E}\left[\mathrm{x}|\mu\right]=\sum\limits _{x}p\left(\mathrm{x}|\mu\right)\mathrm{x}=\mu$
\end_inset


\end_layout

\begin_layout Standard
Given a data set 
\begin_inset Formula $D$
\end_inset

 of 
\begin_inset Formula $N$
\end_inset

 observations, to use KKT conditions to maximize its MAL
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\sum_{k=1}^{K}m_{k}\ln\mu_{k}+\lambda\left(\sum_{k=1}^{K}\mu_{k}-1\right)
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $m_{k}$
\end_inset

is the number of occurrence of 
\begin_inset Formula $x_{k}=1$
\end_inset

, yields 
\begin_inset Formula $\mu_{k}^{ML}=\dfrac{m_{k}}{N}$
\end_inset

.
\end_layout

\begin_layout Standard
The joint distribution of 
\begin_inset Formula $m_{1},\dots,m_{K}$
\end_inset

 conditioned on the parameters 
\begin_inset Formula $\mu$
\end_inset

 and the total number of observations 
\begin_inset Formula $N$
\end_inset


\begin_inset Formula 
\begin{align}
\text{Mult}\left(m_{1},\dots,m_{k}|\mu,N\right) & =\frac{N!}{m_{1}!m_{2}!\dots m_{k}!}\prod_{k=1}^{K}\mu_{k}^{m_{k}}
\end{align}

\end_inset


\end_layout

\begin_layout Standard
subject to
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\prod_{k=1}^{K}m_{k}=N
\]

\end_inset


\end_layout

\begin_layout Subsection*
2.2.1 The Dirichlet distribution
\end_layout

\begin_layout Standard
A family of prior distribution for the parameters 
\begin_inset Formula $\left\{ \mu_{k}\right\} $
\end_inset

 of the multinomial distribution, the 
\shape italic
dirichlet distribution
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{equation}
\text{Dir}(\mu|\alpha)=\frac{\Gamma\left(\sum_{k}\alpha_{k}\right)}{\Gamma\left(\alpha_{1}\right)\dots\Gamma\left(\alpha_{K}\right)}\prod_{k=1}^{K}\mu_{k}^{\alpha_{k}-1}
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
Multiply (0.7) by (0.6)
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align}
p\left(\mu|D,\alpha\right) & \propto\prod_{k=1}^{K}\mu_{k}^{\alpha_{k}+m_{k}+1}\\
p\left(\mu|D,\alpha\right) & =\text{Dir}\left(\mu|\alpha+m\right)=\frac{\Gamma\left(N+\sum_{k}\alpha_{k}\right)}{\Gamma\left(\alpha_{1}+m_{1}\right)\dots\Gamma\left(\alpha_{K}+m_{k}\right)}\prod_{k=1}^{K}\mu_{k}^{\alpha_{k}+m_{k}-1}
\end{align}

\end_inset


\end_layout

\begin_layout Section*
2.3 The Gaussian Distribution
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{equation}
N\left(x|\mu,\sigma^{2}\right)=\dfrac{1}{\left(2\pi\right){}^{D/2}}\dfrac{1}{\left|\Sigma\right|{}^{1/2}}\exp\left\{ -\dfrac{1}{2}\left(x-\mu\right){}^{T}\Sigma^{-1}\left(x-\mu\right)\right\} 
\end{equation}

\end_inset


\end_layout

\begin_layout Standard

\emph on
Mahalanobis distance
\family roman
\series medium
\shape up
\size normal
\emph off
\bar no
\strikeout off
\xout off
\uuline off
\uwave off
\noun off
\color none
 
\begin_inset Formula $\left(x-\mu\right){}^{T}\Sigma^{-1}\left(x-\mu\right)$
\end_inset


\end_layout

\begin_layout Subsection*
2.3.1 Conditional and Marginal Gaussian distributions
\end_layout

\begin_layout Standard
An important property of the multivariate Gaussian distribution is that
 if two sets of variables are jointly Gaussian, then the conditional distributio
n of one set conditoned on the other is again Gaussian.
 Similarly, the marginal distribution of either set is also Gaussian.
\end_layout

\begin_layout Subsection*
2.3.3 Bayes' theorem for Gaussian variables
\end_layout

\begin_layout Standard
Given a marginal Gaussian distribution 
\begin_inset Formula $p\left(x\right)$
\end_inset

 for 
\begin_inset Formula $x$
\end_inset

 and a condition Gaussian distribution 
\begin_inset Formula $p\left(y|x\right)$
\end_inset

 for 
\begin_inset Formula $y$
\end_inset

 given 
\begin_inset Formula $x$
\end_inset

, the marginal distribution of 
\begin_inset Formula $y$
\end_inset

 and the conditional distribution of 
\begin_inset Formula $x$
\end_inset

 given 
\begin_inset Formula $y$
\end_inset

 are also Gaussian.
\end_layout

\begin_layout Subsection*
2.3.4 Maximum likelihood for the Gaussian
\end_layout

\begin_layout Standard
Given a data set 
\begin_inset Formula $X=\left(x_{1},...,x_{N}\right){}^{T}$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{equation}
\ln p\left(X|\mu,\Sigma\right)=-\dfrac{ND}{2}\ln\left(2\pi\right)-\dfrac{N}{2}\ln\left|\Sigma\right|-\dfrac{1}{2}\sum_{n=1}^{N}(x_{n}-\mu)^{T}\Sigma^{-1}(x_{n}-\mu)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
Take the derivative w.r.t.
 
\begin_inset Formula $\mu$
\end_inset

and set it to zero
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\mu_{ML}=\dfrac{1}{N}\sum_{n=1}^{N}x_{n}
\]

\end_inset


\end_layout

\begin_layout Standard
And the covariance (Magnus and Neudecker (1999))
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\Sigma_{ML}=\dfrac{1}{N}\sum_{n=1}^{N}(x_{n}-\mu_{ML})(x_{n}-\mu_{ML})^{T}
\]

\end_inset


\end_layout

\begin_layout Standard
which is biased.
\end_layout

\begin_layout Subsection*
2.3.5 Sequential estimation
\end_layout

\begin_layout Standard
Robbins-Monro algorithm (Robbins and Monro,1951; Fukunaga,1990)
\end_layout

\begin_layout Subsection*
2.3.6 Bayesian inference for the Gaussian
\end_layout

\begin_layout Paragraph
Given variance 
\begin_inset Formula $\sigma^{2}$
\end_inset

 and a set of 
\begin_inset Formula $N$
\end_inset

 observations 
\begin_inset Formula $X=\left\{ x_{1},\dots,x_{N}\right\} $
\end_inset

, the mean 
\begin_inset Formula $\mu$
\end_inset

 to be inferred
\end_layout

\begin_layout Standard
Take the prior distribution to be 
\begin_inset Formula 
\begin{equation}
p\left(\mu\right)=\mathcal{N}\left(\mu|\mu_{0},\sigma_{0}^{2}\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
The posterior distribution becomes
\begin_inset Formula 
\begin{equation}
p\left(\mu|\mathrm{X}\right)=\mathcal{N}\left(\mu|\mu_{N},\sigma_{N}^{2}\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where
\begin_inset Formula 
\begin{align}
\mu_{N} & =\frac{\sigma^{2}}{N\sigma_{0}^{2}+\sigma^{2}}\mu_{0}+\frac{N\sigma_{0}^{2}}{N\sigma_{0}^{2}+\sigma^{2}}\mu_{ML}\\
\frac{1}{\sigma_{N}^{2}} & =\frac{1}{\sigma_{0}^{2}}+\frac{N}{\sigma^{2}}
\end{align}

\end_inset


\end_layout

\begin_layout Standard
Note how the paramter change as 
\begin_inset Formula $N\rightarrow0/\infty,\sigma_{0}^{2}\rightarrow\infty$
\end_inset

: the precision increases and the posterior distribution becomes infintely
 peaked around the MAL solution.
\end_layout

\begin_layout Subparagraph
Sequential learning 
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{equation}
p\left(\mu|D_{N}\right)\propto\underbrace{\left[p\left(\mu\right)\prod_{n=1}^{N-1}p\left(\mathrm{x_{n}|\mu}\right)\right]}_{p\left(\mu|D_{N-1}\right)}p\left(\mathrm{x_{N}|\mu}\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Paragraph
Given variance 
\begin_inset Formula $\mu$
\end_inset

 and a set of 
\begin_inset Formula $N$
\end_inset

 observations 
\begin_inset Formula $X=\left\{ x_{1},\dots,x_{N}\right\} $
\end_inset

, the mean 
\begin_inset Formula $\sigma^{2}$
\end_inset

 or 
\begin_inset Formula $\lambda\equiv\dfrac{1}{\sigma^{2}}$
\end_inset

 to be inferred
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Gamma distribution"
target "https://en.wikipedia.org/wiki/Gamma_distribution"
literal "false"

\end_inset


\end_layout

\begin_layout Standard
The likelihood
\begin_inset Formula 
\begin{equation}
p\left(\mathrm{X|\lambda}\right)\propto\lambda^{N/2}\exp\left\{ -\frac{\lambda}{2}\sum_{n=1}^{N}\left(x_{n}-\mu\right)^{2}\right\} 
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
The prior is a 
\emph on
gamma
\emph default
 distribution
\begin_inset Formula 
\begin{equation}
\text{Gam}\left(\lambda|a,b\right)=\frac{1}{\Gamma\left(a\right)}b^{a}\lambda^{a-1}\exp\left(-b\lambda\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
The posterior 
\begin_inset Formula 
\begin{equation}
p\left(\lambda|\mathrm{X}\right)\propto\text{Gam}\left(\lambda|a_{N},b_{N}\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where
\begin_inset Formula 
\begin{align}
a_{N} & =a_{0}+\frac{N}{2}\\
b_{N} & =b_{0}+\frac{N}{2}\sigma_{ML}^{2}
\end{align}

\end_inset


\end_layout

\begin_layout Standard
Another prior distribution 
\emph on
can be inverse gamma
\end_layout

\begin_layout Paragraph
Consider the case where both parameters are unknown
\end_layout

\begin_layout Standard
The likelihood can be written as
\begin_inset Formula 
\begin{equation}
p\left(\mu,\lambda\right)=\mathcal{N}\left(\mu|\mu_{0},\left(\beta\lambda\right)^{-1}\right)\text{Gam}\left(\lambda|a,b\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mu_{0}=c/\beta,a=1+\beta/2,b=d-c^{2}/2\beta$
\end_inset

.
 This distribution is called the 
\emph on
normal-gamma
\emph default
 or 
\emph on
Gaussian-gamma
\emph default
 distribution.
\end_layout

\begin_layout Paragraph
In the case the multivariate Gaussian distribution
\end_layout

\begin_layout Standard
For known mean and unknown precision matrix, the conjugata prior is the
 
\emph on
Wishart 
\emph default
distribution.
 If both are unknown, this goes to the 
\emph on
normal-Wishart
\emph default
 or 
\emph on
Gaussian-Wishart
\emph default
 distribution.
\end_layout

\begin_layout Subsection*
2.3.9 Mixtures of Gaussians
\end_layout

\begin_layout Standard

\emph on
Mixture distributions
\emph default
: probabilistic models formed by taking linear combinations of more basic
 distributions.
\end_layout

\begin_layout Standard

\emph on
mixture of Gaussians
\emph default
: 
\begin_inset Formula 
\begin{equation}
p\left(x\right)=\sum_{k=1}^{K}\pi_{k}N\left(x|\mu_{k},\Sigma_{k}\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where each Gaussian density is called a 
\emph on
component 
\emph default
which has its own mean and covariance, and 
\begin_inset Formula $\pi_{k}$
\end_inset

's are called 
\emph on
mixing coefficients 
\emph default
with 
\begin_inset Formula 
\begin{align}
\sum_{k=1}^{K}\pi_{k} & =1\\
0\leq\pi_{k} & \leq1
\end{align}

\end_inset


\end_layout

\begin_layout Standard
From the sum and product rules
\begin_inset Formula 
\begin{equation}
p\left(x\right)=\sum_{k=1}^{K}p\left(k\right)p\left(x|k\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
equivalent the definition.
 The posterior distribution 
\begin_inset Formula $p\left(x|k\right)$
\end_inset

 are called 
\emph on
responsibilities
\emph default
 and from the Bayes' theorem
\begin_inset Formula 
\[
p\left(x|k\right)=\frac{\pi_{k}N\left(x|\mu_{k},\Sigma_{k}\right)}{\sum_{l}\pi_{l}N\left(x|\mu_{l},\Sigma_{l}\right)}
\]

\end_inset


\end_layout

\begin_layout Standard
The MAL for mixture Gaussians has no closed-form analytical solution.
 iterative numerical optimzation techniques may be used or 
\emph on
expectation maximization.
\end_layout

\begin_layout Section*
2.4 The Expoential Family
\end_layout

\begin_layout Standard
The exponential family of distributions over 
\begin_inset Formula $x$
\end_inset

, given parameters 
\begin_inset Formula $\eta$
\end_inset

, is defined to be the set of distributions of the form
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{equation}
p(x|\eta)=h\left(x\right)g\left(\eta\right)\exp\left\{ \eta^{T}u(x)\right\} 
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\eta$
\end_inset

 are called the 
\emph on
natural parameters.
 
\emph default
Since the distribution is normalized 
\begin_inset Formula 
\begin{equation}
g\left(\eta\right)\int h\left(x\right)\exp\left\{ \eta^{T}u(x)\right\} dx=1
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
For the Bernoulli distribution
\begin_inset Formula 
\begin{equation}
p(x|\eta)=\text{sigmoid}\left(-\eta\right)\exp\left(\eta x\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\eta=\ln\dfrac{\mu}{1-\mu}.$
\end_inset


\end_layout

\begin_layout Standard
For the multinomial distribution
\begin_inset Formula 
\begin{equation}
p\left(x|\eta\right)=\exp\left(\eta^{T}x\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\eta_{k}=\ln\mu_{k}$
\end_inset


\end_layout

\begin_layout Standard
Or 
\begin_inset Formula 
\begin{equation}
p(x|\eta)=\left(1+\sum_{k=1}^{M-1}\exp\left(\eta_{k}\right)\right)^{-1}\exp\left(\eta^{T}x\right)
\end{equation}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mu_{k}=\dfrac{\exp\left(\eta_{k}\right)}{1+\sum_{j}\exp\left(\eta_{j}\right)}$
\end_inset

, called 
\emph on
softmax 
\emph default
or 
\emph on
normalized exponential.
\end_layout

\begin_layout Subsection*
2.4.1 Maximum likelihood and sufficient statistics
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Sufficient statistics"
target "https://en.wikipedia.org/wiki/Sufficient_statistic"
literal "false"

\end_inset


\end_layout

\begin_layout Standard
Take the gradient of both sides of the integrated exponential distribution
\begin_inset Formula 
\[
-\triangledown\ln g\left(\eta\right)=\mathbb{E}\left[u\left(x\right)\right]
\]

\end_inset


\end_layout

\begin_layout Standard
By MAL
\begin_inset Formula 
\[
-\triangledown\ln g\left(\eta\right)=\frac{1}{N}\sum_{n=1}^{N}u\left(x_{n}\right)
\]

\end_inset


\end_layout

\begin_layout Subsection*
2.4.1 Conjugate priors
\end_layout

\begin_layout Standard
For any member of the exponential family, there exists a conjugate prior
 
\begin_inset Formula 
\[
p\left(\eta|\chi,\nu\right)=f\left(\chi,\nu\right)g\left(\eta\right)^{\nu}\exp\left\{ \nu\eta^{T}\chi\right\} 
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $f\left(\chi,\nu\right)$
\end_inset

 is a normalization coefficients.
\end_layout

\begin_layout Standard
The posterior distribution becomes
\begin_inset Formula 
\[
p\left(\eta|X,\chi,\nu\right)\propto g\left(\eta\right)^{\nu+N}\exp\left\{ \eta^{T}\left(\sum_{n=1}^{N}u\left(x_{n}\right)+\nu\chi\right)\right\} 
\]

\end_inset


\end_layout

\begin_layout Standard
taking the same form, confirming conjugacy.
\end_layout

\begin_layout Chapter*
3.
 Linear Models for Regression
\end_layout

\begin_layout Standard
The goal of regression is to predict the value of one or more continuous
 
\emph on
target
\emph default
 variables 
\begin_inset Formula $t$
\end_inset

 given the value of a 
\begin_inset Formula $D$
\end_inset

-dimensional vector 
\begin_inset Formula $\mathrm{x}$
\end_inset

 
\emph on
input
\emph default
 variables.
\end_layout

\begin_layout Section*
3.1 Linear Basis Function Models
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
y(x,w)= & w_{0}+\sum\limits _{j=1}^{M-1}w_{j}\phi_{j}(x)\\
= & w^{T}\phi(x)
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\phi_{j}(\mathrm{x})$
\end_inset

 are known as 
\emph on
basis functions
\emph default
, and 
\begin_inset Formula $w_{0}$
\end_inset

 
\emph on
bias
\emph default
 parameter
\end_layout

\begin_layout Standard
\begin_inset Formula $w=(w_{0},...,w_{M-1})^{T}$
\end_inset

 and 
\begin_inset Formula $\phi=(\phi_{0},...,\phi_{M-1})^{T}$
\end_inset

.
\end_layout

\begin_layout Standard
By using nonlinear basis functions, we allow the function 
\begin_inset Formula $y(x,w)$
\end_inset

 to be a non-linear function of the input vector 
\begin_inset Formula $x$
\end_inset

.
 Thus the equation above is called a 
\emph on
linear model
\emph default
.
\end_layout

\begin_layout Standard

\series bold
Choices for the basis functions
\end_layout

\begin_layout Standard

\emph on
Gaussian
\emph default
:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\phi_{j}(x)=\exp\Bigg\{-\dfrac{(x-\mu_{j})^{2}}{2s^{2}}\Bigg\}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mu_{j}$
\end_inset

 govern the locations of the basis functions in input space.
\end_layout

\begin_layout Standard

\emph on
Sigmoidal basis function
\emph default
:
\begin_inset Formula 
\[
\phi_{j}(x)=\sigma(\dfrac{x-\mu_{j}}{s}):\sigma(a)=\dfrac{1}{1+\exp(-a)}
\]

\end_inset


\end_layout

\begin_layout Standard
and 
\begin_inset Formula $tanh(a)=2\sigma(a)-1$
\end_inset


\end_layout

\begin_layout Standard

\emph on
the Fourier basis
\emph default
: Each basis function represents a specific frequency and has infinite spatial
 extent.
\end_layout

\begin_layout Paragraph
Maximum Likelihood and least squares
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Mode"
target "https://en.wikipedia.org/wiki/Mode_(statistics)"
literal "false"

\end_inset


\end_layout

\begin_layout Standard
Assume 
\begin_inset Formula $p(t,|X,w,\beta)=\mathcal{N}(t|y,\beta^{-1})$
\end_inset

, where 
\begin_inset Formula $t=y\left(w,x\right)+\epsilon$
\end_inset

 and 
\begin_inset Formula $y=w^{T}\phi(x)$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
\ln p(t|w,\beta)= & \sum_{n=1}^{N}\ln N\left(t_{n}|w^{T}\phi\left(x_{n}\right),\beta^{-1}\right)\\
= & \dfrac{N}{2}\ln\beta-\dfrac{N}{2}\ln(2\pi)-\beta E_{D}(w)
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mathrm{t}$
\end_inset

 is the column vector of targets and 
\begin_inset Formula $E_{D}(w)=\dfrac{1}{2}\sum\limits _{n=1}^{N}\left\{ t_{n}-w^{T}\phi(x_{n})\right\} {}^{2}$
\end_inset

.
 It is easy to see that maximization of the likelihood function under a
 conditional Gaussian noise distribution for a linear model is equivalent
 to minimizing a sum-of-squares error function.
 Take the gradient and set it to zero:
\begin_inset Foot
status open

\begin_layout Plain Layout
\begin_inset Formula $y=Ax\rightarrow\triangledown y=x$
\end_inset

, 
\begin_inset Formula $W=x^{T}Ax\rightarrow\triangledown W=2x^{T}A$
\end_inset

.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
w_{ML}=(\Phi^{T}\Phi)^{-1}\Phi^{T}\mathrm{t}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\emph on
design matrix
\emph default
 
\begin_inset Formula $\Phi$
\end_inset

 is given by 
\begin_inset Formula $\Phi_{nj}=\phi_{j}(x_{n})$
\end_inset


\end_layout

\begin_layout Standard
Take the derivative w.r.t.
 
\begin_inset Formula $w_{0}$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
w_{0}=\bar{t}-\sum\limits _{j=1}^{N}w_{j}\bar{\phi}_{j}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\bar{t}$
\end_inset

 and 
\begin_inset Formula $\bar{\phi_{j}}$
\end_inset

 are the arithmetic mean of their elements.
\end_layout

\begin_layout Standard
Maximize the log likehood w.r.t.
 the noise precision parameter 
\begin_inset Formula $\beta$
\end_inset

:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\dfrac{1}{\beta_{ML}}=\dfrac{1}{N}\sum\limits _{n=1}^{N}[t_{n}-w_{ML}^{T}\phi(x_{n})]^{2}
\]

\end_inset


\end_layout

\begin_layout Standard
The least-squares regression function (Euclidean distance) is obtained by
 finding the orthogonal projection of the data vector 
\begin_inset Formula $t$
\end_inset

 onto the subspace spanned by the basis functions 
\begin_inset Formula $\phi_{j}(x)$
\end_inset

 in which each basis function is viewed as a vector 
\begin_inset Formula $\phi_{j}$
\end_inset

 of length 
\emph on
N
\emph default
 with elements 
\begin_inset Formula $\phi_{j}(x_{n})$
\end_inset

.
\end_layout

\begin_layout Paragraph
Sequential learning (online algorithms)
\end_layout

\begin_layout Standard

\series bold
Stochastic graidient descent (sequential gradient descent)
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Gradient descent"
target "https://en.wikipedia.org/wiki/Gradient_descent"
literal "false"

\end_inset


\emph default
, see the description.
\end_layout

\begin_layout Standard
Given an error function 
\begin_inset Formula $E=\sum_{n}E_{n}$
\end_inset

 , a sum over data points, after presentation of pattern 
\begin_inset Formula $n$
\end_inset


\begin_inset Newline newline
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
w^{(t+1)}=w^{(t)}-\eta\triangledown E_{n}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $t$
\end_inset

 is the iteration number and 
\begin_inset Formula $\eta$
\end_inset

 is a 
\emph on
learning rate
\emph default
.
\end_layout

\begin_layout Standard

\emph on
LMS (least-mean-squares) algorithm
\end_layout

\begin_layout Standard
For the case of the sum-of-squares error function:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\triangledown E_{n}=(t_{n}-w^{(t)T}\phi_{n})\phi_{n}
\]

\end_inset


\end_layout

\begin_layout Paragraph
Regularized least squares
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
E(x)=E_{D}+\lambda E_{W}
\]

\end_inset


\end_layout

\begin_layout Standard
In general 
\begin_inset Formula $E_{W}=\dfrac{\lambda}{2}\sum\limits _{j=1}^{M}|w_{j}|^{q}$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula $q=1$
\end_inset

: (lasso) if is large enough, some of the coefficients are driven to zero
\begin_inset Newline newline
\end_inset

 
\begin_inset Formula $q=2$
\end_inset

: 
\begin_inset Formula $E_{w}(q=2)$
\end_inset

 is known in ML as 
\emph on
weight decay
\emph default
, because in sequential learning algorithms, it encourages weight values
 to decay towards zero.
 In statistics, it is an example of a 
\emph on
parameter shrinkage
\emph default
 method.
 
\begin_inset Formula $w=(\lambda I+\Phi^{T}\Phi)^{-1}\Phi^{T}t$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Graphics
	filename /home/djn_dl/Desktop/GitHub/Commentarii/PRML/1524666731026.png
	lyxscale 50

\end_inset

 
\end_layout

\begin_layout Standard
Think about this figure in a Langrange-multipliers way.
\end_layout

\begin_layout Paragraph
Multiple outputs
\end_layout

\begin_layout Standard
Of course we can decouple into multiple independent regression problems,
 however there is an approach using the same set of basis functions so that
 y
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
y(x,w)=W^{T}\phi(x)
\]

\end_inset


\end_layout

\begin_layout Standard
and
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
p(\mathrm{t}|x,W,\beta)=\mathcal{N}(\mathrm{t}|W^{T}\phi(x),\beta^{-1}I)
\]

\end_inset


\end_layout

\begin_layout Standard
which yields
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
W_{ML}=\Phi^{\dagger}T
\]

\end_inset


\end_layout

\begin_layout Standard
For the case of arbitrary covariance matrices, see MAL of multi-variate
 Gaussian distribution.
\end_layout

\begin_layout Chapter*
4.
 Linear Models for Classification
\end_layout

\begin_layout Standard
The goal in classification is to take an input vector 
\begin_inset Formula $\mathrm{x}$
\end_inset

 and to assign it to one of 
\begin_inset Formula $K$
\end_inset

 discrete classes 
\begin_inset Formula $\mathcal{C}_{k}$
\end_inset

where 
\begin_inset Formula $k=1,\dots,K$
\end_inset

.
 The input space is divided into 
\emph on
decision regions
\emph default
 whose boundaries are called 
\emph on
decision boundaries
\emph default
 or 
\emph on
decision surfaces
\emph default
.
 Data sets whose classes can be separated exactly by linear decision surfaces
 are said to be 
\emph on
linearly separable.
 
\emph default
For probabilistic models, the most convenient is 1-of-
\begin_inset Formula $K$
\end_inset

 coding scheme.
\end_layout

\begin_layout Standard
There are three distinct approaches to the classification problem: the simplest
 constructs a 
\emph on
discriminant function 
\emph default
that directly assigns each vector 
\begin_inset Formula $\mathrm{x}$
\end_inset

 to a specfic class.
 A more powerful approach models the conditional probability 
\begin_inset Formula $p\left(\mathcal{C}_{k}|\mathrm{x}\right)$
\end_inset

 in an inference stage and then uses this distribution to make optimal decisions.
 One way to the model is to model directly, the other is to adopt a generative
 approach, i.e., to model the class-conditional densities 
\begin_inset Formula $p\left(x|C_{k}\right)$
\end_inset

 and the prior probabilities 
\begin_inset Formula $p\left(C_{k}\right)$
\end_inset

 for the classes and use Bayes' theorem
\begin_inset Formula 
\[
p\left(C_{k}|x\right)=\frac{p\left(x|C_{k}\right)p\left(C_{k}\right)}{p\left(x\right)}
\]

\end_inset


\end_layout

\begin_layout Paragraph
Generalized Linear Models
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
y\left(x\right)=f\left(w^{T}x+w_{0}\right)
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $f\left(\cdot\right)$
\end_inset

 is an nonlinear 
\emph on
activatiion function
\emph default
 whereas its inverse is called a 
\emph on
link function
\emph default
.
\end_layout

\begin_layout Section*
4.1 Discriminant Functions
\end_layout

\begin_layout Subsection*
4.1.1 Two classes
\end_layout

\begin_layout Standard
A linear dicriminant function
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
y\left(x\right)=w^{T}x+w_{0}=\tilde{w}^{T}\tilde{x}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\tilde{x}=\left(x_{0},\mathrm{x}\right)$
\end_inset

 and 
\begin_inset Formula $\tilde{w}=\left(w_{0},\mathrm{w}\right)$
\end_inset


\end_layout

\begin_layout Standard
If 
\begin_inset Formula $y\left(x\right)\geq0,$
\end_inset

then 
\begin_inset Formula $x\in C_{1}$
\end_inset

 otherwise 
\begin_inset Formula $x\in C_{2}$
\end_inset

.
 The decision boundary is defined by 
\begin_inset Formula $y\left(x\right)=0$
\end_inset

, a 
\begin_inset Formula $\left(D-1\right)$
\end_inset

-dimensional hyperplane within the 
\begin_inset Formula $D$
\end_inset

-dimensional input space.
 The 
\emph on
weight vector 
\emph default

\begin_inset Formula $w$
\end_inset

 is orthogonal to the hyperplane.
 
\end_layout

\begin_layout Standard
Set the distance of a point from the hyperplane to zero, we have
\begin_inset Formula 
\[
\frac{w^{T}x}{\left\Vert w\right\Vert }=-\frac{w_{0}}{\left\Vert w\right\Vert }
\]

\end_inset

 Thus 
\begin_inset Formula $w_{0}$
\end_inset

 determines the location of the decision surface.
\end_layout

\begin_layout Subsection*
4.1.3 Least squares for classification
\end_layout

\begin_layout Standard
One justification for using least squares in a general classfication problem
 with 
\begin_inset Formula $K$
\end_inset

 classes, with a 1-of-
\begin_inset Formula $K$
\end_inset

 binary coding scheme for the target vector 
\begin_inset Formula $t$
\end_inset

, is that it approximates the conditional expectation 
\begin_inset Formula $E\left[t|x\right]$
\end_inset

 of the target values given the input vector.
 For binary coding scheme, this conditional expectation expectation is given
 by the vector of posterior class probabilities which, unfortuately, give
 poor approximation.
\end_layout

\begin_layout Standard
Each class 
\begin_inset Formula $C_{k}$
\end_inset

 is described by ots own linear model so that 
\begin_inset Formula 
\[
y_{k}\left(x\right)=w_{k}^{T}x+w_{k0}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $k=1,\dots,K$
\end_inset


\begin_inset Formula 
\[
\mathrm{y}\left(\mathrm{x}\right)=\tilde{\mathrm{W}}^{T}\tilde{\mathrm{x}}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\tilde{\mathrm{W}}_{\cdot,k}=\left(w_{k0},\mathrm{w}_{k}^{T}\right)^{T}$
\end_inset

and 
\begin_inset Formula $\tilde{\mathrm{x}}=\left(1,\mathrm{\mathrm{x}^{T}}\right)^{T}$
\end_inset


\end_layout

\begin_layout Standard
Consider a training data set 
\begin_inset Formula $\left\{ x_{n},t_{n}\right\} $
\end_inset

 where 
\begin_inset Formula $n=1,\dots,N$
\end_inset

, target matrix 
\begin_inset Formula $T_{n,\cdot}=t_{n}^{T}$
\end_inset

, matrix 
\begin_inset Formula $\tilde{\mathrm{X}}_{n,\cdot}=\tilde{\mathrm{x}_{n}^{T}}$
\end_inset

.
 The sum-of-squares error function can be written as 
\begin_inset Formula 
\[
E_{D}\left(\tilde{\mathrm{W}}\right)=\frac{1}{2}\text{Tr}\left\{ \left(\tilde{\mathrm{X}}\tilde{\mathrm{W}}-T\right)^{T}\left(\tilde{\mathrm{X}}\tilde{\mathrm{W}}-T\right)\right\} 
\]

\end_inset


\end_layout

\begin_layout Standard
Minimizing it, we obtain the solution for the coefficients
\begin_inset Formula 
\[
\tilde{\mathrm{W}}=\left(\tilde{\mathrm{X}}^{T}\tilde{\mathrm{X}}\right)^{-1}\tilde{\mathrm{X}}^{T}T
\]

\end_inset


\end_layout

\begin_layout Standard
The least-squares solutions lack robustness to outliers.
 The failure of least squares results from the assumption of the target
 being Gaussian, while in a classification problem this is not the case.
\end_layout

\begin_layout Section*
4.2 Probabilistic Generative Models
\end_layout

\begin_layout Standard
We model the class-conditional densities 
\begin_inset Formula $p\left(x|C_{k}\right)$
\end_inset

 as well as the class prior 
\begin_inset Formula $p\left(C_{k}\right)$
\end_inset

, and then use them to compute the posterior probabilities 
\begin_inset Formula $p\left(C_{k}|x\right)$
\end_inset

 through Bayes' theorem.
\end_layout

\begin_layout Paragraph
The case of two classes
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Logistic function"
target "https://en.wikipedia.org/wiki/Logistic_function#Mathematical_properties"
literal "false"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
p\left(C_{1}|x\right) & =\frac{p\left(x|C_{1}\right)p\left(C_{1}\right)}{p\left(x|C_{1}\right)p\left(C_{1}\right)+p\left(x|C_{2}\right)p\left(C_{2}\right)}\\
 & =\frac{1}{1+\exp\left(-a\right) }=\text{sigmoid}\left(a\right)
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula 
\[
a=\ln\frac{p\left(x|C_{1}\right)p\left(C_{1}\right)}{p\left(x|C_{2}\right)p\left(C_{2}\right)}
\]

\end_inset


\end_layout

\begin_layout Paragraph
The case of 
\begin_inset Formula $K>2$
\end_inset

 classes
\end_layout

\begin_layout Standard

\emph on
\begin_inset CommandInset href
LatexCommand href
name "Softmax function"
target "https://en.wikipedia.org/wiki/Softmax_function"
literal "false"

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
p\left(C_{k}|x\right) & =\frac{p\left(x|C_{1}\right)p\left(C_{1}\right)}{\sum_{j}p\left(x|C_{j}\right)p\left(C_{j}\right)}\\
 & =\underbrace{\frac{\exp\left(a_{k}\right)}{\sum_{j}\exp\left(a_{j}\right)}}_{\text{softmax}}
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula 
\[
a_{k}=\ln p\left(x|C_{k}\right)p\left(C_{k}\right)
\]

\end_inset


\end_layout

\begin_layout Subsection*
4.2.1 Continuous inputs
\end_layout

\begin_layout Standard
Assume the class-conditional probabilities are all Gaussian
\begin_inset Formula 
\[
p\left(x|C_{k}\right)=N\left(x|\mu_{k},\Sigma\right)
\]

\end_inset


\end_layout

\begin_layout Standard
Consider the case of two-classes, the posterior becomes
\begin_inset Formula 
\[
p\left(C_{1}|x\right)=\sigma\left(w^{T}x+w_{0}\right)
\]

\end_inset


\end_layout

\begin_layout Standard
where
\begin_inset Formula 
\begin{align*}
w & =\Sigma^{-1}\left(\mu_{1}-\mu_{2}\right)\\
w_{0} & =-\frac{1}{2}\mu_{1}\Sigma^{-1}\mu_{1}+\frac{1}{2}\mu_{2}\Sigma^{-1}\mu_{2}+\ln\frac{p\left(C_{1}\right)}{p\left(C_{2}\right)}
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
And in the case of 
\begin_inset Formula $K$
\end_inset

 classes
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
a_{k}(x)=w_{k}^{T}x+w_{k0}
\]

\end_inset


\end_layout

\begin_layout Standard
where
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
w_{k} & =\Sigma^{-1}\mu_{k}\\
w_{k0} & =-\frac{1}{2}\mu_{k}^{T}\Sigma^{-1}\mu_{k}+\ln p\left(C_{k}\right)
\end{align*}

\end_inset


\end_layout

\begin_layout Standard
Note that the 
\begin_inset Formula $a_{k}$
\end_inset

 here is not directly obtain from the second equation rather than the posterior
 equation, thus the quadratic forms are cancelled.
\end_layout

\begin_layout Standard
If weach class-conditional density 
\begin_inset Formula $p\left(x|C_{k}\right)$
\end_inset

 has its own covariance matrix, the cancellations no longer occurs, giving
 rise to a 
\emph on
quadratic discriminant.
\end_layout

\begin_layout Subsection*
4.2.2 Maximum likelihood solution
\end_layout

\begin_layout Standard
Here we maximize the likehood of the posterior.
 Consider the case of two classes where the targets are scalars, either
 1 or 0.
 Denoting the prior 
\begin_inset Formula $p\left(C_{1}\right)=\pi$
\end_inset

 and 
\begin_inset Formula $p\left(C_{2}\right)=1-\pi$
\end_inset

, the likelihood function is given by 
\begin_inset Formula 
\[
p\left(\mathrm{t}|\pi,\mu_{1},\mu_{2},\Sigma\right)=\prod_{n=1}^{N}\left[\pi N\left(\mathrm{\mathrm{x}_{n}|\mu_{1},\Sigma}\right)\right]^{t_{n}}\left[\left(1-\pi\right)N\left(\mathrm{\mathrm{x}_{n}|\mu_{2},\Sigma}\right)\right]^{1-t_{n}}
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $\mathrm{t}=\left(t_{1},\dots,t_{N}\right)^{T}.$
\end_inset

 Since both Gaussians do not contain 
\begin_inset Formula $\pi$
\end_inset

, it is easy to maximize w.r.t.
 
\begin_inset Formula $\pi$
\end_inset

 which yields
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
\pi=\frac{N_{1}}{N_{1}+N_{2}}
\]

\end_inset


\end_layout

\begin_layout Standard
w.r.t.
 
\begin_inset Formula $\mu_{1}$
\end_inset

 and 
\begin_inset Formula $\mu_{2}$
\end_inset

, we have
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{align*}
\mu_{1} & =\frac{1}{N_{1}}\sum_{n=1}^{N}t_{n}x_{n}\\
\mu_{2} & =\frac{1}{N_{2}}\sum_{n=1}^{N}\left(1-t_{n}\right)x_{n}
\end{align*}

\end_inset

 which are the arithmetic mean of their respective class.
\end_layout

\begin_layout Standard
w.r.t.
 
\begin_inset Formula 
\[
\Sigma=S=\frac{N_{1}}{N}S_{1}+\frac{N_{2}}{N}S_{2}
\]

\end_inset


\end_layout

\end_body
\end_document