chapter_1.2.lyx

#LyX 2.0 created this file. For more info see http://www.lyx.org/
\lyxformat 413
\begin_document
\begin_header
\textclass scrbook
\begin_preamble
\setcounter{chapter}{1}
\usepackage{graphicx}
\usepackage{pict2e}
\usepackage{graphpap}
\usepackage{color}
\usepackage{bm}
\end_preamble
\use_default_options false
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans default
\font_typewriter default
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100

\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize default
\spacing single
\use_hyperref false
\papersize default
\use_geometry false
\use_amsmath 1
\use_esint 1
\use_mhchem 1
\use_mathdots 1
\cite_engine basic
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\use_refstyle 0
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Section
Basic Calculus
\end_layout

\begin_layout Standard
In this section we will cover basic rules for calculating derivatives and
 simple integrals.
 Along with introducing the different rules we will also introduce the derivativ
es of all the functions covered in the section before.
 In order to keep the equations simple, we will from now on leave out the
 brackets for functions like 
\begin_inset Formula $\sin$
\end_inset

, 
\begin_inset Formula $\cos$
\end_inset

, 
\begin_inset Formula $\log$
\end_inset

, ...
 as long as the argument of the function is clear from the context.
 
\end_layout

\begin_layout Subsection
Derivatives
\end_layout

\begin_layout Standard
The derivative 
\begin_inset Formula $f'(x_{0})=\frac{df}{dx}(x_{0})$
\end_inset

 (denoted with a ``
\begin_inset Formula $'$
\end_inset

'', 
\begin_inset Formula $\frac{df}{dx}$
\end_inset

, or 
\begin_inset Formula $\frac{d}{dx}f$
\end_inset

) of a function 
\begin_inset Formula $f(x)$
\end_inset

 has two intuitive meanings: 
\end_layout

\begin_layout Enumerate
It measures the rate of change of a function at a certain location 
\begin_inset Formula $x_{0}$
\end_inset

.
 
\end_layout

\begin_layout Enumerate
It is the slope of the line touching the function 
\begin_inset Formula $f$
\end_inset

 at a the point 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

.
 This line is called 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
tangent line
\end_layout

\end_inset

tangent line
\emph default
 or simply 
\emph on
tangent
\emph default
.
\end_layout

\begin_layout Standard
Using the first intuition, 
\begin_inset Formula $f'(x_{0})$
\end_inset

 is an approximation of how the function value of 
\begin_inset Formula $f(x)$
\end_inset

 changes when going from 
\begin_inset Formula $x_{0}$
\end_inset

 to 
\begin_inset Formula $x_{0}+1$
\end_inset

.
 If 
\begin_inset Formula $f$
\end_inset

 is a linear function, the approximation will be exact, that means 
\begin_inset Formula $f$
\end_inset

 will change exactly by 
\begin_inset Formula $f'(x_{0})$
\end_inset

.
 If 
\begin_inset Formula $f$
\end_inset

 is not linear we will make some error, but we still can use 
\begin_inset Formula $f'(x)$
\end_inset

 to construct the best linear approximation of 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
 But first, we will start with an example of the exact case.
\end_layout

\begin_layout Standard
\begin_inset Note Comment
status open

\begin_layout Plain Layout
Note that the tangent at the graph of 
\begin_inset Formula $f(x)$
\end_inset

 involves an offset which makes the tangent an affine and not necessarily
 a linear function.
 There is some ambiguity here.
 Often, people say 
\begin_inset Quotes eld
\end_inset

linear function
\begin_inset Quotes erd
\end_inset

 when the mean 
\begin_inset Quotes eld
\end_inset

function of a line
\begin_inset Quotes erd
\end_inset

.
\end_layout

\end_inset


\end_layout

\begin_layout Paragraph
Example
\end_layout

\begin_layout Standard
Consider the linear function 
\begin_inset Formula $f(x)=3x$
\end_inset

.
 According to the first intuition, the derivative 
\begin_inset Formula $f'(x)$
\end_inset

 is the rate of change of 
\begin_inset Formula $f$
\end_inset

, i.e.
 the change in the function value of 
\begin_inset Formula $f$
\end_inset

 divided by the change in the value of 
\begin_inset Formula $x$
\end_inset

.
 Consider two arbitrary points 
\begin_inset Formula $x_{0}$
\end_inset

 and 
\begin_inset Formula $x_{1}$
\end_inset

.
 The rate of change is then given by:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \frac{f(x_{1})-f(x_{0})}{x_{1}-x_{0}}\\
 & = & \frac{3x_{1}-3x_{0}}{x_{1}-x_{0}}\\
 & = & \frac{3(x_{1}-x_{0})}{(x_{1}-x_{0})}\\
 & = & 3.
\end{eqnarray*}

\end_inset


\begin_inset Float figure
placement h
wide false
sideways false
status collapsed

\begin_layout Plain Layout
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{center}
\end_layout

\begin_layout Plain Layout


\backslash
setlength{
\backslash
unitlength}{0.04cm}
\end_layout

\begin_layout Plain Layout


\backslash
begin{picture}(200,200)
\end_layout

\begin_layout Plain Layout


\backslash
put(70,63){
\backslash
makebox(0,0){$
\backslash
underbrace{
\backslash
hspace{1cm}}_{x_1-x_0}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(82,69){
\backslash
rotatebox{90}{$
\backslash
underbrace{
\backslash
hspace{3cm}}_{f(x_1)-f(x_0)}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(57,69){
\backslash
line(1,0){25}}
\end_layout

\begin_layout Plain Layout


\backslash
put(82,69){
\backslash
line(0,1){78}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
thicklines  
\end_layout

\begin_layout Plain Layout


\backslash
put(50,50){
\backslash
vector(1,0){100}}
\end_layout

\begin_layout Plain Layout


\backslash
put(50,50){
\backslash
vector(0,1){100}}
\end_layout

\begin_layout Plain Layout


\backslash
put(34,0){
\backslash
line(1,3){50}}
\end_layout

\begin_layout Plain Layout


\backslash
put(150,55){
\backslash
makebox(0,0){$x$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(55,150){
\backslash
makebox(0,0){$y$}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
put(57,35){
\backslash
makebox(0,0){$f(x)$}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
end{picture}
\end_layout

\begin_layout Plain Layout


\backslash
end{center}
\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:DerivativeLinearFunc"

\end_inset

Geometrical picture for calculating derivatives of linear functions.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
Since any linear function can be written as 
\begin_inset Formula $f(x)=ax$
\end_inset

, we just showed that the first derivative 
\begin_inset Formula $f'(x)$
\end_inset

 of a linear function does not depend on 
\begin_inset Formula $x$
\end_inset

.
 This means that it is the same everywhere.
 This is what we expect intuitively from a line.
 Secondly we verified the second intuition for linear functions.
 The first derivative at a point 
\begin_inset Formula $x_{0}$
\end_inset

 is the slope of the tangent line of 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

.
 Since the tangent line is simply the linear function itself, the first
 derivative 
\begin_inset Formula $f'(x)$
\end_inset

 is the slope of the linear function.
 
\end_layout

\begin_layout Standard
\begin_inset Note Comment
status open

\begin_layout Plain Layout
There is lot of technical stuff coming in the next section, like limits,
 definitions of differentiability and continuity etc.
 While it is very good to have it in the script, we should be very careful
 not to load the lectures with it too much.
 We should really cover calculus (except integration) in two lectures, so
 it is much more important that people can simply do the calculations, and
 not necessarily understand the theory behind it.
 I know this is sad, but at the end of the day, we have to cover all of
 mathematics in 10 lectures....
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
The situation changes when we consider arbitrary functions 
\begin_inset Formula $f(x)$
\end_inset

.
 In that case the value of the rate of change, given by the quotient 
\begin_inset Formula $\frac{f(x_{1})-f(x_{0})}{x_{1}-x_{0}}$
\end_inset

, will depend on the choices of 
\begin_inset Formula $x_{1}$
\end_inset

 and 
\begin_inset Formula $x_{0}$
\end_inset

.
 This raises the question how we could define the rate of change in a meaningful
 way? The second intuition can help us here: In order to get the first derivativ
e at a point 
\begin_inset Formula $x_{0}$
\end_inset

 we approximate 
\begin_inset Formula $f(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

 with a line and define the first derivative to be its slope.
 Since we are merely interested in computing the slope of that line, we
 do not need to compute the full line equation but start with the slope
 right away.
 Remember, given a line 
\begin_inset Formula $g(x)=ax+t$
\end_inset

, we can compute its slope via 
\begin_inset Formula $\frac{g(x_{1})-g(x_{0})}{(x_{1}-x_{0})}=\frac{ax_{1}+t-ax_{0}-t}{(x_{1}-x_{0})}=a$
\end_inset

 where 
\begin_inset Formula $x_{1}$
\end_inset

 and 
\begin_inset Formula $x_{0}$
\end_inset

 are two arbitrary points.
 Now imagine we have a line 
\begin_inset Formula $g(x)=ax+t$
\end_inset

 that contains the two points 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

 and 
\begin_inset Formula $(x_{1},f(x_{1}))$
\end_inset

 (see Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:DerivativeArbrFunc"

\end_inset

).
 In order to compute its slope we do not need the full line equation.
 Instead we can simply use the quotient from above and compute 
\begin_inset Formula 
\begin{eqnarray*}
a & = & \frac{g(x_{1})-g(x_{0})}{x_{1}-x_{0}}\\
 & = & \frac{f(x_{1})-f(x_{0})}{x_{1}-x_{0}}.
\end{eqnarray*}

\end_inset

The slope 
\begin_inset Formula $a$
\end_inset

 of this line is not yet the first derivative since 
\begin_inset Formula $g$
\end_inset

 contains 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

 
\emph on
and
\emph default
 
\begin_inset Formula $(x_{1},f(x_{1}))$
\end_inset

.
 This means, that (in most cases) it will intersect with 
\begin_inset Formula $f$
\end_inset

 and not 
\emph on
touch
\emph default
 it at 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

.
 However, we can achieve this goal by moving 
\begin_inset Formula $x_{1}$
\end_inset

 close to 
\begin_inset Formula $x_{0}$
\end_inset

.
 Once it is infinitely, called 
\emph on
infinitesimally
\emph default
, close to 
\begin_inset Formula $x_{0}$
\end_inset

, 
\begin_inset Formula $a=\frac{f(x_{1})-f(x_{0})}{x_{1}-x_{0}}$
\end_inset

 will be the slope of the tangent line, i.e.
 
\begin_inset Formula $a=f'(x_{0})$
\end_inset

.
 Mathematically this is expressed in terms of a limit.
 We do not go into the details of limits here, but merely demonstrate, how
 the derivative of a function 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

 is defined.
 The rate of change with a infinitesimal close point 
\begin_inset Formula $x_{1}=x_{0}+h$
\end_inset

 can be written in terms of limits as
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \lim_{h\rightarrow0}\frac{f(\overbrace{x_{0}+h}^{=x_{1}})-f(x_{0})}{\underbrace{h}_{=x_{1}-x_{0}}}.
\end{eqnarray*}

\end_inset

The ''
\begin_inset Formula $\lim$
\end_inset

'' expresses that we let 
\begin_inset Formula $h$
\end_inset

 come infinitesimally close to zero and therefore 
\begin_inset Formula $x_{1}=x_{0}+h$
\end_inset

 infinitesimally close to 
\begin_inset Formula $x_{0}$
\end_inset

.
 The expression 
\begin_inset Formula $\lim_{h\rightarrow0}\frac{f(x_{0}+h)-f(x_{0})}{h}$
\end_inset

 is called 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
differential quotient
\end_layout

\end_inset

differential quotient
\emph default
 of 
\begin_inset Formula $f$
\end_inset

.
 Note that, in order to obtain a unique notion of a derivative at a point
 
\begin_inset Formula $x_{0}$
\end_inset

, the direction from which 
\begin_inset Formula $x_{1}=x_{0}+h$
\end_inset

 approaches 
\begin_inset Formula $x_{0}$
\end_inset

 should not matter.
 This means that 
\begin_inset Formula $h$
\end_inset

 could be negative (
\begin_inset Formula $x_{1}$
\end_inset

 approaches 
\begin_inset Formula $x_{0}$
\end_inset

 from the left) or positive (
\begin_inset Formula $x_{1}$
\end_inset

 approaches 
\begin_inset Formula $x_{0}$
\end_inset

 from the right).
\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{center}
\end_layout

\begin_layout Plain Layout


\backslash
setlength{
\backslash
unitlength}{0.04cm}
\end_layout

\begin_layout Plain Layout


\backslash
begin{picture}(200,200)
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

%
\backslash
put(70,63){
\backslash
makebox(0,0){$
\backslash
underbrace{
\backslash
hspace{1cm}}_{x_1-x_0}$}}
\end_layout

\begin_layout Plain Layout

%
\backslash
put(82,69){
\backslash
rotatebox{90}{$
\backslash
underbrace{
\backslash
hspace{3cm}}_{f(x_1)-f(x_0)}$}}
\end_layout

\begin_layout Plain Layout

%
\backslash
put(57,69){
\backslash
line(1,0){25}}
\end_layout

\begin_layout Plain Layout

%
\backslash
put(82,69){
\backslash
line(0,1){78}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
put(65,62){
\backslash
line(4,6){90}}
\end_layout

\begin_layout Plain Layout


\backslash
put(108,184){
\backslash
makebox(0,0){$
\backslash
overbrace{
\backslash
hspace{2.6cm}}^{h_1}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(61,78){
\backslash
rotatebox{90}{$
\backslash
overbrace{
\backslash
hspace{4cm}}^{f(x_0+h_1)-f(x_0)}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(75,76){
\backslash
line(0,1){102}}
\end_layout

\begin_layout Plain Layout


\backslash
put(142,178){
\backslash
line(-1,0){67}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
put(65,63){
\backslash
line(3,4){90}}
\end_layout

\begin_layout Plain Layout


\backslash
put(102,70){
\backslash
makebox(0,0){$
\backslash
underbrace{
\backslash
hspace{2.15cm}}_{h_2}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(129,76.5){
\backslash
rotatebox{90}{$
\backslash
underbrace{
\backslash
hspace{2.85cm}}_{f(x_0+h_2)-f(x_0)}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(129,148){
\backslash
line(0,-1){72}}
\end_layout

\begin_layout Plain Layout


\backslash
put(75,76){
\backslash
line(1,0){54}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
thicklines  
\end_layout

\begin_layout Plain Layout


\backslash
put(77,40){
\backslash
makebox(0,0){$x_0$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(77,48){
\backslash
line(0,1){4}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
put(50,50){
\backslash
vector(1,0){100}}
\end_layout

\begin_layout Plain Layout


\backslash
put(50,50){
\backslash
vector(0,1){100}}
\end_layout

\begin_layout Plain Layout


\backslash
qbezier(0,50)(100,50)(150,200) % function
\end_layout

\begin_layout Plain Layout


\backslash
put(150,55){
\backslash
makebox(0,0){$x$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(55,150){
\backslash
makebox(0,0){$y$}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

%
\backslash
put(57,35){
\backslash
makebox(0,0){$f(x)$}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
end{picture}
\end_layout

\begin_layout Plain Layout


\backslash
end{center}
\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:DerivativeArbrFunc"

\end_inset

Geometrical picture of the differential quotient for arbitrary functions.
 As 
\begin_inset Formula $h$
\end_inset

 becomes smaller, slope of the line through 
\begin_inset Formula $(x_{0},f(x_{0}))$
\end_inset

 and 
\begin_inset Formula $(x_{0}+h,f(x_{0}+h))$
\end_inset

 converges to the derivative of 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
Looking at Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:DerivativeArbrFunc"

\end_inset

, we can see that not every function has a derivative at any point 
\begin_inset Formula $x_{0}$
\end_inset

.
 If the function makes a step at 
\begin_inset Formula $x_{0}$
\end_inset

 such that there is a gap between the function value at 
\begin_inset Formula $x_{0}$
\end_inset

 and the function value at 
\begin_inset Formula $x_{0}$
\end_inset

 plus or minus an infinitesimal 
\begin_inset Formula $h$
\end_inset

, then the value of the differential quotient at 
\begin_inset Formula $x_{0}$
\end_inset

 depends on the direction from which 
\begin_inset Formula $x_{0}+h$
\end_inset

 approaches 
\begin_inset Formula $x_{0}$
\end_inset

 and the derivative would not be unique.
 Therefore, functions are only differentiable at points where the function
 does not make such a step.
 This property is expressed in the concept of 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
continuity
\end_layout

\end_inset

continuity
\emph default
.
\end_layout

\begin_layout Paragraph
Definition (Continuous Function)
\end_layout

\begin_layout Standard
A function 
\begin_inset Formula $f$
\end_inset

 is said to be 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
continuous
\end_layout

\end_inset

continuous
\emph default
 
\emph on
at a point
\emph default
 
\begin_inset Formula $x_{0}$
\end_inset

 if 
\begin_inset Formula 
\begin{eqnarray*}
\lim_{h\rightarrow0}f(x_{0}+h)=\lim_{h\rightarrow0}f(x_{0}-h) & = & f(x_{0}),
\end{eqnarray*}

\end_inset

i.e.
 if getting infinitesimal close to 
\begin_inset Formula $x_{0}$
\end_inset

 (no matter from which side) implies getting infinitesimal close to 
\begin_inset Formula $f(x_{0})$
\end_inset

.
\end_layout

\begin_layout Standard
A function 
\begin_inset Formula $f$
\end_inset

 is said to be 
\emph on
continuous
\emph default
, if it is continuous in every point of its domain.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\Diamond$
\end_inset


\end_layout

\begin_layout Standard
If a function is continuous, we can calculate the value of the differential
 quotient 
\begin_inset Formula $\lim_{h\rightarrow0}\frac{f(x_{0}+h)-f(x_{0})}{h}$
\end_inset

.
 If the function is not continuous, the derivative at that point is classified
 as 
\emph on
not defined
\emph default
.
 In a similar fashion, functions that have a kink at 
\begin_inset Formula $x_{0}$
\end_inset

 give rise to an undefined derivative at 
\begin_inset Formula $x_{0}$
\end_inset

.
 The reason is that because of the kink approaching 
\begin_inset Formula $x_{0}$
\end_inset

 from the left yields a different slope than approaching 
\begin_inset Formula $x_{0}$
\end_inset

 from the right.
 For example 
\begin_inset Formula $f(x)=|x|=\mbox{abs}(x)$
\end_inset

 is not differentiable at 
\begin_inset Formula $x_{0}=0$
\end_inset

.
 At all other points, however, it is.
 We will see how to differentiate 
\begin_inset Formula $f(x)=|x|$
\end_inset

 in an example below.
\end_layout

\begin_layout Standard
However, if we can find a linear function that with its slope equal to the
 value of the differential quotient, no matter if 
\begin_inset Formula $h$
\end_inset

 approaches zero from the left (
\begin_inset Formula $h<0$
\end_inset

) or from the right 
\begin_inset Formula $(h>0)$
\end_inset

, the function is called 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
differentiable function
\end_layout

\end_inset

differentiable.
 
\end_layout

\begin_layout Paragraph
Definition (Differentiable Function)
\end_layout

\begin_layout Standard
A function 
\begin_inset Formula $f$
\end_inset

 is said to be 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
differentiable
\end_layout

\end_inset

differentiable
\emph default
 
\emph on
at a point
\emph default
 
\begin_inset Formula $x_{0}$
\end_inset

 if there exists a linear function 
\begin_inset Formula $L(x)$
\end_inset

 such that 
\begin_inset Formula 
\begin{eqnarray*}
L(x_{0})-\lim_{h\rightarrow0}\frac{f(x_{0}+h)-f(x_{0})}{h} & = & 0,
\end{eqnarray*}

\end_inset

no matter from which side 
\begin_inset Formula $h$
\end_inset

 approaches zero.
 
\end_layout

\begin_layout Standard
A function 
\begin_inset Formula $f$
\end_inset

 is said to be 
\emph on
differentiable
\emph default
, if it is differentiable in every point of its domain.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\Diamond$
\end_inset


\end_layout

\begin_layout Standard
Fortunately, we do not need to go through the tedious process of calculating
 the value of the differential quotient every time.
 There are easier ways to compute the slope of the tangent at a point 
\begin_inset Formula $x_{0}$
\end_inset

.
 In the following we will review the most important rules how to compute
 derivatives.
 In most cases those rules are sufficient to compute derivatives of functions
 that you are dealing with.
 
\end_layout

\begin_layout Standard
We start with the simplest rule: Calculating the derivatives of powers.
 If 
\begin_inset Formula $f(x)=ax^{b}$
\end_inset

, then the derivative of 
\begin_inset Formula $f$
\end_inset

 is given by 
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & abx^{b-1}.
\end{eqnarray*}

\end_inset

As we can see, the rule is fairly easy: We just premultiply the exponent
 
\begin_inset Formula $b$
\end_inset

, subtract 
\begin_inset Formula $1$
\end_inset

 from the exponent of 
\begin_inset Formula $x$
\end_inset

 and leave the factor 
\begin_inset Formula $a$
\end_inset

 untouched.
 Before looking at a small example, we just introduce another rule: The
 derivative of a sum equals the derivatives of the single terms:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
f(x)=g_{1}(x)+g_{2}(x)\Rightarrow f'(x)=g_{1}'(x)+g_{2}'(x)
\]

\end_inset


\end_layout

\begin_layout Standard
Or in general:
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{eqnarray*}
f(x)=\sum_{k=1}^{n}g_{k}(x) & \Rightarrow & f'(x)=\left(\sum_{k=1}^{n}g_{k}(x)\right)'=\sum_{k=1}^{n}g'_{k}(x).
\end{eqnarray*}

\end_inset

Here, 
\begin_inset Formula $g_{k}$
\end_inset

 are arbitrary differentiable functions.
\end_layout

\begin_layout Paragraph
Example
\end_layout

\begin_layout Standard
Assume that 
\begin_inset Formula $f(x)=\frac{1}{2}x^{2}+5x+10$
\end_inset

.
 In order to apply the rule, we must rewrite the constant 
\begin_inset Formula $10$
\end_inset

.
 Since we already know that 
\begin_inset Formula $x^{0}=1$
\end_inset

 we can write 
\begin_inset Formula $10=10x^{0}$
\end_inset

 and 
\begin_inset Formula $f$
\end_inset

 becomes 
\begin_inset Formula $f(x)=\frac{1}{2}x^{2}+5x+10x^{0}$
\end_inset

.
 Now we can apply our rules step by step:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \left(\frac{1}{2}x^{2}+5x+10x^{0}\right)'\\
 & = & \left(\frac{1}{2}x^{2}\right)'+(5x)'+(10x^{0})'\\
 & = & x+5.
\end{eqnarray*}

\end_inset

As we can see, the last term vanishes since premultiplying 
\begin_inset Formula $0$
\end_inset

 cancels the whole term.
 Since we can write any constant 
\begin_inset Formula $b$
\end_inset

 that does not depend on 
\begin_inset Formula $x$
\end_inset

 as 
\begin_inset Formula $b=bx^{0}$
\end_inset

, constants always vanish when calculating the derivative.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
We can generalize our example to arbitrary polynomials: 
\begin_inset Formula 
\begin{eqnarray*}
f(x)=\sum_{k=0}^{n}a_{k}x^{k} & \Rightarrow & f'(x)=\sum_{k=1}^{n}a_{k}kx^{k-1}.
\end{eqnarray*}

\end_inset

Note, that the index 
\begin_inset Formula $k$
\end_inset

 of 
\begin_inset Formula $f'(x)$
\end_inset

 starts at one instead of zero.
 This is because the last term 
\begin_inset Formula $a_{0}x^{0}=a_{0}$
\end_inset

 vanishes when differentiating.
 
\end_layout

\begin_layout Standard
We can also use this rule to calculate the derivative of roots and ratios.
\end_layout

\begin_layout Paragraph
Examples
\end_layout

\begin_layout Enumerate
Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=\sqrt{x}$
\end_inset

.
 Since we can write 
\begin_inset Formula $f$
\end_inset

 as 
\begin_inset Formula $f(x)=x^{\frac{1}{2}}$
\end_inset

, the derivative of 
\begin_inset Formula $f$
\end_inset

 is given by 
\begin_inset Formula $f'(x)=\frac{1}{2}x^{-\frac{1}{2}}=\frac{1}{2\sqrt{x}}$
\end_inset

.
\end_layout

\begin_layout Enumerate
Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=\frac{1}{x^{a}}$
\end_inset

.
 Since we can write 
\begin_inset Formula $f$
\end_inset

 as 
\begin_inset Formula $f(x)=x^{-a}$
\end_inset

, the derivative is given by 
\begin_inset Formula $f'(x)=-ax^{-a-1}=-\frac{a}{x^{a+1}}$
\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
As those examples show, we can already calculate the derivative of a fair
 amount of functions.
 However, so far we cannot differentiate functions that are compositions
 of other functions.
 For example, we cannot get 
\begin_inset Formula $f'(x)$
\end_inset

 for function 
\begin_inset Formula $f(x)=\sqrt{\frac{1}{2}x^{2}+5x+10}$
\end_inset

 with our current set of rules, since 
\begin_inset Formula $f(x)=g_{2}(g_{1}(x))$
\end_inset

 is the composition of 
\begin_inset Formula $g_{2}(y)=\sqrt{y}$
\end_inset

 and 
\begin_inset Formula $y=g_{1}(x)=\frac{1}{2}x^{2}+5x+10$
\end_inset

.
 Therefore, we introduce a new rule, called 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
chain rule
\end_layout

\end_inset

chain rule:
\emph default
 Let 
\emph on

\begin_inset Formula $f$
\end_inset


\emph default
 be 
\emph on

\begin_inset Formula $f(x)=g_{2}(g_{1}(x))$
\end_inset


\emph default
 with two arbitrary differentiable functions 
\begin_inset Formula $g_{1}$
\end_inset

 and 
\begin_inset Formula $g_{2}$
\end_inset

, then the derivative 
\emph on

\begin_inset Formula $f'(x)$
\end_inset


\emph default
 is given by 
\emph on

\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & g_{2}'(g_{1}(x))\cdot g_{1}'(x).
\end{eqnarray*}

\end_inset


\emph default
According to this rule we first differentiate 
\emph on

\begin_inset Formula $g_{2}(y)$
\end_inset


\emph default
 while treating 
\begin_inset Formula $y=g_{1}(x)$
\end_inset

 as a variable on its own.
 After that, we multiply the result with the derivative 
\begin_inset Formula $g_{1}'(x)$
\end_inset

 of 
\begin_inset Formula $g_{1}$
\end_inset

 with respect to 
\begin_inset Formula $g_{1}$
\end_inset

.
 Let us illustrate this rule by differentiating 
\begin_inset Formula $f(x)=\sqrt{\frac{1}{2}x^{2}+5x+10}$
\end_inset

.
\end_layout

\begin_layout Paragraph
Example
\end_layout

\begin_layout Standard
Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=\sqrt{\frac{1}{2}x^{2}+5x+10}$
\end_inset

.
 As already mentioned before, 
\begin_inset Formula $f$
\end_inset

 is the composition of 
\begin_inset Formula $g_{2}(y)=\sqrt{y}$
\end_inset

 and 
\begin_inset Formula $y=g_{1}(x)=\frac{1}{2}x^{2}+5x+10$
\end_inset

.
 We already calculated the derivative of those functions:
\begin_inset Formula 
\begin{eqnarray*}
g_{2}'(y) & = & \frac{1}{2\sqrt{y}}\\
g_{1}(x) & = & x+5.
\end{eqnarray*}

\end_inset

Applying the chain rule therefore yields
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & g_{2}'(g_{1}(x))\cdot g_{1}'(x)\\
 & = & \frac{1}{2\sqrt{\frac{1}{2}x^{2}+5x+10}}\cdot g_{1}'(x)\\
 & = & \frac{1}{2\sqrt{\frac{1}{2}x^{2}+5x+10}}\cdot(x+5).
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
Now, there is only one rule left: How to differentiate products 
\begin_inset Formula $f(x)=g_{1}(x)\cdot g_{2}(x)$
\end_inset

 of two differentiable functions 
\begin_inset Formula $g_{1}$
\end_inset

 and 
\begin_inset Formula $g_{2}$
\end_inset

.
 If 
\begin_inset Formula $f$
\end_inset

 is the product of two functions 
\begin_inset Formula $g_{1}$
\end_inset

 and 
\begin_inset Formula $g_{2}$
\end_inset

, the derivative is given by 
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & g_{1}'(x)\cdot g_{2}(x)+g_{1}(x)\cdot g_{2}'(x).
\end{eqnarray*}

\end_inset

This rule is called 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
product rule
\end_layout

\end_inset

product rule
\emph default
.
 It tells us to first differentiate 
\begin_inset Formula $g_{1}(x)$
\end_inset

 and multiply the result with 
\begin_inset Formula $g_{2}(x)$
\end_inset

, then do it the other way round and sum the results in the end.
 Again, an example will illustrate this rule.
\end_layout

\begin_layout Paragraph
Example
\end_layout

\begin_layout Standard
Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=(x+1)(\frac{1}{2}x^{2}+5)$
\end_inset

.
 Apparently, 
\begin_inset Formula $f$
\end_inset

 is the product of 
\begin_inset Formula $g_{1}(x)=(x+1)$
\end_inset

 and 
\begin_inset Formula $g_{2}(x)=(\frac{1}{2}x^{2}+5)$
\end_inset

.
 It is easy to calculate their derivatives:
\begin_inset Formula 
\begin{eqnarray*}
g_{1}'(x) & = & 1\\
g_{2}'(x) & = & x.
\end{eqnarray*}

\end_inset

Therefore, by applying the product rule we get the derivative of 
\begin_inset Formula $f$
\end_inset

:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & (x+1)'\cdot\left(\frac{1}{2}x^{2}+5\right)+(x+1)\cdot\left(\frac{1}{2}x^{2}+5\right)'\\
 & = & \frac{1}{2}x^{2}+5+(x+1)x\\
 & = & \frac{3}{2}x^{2}+x+5.
\end{eqnarray*}

\end_inset

In this particular case, we can check the rule by expanding 
\begin_inset Formula 
\begin{eqnarray*}
f(x) & = & (x+1)(\frac{1}{2}x^{2}+5)\\
 & = & \frac{1}{2}x^{3}+\frac{1}{2}x^{2}+5x+5.
\end{eqnarray*}

\end_inset

 Using the rule for polynomials, yields:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \left(\frac{1}{2}x^{3}+\frac{1}{2}x^{2}+5x+5\right)'\\
 & = & \frac{3}{2}x^{2}+x+5,
\end{eqnarray*}

\end_inset

which is the same results as the one from the product rule.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
In most cases, a fourth rule for differentiating quotients of functions
 is specified.
 However, we can derive the 
\emph on
quotient rule
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
quotient rule
\end_layout

\end_inset


\emph default
 from the rules we already know.
 Before stating the general quotient rule, we look at a small example.
\end_layout

\begin_layout Paragraph
Example
\end_layout

\begin_layout Standard
Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=\frac{(x+1)}{(\frac{1}{2}x^{2}+5)}$
\end_inset

.
 For calculating the derivative of 
\begin_inset Formula $f$
\end_inset

 we reformulate it first into 
\begin_inset Formula $f(x)=(x+1)(\frac{1}{2}x^{2}+5)^{-1}$
\end_inset

.
 Now we can apply the product rule and the chain rule to differentiate 
\begin_inset Formula $f$
\end_inset

.
 Let us look at the calculation step by step:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & \stackrel{\mbox{Product Rule}}{=} & (x+1)'\left(\frac{1}{2}x^{2}+5\right)^{-1}+(x+1)\left(\left(\frac{1}{2}x^{2}+5\right)^{-1}\right)'\\
 & \stackrel{\mbox{Chain Rule}}{=} & (x+1)'\left(\frac{1}{2}x^{2}+5\right)^{-1}-(x+1)\left(\frac{1}{2}x^{2}+5\right)^{-2}\left(\frac{1}{2}x^{2}+5\right)'\\
 & = & \frac{(x+1)'}{\left(\frac{1}{2}x^{2}+5\right)}-\frac{(x+1)\left(\frac{1}{2}x^{2}+5\right)'}{\left(\frac{1}{2}x^{2}+5\right)^{2}}\\
 & = & \frac{(x+1)'(\frac{1}{2}x^{2}+5)-(x+1)\left(\frac{1}{2}x^{2}+5\right)'}{\left(\frac{1}{2}x^{2}+5\right)^{2}}\\
 & = & \frac{-\frac{1}{2}x^{2}-x+5}{\left(\frac{1}{2}x^{2}+5\right)^{2}}.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
In general, it might be easier to calculate all the derivatives first.
 In this example, however, the derivatives were only resolved at the end
 to show an important aspect: When looking at the fourth line of the calculation
, we see that it consists exclusively of terms that already appeared in
 the product.
 We can generalize this, to get the quotient rule: Let 
\begin_inset Formula $f$
\end_inset

 be a quotient of two functions 
\begin_inset Formula $f(x)=\frac{g_{1}(x)}{g_{2}(x)}.$
\end_inset

 By rewriting 
\begin_inset Formula $f(x)=g_{1}(x)\cdot g_{2}(x)^{-1}$
\end_inset

 and applying the product rule, the chain rule and the rule for powers,
 we arrive at 
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \frac{g_{1}'(x)g_{2}(x)-g_{1}(x)g_{2}'(x)}{g_{2}(x)^{2}}.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
Here is a summary of all the rules we just saw:
\end_layout

\begin_layout Standard
\align center
\begin_inset Box Boxed
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "100col%"
special "none"
height "1in"
height_special "totalheight"
status open

\begin_layout Plain Layout

\series bold
Differentiation Rules
\end_layout

\begin_layout Itemize

\bar under
Derivatives of constant functions
\bar default
: The derivative of any constant function 
\begin_inset Formula $f(x)=a$
\end_inset

 is 
\begin_inset Formula $f'(x)=0$
\end_inset

.
\end_layout

\begin_layout Itemize

\bar under
Summation Rule:
\bar default
 Let 
\begin_inset Formula $f(x)=\sum_{k=1}^{n}g_{k}(x)$
\end_inset

 a sum of arbitrary differentiable functions.
 Then 
\begin_inset Formula $f'(x)$
\end_inset

 is given by:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \sum_{k=1}^{n}g_{k}'(x).
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Itemize

\bar under
Power Rule:
\bar default
 Let 
\begin_inset Formula $f$
\end_inset

 be 
\begin_inset Formula $f(x)=ax^{b}$
\end_inset

, then 
\begin_inset Formula $f'(x)$
\end_inset

 is give by:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & abx^{b-1}.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Itemize

\bar under
Chain Rule:
\bar default
 Let 
\begin_inset Formula $f(x)=g_{2}(g_{1}(x))$
\end_inset

 be a composition of arbitrary differentiable functions.
 Then 
\begin_inset Formula $f'(x)$
\end_inset

 is given by:
\emph on

\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & g_{2}'(g_{1}(x))\cdot g_{1}'(x).
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Itemize

\bar under
Product Rule:
\bar default
 Let 
\begin_inset Formula $f(x)=g_{1}(x)g_{2}(x)$
\end_inset

 be a product of arbitrary differentiable functions.
 Then 
\begin_inset Formula $f'(x)$
\end_inset

 is given by:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & g_{1}'(x)\cdot g_{2}(x)+g_{1}(x)\cdot g_{2}'(x).
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Itemize

\bar under
Quotient Rule:
\bar default
 Let 
\begin_inset Formula $f(x)=\frac{g_{1}(x)}{g_{2}(x)}$
\end_inset

 be a quotient of two arbitrary differentiable functions.
 Then 
\begin_inset Formula $f'(x)$
\end_inset

 is given by:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \frac{g_{1}'(x)g_{2}(x)-g_{1}(x)g_{2}'(x)}{g_{2}(x)^{2}}.
\end{eqnarray*}

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Subsubsection*

\series medium
We conclude this section by stating derivatives of important functions that
 you cannot compute using the rules above.
 Since they occur quite often, you should know them by heart.
\end_layout

\begin_layout Standard
\align center
\begin_inset Box Boxed
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "100col%"
special "none"
height "1in"
height_special "totalheight"
status open

\begin_layout Plain Layout

\series bold
Derivatives of important functions
\end_layout

\begin_layout Itemize
\begin_inset Formula $f(x)=\sin(x)\Rightarrow f'(x)=\cos(x)$
\end_inset


\end_layout

\begin_layout Itemize
\begin_inset Formula $f(x)=\cos(x)\Rightarrow f'(x)=-\sin(x)$
\end_inset


\end_layout

\begin_layout Itemize
\begin_inset Formula $f(x)=e^{x}\Rightarrow f'(x)=e^{x}$
\end_inset


\end_layout

\begin_layout Itemize
\begin_inset Formula $f(x)=\ln(x)=\log_{e}(x)\Rightarrow f'(x)=\frac{1}{x}$
\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
Let us calculate the derivative of a little bit more advanced case: 
\begin_inset Formula $f(x)=|x|=\mbox{abs}(x)$
\end_inset

.
 Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:abs"

\end_inset

 show the graph of the function.
\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/abs.eps
	width 10cm
	keepAspectRatio

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:abs"

\end_inset

Graph of the function 
\begin_inset Formula $f(x)=|x|$
\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
We can either use the definition of 
\begin_inset Formula $|x|$
\end_inset

 as a piecewise defined function, or use the fact that 
\begin_inset Formula $f(x)=\sqrt{x^{2}}.$
\end_inset

 In both cases the derivative is 
\begin_inset Formula $f'(x)=-1$
\end_inset

 for 
\begin_inset Formula $x<0$
\end_inset

 and 
\begin_inset Formula $f'(x)=1$
\end_inset

 for 
\begin_inset Formula $x>0$
\end_inset

.
 But what happens at 
\begin_inset Formula $x_{0}=0$
\end_inset

.
 If we use the piecewise definition 
\begin_inset Formula 
\[
|x|=\left\{ \begin{array}{cc}
-x & x\leq0\\
x & x>0
\end{array}\right.,
\]

\end_inset

then the value of the differential quotient 
\begin_inset Formula $\lim_{h\rightarrow0}\frac{|x_{0}+h|-|x_{0}|}{h}$
\end_inset

 is 
\begin_inset Formula $-1$
\end_inset

 if we approach 
\begin_inset Formula $x_{0}=0$
\end_inset

 from the left (i.e.
 
\begin_inset Formula $h<0$
\end_inset

) and 
\begin_inset Formula $1$
\end_inset

 if we approach 
\begin_inset Formula $x_{0}=0$
\end_inset

 from the right (i.e.
 
\begin_inset Formula $h>0$
\end_inset

).
 Since there cannot be a linear function which has the slope 
\begin_inset Formula $-1$
\end_inset

 and 
\begin_inset Formula $1$
\end_inset

 at the same time, the derivative is not defined at 
\begin_inset Formula $x_{0}=0$
\end_inset

.
 A similar thing happens when we use 
\begin_inset Formula $|x|=\sqrt{x^{2}}$
\end_inset

.
 Since 
\begin_inset Formula $\left(\sqrt{x^{2}}\right)'=\frac{x}{\sqrt{x^{2}}}$
\end_inset

 we cannot choose 
\begin_inset Formula $x$
\end_inset

 to be zero since this would make the denominator zero and the fraction
 would not be defined.
 Therefore the derivative of 
\begin_inset Formula $f(x)=|x|$
\end_inset

 is given by
\begin_inset Formula 
\[
f(x)=\left\{ \begin{array}{cc}
-1 & x<0\\
\mbox{undefined} & x=0\\
1 & x>0
\end{array}\right..
\]

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
If you do use this example in lectures, I would just hands and pointing
 to show that the derivative is not defined at zero.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
There is an easy numerical way to check and compute derivatives with matlab
 based on the differential quotient:
\begin_inset Formula 
\begin{eqnarray*}
f'(x) & = & \lim_{h\rightarrow0}\frac{f(\overbrace{x_{0}+h}^{=x_{1}})-f(x_{0})}{\underbrace{h}_{=x_{1}-x_{0}}}.
\end{eqnarray*}

\end_inset

The idea is that 
\begin_inset Formula $\frac{f(\overbrace{x_{0}+h}^{=x_{1}})-f(x_{0})}{\underbrace{h}_{=x_{1}-x_{0}}}$
\end_inset

 already sufficiently approximates 
\begin_inset Formula $f'(x)$
\end_inset

 for really small 
\begin_inset Formula $h$
\end_inset

.
 This means, that you choose at a set of closely spaced points 
\begin_inset Formula $x_{1},...,x_{i},x_{i+1},...,x_{n}$
\end_inset

 and compute the function values 
\begin_inset Formula $f(x_{1}),...,f(x_{i}),f(x_{i+1}),...,f(x_{n})$
\end_inset

 at those points and compute 
\begin_inset Formula $f'(x_{i})\approx\frac{f(x_{i+1})-f(x_{i})}{x_{i+1}-x_{i}}$
\end_inset

.
 Fortunately, there is a built-in matlab function that takes the differences
 for you.
 Here is an example for how to compute the numerical derivative of 
\begin_inset Formula $f(x)=\sin(x)$
\end_inset

:
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> h = .001; x = 0:h:2*pi; % define h and the base points
\end_layout

\begin_layout Plain Layout

>> f = sin(x); % sample the function at x
\end_layout

\begin_layout Plain Layout

>> df = diff(f)/h; % compute the differences
\end_layout

\begin_layout Plain Layout

>> plot(x,f,'k'), hold on; % plot the function
\end_layout

\begin_layout Plain Layout

>> plot(x(1:end-1),df,'r'); % plot the approximation
\end_layout

\begin_layout Plain Layout

>> plot(x(1:end-1),cos(x(1:end-1)),'g'); % plot the true derivative
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
You surely noticed that we were only able to compute 
\begin_inset Formula $f'(x)$
\end_inset

 for all but the last 
\begin_inset Formula $x$
\end_inset

.
 The reason is, that there is no 
\begin_inset Formula $x_{n+1}$
\end_inset

 and 
\begin_inset Formula $f(x_{n+1})$
\end_inset

 that we could have used for computing 
\begin_inset Formula $f'(x_{n})$
\end_inset

.
 However, there is also a little more involved matlab function that computed
 the derivatives at all base points.
 Using that function the example becomes:
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> h = .001; x = 0:h:2*pi; % define h and the base points
\end_layout

\begin_layout Plain Layout

>> f = sin(x); % sample the function at x
\end_layout

\begin_layout Plain Layout

>> dfdh = gradient(f,h); % compute the first derivative
\end_layout

\begin_layout Plain Layout

>> plot(x,f,'k'), hold on; % plot the function
\end_layout

\begin_layout Plain Layout

>> plot(x,dfdh,'r'); % plot the approximation
\end_layout

\begin_layout Plain Layout

>> plot(x,cos(x),'g'); % plot the true derivative
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
You can (and should) use this function to check derivatives that you computed
 analytically.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsection
Higher-Order Derivatives
\end_layout

\begin_layout Standard
The term 
\emph on
higher-order derivatives
\emph default

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
higher-order derivatives
\end_layout

\end_inset

 comprises the result of iterated differentiation.
 Since we can treat 
\begin_inset Formula $f'(x)$
\end_inset

 as a function on its own, there is really nothing new here.
 If 
\begin_inset Formula $f'(x)$
\end_inset

 is differentiable, then 
\begin_inset Formula $f(x)$
\end_inset

 twice differentiable.
 This can be extended to higher-orders: The third order derivative which
 is written 
\begin_inset Formula $f'''(x)$
\end_inset

 or sometimes 
\begin_inset Formula $f^{(3)}(x)$
\end_inset

 is just the derivative of the second order derivative.
 
\end_layout

\begin_layout Standard
The second derivative, however, has a special meaning.
 Geometrically, it is the slope of the slope, i.e.
 how do the linear approximations at 
\begin_inset Formula $f(x)$
\end_inset

 vary with 
\begin_inset Formula $x$
\end_inset

.
 This give us a notion of 
\emph on
curvature
\emph default

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
curvature
\end_layout

\end_inset

 of 
\begin_inset Formula $f(x)$
\end_inset

.
 Intuitively, if the linear approximations at 
\begin_inset Formula $f(x_{0})$
\end_inset

 bend away very quickly in the area around 
\begin_inset Formula $x_{0}$
\end_inset

, then 
\begin_inset Formula $f(x)$
\end_inset

 must have a high curvature at 
\begin_inset Formula $x_{0}$
\end_inset

.
\end_layout

\begin_layout Standard
\begin_inset Note Note
status open

\begin_layout Plain Layout
Draw picture to explain the curvature
\end_layout

\end_inset


\end_layout

\begin_layout Standard
The geometric intuition behind higher-order derivatives is somewhat harder.
 In particular, our visual system is not very good at judging higher-order
 derivatives.
 By visual inspection, it is pretty hard to say if e.g.
 the fourth derivative is positive or negative.
\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
Suppose that the function 
\begin_inset Formula $f(t)$
\end_inset

 is the position of an object in space at time 
\begin_inset Formula $t$
\end_inset

.
 Then, 
\begin_inset Formula $f'(t)$
\end_inset

 gives us the rate at which the object changes its position, i.e.
 the object's speed at time 
\begin_inset Formula $t$
\end_inset

.
 The second order derivative 
\begin_inset Formula $f''(t)$
\end_inset

 gives us the rate at which the object changes its speed, which is simply
 the acceleration of the object.
 Similarly, if 
\begin_inset Formula $f(t)$
\end_inset

 is the concentration of 
\emph on

\begin_inset Formula $Ca^{2+}$
\end_inset


\emph default
-ions in a neuron at time 
\begin_inset Formula $t$
\end_inset

, then 
\begin_inset Formula $f'(t)$
\end_inset

 gives you the rate at which ions enter or leave the cell at time 
\begin_inset Formula $t,$
\end_inset

 and 
\begin_inset Formula $f''(t)$
\end_inset

 indicates whether this rate is getting bigger or smaller.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsection
Finding Maxima/Minima of a Function
\end_layout

\begin_layout Standard
Derivatives can be used to find global minima or maxima of functions.
 If 
\begin_inset Formula $f(x)$
\end_inset

 has a local maximum or minimum at a point 
\begin_inset Formula $x_{0}$
\end_inset

, the tangent line is horizontal.
 This means that the slope of the tangent and, therefore, the derivative
 of the function at that point 
\begin_inset Formula $x_{0}$
\end_inset

 must be 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

.
 Mathematically speaking, 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

 is a 
\emph on
necessary condition
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
necessary condition
\end_layout

\end_inset


\emph default
 for the function 
\begin_inset Formula $f$
\end_inset

 having a maximum or a minimum, i.e.
 ``
\begin_inset Formula $x_{0}$
\end_inset

 is maximum/minimum'' 
\begin_inset Formula $\Rightarrow$
\end_inset


\begin_inset Formula $f'(x_{0})$
\end_inset

.
 However, it can happen, that 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

 but not 
\begin_inset Formula $x_{0}$
\end_inset

 is not a maximum or a minimum.
 Those points are called saddle-points.
 They arise for example, when 
\begin_inset Formula $f(x)$
\end_inset

 increases, becomes more and more flat when approaching 
\begin_inset Formula $x_{0}$
\end_inset

, is completely flat at 
\begin_inset Formula $x_{0}$
\end_inset

, and increases again afterwards.
 Therefore, since 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

 does not imply a maximum or a minimum, we must find a condition that does
 that, i.e.
 we need to find a 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
sufficient condition
\end_layout

\end_inset

sufficient condition, 
\emph default
one for which ``condition''
\begin_inset Formula $\Rightarrow$
\end_inset

``
\begin_inset Formula $x_{0}$
\end_inset

 is maximum/minimum''.
 
\end_layout

\begin_layout Standard
Since we already know that we only need to look at points 
\begin_inset Formula $x_{0}$
\end_inset

 where 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

, we can think about what must happen for 
\begin_inset Formula $x_{0}$
\end_inset

 to be
\begin_inset ERT
status open

\begin_layout Plain Layout

---
\end_layout

\end_inset

say
\begin_inset ERT
status open

\begin_layout Plain Layout

---
\end_layout

\end_inset

a maximum.
 If 
\begin_inset Formula $x_{0}$
\end_inset

 is a maximum, then the slope of tangent lines of points to the right of
 
\begin_inset Formula $x_{0}$
\end_inset

 must become more and more negative.
 At the same time the slope of tangent lines at points left to 
\begin_inset Formula $x_{0}$
\end_inset

 must become less and less negative and approach zero at 
\begin_inset Formula $x_{0}$
\end_inset

.
 If you think about it, this is exactly the same as requiring 
\begin_inset Formula $f''(x_{0})<0$
\end_inset

.
 Similarly, 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

 and 
\begin_inset Formula $f''(x_{0})>0$
\end_inset

 imply a minimum of 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
 If 
\begin_inset Formula $f'(x_{0})=0$
\end_inset

, 
\begin_inset Formula $f''(x_{0})=0$
\end_inset

, but 
\begin_inset Formula $f'''(x_{0})\not=0$
\end_inset

 then 
\begin_inset Formula $f$
\end_inset

 has a saddle-point at 
\begin_inset Formula $x_{0}$
\end_inset

.
 In general, if the first 
\begin_inset Formula $n$
\end_inset

 derivatives are zero and the 
\begin_inset Formula $(n+1)$
\end_inset

th derivative 
\begin_inset Formula $f^{(n+1)}(x_{0})$
\end_inset

 is not equal to zero, then 
\begin_inset Formula $x_{0}$
\end_inset

 is a minimum, if 
\begin_inset Formula $n+1$
\end_inset

 is even and 
\begin_inset Formula $f^{(n+1)}(x_{0})>0$
\end_inset

, a maximum if 
\begin_inset Formula $n+1$
\end_inset

 is even and 
\begin_inset Formula $f^{(n+1)}(x_{0})<0$
\end_inset

, and a saddle-point if 
\begin_inset Formula $n+1$
\end_inset

 is odd and 
\begin_inset Formula $f^{(n+1)}(x_{0})\not=0$
\end_inset

.
\end_layout

\begin_layout Standard
\align center
\begin_inset Box Boxed
position "t"
hor_pos "c"
has_inner_box 1
inner_pos "t"
use_parbox 0
use_makebox 0
width "100col%"
special "none"
height "1in"
height_special "totalheight"
status open

\begin_layout Plain Layout

\series bold
Finding Maxima, Minima, and Saddle-Points
\end_layout

\begin_layout Plain Layout
Let 
\begin_inset Formula $f$
\end_inset

 be a 
\begin_inset Formula $(n+1)$
\end_inset

 times differentiable function.
 Then 
\begin_inset Formula $f$
\end_inset

 has a 
\end_layout

\begin_layout Itemize

\emph on
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
minimum
\end_layout

\end_inset

minimum
\emph default
 at 
\begin_inset Formula $x_{0}$
\end_inset

 if and only if 
\begin_inset Formula $f^{(k)}(x_{0})=0$
\end_inset

 for 
\begin_inset Formula $k=1,...,n$
\end_inset

 and 
\begin_inset Formula $f^{(n+1)}(x_{0})>0$
\end_inset

 with 
\begin_inset Formula $(n+1)$
\end_inset

 even.
\end_layout

\begin_layout Itemize

\emph on
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
maximum
\end_layout

\end_inset

maximum
\emph default
 at 
\begin_inset Formula $x_{0}$
\end_inset

 if and only if 
\begin_inset Formula $f^{(k)}(x_{0})=0$
\end_inset

 for 
\begin_inset Formula $k=1,...,n$
\end_inset

 and 
\begin_inset Formula $f^{(n+1)}(x_{0})<0$
\end_inset

 with 
\begin_inset Formula $(n+1)$
\end_inset

 even.
\end_layout

\begin_layout Itemize

\emph on
saddle-point
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
saddle-point
\end_layout

\end_inset


\emph default
 
\begin_inset Formula $x_{0}$
\end_inset

 if and only if 
\begin_inset Formula $f^{(k)}(x_{0})=0$
\end_inset

 for 
\begin_inset Formula $k=1,...,n$
\end_inset

 and 
\begin_inset Formula $f^{(n+1)}(x_{0})\not=0$
\end_inset

 with 
\begin_inset Formula $(n+1)$
\end_inset

 odd.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
It should be pointed out that this procedure only yields 
\begin_inset Formula $local$
\end_inset

 maxima, and not necessarily 
\begin_inset Formula $global$
\end_inset

 ones.
 
\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
Consider the function 
\begin_inset Formula $f(x)=(x-1)^{2}$
\end_inset

.
 Then 
\begin_inset Formula $f'(x)=2(x-1)$
\end_inset

 and 
\begin_inset Formula $f''(x)=2$
\end_inset

.
 In order to find candidates for a maximum or a minimum, we set 
\begin_inset Formula $f'(x)=0$
\end_inset

 and solve for 
\begin_inset Formula $x$
\end_inset

:
\begin_inset Formula 
\begin{eqnarray*}
f'(x)=0 & \Leftrightarrow & 2(x-1)=2\\
 & \Leftrightarrow & x=1.
\end{eqnarray*}

\end_inset

Since 
\begin_inset Formula $f''(x)>0$
\end_inset

 for all 
\begin_inset Formula $x$
\end_inset

 (therefore also for 
\begin_inset Formula $x=1$
\end_inset

), 
\begin_inset Formula $x=1$
\end_inset

 must be a minimum of 
\begin_inset Formula $f$
\end_inset

.
 
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsubsection*
Example: Estimating the rate of a Poisson distribution
\end_layout

\begin_layout Standard
A probabilistic model for a neuron that generates completely random spike
 trains with no temporal structure is the homogeneous 
\emph on
Poisson process
\emph default
.
 Since there is no temporal structure, it serves as a baseline model that
 other
\begin_inset ERT
status open

\begin_layout Plain Layout

---
\end_layout

\end_inset

more advanced
\begin_inset ERT
status open

\begin_layout Plain Layout

---
\end_layout

\end_inset

models for spike trains can compare to.
 You can generate spike trains from that model as follows: Bin the time
 axis into sufficiently small bins (say 
\begin_inset Formula $1ms$
\end_inset

).
 For each bin you randomly place a spike with a certain small probability.
 The matlab code looks like this
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> t = [0:0.001:10]; 
\end_layout

\begin_layout Plain Layout

>> p = 0.05; % spiking probability in each bin
\end_layout

\begin_layout Plain Layout

>> ind = find( rand(size(t)) <= p); % sample spike times 
\end_layout

\begin_layout Plain Layout

>> s = zeros(size(t)); % generate spike train 
\end_layout

\begin_layout Plain Layout

>> s(ind) = 1; % insert spikes 
\end_layout

\begin_layout Plain Layout

>> plot(t,s,'k')
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset

The reason, why it is called Poisson process it that the probability of
 observing 
\begin_inset Formula $k$
\end_inset

 spikes in a time window of length 
\begin_inset Formula $\Delta t=1$
\end_inset

 is given by the Poisson distribution 
\begin_inset Formula 
\begin{eqnarray*}
p(k\mbox{ spikes}) & = & \frac{\lambda^{k}e^{-\lambda}}{k!}
\end{eqnarray*}

\end_inset

 or simply 
\begin_inset Formula 
\begin{eqnarray*}
p(k) & = & \frac{\lambda^{k}e^{-\lambda}}{k!}.
\end{eqnarray*}

\end_inset


\begin_inset Formula $\lambda$
\end_inset

 is called the rate of the Poisson distribution.
 It tells us how many spike we expect in one second on average.
 The question that we want to solve in this example is how to estimate 
\begin_inset Formula $\lambda$
\end_inset

 from a number of observed spike trains.
 But lets first look at our matlab example again and check, whether our
 spike counts in one second really follow a Poisson distribution.
 For that reason, we generate a large number of spike trains of length 
\begin_inset Formula $1s$
\end_inset

, count the spikes in each spike train and look at the empirical histogram.
 The matlab code looks like this:
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> t = [0:0.001:1];
\end_layout

\begin_layout Plain Layout

>> p = 0.05; % probability of spike in each bin
\end_layout

\begin_layout Plain Layout

>> m = 5000; % number of spike trains
\end_layout

\begin_layout Plain Layout

>> U = rand(m,length(t)); 
\end_layout

\begin_layout Plain Layout

>> S = zeros(size(U)); % generate spike train matrix (trains X time) 
\end_layout

\begin_layout Plain Layout

>> S(U <= p) = 1;
\end_layout

\begin_layout Plain Layout

>> figure 
\end_layout

\begin_layout Plain Layout

>> imagesc(1-S); colormap gray; 
\end_layout

\begin_layout Plain Layout

>> xlabel('time bins'); ylabel('spike trains')
\end_layout

\begin_layout Plain Layout

>> figure, C = sum(S,2); % get spike counts 
\end_layout

\begin_layout Plain Layout

>> K = [1:max(C)]; 
\end_layout

\begin_layout Plain Layout

>> hist(C,K); % plot histogram 
\end_layout

\begin_layout Plain Layout

>> hold on
\end_layout

\begin_layout Plain Layout

>> lambda = p*length(t); 
\end_layout

\begin_layout Plain Layout

>> plot(K, m*poisspdf(K,lambda),'r','LineWidth',2); hold off
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/spiketrain.eps
	width 6cm

\end_inset


\begin_inset Graphics
	filename figs/Poisson.eps
	width 6cm

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:PoissonSpikes"

\end_inset

Poisson spikes with rate 
\begin_inset Formula $\lambda=50.05$
\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
A typical output of the matlab code fragment is shown in Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:PoissonSpikes"

\end_inset

.
 Before we see how to estimate 
\begin_inset Formula $\lambda$
\end_inset

 from the data, let us first spend a thought about what values we expect.
 The answer is not too difficult.
 Since we see a spike with probability 
\begin_inset Formula $p=0.05$
\end_inset

 in each bin, all bins are independent, we expect to see a total number
 of 
\begin_inset Formula $\lambda=n_{bins}\cdot p$
\end_inset

 spikes per second where 
\begin_inset Formula $n_{bins}$
\end_inset

 is the number of bins per second.
 Strictly speaking, the rate 
\begin_inset Formula $\lambda$
\end_inset

 is defined as the number of bins times the probability of spiking per bin
 in the limit of infinite 
\begin_inset Formula $n_{bins}$
\end_inset

.
 The idea is, that the larger the number of bins gets, the lower the probability
 of observing a spike in a specific bin becomes, i.e.
 it goes to zero for 
\begin_inset Formula $n_{bins}$
\end_inset

 to infinity.
 Therefore, the product of both number can be something finite, i.e.
 the rate 
\begin_inset Formula $\lambda$
\end_inset

.
\end_layout

\begin_layout Standard
Let us know turn to the question of how to estimate 
\begin_inset Formula $\lambda$
\end_inset

.
 Assume that you are only given the number of spikes 
\begin_inset Formula $k_{1},...,k_{m}$
\end_inset

 in each of the spike trains of length 
\begin_inset Formula $1s$
\end_inset

.
 Certainly you also do not know 
\begin_inset Formula $p$
\end_inset

.
 However, you do know (or do assume) that the spikes have been generated
 as described above and therefore the spike counts 
\begin_inset Formula $k$
\end_inset

 must be Poisson distributed.
 We further assume that each spike train has been generated independently
 of the others.
\end_layout

\begin_layout Standard
Given a distribution and a number of observations, one principled way of
 estimating the parameters of that distribution (in our case the rate 
\begin_inset Formula $\lambda$
\end_inset

) is the 
\emph on

\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
maximum likelihood
\end_layout

\end_inset

maximum likelihood
\emph default
 method.
 The idea behind that method is simple: We choose 
\begin_inset Formula $\lambda$
\end_inset

 such that the probability of our observations is maximized:
\begin_inset Formula 
\begin{eqnarray*}
\hat{\lambda} & = & \mbox{argmax}_{\lambda\in\mathbb{R}^{+}}p(k_{1},...,k_{m}|\lambda).
\end{eqnarray*}

\end_inset

Since we assumed that the observations are independent from each other,
 the probability of observing all spike counts jointly, is the product of
 the probabilities of observing each single spike count:
\begin_inset Formula 
\begin{eqnarray*}
p(k_{1},...,k_{m}|\lambda) & = & \prod_{i=1}^{m}p(k_{i}|\lambda).
\end{eqnarray*}

\end_inset

For each single spike count, we know that 
\begin_inset Formula $k_{i}$
\end_inset

 is Poisson distributed and therefore 
\begin_inset Formula 
\begin{eqnarray*}
p(k_{1},...,k_{m}|\lambda) & = & \prod_{i=1}^{m}p(k_{i}|\lambda)\\
 & = & \prod_{i=1}^{m}\frac{\lambda^{k_{i}}e^{-\lambda}}{k_{i}!}.
\end{eqnarray*}

\end_inset

Now, we only need to find that maximum of 
\begin_inset Formula $f(\lambda)=\prod_{i=1}^{m}\frac{\lambda^{k_{i}}e^{-\lambda}}{k_{i}!}$
\end_inset

 with respect to 
\begin_inset Formula $\lambda$
\end_inset

.
 Unfortunately, taking the derivative of 
\begin_inset Formula $f(\lambda)=\prod_{i=1}^{m}\frac{\lambda^{k_{i}}e^{-\lambda}}{k_{i}!}$
\end_inset

 is very complicated.
 Luckily, there is a simple trick that we can apply: By taking the 
\begin_inset Formula $\log$
\end_inset

 of both sides changes the function values, but leaves the position of the
 maxima unchanged (the reason for that is, that 
\begin_inset Formula $\log$
\end_inset

 is a strictly increasing continuous function).
 However, taking the 
\begin_inset Formula $\log$
\end_inset

 makes the right hand side considerably easier to deal with:
\begin_inset Formula 
\begin{eqnarray*}
\log f(\lambda) & = & \log\left(\prod_{i=1}^{m}\frac{\lambda^{k_{i}}e^{-\lambda}}{k_{i}!}\right)\\
 & = & \sum_{i=1}^{m}\log\frac{\lambda^{k_{i}}e^{-\lambda}}{k_{i}!}\\
 & = & \sum_{i=1}^{m}\left(\log\lambda^{k_{i}}+\log e^{-\lambda}-\log k_{i}!\right)\\
 & = & \sum_{i=1}^{m}\left(k_{i}\log\lambda-\lambda-\log k_{i}!\right).
\end{eqnarray*}

\end_inset

Let us now compute the maximum of 
\begin_inset Formula $\sum_{i=1}^{m}k_{i}\log\lambda-\lambda-\log k_{i}!.$
\end_inset

 For that reason, we first compute the first derivative
\begin_inset Formula 
\begin{eqnarray*}
\frac{d}{d\lambda}\left(\sum_{i=1}^{m}k_{i}\log\lambda-\lambda-\log k_{i}!.\right) & = & \sum_{i=1}^{m}\left(k_{i}\frac{d}{d\lambda}\log\lambda-\frac{d}{d\lambda}\lambda-\frac{d}{d\lambda}\log k_{i}!\right)\\
 & = & \sum_{i=1}^{m}\left(k_{i}\frac{1}{\lambda}-1\right)\\
 & = & -m+\frac{1}{\lambda}\sum_{i=1}^{m}k_{i}.
\end{eqnarray*}

\end_inset

Setting the first derivative to zero and solving for 
\begin_inset Formula $\lambda$
\end_inset

 yields
\begin_inset Formula 
\begin{eqnarray*}
-m+\frac{1}{\lambda}\sum_{i=1}^{m}k_{i}=0 & \Leftrightarrow & \frac{1}{\lambda}\sum_{i=1}^{m}k_{i}=m\\
 & \Leftrightarrow & \lambda=\frac{1}{m}\sum_{i=1}^{m}k_{i}.
\end{eqnarray*}

\end_inset

In order to check that it is really a maximum, we compute the second derivative
\begin_inset Formula 
\begin{eqnarray*}
\frac{d}{d\lambda}\left(-m+\frac{1}{\lambda}\sum_{i=1}^{m}k_{i}\right) & = & -\frac{1}{\lambda^{2}}\sum_{i=1}^{m}k_{i}.
\end{eqnarray*}

\end_inset

Since rates are always positive (i.e.
 
\begin_inset Formula $\lambda>0$
\end_inset

) the second derivative is always smaller than zero and, therefore, 
\begin_inset Formula $\hat{\lambda}=\frac{1}{m}\sum_{i=1}^{m}k_{i}$
\end_inset


\family roman
\series medium
\shape up
\size normal
\emph off
\bar no
\noun off
\color none
 is really a maximum.
 It also has a very intuitive interpretation.
 The maximum likelihood estimate for the rate of a Poisson distribution
 is the mean of the observed spike counts.
 In that sense it is also not surprising, that the mean of a Poisson distributio
n is 
\begin_inset Formula $\lambda$
\end_inset

.
\end_layout

\begin_layout Standard
You should include the maximum likelihood estimate in the matlab example
 from above and compare it to the expected rate 
\begin_inset Formula $\lambda=p\cdot n_{bins}$
\end_inset

.
 The code fragment, you have to add, is:
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> lambda_est = mean(C);
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsection
Approximating Functions Locally by Lines and Polynomials
\end_layout

\begin_layout Subsubsection*
Example: Finding a linear approximation to the function sin(x) for x near
 0
\end_layout

\begin_layout Standard
Suppose 
\begin_inset Formula $f(x)=\sin(x)$
\end_inset

.
 If we want to approximate 
\begin_inset Formula $f(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}=0$
\end_inset

 by a line, i.e.
 a function of the form 
\begin_inset Formula $g(x)=wx+b$
\end_inset

, such that 
\begin_inset Formula $g(x)$
\end_inset

 is a good approximation for 
\begin_inset Formula $x$
\end_inset

 near 
\begin_inset Formula $0$
\end_inset

.
 From the definition of the derivative, we saw that the derivate of a function
 at 
\begin_inset Formula $x_{0}$
\end_inset

 is really the slope of a linear approximation of 
\begin_inset Formula $f(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
 Therefore we can just compute 
\begin_inset Formula $f'(x)=\cos(x)$
\end_inset

 and evaluate it at 
\begin_inset Formula $x_{0}=0$
\end_inset

.
 This yields the value of 
\begin_inset Formula $w$
\end_inset

, since 
\begin_inset Formula $w$
\end_inset

 if the derivative of 
\begin_inset Formula $g(x)$
\end_inset

 and we want the derivatives of the function 
\begin_inset Formula $f$
\end_inset

 and its linear approximation 
\begin_inset Formula $g$
\end_inset

 to be the same.
 In our case, 
\begin_inset Formula 
\begin{eqnarray*}
w & = & f'(x_{0})=\cos(0)=1.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
Now, we must ensure, that 
\begin_inset Formula $g(x_{0})=f(x_{0})$
\end_inset

.
 We do that by adjusting 
\begin_inset Formula $b$
\end_inset

.
 The offset 
\begin_inset Formula $b$
\end_inset

 shifts the function horizontally.
 If we want 
\begin_inset Formula $g$
\end_inset

 to have the same value at 
\begin_inset Formula $x_{0}$
\end_inset

 as 
\begin_inset Formula $f$
\end_inset

, we just need to add 
\begin_inset Formula $f(x_{0})$
\end_inset

.
 Therefore 
\begin_inset Formula $b=f(x_{0})$
\end_inset

.
 Since 
\begin_inset Formula $f(x_{0})=\sin(0)=0$
\end_inset

, we set 
\begin_inset Formula $b=0$
\end_inset

 and get 
\begin_inset Formula $g(x)=wx$
\end_inset

 as linear approximation of 
\begin_inset Formula $f(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
 Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:linearApprox"

\end_inset

 plots the linear approximation.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/linearApprox.eps
	width 13cm
	keepAspectRatio

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:linearApprox"

\end_inset

Graph of the function 
\begin_inset Formula $f(x)=\sin(x)$
\end_inset

 and its linear approximation 
\begin_inset Formula $g(x)=x$
\end_inset

 at 
\begin_inset Formula $x_{0}=0$
\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
In the example, we implicitly used that the function is approximated at
 
\begin_inset Formula $x_{0}=0$
\end_inset

 when computing the offset 
\begin_inset Formula $b$
\end_inset

.
 In general, if 
\begin_inset Formula $x_{0}\not=0$
\end_inset

, we cannot simply use 
\begin_inset Formula $b=f(x_{0})$
\end_inset

, since
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{eqnarray*}
g(x_{0}) & = & wx_{0}+b\\
 & = & f'(x_{0})x_{0}+f(x_{0})\\
 & \not= & f(x_{o})\mbox{ \mbox{for }}w,x_{0}\not=0.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
How can we compute the offset 
\begin_inset Formula $b$
\end_inset

 in the general case 
\begin_inset Formula $x_{0}\not=0$
\end_inset

? We can answer this question by some geometrical thoughts.
 First of all, imagine we set 
\begin_inset Formula $b=0$
\end_inset

.
 The approximating function 
\begin_inset Formula $g(x)=wx=f'(x_{0})\cdot x$
\end_inset

 would be a true linear function that had the same slope as 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

 but not the same function value 
\begin_inset Formula $g(x_{0})=wx_{0}\not=f(x_{0})$
\end_inset

.
 Since we know that 
\begin_inset Formula $b$
\end_inset

 lifts 
\begin_inset Formula $g$
\end_inset

 along the 
\begin_inset Formula $y$
\end_inset

-axis we need to find out by how much we must lift it in order to obtain
 
\begin_inset Formula $g(x_{0})=f(x_{0})$
\end_inset

.
 The answer is: We need to lift it by the difference between the true function
 value 
\begin_inset Formula $f(x_{0})$
\end_inset

 and the linear function 
\begin_inset Formula $g(x_{0})=wx_{0}$
\end_inset

 that has the same slope but the wrong offset, i.e.
 
\begin_inset Formula $b=f(x_{0})-wx_{0}$
\end_inset

.
 By substituting 
\begin_inset Formula $w=f'(x_{0})$
\end_inset

 and 
\begin_inset Formula $b=f(x_{0})-wx_{0}$
\end_inset

 in the general line equation, we end up with 
\begin_inset Formula 
\begin{eqnarray*}
g(x) & = & f'(x_{0})x+f(x_{0})-f'(x_{0})x_{0}\\
 & = & f(x_{0})+f'(x_{0})(x-x_{0})
\end{eqnarray*}

\end_inset

as the best ``linear'' approximation (it is not linear, is is a line) of
 
\begin_inset Formula $f$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
\end_layout

\begin_layout Standard
The quality of the approximation depends, of course, on the properties of
 
\begin_inset Formula $f(x)$
\end_inset

.
 Clearly, in the example above, the approximation is really bad for e.g.
 
\begin_inset Formula $x=\pi.$
\end_inset


\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
The linear approximation to 
\begin_inset Formula $f(x)=e^{x}$
\end_inset

 at 
\begin_inset Formula $x_{o}=0$
\end_inset

 is given by 
\begin_inset Formula 
\begin{eqnarray*}
g_{0}(x) & = & f(x)+f'(x_{o})(x-x_{o})\\
 & = & e^{0}+e^{0}\cdot x\\
 & = & 1+x.
\end{eqnarray*}

\end_inset

The approximation at 
\begin_inset Formula $x_{o}=1$
\end_inset

 is given by:
\begin_inset Formula 
\begin{eqnarray*}
g_{1}(x) & = & f(x)+f'(x_{o})(x-x_{o})\\
 & = & e^{1}+(x-1)\cdot e^{1}\\
 & = & xe.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
Figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:expLinApprox"

\end_inset

 shows the graphs of these functions.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
placement h
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset Graphics
	filename figs/expLinApprox.eps
	width 8cm
	keepAspectRatio

\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:expLinApprox"

\end_inset

Graph of the functions 
\begin_inset Formula $f(x)=\exp(x)$
\end_inset

 and its linear approximation 
\begin_inset Formula $g_{0}(x)=1+x$
\end_inset

 and 
\begin_inset Formula $g_{1}(x)=ex$
\end_inset

 at 
\begin_inset Formula $x_{0}=0$
\end_inset

 and 
\begin_inset Formula $x_{0}=1$
\end_inset

.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
As we can see in figure 
\begin_inset CommandInset ref
LatexCommand ref
reference "fig:expLinApprox"

\end_inset

, the linear approximations to 
\begin_inset Formula $e^{x}$
\end_inset

 are not very good.
 Can we find a better approximation by using a quadratic approximation,
 i.e.
 one that uses a polynomial of degree 
\begin_inset Formula $2$
\end_inset

? 
\end_layout

\begin_layout Standard
The answer is yes.
 We simply add a quadratic term to our approximating function 
\begin_inset Formula $g$
\end_inset

.
 So far 
\begin_inset Formula $g$
\end_inset

 had the form 
\begin_inset Formula $g(x)=w(x-x_{0})+b$
\end_inset

.
 To turn it into an quadratic approximation, we need to add a term, that
 is quadratic in 
\begin_inset Formula $(x-x_{0})$
\end_inset

.
 For reasons that will become clear soon this additional term is 
\begin_inset Formula $\frac{1}{2}v(x-x_{0})^{2}$
\end_inset

 yielding 
\begin_inset Formula $g(x)=\frac{1}{2}v(x-x_{0})^{2}+w(x-x_{0})+b$
\end_inset

.
 The additional work we have to do in comparison to a simple linear approximatio
n is to determine the value of 
\begin_inset Formula $v$
\end_inset

.
 Analogously to determining the value of 
\begin_inset Formula $w$
\end_inset

, the value of 
\begin_inset Formula $v$
\end_inset

 is simply 
\begin_inset Formula $v=f''(x_{0})$
\end_inset

.
 By making this choice, we enforce that 
\emph on
the value
\emph default
, 
\emph on
the slope
\emph default
 and 
\emph on
the curvature
\emph default
 of 
\begin_inset Formula $g$
\end_inset

 and 
\begin_inset Formula $f$
\end_inset

 match at 
\begin_inset Formula $x_{0}$
\end_inset

, i.e.
\end_layout

\begin_layout Standard
\begin_inset Formula 
\begin{eqnarray*}
f(x_{0}) & = & g(x_{0})\\
f'(x_{0}) & = & g'(x_{0})\\
f''(x_{0}) & = & g''(x_{0}).
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Standard
The reason why the quadratic term is multiplied with 
\begin_inset Formula $\frac{1}{2}$
\end_inset

 is to cancel the exponent 
\begin_inset Formula $2$
\end_inset

 when taking the derivatives.
 It is an instructive exercise to check that the function value and the
 first two derivatives really match at 
\begin_inset Formula $x_{0}$
\end_inset

 for 
\begin_inset Formula $w=f'(x_{0})$
\end_inset

 and 
\begin_inset Formula $v=f''(x_{0})$
\end_inset

.
\end_layout

\begin_layout Subsubsection*
Example
\end_layout

\begin_layout Standard
For our example 
\begin_inset Formula $f(x)=e^{x}$
\end_inset

, the best quadratic approximation is 
\begin_inset Formula $g(x)=1+x+\frac{x^{2}}{2}.$
\end_inset

 
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
This example can be generalized to polynomials of arbitrary order.
 We can find better local approximations by matching the (higher order)
 derivatives.
 If we want to match the first three derivatives, then we have to use approximat
ions by a polynomial of degree 
\begin_inset Formula $3$
\end_inset

, 
\begin_inset Formula $4$
\end_inset

 and so on.
 It is a remarkable fact that any function that is sufficiently differentiable
 can (locally) be approximated arbitrarily well by polynomials:
\end_layout

\begin_layout Subsubsection*
Theorem (Taylor/MacLaurin) 
\end_layout

\begin_layout Standard
For a differentiable function 
\begin_inset Formula $f$
\end_inset

 and a 
\begin_inset Formula $x$
\end_inset

 near 
\begin_inset Formula $x_{0}$
\end_inset

, 
\begin_inset Formula $f(x)$
\end_inset

 can be approximated by
\end_layout

\begin_layout Subsubsection*
\begin_inset Formula 
\[
f(x)\approx f(x_{0})+(x-x_{0})f'(x_{0})+\frac{1}{2}(x-x_{0})^{2}f''(x_{0})+\frac{1}{6}(x-x_{0})^{3}f'''(x_{0})+\ldots
\]

\end_inset


\end_layout

\begin_layout Standard
or in general
\end_layout

\begin_layout Standard
\begin_inset Formula 
\[
f(x)\approx f_{(n)}(x)=\sum_{k=0}^{n}\frac{1}{k!}(x-x_{0})^{k}f^{(k)}(x_{0}),
\]

\end_inset


\end_layout

\begin_layout Standard
where 
\begin_inset Formula $f^{(k)}(x_{0})$
\end_inset

 denotes the 
\begin_inset Formula $k$
\end_inset


\family roman
\series medium
\shape up
\size normal
\emph off
\bar no
\noun off
\color none
-th
\family default
\series default
\shape default
\size default
\emph default
\bar default
\noun default
 order derivative of 
\begin_inset Formula $f$
\end_inset

, and 
\begin_inset Formula $f_{(n)}$
\end_inset

is referred to as the 
\emph on
Taylor series
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
Taylor series
\end_layout

\end_inset


\emph default
 approximation to 
\begin_inset Formula $f(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

, or simply the 
\emph on
Taylor approximation
\begin_inset Index idx
status collapsed

\begin_layout Plain Layout
Taylor approximation
\end_layout

\end_inset

 of order
\emph default
 
\begin_inset Formula $k$
\end_inset

 at 
\begin_inset Formula $x_{0}$
\end_inset

.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\Diamond$
\end_inset


\end_layout

\begin_layout Standard
\begin_inset Note Comment
status open

\begin_layout Plain Layout
Computer demo: Pick f(x), draw approximations of higher and higher degree.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
The theorem states, that every function can locally be approximated by a
 polynomial.
 The higher the order of the polynomial, the more precise is the approximation.
 The linear approximation we had above is therefore really a special case
 for a polynomial of degree 1.
 
\end_layout

\begin_layout Subsubsection*
Examples
\end_layout

\begin_layout Enumerate
The cubic approximation to 
\begin_inset Formula $f(x)=e^{x}$
\end_inset

at 
\begin_inset Formula $x_{0}=0$
\end_inset

 is given by 
\begin_inset Formula 
\begin{eqnarray*}
g(x) & = & f(x_{0})+f'(x_{0})(x-x_{0})+f''(x_{0})\frac{1}{2}(x-x_{0})^{2}+f'''(x_{0})\frac{1}{6}(x-x_{0})^{3}\\
 & = & e^{0}+e^{0}x+\frac{1}{2}e^{0}x^{2}+\frac{1}{6}e^{0}x^{3}\\
 & = & 1+x+\frac{1}{2}x^{2}+\frac{1}{6}x^{3}.
\end{eqnarray*}

\end_inset

We can generalize that to polynomials of any given order 
\begin_inset Formula $n$
\end_inset

: The 
\begin_inset Formula $n$
\end_inset

-th order approximation to 
\begin_inset Formula $f(x)=e^{x}$
\end_inset

 is given by
\begin_inset Formula 
\begin{eqnarray*}
g_{n}(x) & = & \sum_{k=0}^{n}\frac{1}{k!}(x-x_{0})^{k}.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Enumerate
The linear and the quadratic approximation to 
\begin_inset Formula $\cos(x)$
\end_inset

 for 
\begin_inset Formula $x_{0}=0$
\end_inset

 are given by
\begin_inset Formula 
\begin{eqnarray*}
g_{lin}(x) & = & \cos(x_{0})-\sin(x_{0})\cdot x\\
 & = & 1
\end{eqnarray*}

\end_inset

and
\begin_inset Formula 
\begin{eqnarray*}
g_{quadr}(x) & = & \cos(x_{0})-\sin(x_{0})\cdot x-\frac{1}{2}\cos(x_{0})x^{2}\\
 & = & 1-x^{2}.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Enumerate
The quadratic approximation to 
\begin_inset Formula $\sin(x)$
\end_inset

 at 
\begin_inset Formula $x_{0}=0$
\end_inset

 is given by
\begin_inset Formula 
\begin{eqnarray*}
g(x) & = & \sin(x_{0})+\cos(x_{0})\cdot x-\frac{1}{2}\sin(x_{0})\cdot x^{2}\\
 & = & x.
\end{eqnarray*}

\end_inset


\end_layout

\begin_layout Enumerate
Here is a small piece of matlab code that plots the Taylor approximation
 of 
\begin_inset Formula $e^{x}$
\end_inset

 at 
\begin_inset Formula $x_{0}=1$
\end_inset

 up to the order of 
\begin_inset Formula $n=10$
\end_inset

.
 
\family typewriter

\begin_inset Newline newline
\end_inset


\family default

\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{verbatim}
\end_layout

\begin_layout Plain Layout

>> t = [-1:0.01:2]; % define the range where we want to plot exp
\end_layout

\begin_layout Plain Layout

>> x0 = 1; % set x0 = 1; you can change that 
\end_layout

\begin_layout Plain Layout

>> plot(t,exp(t),'k-','LineWidth',3), hold on % plot exp
\end_layout

\begin_layout Plain Layout

>> fk = exp(x0); % compute constant part
\end_layout

\begin_layout Plain Layout

>> n = 1; % set the polynomial order to 1
\end_layout

\begin_layout Plain Layout

>> for k = 1:10 % loop over 
\end_layout

\begin_layout Plain Layout

     plot(t,fk,'-r','LineWidth',3); % plot the best approx.
 in red
\end_layout

\begin_layout Plain Layout

     pause % wait until someone presses a key
\end_layout

\begin_layout Plain Layout

     plot(t,fk,'-b','LineWidth',3); % plot the same curve in blue
\end_layout

\begin_layout Plain Layout

     n = n*k; % update the factorial function 
\end_layout

\begin_layout Plain Layout

     fk = fk + 1/n*(t-x0).^k.*exp(x0); % get the next polynomial
\end_layout

\begin_layout Plain Layout

   end
\end_layout

\begin_layout Plain Layout

>> hold off
\end_layout

\begin_layout Plain Layout


\backslash
end{verbatim}
\end_layout

\end_inset


\begin_inset Newline newline
\end_inset

Note that this code only works for the function 
\begin_inset Formula $f(x)=\exp(x)$
\end_inset

 since 
\begin_inset Formula $\exp$
\end_inset

 is its own derivative.
\begin_inset Newline newline
\end_inset


\begin_inset Note Comment
status open

\begin_layout Plain Layout
Note that the quadratic approximation is actually a linear function!
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsubsection*
Exercise
\end_layout

\begin_layout Standard
\begin_inset Marginal
status collapsed

\begin_layout Plain Layout
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
fbox{
\backslash
bf{
\backslash
Large 
\backslash
sf E}}
\end_layout

\end_inset


\end_layout

\end_inset

Find the quadratic approximation to 
\begin_inset Formula $f(x)$
\end_inset

=
\begin_inset Formula $x^{4}+3x^{3}+x^{2}+x$
\end_inset

 at 
\begin_inset Formula $x_{0}=1$
\end_inset

.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Standard
Being able to approximate functions by polynomials is extremely useful,
 and done very often.
 Because many functions are hard to analyze, people work with their linear
 or quadratic approximations in many cases .
 By using these approximations, one can often get away by just using polynomials.
 Furthermore, finding Taylor approximations is often not hard, you just
 have to be able to differentiate.
 You should keep in mind though that the approximation is only local, i.e.
 for 
\begin_inset Formula $x$
\end_inset

 near 
\begin_inset Formula $x_{0},$
\end_inset

 and you should be careful that the approximation is really appropriate
 for your particular application.
\end_layout

\begin_layout Standard
Here are two applications where the Taylor expansion is used.
\end_layout

\begin_layout Subsubsection*
Example: Estimating small angles
\end_layout

\begin_layout Standard
A calculation that has to be done for many psychophysical experiments it
 to compute the degree of visual angle.
 A part of the problem (it is still part of the exercises which) is to figure
 out the angle 
\begin_inset Formula $\vartheta$
\end_inset

 in the following setup:
\end_layout

\begin_layout Standard
\begin_inset ERT
status open

\begin_layout Plain Layout


\backslash
begin{center}
\end_layout

\begin_layout Plain Layout


\backslash
setlength{
\backslash
unitlength}{0.04cm}
\end_layout

\begin_layout Plain Layout


\backslash
begin{picture}(200,230)
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
put(125,184){
\backslash
makebox(0,0){$
\backslash
overbrace{
\backslash
hspace{1.3cm}}^{y}$}}
\end_layout

\begin_layout Plain Layout


\backslash
put(95,78){
\backslash
rotatebox{90}{$
\backslash
overbrace{
\backslash
hspace{4cm}}^{x}$}}
\end_layout

\begin_layout Plain Layout

%
\backslash
put(108,76){
\backslash
line(0,1){102}}
\end_layout

\begin_layout Plain Layout


\backslash
put(142,178){
\backslash
line(-1,0){34}}
\end_layout

\begin_layout Plain Layout


\backslash
put(108,76){
\backslash
line(2,6){40}}
\end_layout

\begin_layout Plain Layout


\backslash
put(108,76){
\backslash
line(0,1){120}}
\end_layout

\begin_layout Plain Layout


\backslash
qbezier(108,116)(115,126)(121,116)
\end_layout

\begin_layout Plain Layout


\backslash
put(114,115){
\backslash
makebox(0,0){$
\backslash
vartheta$}}
\end_layout

\begin_layout Plain Layout


\backslash
end{picture}
\end_layout

\begin_layout Plain Layout


\backslash
end{center}
\end_layout

\begin_layout Plain Layout

\end_layout

\end_inset


\end_layout

\begin_layout Standard
The natural way to do so is simply by 
\begin_inset Formula $\tan\vartheta=\frac{y}{x}$
\end_inset

 and, therefore, 
\begin_inset Formula $\vartheta=\mbox{atan}\frac{y}{x}$
\end_inset

.
 However, if you do not have a computer at your disposal (e.g.
 when reading the methods section of a paper on much more comfortable couch)
 you would still like to have an idea what 
\begin_inset Formula $\vartheta$
\end_inset

 is, even if you cannot invert the 
\begin_inset Formula $\tan$
\end_inset

 function in your head.
 Since the usual setup involves large 
\begin_inset Formula $x$
\end_inset

 and small 
\begin_inset Formula $y$
\end_inset

, we expect 
\begin_inset Formula $\vartheta$
\end_inset

 to be very small, i.e.
 close to zero.
 Therefore, we can simply start by computing the first order Taylor approximatio
n of 
\begin_inset Formula $\tan$
\end_inset

 around 
\begin_inset Formula $0$
\end_inset

.
 First, we need the derivative of course:
\begin_inset Formula 
\begin{eqnarray*}
(\tan\vartheta)' & = & \left(\frac{\sin\vartheta}{\cos\vartheta}\right)'\\
 & = & \frac{\cos^{2}\vartheta+\sin^{2}\vartheta}{\cos^{2}\vartheta}\\
 & = & \frac{1}{\cos^{2}\vartheta}.
\end{eqnarray*}

\end_inset

At zero, the first derivative is simply one.
 Since 
\begin_inset Formula $\tan0=0$
\end_inset

, the first order Taylor approximation has the simple form 
\begin_inset Formula 
\begin{eqnarray*}
\tan\vartheta & \approx & f_{(1)}(\vartheta)=\vartheta.
\end{eqnarray*}

\end_inset

Fortunately, this function is really easy to invert: It is simply the function
 itself 
\begin_inset Formula $f_{(1)}^{-1}(\vartheta)=\vartheta$
\end_inset

.
 This shows that we can use 
\begin_inset Formula $\frac{y}{x}$
\end_inset

 for small 
\begin_inset Formula $y$
\end_inset

 and large 
\begin_inset Formula $x$
\end_inset

 itself as good estimate for the angle 
\begin_inset Formula $\vartheta$
\end_inset

.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\begin_layout Subsubsection*
Example: An error analysis of depth perception
\end_layout

\begin_layout Standard
Very often a physical system shows a very specific error behavior, i.e.
 how the error in the inputs shows up in the outputs of a system.
 This error transformation is important to know, since it tells us how an
 error in the input will affect the quality of the system's output.
 A good example for that is depth perception from disparity.
 In this case the error in the estimated depth will grow quadratically with
 depth.
 This is what we will show in this example.
\end_layout

\begin_layout Standard
In the simplest case of two parallel pinhole cameras with a focal length
 
\begin_inset Formula $f$
\end_inset

 a distance of 
\begin_inset Formula $b$
\end_inset

 between them, the distance of a point 
\begin_inset Formula $\mathbf{x}$
\end_inset

 space to the view planes of the cameras can be estimated by 
\begin_inset Formula $d(\varrho)=\frac{fb}{\varrho}$
\end_inset

 where 
\begin_inset Formula $\varrho=x_{l}-x_{r}$
\end_inset

 the disparity of the two images of 
\begin_inset Formula $\mathbf{x}$
\end_inset

, i.e.
 the distance between the x-coordinates of the image of 
\begin_inset Formula $\mathbf{x}$
\end_inset

 in the two view planes of the cameras.
 Consider we have measured a certain disparity 
\begin_inset Formula $\hat{\varrho}$
\end_inset

 which is 
\begin_inset Formula $\delta$
\end_inset

 away from the true disparity 
\begin_inset Formula $\varrho$
\end_inset

, i.e.
 our measurement error was 
\begin_inset Formula $\delta$
\end_inset

, or 
\begin_inset Formula $\hat{\varrho}=\varrho+\delta$
\end_inset

.
 To see how this error affects the depth estimation 
\begin_inset Formula $d$
\end_inset

 we make a first order Taylor expansion around the true value 
\begin_inset Formula $\varrho$
\end_inset

:
\begin_inset Formula 
\begin{eqnarray*}
d(\hat{\varrho}) & \approx & d_{(1)}(\hat{\varrho})\\
 & = & \frac{fb}{\varrho}-\frac{fb}{\varrho^{2}}(\hat{\varrho}-\varrho)\\
 & = & \frac{fb}{\varrho}-\frac{fb}{\varrho^{2}}(\varrho+\delta-\varrho)\\
 & = & \frac{fb}{\varrho}-\frac{fb}{\varrho^{2}}\delta.
\end{eqnarray*}

\end_inset

The true depth error is 
\begin_inset Formula $d(\hat{\varrho})-d(\varrho)$
\end_inset

.
 Using the Taylor expansion instead of 
\begin_inset Formula $d$
\end_inset

, we get 
\begin_inset Formula $d(\hat{\varrho})-d(\varrho)\approx\frac{fb}{\varrho^{2}}\delta.$
\end_inset

 From 
\begin_inset Formula $d(\varrho)=\frac{fb}{\varrho}$
\end_inset

 we can read off that the depth 
\begin_inset Formula $d$
\end_inset

 is inversely proportional to the true disparity, i.e.
 
\begin_inset Formula $d\sim\frac{1}{\rho}$
\end_inset

.
 Plugging that into our expression for the depth estimation error, we obtain
 
\begin_inset Formula $d(\hat{\varrho})-d(\varrho)\sim fbd^{2}\rho$
\end_inset

, which shows that the influence of an error in the inputs grows quadratically
 with the depth that we want to estimate.
\end_layout

\begin_layout Standard
\align right
\begin_inset Formula $\lhd$
\end_inset


\end_layout

\end_body
\end_document