%begin-include

\section{Propositions}

\begin{para}[Propositions]
The central concept in propositional logic is (you guessed it!) that of a proposition.
A \emph{proposition} is a statement that can be either true or false.
If a proposition is true, we say that its \emph{truth value} is $1$; if it is false, we say that its truth value is $0$.

Propositions can be modified and joined through the use of \emph{connectives}.
A connective is said to be $n$-ary if it takes $n$ propositions as input to return a new one.
The most basic example of a connective is the unary (1-ary) \emph{negation} connective. As its name suggests, it takes a proposition $P$ and transforms it into a proposition $(\lnot P)$ that is true if and only if $P$ is false.
For instance, let us consider the proposition ``I am human''. Since that proposition is true, the proposition $(\lnot(\tx{I am human}))$, which stands for ``I am not human'', is false.

Let $*$ be an arbitrary binary connective that, when acting on two propositions $P$ and $Q$, yields a proposition $P* Q$. If $P * Q$ has the same truth value as $Q * P$ for any propositions $P$ and $Q$, we say that $*$ is \emph{symmetric}.
Moreover, if, for any propositions $P$, $Q$ and $R$, the truth value of $P * (Q * R)$ is the same as that of $(P* Q) * R$, we say that $*$ is \emph{associative}.
\end{para}

\begin{para}[Propositional variables and forms]
Our attention should not be focused on particular propositions, but on the way connectives act on them in an abstract way.
For that purpose, we shall use \emph{propositional variables}, which are nothing more than symbols representing arbitrary propositions.
Keep in mind, however, that propositional variables are not propositions by themselves: they only ``become'' propositions when they have been assigned a particular proposition, this is, a particular truth value.
A statement involving only propositional variables and connectives (such as $(\lnot p)$) is said to be a \emph{propositional form}.
Notice that propositional variables may well be used to represent arbitrary propositional forms.

It should be clear that there is no point in talking about the truth value of a propositional form in an absolute manner: we can only talk about it under a particular assignment of truth values to its propositional variables.
Nevertheless, there are two special cases that deserve some attention.
If a propositional form is true for any possible assignment of truth values to its variables, it is said to be a \emph{tautology}; if it is false for every possible assignment, it is said to be a \emph{contradiction}.

In order to better distinguish propositions from propositional variables, we will consistently use lower-case letters to represent propositional variables and upper-case letters for propositions.
\end{para}

\begin{para}[Conjunction and disjunction]
Moving on to more sophisticated connectives, the \emph{conjunction} connective ``and'' is a binary (2-ary) connective: if we consider the propositions ``I am human'' and ``I like cheese'', we can construct the proposition ``I am human \und{and} I like cheese''.
As was to be expected, if $p$ and $q$ are propositional variables, the propositional form ``$p$ and $q$'' (written $(p\land q)$) is true if and only if $p$ and $q$ are both true. This can be represented using what is known as a \emph{truth table}.
\begin{center}
\begin{tabular}[]{|c|c||c|}
\hline $p$ & $q$ & $p\land q$ \\
\hline\hline 0 & 0 & 0 \\
\hline 0 & 1 & 0 \\
\hline 1 & 0 & 0 \\
\hline 1 & 1 & 1 \\
\hline
\end{tabular}
\end{center}
As you can see, a truth table is just a convenient way of writing down the truth value of a propositional form for each and every possible assignment of truth values to the propositional variables it consists of.

With the two connectives that we already have, we can now formulate our first contradiction! The propositional form $(p\land (\lnot p))$ is false for any possible truth value of $p$.

Another example of a binary connective is the \emph{disjunction} connective ``or'', which, when applied on two propositional variables $p$ and $q$, transforms them into a propositional form $(p\lor q)$ that is true if and only if at least one of $p$ and $q$ is true. Its corresponding truth table is the following.
\begin{center}
\begin{tabular}[]{|c|c||c|}
\hline $p$ & $q$ & $p\lor q$ \\
\hline\hline 0 & 0 & 0 \\
\hline 0 & 1 & 1 \\
\hline 1 & 0 & 1 \\
\hline 1 & 1 & 1 \\
\hline
\end{tabular}
\end{center}
Using the disjunction connective, we can now write down our first tautology: $(p\lor \lnot p)$. To be or not to be? That's the tautology!
\end{para}

\begin{para}

\label{hierarchy}
When combining a group of propositions or propositional forms with connectives, it is important to use parentheses to define the order in which the connectives have to be applied.
For instance, given three propositional variables $p_1$, $p_2$ and $p_3$, we can construct the propositional form $(p_1 \land (p_2 \lor p_3))$. Notice how failing to use parentheses would lead to an ambiguous expression, for $p_1 \land (p_2\lor p_3)$ is not the same as $(p_1 \land p_2) \lor p_3$.

There are, nevertheless, a few exceptions where parentheses are unnecessary.
The first of them is result of convention (such as the order of operations that we all learn in elementary school):  whenever a unitary connective is acting on a propositional variable, there is no need to write a parentheses.
In this way, $\lnot p \land q$ is the same as $(\lnot p)\land q$ and should not be mistaken with $\lnot(p\land q)$.
The second exception is more natural: whenever we have a sequence of propositional variables joined by an associative binary connective, parentheses are not necessary within that sequence because, regardless of how we wrote them, the resulting proposition would always yield the same truth values.
Thus, for example, we can safely write $p_1\land p_2 \land p_3$ instead of $p_1\land (p_2 \land p_3)$ or $(p_1\land p_2) \land p_3$.
Lastly, there is an obvious exception: the parentheses that surround a full symbolic expression are completely dispensable in propositional logic, so $(p\land q)$ can perfectly be written as $p\land q$.

As a rule of thumb --- not just for propositions, but for everything in mathematics --- you can omit parentheses whenever doing so leads to no ambiguity, whenever the parentheses add no meaning or whenever a convention removes any possible ambiguity.

\end{para}

\begin{para}[Conditional connective]
Let us now introduce one of the most important binary connectives --- and, unfortunately, one of the most problematic for newcomers, --- the conditional connective.
This connective takes two propositions (a \emph{condition} $P$ and a \emph{consequence} $Q$) and produces a new proposition $P\limplies Q$, which is read ``if $P$, then $Q$''.
The truth table associated to the propositional form $p\limplies q$  is the following:
\begin{center}
\begin{tabular}[]{|c|c||c|}
\hline $p$ & $q$ & $p\limplies q$ \\
\hline\hline 0 & 0 & 1 \\
\hline 0 & 1 & 1 \\
\hline 1 & 0 & 0 \\
\hline 1 & 1 & 1 \\
\hline
\end{tabular}
\end{center}
Please, take your time to digest what this means. What we are saying is that $p\limplies q$ is true if and only if ``if the condition $p$ is true, so is the consequence $q$''.

If the condition is false, we do not care about the consequence: $p\limplies q$ is automatically true.
But if the condition is true, we need the consequence to be true in order for $p\limplies q$ to be true.
\end{para}

\begin{example}
In a attempt to make things a little bit clearer, let us consider a simple example.
Both of us, at some point in our lives, have heard the phrase ``if you study hard, you will pass the exam''. According to the way in which we have defined the conditional operator, the truth table corresponding to all the possible scenarios is the following:
\begin{center}
\begin{tabular}[]{|c|c||c|}
\hline Study hard & Pass the exam & Study hard $\limplies$ Pass the exam \\
\hline\hline 0 & 0 & 1 \\
\hline 0 & 1 & 1 \\
\hline 1 & 0 & 0 \\
\hline 1 & 1 & 1 \\
\hline
\end{tabular}
\end{center}
Let us then examine each case in detail. The first case should be easy: if I do not study hard and I do not pass the exam, is the phrase still true? Of course it is! I did not study hard for the exam, so there was no reason to believe I should have passed it.

Now, what about the second case? If I do not study for the exam but I manage to pass it, is it true that if I study, I will pass the exam? Yes! It is still true. What the statement ``if you study, you will pass the exam'' tells us is that, provided I have studied, I will pass the exam, but if I did not study, the statement says nothing about what will happen. Nevertheless, the situation would have been different had the phrase been ``only if you study, will you pass the exam''. Can you spot the difference? How would you express this last statement in an ``if\ldots then\ldots'' form?

The third case is easy: if I studied but I did not pass the exam, the phrase is, obviously, false. The last statement is equally trivial: if I study hard and I pass the exam, the statement ``if you study hard, you will pass the exam'' is, clearly, true.
\end{example}

\begin{remark}
There is something very significant that, at this point, should be highlighted.
The fact that, for any particular propositions $P$ and $Q$, the statement ``if $P$, then $Q$'' is true does not imply in any way the existence of a cause-effect relation between $P$ and $Q$.
Connectives, such as the conditional connective, combine statements to create new statements.
Thus, the real meaning of a sentence is meaningless (no pun intended); this is all about whether things are true or false.
For instance, the statement ``if zero equals one, the Earth is flat'' is perfectly valid and true (in fact, it is true regardless of your ``beliefs'' concerning the roundness of our planet!).

Nonetheless, it is true that conditional connectives have something to do with deductions.
If we know the propositions $P$ and $P\limplies Q$ to be true, we can indeed deduce $Q$ to be true, but, as I mentioned earlier, this does not imply the existence of any cause-effect relation between $P$ and $Q$.
Let us, for example, take $P$ to be the statement ``humans need water'' and $Q$ to be ``the sun is a star''.
Is the statement $P\limplies Q$ true? Sure it is!
Both $P$ and $Q$ are true, hence so must be $P\limplies Q$.
Then, from a purely formal point of view, we can deduce that ``the sun is a star'' from the fact that ``humans need water'' and ``if humans need water, then the sun is a star''.
Everything we have done is completely meaningless, but, from the perspective of formal logic, it is correct.
Notice, by the way, how logic did not allow us to do anything suspicious: if we were able to deduce that ``the sun is a star'' from ``humans need water'' it was because, in order to show that ``if humans need water, then the sun is a star'', we had to assume that ``the sun is a star'' in the first place.

\end{remark}

\begin{para}
Some people like to extend the conventions in \ref{hierarchy} to give the disjunction connective precedence over the conjunction connective, and the conjunction connective precedence over the conditional one.
In this way, $p\limplies q \land r \lor s$ would be interpreted as $p\limplies ((q \land r) \lor s)$.
Although this conventions are widespread, we shall not use them in this book.
\end{para}

\begin{para}[Biconditional connective]
By this point, you should have noticed a crucial fact: the conditional connective, unlike the other ones we have studied, is not symmetric, which is to say that he truth value of $p\limplies q$ says nothing about that of its \emph{converse} $q\limplies p$.
Trust me, this is a crucial bit.

This very asymmetry leads to the definition of the symmetric \emph{biconditional} connective, which, when applied on some variables $p$ and $q$, yields a propositional form $p\liff q$ that is defined to take the same truth value as $(p\limplies q)\land(q\limplies p)$ for the same assignments on the propositional variables $p$ and $q$.
\end{para}

\begin{para}[Implication and equivalence]
If, for any propositional forms $p$ and $q$, $p\limplies q$ is a tautology, it is said that $p$ \emph{implies} $q$; if, in addition, so is $q\limplies p$ (and, therefore, $p\liff q$), then $p$ and $q$ are said \emph{equivalent}.
The important thing here is that, if $p$ implies $q$, $q$ is true whenever $p$ is. Consequently, if $p$ and $q$ are equivalent, $p$ is true if $q$ is true and $q$ is true if $p$ is true.
It is then needless to say that, when two propositional forms are equivalent, their truth tables are identical.

Just to have some examples, the propositional form $p\land (p\limplies q)$ implies $q$, and the propositional form $\lnot(p\land q)$ is equivalent to $\lnot p \lor \lnot q$.
\end{para}

\begin{para}
At this point, our language has gotten a little bit confusing, so let us introduce some new expressions to make it simpler. We already know that ``if $p$ then $q$'' stands for $p\limplies q$. Nevertheless, based on what we know, $p\limplies q$ could also be read ``$p$ only if $q$''. Take some time to think about this.
Then, if we want to say ``if $p$ then $q$ \und{and} if $q$ then $p$'', we could say ``only if $q$ then $p$ \und{and} if $q$ then $p$'' or, in other words, ``$p$ if and only if $q$''. There you have it! Saying that $p\liff q$ is the same as saying ``$p$ if and only if $q$''.

Just to finish with all this language overload, let me give you one more definition. If $p\limplies q$ is true, then $q$ is said to be a necessary condition for $p$ because, according to what we know, for $p\limplies q$ to hold, $p$ cannot be true if $q$ is false. Analogously, $p$ is said to be a sufficient condition for $q$ because if $p$ is true, taking into account that $p\limplies q$ is true, we know, for sure, that $q$ is true too.
Thus, another fancy way of saying that $p\liff q$ is stating that ``$p$ is a necessary and sufficient condition for $q$'' or vice-versa.
\end{para}

\begin{para}
Truth tables are not only used to describe the behaviour of logical connectives in propositional logic; they can also be used to define them. So much so that --- as you probably had concluded on your own by now --- there exists a perfect correspondence between truth tables and connectives.
The reasons for this are obvious: every connective has its own truth table, every truth table can be used to define a connective, and any two connectives with the same truth table are equivalent.

Another way of introducing new connectives is defining them to be equivalent to some combination of known connectives.
The way in which we defined the biconditional connective is a good example.
The problem with this method is that, unlike with truth tables, we do not have a ``correspondence'' guaranteeing us that any connective can be expressed as a combination of others. Well, we did not have one\ldots until now.
\end{para}

\begin{theorem}
\label{ads1}
Any propositional form $p$ involving $n$ propositional variables $p_1,\ldots,p_n$ is equivalent to a propositional form $q$ involving only those variables and the connectives $\land$, $\lor$ and $\lnot$.

In particular, this shows that, given any $n$-ary connective $*$, the propositional form $p$ resulting from its application on $n$ distinct propositional variables can be written, equivalently, as a propositional form involving only the connectives $\lor$, $\land$ and $\lnot$, which is to say that $*$ can be defined in terms of $\land$, $\lor$ and $\lnot$.
\end{theorem}

\begin{proof}
The way we will prove this is by providing an effective algorithm for constructing the equivalent propositional form $q$ from the truth table of the propositional form $p$.
If $p$ is a contradiction, it suffices to take $q$ to be any contradiction such as $(p_1\land \lnot p_1) \lor \cdots \lor (p_n \land \lnot p_n)$.
If, on the other hand, there exists at least a particular assignment of truth values to $p_1,\ldots,p_n$ which makes $p$ true, we list all such assignments (mark them), and proceed as follows:
\begin{enumerate}
\item Set $q$ to be an empty propositional form.
\item \label{stepcontinuepropalg} Find one marked assignment of truth values. Let $p_{i_1},\ldots,p_{i_r}$ with $\{i_1,\ldots,i_r\}\subseteq \{1,\ldots,n\}$ be all the elements that should be set to $1$ in this particular assignment and, analogously, let $p_{j_1},\ldots,p_{j_s}$ with $\{j_1,\ldots,j_s\}\subseteq \{1,\ldots,n\}$ be all the elements that, in this assignment, need to be set to $0$.
\item If $q$ is not the empty propositional form, set it to
\[(q) \lor (p_{i_1} \land \cdots \land  p_{i_r} \land \lnot p_{j_1} \land \cdots \land \lnot p_{j_s}).\]
Otherwise, if $q$ is empty, set it to\
\[p_{i_1} \land \cdots \land  p_{i_r} \land \lnot p_{j_1} \land \cdots \land \lnot p_{j_s}.\]
\item Unmark the assignment we have been considering. If there are no marked assignments left, $p$ is already equivalent to $q$, so we have finished. Otherwise, go back to step \ref{stepcontinuepropalg}.
\end{enumerate}

The reasoning we have followed is by all means valid and, with some thought on your part, should have already convinced you that what the result is true. Nevertheless, as we dive deeper into the world of mathematics, you will see that this same argument can be written in a much more elegant and clear manner.
\end{proof}

\begin{definition}
An \emph{adequate set of connectives} is a collection of connectives such that, for any propositional form involving a certain number of propositional variables, there exists an equivalent propositional form involving only those variables and the connectives in that set.

For example, $\{\land,\lor,\lnot\}$ is an adequate set of connectives.
\label{<+label+>}
\end{definition}

\begin{corollary}
The set $\{\lnot,\limplies\}$ is an adequate set of connectives.
\label{adequateni}
\end{corollary}

\begin{proof}
Since, according to \ref{ads1}, $\{\land,\lor,\lnot\}$ is an adequate set of connectives, it suffices to show that, given any propositional variables $p$ and $q$, each of $p\land q$ and $p\lor q$ is equivalent to a propositional form that only uses the negation and implication connectives.

It is easy to see that $p\land q$ is equivalent to $\lnot(A\limplies(\lnot B))$ and that $p\lor q$ is equivalent to $(\lnot A)\limplies B$.
\end{proof}


\subsection{Overfitting}

Role of lambda: high makes impact of more variables lower => high bias

Low makes impacts of more variables strong => high variance

Can trade off using cut off. only make positive if above \(0.7\)

How to use? difficult, as lambda within cost!

Can do similarly to d:

Run for a range of lambda (eg 0, 0.01, 0.02, 0.04, 0.08:~10), then pick from cross validation set

Low lambda always has low cost for training set, but not for cv set..

Regularisation: add to error term the size of the term. penalised large parameters

May not fit outside sample

High bias: eg house prices and size. linear would have high bias for out of scope sample (underfitting)

High variance: making polynomial passing through all data (overfitting)

Can reduce overfitting by reducing features either manaually or using models

OR regularisation: keep all features, but reduce magnitude of theta

\subsection{Regularisation}

Make cost function include size of \(\theta^2\) values

\(\min \dfrac{1}{2m} [\sum (h(x)-y)^2 + 1000 \theta 3 ^2 + 1000 \theta 4 ^2]\)

or more broadly:

\(\min \dfrac{1}{2m}[\sum ..... + \lambda \sum \theta j^2]\)

Tend to not include theta 0 as convention, no regularisation

Update for linear regression is

\(\theta j = \theta j -\alpha{(\dfrac{1}{m})* sum(h(x)-y)xj + (\lambda/m \theta j)}\)

\(\theta j = \theta j (1- \alpha \lambda / m) -alpha {(1/m)*\sum(h(x)-y)xj}\)

This is the same as before, but theta \(j\) updates from a smaller \(\theta \) \(j\) each time.

Normal equation needs a change

\((X'X)^-1X'y=\theta''\)

Now is

\((X'X+\lambda I)^-1X y'\)

although for theta 0, lambda zero, so indentiy matrix, but first element 0

REGULARISATION FOR REGULARISATION

add to end of \(J(\theta)\):

\(\dfrac{\lambda }{2m} \sum \theta j^2\)

update for \(\theta \)\(j\) \(j>0\):
is a as linear regression, but \(h(x)\) is a different function


\section{Swap Specifications}

For the treatment of swaps within analytics we define the following quantities:

\begin{itemize}
\item $t_1^{fix} \ldots t_M^{fix}$ is the set of fixed payment times
\item $t_1^{flt} \ldots t_N^{flt}$ is the set of floating payment times
\item  $F_i$ is the reference rate for the floating payment at $t_i^{flt}$
\item  $\alpha^{fix}_i = \alpha^{fix}(t_{i-1}^{fix},t_{i}^{fix})$ is the period (accrual fraction), according to the day count convention, between fixed payments $i-1$ and $i$
\item  $\alpha^{flt}_i = \alpha^{flt}(t_{i-1}^{flt},t_{i}^{flt})$ is the period (accrual fraction), according to the day count convention, between floating payments $i-1$ and $i$
\item  $\alpha^{F}_i$ is the accrual fraction for reference rate  $F_i$
\item  $\Delta^s_i$ is the difference between the start (effective or fixing) time of $F_i$ and $t^{flt}_{i-1}$
\item $\Delta^e_i$ is the difference between the end (maturity) time of $F_i$ and $t^{flt}_{i}$
\item $p^{fund}(T)$ is the funding discount curve
\end{itemize}

All times are expressed in years (i.e 0.5 is half a year). $t_0^{flt}$ is taken as the effective (start) date of the swap (not a payment date), which is usually two days after the trade date.  In general, one would expect  the start date of some reference rate to be the same as the end date of the previous reference rate, i.e.  $\Delta^s_{i+1}=\Delta^e_{i}$ which are typically of order a few days.

The present value of the fixed leg is

\begin{equation}
PV(\text{fixed}) = \sum^M_{i=1} k \alpha^{fix}_ip^{fund}(t^{fix}_i)
\end{equation}

where $k$ is the fixed rate. The PV of the floating leg is

\begin{equation}
PV(\text{float}) = \sum^N_{i=1} F_i \alpha^{flt}_ip^{fund}(t^{flt}_i)
\end{equation}

The PV of a payer swap (i.e. pay the fixed leg) is
\begin{equation}
\begin{split}
PV(\text{payer swap}) &= PV(\text{float}) - PV(\text{fixed}) \\
&=  \sum^N_{i=1} F_i \alpha^{flt}_ip^{fund}(t^{flt}_i)-\sum^M_{i=1} k \alpha^{fix}_ip^{fund}(t^{fix}_i)
\end{split}
\end{equation}

A par swap has a swap rate such that its PV is zero. With this condition, the par-swap rate is
\begin{equation}
\label{eqn:SwapRate}
k = \frac{\sum^N_{i=1} F_i \alpha^{flt}_ip^{fund}(t^{flt}_i)}{ \sum^M_{i=1} \alpha^{fix}_ip^{fund}(t^{fix}_i)}
\end{equation}

\subsection{Libor Reference Rate}

We consider the (usual) case where the reference rate is Libor (with some tenor).  To keep things completely general, we assume that funding is not at Libor, and we define  $p^{libor}(T)$ as the Libor discount curve. The (forward) Libor reference rates can be expressed in terms of the discount factors as

\begin{equation}
\label{eqn:FowardRate}
F_i = \frac{1}{\alpha^{F}_i }\left(\frac{p^{libor}(t^{flt}_{i-1}+\Delta^s_i)} {p^{libor}(t^{flt}_i+\Delta^e_i)} -1\right)
\end{equation}

with the expression for the par-swap rate becoming

\begin{equation}
\label{eqn:ParSwapRate}
k = \frac{\sum^N_{i=1}  \frac{\alpha^{flt}_i}{\alpha^{F}_i} \left(\frac{p^{libor}(t^{flt}_{i-1}+\Delta^s_i)} {p^{libor}(t^{flt}_i+\Delta^e_i)} -1\right)p^{fund}(t^{flt}_i)}{ \sum^M_{i=1} \alpha^{fix}_ip^{fund}(t^{fix}_i)}
\end{equation}

Almost certainly we have $\alpha^{flt}_i = \alpha^{F}_i$, but it is left in the expression for maximum generality. In the case that all the $\Delta$s are zero and the funding curve is Libor, equation  \ref{eqn:SwapRate} collapses to the usual textbook form

\begin{equation}
k = \frac{p(t^{flt}_0)-p(t^{flt}_N)}{ \sum^M_{i=1} \alpha^{fix}_ip(t^{fix}_i)}
\end{equation}

In both cases the swap rate is a function of the discount factors (possibly from multiple curves) at fixed times.

\section{Swap Rate Sensitivities}
\label{Sec:SwapRateSense}
Let the fixed leg annuity be $A$ (i.e. $PV(fixed)$ with $k=1$) and $PV(float)=B$, then

\begin{equation}
\label{eqn:SwaprateSense}
\frac{\partial k}{\partial p^x(t)}=\frac{1}{A}\frac{\partial B}{\partial p^x(t)} - \frac{B}{A^2}\frac{\partial A}{\partial p^x(t)}
\end{equation}

where the superscript $x$ indicates with respect to which curve we are taking the derivative. The fixed leg part is

\begin{equation}
\frac{\partial A}{\partial p^{x}(t)}=
\begin{cases}
\alpha^{fix}_k \mathbf{1}(t^{fix}_k-t)& \text{if $x$ is the funding curve},\\
0 \quad \forall t & \text{if $x$ is any other curve}
\end{cases}
\end{equation}

where $\mathbf{1}(x)$ is the indicator function which equals 1 if  $x=0$ and is 0 otherwise. The most general form of the floating leg is

\begin{equation}
\label{eqn:FloatSense}
\frac{\partial B}{\partial p^{x}(t)}=
\begin{cases}
\alpha^{flt}_k F_k\mathbf{1}(t^{flt}_k-t)& \text{if $x$ is the funding curve},\\
\sum^N_{i=1} \frac{\partial F_i}{\partial p^{x}(t)} \alpha^{flt}_ip^{fund}(t^{flt}_i) \quad \forall t & \text{if $x$ is any other curve}
\end{cases}
\end{equation}

If the reference rate is linked to the funding curve (e.g. the reference is Libor and funding is at Libor) then both terms in the above equation must be considered. Returning to the case of a Libor reference rate, we have

\begin{equation}
\label{eqn:ForwardRateSense}
\begin{split}
\frac{\partial F_i}{\partial p(t)} &= \frac{1}{\alpha^{F}_i }\frac{\partial}{\partial p(t)} \left[ \frac{p(t^{flt}_{i-1}+\Delta^s_i)} {p(t^{flt}_i+\Delta^e_i)} -1\right]\\
&=\frac{\mathbf{1}(t^{flt}_{i-1}+\Delta^s_i-t)} {p(t^{flt}_i+\Delta^e_i)} - \frac{p(t^{flt}_{i-1}+\Delta^s_i)\mathbf{1}(t^{flt}_{i}+\Delta^e_i-t)} {\left(p(t^{flt}_i+\Delta^e_i)\right)^2}
\end{split}
\end{equation}

with $p(t)$ understood to be the discount factors for the Libor curve.  Putting this together with equation \ref{eqn:FloatSense} we finally have

\begin{equation}
\label{eqn:FloatLegSense}
\frac{\partial B}{\partial p^{libor}(t)}=
\begin{cases}
 \frac{\alpha^{flt}_k}{\alpha^{F}_k }\frac{p^{fund}(t^{flt}_{k})} {p^{libor}(t^{flt}_k+\Delta^e_k)}& \text{for } t=t^{flt}_{k-1}+\Delta^s_k,\\
 -\frac{\alpha^{flt}_k}{\alpha^{F}_k }p^{fund}(t^{flt}_{k})\frac{p^{libor}(t^{flt}_{k-1}+\Delta^s_k)} {\left(p^{libor}(t^{flt}_k+\Delta^e_k)\right)^2}& \text{for } t=t^{flt}_{k}+\Delta^e_k,\\
 0 & \text{for $t$ taking any other value}
 \end{cases}
\end{equation}

It is clear from equation  \ref{eqn:SwapRate} that the swap rate only depends on the values of discount factors (possibly from multiple curves) at a set of predefined times - the values of the curve away from these times are irrelevant (hence the indicator function popping up in the sensitivities).

\subsection{Present Value Sensitivity}
Using the same notation as above, the PV of a payer swap is
%
\begin{equation}
PV(\text{payer}) = B-kA
\end{equation}
%
so
%
\begin{equation}
\frac{\partial PV}{\partial p^{x}(t)}= \frac{\partial B}{\partial p^{x}(t)}-k\frac{\partial A}{\partial p^{x}(t)}
\end{equation}
with the sensitivities of the two legs being the same as above.

\section{Implying Yield Cuves}
A discount curve is a monotonically decreasing function,\footnote{providing the instantaneous forward rates never become negative} so does not provide much visual information nor  the best metric for interpolation. We work with yield curves, with the yield to time $t$ defined as $r^x(t)=-\frac{1}{t}\ln\left(p^x(t)\right)$. Clearly, our sensitivity to a point on yield curve $x$ is
%
\begin{equation}
\frac{\partial \theta}{\partial r^{x}(t)} = -tp^x(t)\frac{\partial \theta}{\partial p^{x}(t)}
\end{equation}
%
where $\theta$ is any curve sensitive quantity (e.g. par swap rate, present value etc).

Given a yield curve (or set of curves), one can find the swap rates for any number of swaps\footnote{with payments out to the last defined time on the curve}  using equation \ref{eqn:SwapRate}.  We are interested in the inverse problem of finding yield curve(s) given a set of market swap rates for reference par swaps. For some list of swaps, adding an additional swap will add (many) more than one point to the list of required times for which a discount factor is needed. For this reason we must interpolate the yield curve through a set of nodes, and it is these nodes that we aim to find.

If we have $N$ swaps then we can in principle find a curve with $N$ nodes (or multiple curves with a total of $N$ nodes) that exactly recovers all the swap rates - this is an $N$-dimensional root finding problem.\footnote{for less than $N$ nodes, this can be formulated as a optimisation  problem where one tries to minimise the sum of squares between the market swap rates and those from the candidate curve.}  Alternatively we can find the curve that gives a zero PV to all our swaps - of course this will give exactly the same answer, and has the advantages of allowing more general instruments in the curve construction (not everything has a rate associated with it) and producing a more useful Jacobian (see below).

\subsection{The Jacobian}

The standard method of inverting the vector equation $\mathbf{y} = f(\mathbf{x})$,  when  $\mathbf{x}$ and  $\mathbf{y}$ are the same length, is a Newton-Raphson approach which iterates from some initial guess $\mathbf{x}_0$ using
\begin{equation*}
\mathbf{x}_{i+1} = \mathbf{x}_i - \mathbf{J}^{-1}(f(\mathbf{x})-\mathbf{y} )
\end{equation*}
until $\mathbf{x}_{i+1} - \mathbf{x}_i= \mathbf{0}$ or equivalently $f(\mathbf{x}_i)-\mathbf{y} =\mathbf{0}$. The Jacobian, $\mathbf{J}$ is

\begin{equation*}
\mathbf{J} = \left(
\begin{array}{cccc}
\frac{\partial y_{1}}{\partial x_1} & \frac{\partial y_1}{\partial x_2} & \ldots & \frac{\partial y_1}{\partial x_N}\\
\frac{\partial y_2}{\partial x_1}&\frac{\partial y_2}{\partial x_2} &\ldots&\frac{\partial y_2}{\partial x_N}\\
\vdots&\ddots&\ddots&\vdots\\
\frac{\partial y_N}{\partial x_1}&\ldots&\ldots&\frac{\partial y_N}{\partial x_N}
\end{array}
\right)
\end{equation*}

Additional heuristics are added to Newton-Raphson to improve stability  and the Broyden method can be used to update the Jacobian rather than recalculate it at every step\footnote{alternatively, one could use Sherman-Morrison to update the inverse Jacobian directly}.

For a set of $N$ swaps, the Jacobian for a single curve is arranged as above, with some instrument value (par swap rates, PV etc) taking the place of the $y$s and the yields taking the place of the $x$s.  For two curves, e.g.  libor and funding, we arrange the Jacobian as follows
%
\begin{equation}
\mathbf{J} = \left(
\begin{array}{cccccc}
\frac{\partial \theta_{1}}{\partial r^{libor}_1} & \ldots & \frac{\partial  \theta_1}{\partial r^{libor}_j} &  \frac{\partial  \theta_1}{\partial r^{fund}_1} &\ldots & \frac{\partial  \theta_1}{\partial r^{fund}_{N-j}}\\
\frac{\partial  \theta_{2}}{\partial r^{libor}_1} & \ldots & \frac{\partial s_2}{\partial r^{libor}_j} &  \frac{\partial  \theta_2}{\partial r^{fund}_1} &\ldots & \frac{\partial  \theta_2}{\partial r^{fund}_{N-j}}\\
\vdots & \ddots & \vdots & \vdots & \ddots  &\vdots\\
\frac{\partial  \theta_{N}}{\partial r^{libor}_1} & \ldots & \frac{\partial  \theta_N}{\partial r^{libor}_j} &  \frac{\partial  \theta_N}{\partial r^{fund}_1} &\ldots & \frac{\partial  \theta_N}{\partial r^{fund}_{N-j}}
\end{array}
\right)
\end{equation}
%
with $j$ nodes on the Libor curve and $N-j$ nodes on the funding curve.\footnote{Clearly the vector $\mathbf{x}$ has the $j$ nodes of the Libor curve as its first $j$ entries and the $N-j$ nodes of the funding curves as its remaining entries.}

When $\theta$ is a PV rather than a rate, the Jacobian provides direct hedging information - the PV01\footnote{Strictly PV01 (present value of a basis point) is the change in PV with respect to a parallel move of the yield curve by one basis point. We have taken the derivative of PV with respect to a parallel shift of the yield curve scaled to one basis point.}  of instrument $i$ is just the sum of terms in the $i^{th}$ row (multiplied by one-thousandth).

\subsection{Interpolator Sensitivity}

As we have already mentioned, not all of the discount factors needed to calculate the swap rate can be nodes of the yield curve. This means we need to know the sensitivity of any point on the interpolated curve to the movement of the nodes. With this additional information we can form the analytical Jacobian.\footnote{it is possible to obtain the Jacobian using central differencing, which is what we originally did - this requires $2N$ calls to $f(\mathbf{x})$. }

For $N$ nodes $(x_1,y_1), (x_2,y_2),\ldots,(x_N,y_N)$ many interpolators can be written in the form
%
\begin{equation}
y=\sum^N_{i=1} a_i(x)y_i
\end{equation}
%
The sensitivity of interest is
\begin{equation}
\frac{\partial y}{\partial y_k}=\sum^N_{i=1}a_i(x)\delta_{i,k} =a_k(x)
\end{equation}

A local interpolator (e.g. linear) will only have sensitivity to the local nodes (i.e. the nodes $k$ and  $k+1$ for $x\in(x_k,x_{k+1})$), while a global method, such as cubic spline, will have sensitivity to all the nodes - this is an important consideration for hedging.
For an instrument  which is sensitive to a set of yields at times $t_1,\ldots,t_M$, the sensitivity to node $j$ is
%
\begin{equation}
\frac{\partial \theta}{\partial r_j}=\sum_{i=1}^M \frac{\partial r(t_i)}{\partial r_j}\frac{\partial  \theta}{\partial r(t_i)}
\end{equation}

We now have all the pieces in place to imply a yield curve(s) - the inputs are the market instruments (par swaps), the positioning of the yield curve nodes and the choice of interpolator. The output is the yields at those nodes, which (together with the choice of interpolator) will exactly recover the swap rates.

In order to back out a single curve, the obvious choice of the node positions is at the last payment time of each of the swaps. If a linear interpolator is used, the Jacobian will be lower triangular, and the nodes could be found by bootstrapping - the value of the first node (yield) is chosen to give the correct swap rate for the first swap, with this fixed the second node is chosen to give the correct rate to the second swap, etc. We tested the system with a cubic spline, where every node to some extent has an effect on every swap rate. The system worked well both in an idealised test case\footnote{We constructed a dummy curve, got swap rates off it for USD 6m, 1y, 2y, 3y, 5y, 7y, 10y, 15y, 20y, 25y \& 30y, then used these rates to recover the curve} and using USD reference par-swap rates from Bloomberg.

It is also possible is recover two curves (funding and Libor) from the swap rates. The nodes cannot be placed at the last payment times of the swaps, but instead must be divided up between the two curves. The sensitivity of the swap rates to the funding curve only is small (that part of the Jacobian has much smaller values than the Libor part), and while for an idealised test case both curves can be recovered, there is no guarantee that a root even exists with market data. In practice, additional instruments are used (cash rates, FRAs, futures, basis swaps) to give the required sensitivity to various curves (and particular time periods on those curves). This is dealt with in the next section.

\section{Adding Additional Instruments}

\subsection{Forward Rate Agreements (FRA)}

A FRA has three principle dates associated with it: the fixing date ($t_f$), when the reference Libor rate is observed, the maturity date of that Libor  ($t_m$) - e.g. three months after the fixing date, and the settlement date ($t_s$) when the net payment is made. Unlike swaps, where the payment is at or near maturity, the settlement date of a FRA is close to the fixing date (normally two days after it).
The actual payment is discounted from a hypothetical payment made at maturity by the observed forward rate.

\begin{equation}
\text{payment at settlement} = \frac{(L(t_f,t_m)-k)\alpha}{1+\tau L(t_f,t_m)}
\end{equation}
%
where the year fraction to calculate the payment ($\alpha$) and that used to calculate the discount ($\tau$) are not necessarily the same.  The PV of this instrument is given by
\begin{equation}
PV = \mathbb{E}^\mathbb{P}\left[e^{-\int^{t_s}_0 r_t dt} \frac{(L(t_f,t_m)-k)\alpha}{1+\tau L(t_f,t_m)}\right]
\end{equation}
%
where $r_t$ is the funding rate. If funding is at Libor, we can change numeraire to the zero coupon bond maturing at $t_m$, and the expression becomes
\begin{equation}
\begin{split}
PV &=p^{libor}(t_m) \mathbb{E}^\mathbb{T}\left[(L(t_f,t_m)-k)\alpha\right] \\
&= p^{libor}(t_{f}) -(1+\alpha k) p^{libor}(t_m)
\end{split}
\end{equation}
%
since
\begin{equation}
\mathbb{E}^\mathbb{T}\left[ L(t_f,t_m)\right] = F(t_f,t_m)= \frac{1}{\alpha}\left(\frac{p^{libor}(t_{f})} {p^{libor}(t_m)} -1\right)
\end{equation}
which is also the strike, $k$, that makes the FRA fair (zero PV).

 In the absence of a model for the joint dynamics of the forward and funding rates, we assume the PV to be given by
 \begin{equation}
 \label{eqn:FRA_PV}
PV = p^{fund}(t_s)\left(\frac{(F(t_f,t_m)-k)\alpha}{1+\tau F(t_f,t_m)}\right)
\end{equation}

For a fair FRA, the sensitivities of the strike purely to the Libor curve given by

\begin{equation}
\frac{\partial k}{\partial p^{libor}(t)}=
\begin{cases}
 \frac{1}{\alpha}\frac{1} {p^{libor}(t_m)}& \text{for } t=t_{f},\\
 - \frac{1}{\alpha }\frac{p^{libor}(t_{f})} {\left(p^{libor}(t_m)\right)^2}& \text{for } t=t_{m},\\
 0 & \text{for $t$ taking any other value}
 \end{cases}
\end{equation}

The PV sensitivity for a Libor-funded FRA is
\begin{equation}
\frac{\partial PV}{\partial p^{libor}(t)}=
\begin{cases}
 1& \text{for } t=t_{f},\\
  -(1+\alpha k)  & \text{for } t=t_{m},\\
 0 & \text{for $t$ taking any other value}
 \end{cases}
\end{equation}
%
while for some other funding curve it is
\begin{equation}
\frac{\partial PV}{\partial p^{libor}(t)}=\frac{\alpha p^{fund}(t_s)}{1+\tau F}\left[ 1-\frac{(F-k)\tau}{1+\tau F}  \right] \frac{\partial F}{\partial p^{libor}(t)}
\end{equation}
where $\frac{\partial F}{\partial p(t)}$ is given by equation \ref{eqn:ForwardRateSense}.  Even though the funding curve cannot affect whether an instrument has zero PV (see equation \ref{eqn:FRA_PV}), there is sensitivity to this curve, which is important for calculating the correct hedge ratio.

\subsection{Futures}
Futures are traded based on price. The price implies a forward  rate given by $F = 1-\frac{\text{price}}{100}$, which is usually 3-month Libor with fixing date the third Wednesday of the contract month. The sensitivity of the implied rate can be treated the same as FRAs above  (i.e. we do not apply any convexity adjustment).

No money is paid upfront for a future. For a tenor of $\alpha$, if the price rises by $\Delta$ (implying a fall in the forward rate of $\Delta/100$) , there is an immediate profit (paid into a margin account) of $N\Delta \alpha/100$ where $N$ is the notional of one contract.\footnote{For Eurodollar futures  $\alpha$ is 0.25 (i.e 90/360) and the notional is \$1M so a 1bp fall in the forward rate gives a gain of \$25. The minimum change of value (tick) is \$12.50 or 0.5 bps}
So the PV can be written in terms of the current implied forward, $F(t_s,t_m)$, and the price of the future when  the contact was entered, $c$.
%
\begin{equation}
PV = \alpha N\left(1-F(t_s,t_m) -\frac{c}{100}\right)
\end{equation}
%
Then
\begin{equation}
\frac{\partial PV}{\partial p^{libor}(t)}=-\alpha N\frac{\partial F}{\partial p^{libor}(t)}
\end{equation}
%
where again $\frac{\partial F}{\partial p(t)}$ is given by equation \ref{eqn:ForwardRateSense}. So a future has the opposite sign of PV sensitivity to a FRA, and also no sensitivity to the funding curve due to mark-to-marketing.

\subsection{Spot Libor}
Today's Libor (set at 11am) is available for a range of tenors (1 month, 2 months and 3 months being the most liquid). To be completely consistent when building a Libor curve, we should only use the tenor used in the swaps and FRAs (3 months for USD swaps and FRAs). In the OpenGamma system, spot Libor is treated as a FRA with fixing date and settlement date equal to the trade date, which is normally two days from now.

\subsection{Cash}
Short term cash deposits (i.e. overnight O/N, 1W, 2W etc) which use simple interest can be treated as spot Libor above, but with sensitivity to the funding curve (assuming cash rates represent short term funding costs).

\subsection{Floating Rate Notes (FRN)}
\label{sec:FRN}
FRNs are bonds that pay a variable coupon based on a reference rate plus a spread. The notional is paid up front and returned at maturity. Using the same notation as for swaps, the present value (with unit notional)  is:
%
\begin{equation}
PV = - p^{fund}(t_0) + p^{fund}(t_N) +  \sum^N_{i=1} (F_i+s) \alpha_ip^{fund}(t_i)
\end{equation}
%
where  $t_0$ is the trade time (which may be zero) and $s$ is the spread. If funding is at Libor, with a zero spread the PV is zero - FRNs are always at par when funding at the reference rate.\footnote{Strictly we  also need $\Delta^s_i$ and  $\Delta^e_i$ to be zero, and $\alpha^{flt}_i=\alpha^F_i$}
To trade at par for any other funding requires a spread of
%
\begin{equation}
s =  \frac{p^{fund}(t_0) - p^{fund}(t_N)- \sum^N_{i=1} F_i \alpha_ip^{fund}(t_i) }{ \sum^N_{i=1} \alpha_ip^{fund}(t_i)}
\end{equation}
%
which will be negative if one can fund at less than Libor (e.g. by posting collateral). The sensitivity of this spread and the PV to the curves will follow through as for swaps.

\subsection{Basis Swaps}

Basis swaps are float-for-float with each leg based on a different reference rate (interest-rate indices), with one leg typically paying a (fixed) spread.

\paragraph{Basis Swap in a Single Currency}

A basis swap exchanges payments based on one index (e.g.Fed-Funds) for another (e.g. 3m-Libor) on the same notional amount. A tenor swap exchanges payments based on different Libor tenors, e.g. 3 month Libor paid quarterly for  6 month Libor paid semi-annually.

If legs of the swap ($a$ and $b$) have reference rates $F^a$ and $F^b$, the PV of the receiver of leg $a$ is
\begin{equation}
PV = \sum^M_{i=1} F^a_i \alpha^{a}_ip^{fund}(t^{a}_i) -  \sum^N_{i=1} (F^b_i+s) \alpha^{b}_ip^{fund}(t^{b}_i)
\end{equation}
 %
where $s$ is the spread, which can take positive or negative values. Making the PV zero, the spread becomes

 \begin{equation}
s = \frac{\sum^M_{i=1} F^a_i \alpha^{a}_ip^{fund}(t^{a}_i)-\sum^N_{i=1} F^b_i \alpha^{b}_ip^{fund}(t^{b}_i)}{ \sum^N_{i=1} \alpha^{b}_ip^{fund}(t^{b}_i)}
\end{equation}

If both leg payments are at the same time, this reduces to
%
 \begin{equation}
s = \frac{\sum^M_{i=1} (F^a_i-F^b_i) \alpha_ip^{fund}(t_i)}{ \sum^M_{i=1} \alpha_ip^{fund}(t_i)}
\end{equation}
%
which is just equation \ref{eqn:SwapRate} for the par swap rate, with $F_i = (F^a_i-F^b_i)$.  In the general case,  we again let the spread $s=B/A$ and proceed as in Section \ref{Sec:SwapRateSense}.

\begin{align}
\frac{\partial A}{\partial p^{x}(t)}&=
\begin{cases}
\alpha^{b}_k \mathbf{1}(t^{b}_k-t)& \text{if $x$ is the funding curve},\\
0 \quad \forall t & \text{if $x$ is any other curve}
\end{cases}\\
%
\frac{\partial B}{\partial p^{fund}(t)}&=
\begin{cases}
\alpha^{a}_k F^a_k& \quad t=t^a_k,\\
-\alpha^{b}_k F^b_k& \quad t=t^b_k,\\
0& \text{any other }t
\end{cases}\\
%
\frac{\partial B}{\partial p^{x}(t)}&=\sum^M_{i=1} \frac{\partial F^a_i}{\partial p^{x}(t)} \alpha^{a}_ip^{fund}(t^{a}_i) -\sum^n_{i=1} \frac{\partial F^b_i}{\partial p^{x}(t)} \alpha^{b}_ip^{fund}(t^{b}_i)
\end{align}
%
If  the reference rates are Libor (with different tenors) then terms like $\frac{\partial F^a_i}{\partial p^{x}(t)}$  are given by equation \ref{eqn:ForwardRateSense}. The PV sensitivity is similarly calculated.

\paragraph*{Example: Implying two curves - 3m Libor and Fed-Funds}
A general tenor swap will have sensitivity to three curves.\footnote{funding and the two different tenor curves} To back out a Libor (e.g. 3m) and funding (e.g. Fed-Funds) curve, we could consider a set of $N$ collateralized swaps (so funding is at Fed-Funds), and a set of $M$ collateralized  Fed-Funds vs 3m Libor tenor swaps (again funding at Fed-Funds) - the swaps are mostly sensitive to the Libor curve, so the $N$ nodes on this curve should be at the swap maturities, while the FRNs are mainly sensitive to the spread between the Libor and Fed-fund curve, in which case the $M$ nodes of the fund curve should be at the maturity of the FRNs. This treats both curves on an equal footing. An alternative is to have a primary Libor curve and a spread curve (funding is Libor - spread) - the Libor curve could use more than $N$ nodes, with the remaining going to the spread curve.\footnote{The point is that the two curves will most likely have very similar macro features that you want to capture with as many nodes as possible, while the spread could be a very simple thing represented by only a few nodes (or one for a constant spread).  The funding curve could equally take the role of the primary curve.}

\paragraph{Cross Currency Swaps (CCS)}
A CCS is effectively an exchange of a domestic FRN on a notional of $N$ for a foreign FRN on a notional of $Nx(0)$ where $x(0)$ is the spot forex rate (units of foreign currency for 1 domestic). If both legs have spreads set up as in section \ref{sec:FRN}, then they will both PV to zero and can be treated as separate FRNs (one being sensitive to domestic funding and a domestic index curve, and the other being sensitive to foreign funding and a foreign index curve).

In reality, a single spread is paid on one of the legs, which renders the whole contract fair. In general neither leg is separately PV zero. Of course, if the funding and index curve are the same for one leg, we can treat the other leg as a FRA provided it is the one that the spread is paid on.

Let $p^d(t)$ and $p^f(t)$ be domestic and foreign discount factors for the respective funding curves and $F^d_i$ and $F^f_i$ be the domestic and foreign index rates which are paid at times $t^d_1\ldots t^d_{N^d}$ and  $t^f_1\ldots t^f_{N^f}$. The PV on the domestic side (i.e. paying the domestic index rate plus spread and receiving the foreign) for unit notional is:
%
\begin{equation}
\begin{split}
PV =  \frac{x(t_0)}{x(0)}\left[-p^f(t_0) + p^f(t^f_{N^f}) +\sum^{N^f}_{i=1} F^f_i \alpha^{f}_ip^{f}(t^{f}_i)\right]\\
-\left[-p^d(t_0) + p^d(t^d_{N^d}) +\sum^{N^d}_{i=1}( F^d_i+s) \alpha^{d}_ip^{d}(t^{d}_i)\right]
\end{split}
\end{equation}
%
The term $\frac{x(t_0)}{x(0)}$ arrises from exchanging notional amounts on the trade date $t_0$ (normally two days) and converting the foreign cash flow back to domestic now. The spread that makes this fair is
%
\begin{equation}
s =  \frac{\frac{x(t_0)}{x(0)}\left[-p^f(t_0) + p^f(t^f_{N^f}) +\sum^{N^f}_{i=1} F^f_i \alpha^{f}_ip^{f}(t^{f}_i)\right]+p^d(t_0) -p^d(t^d_{N^d}) -\sum^{N^d}_{i=1}F^d_i \alpha^{d}_ip^{d}(t^{d}_i)}
{\sum^{N^d}_{i=1}\alpha^{d}_ip^{d}(t^{d}_i)}
\end{equation}
%
The PV (and equivalently the spread) is sensitive to (up to) four curves\footnote{the funding in both currencies, and the Libors in both currencies} - these sensitivities (and that for the spread) can be calculated in the same manner as the previous sections.

\paragraph*{Disscusion}
The tenors used in CCS do not necessarily match those of interest rate swaps (IRS), so one could easily be in a situation of needing strips of foreign and domestic IRSs (plus other instruments to take care of the short end of the curve), strips of overnight rates versus Libor basis swaps, Libor tenor swaps, and CCSs. In this situation one would most likely want base curves of the foreign and domestic funding rates, then all other curves implemented as spreads over these curves.

\section{Hedging}
Let $x_1\ldots x_N$ be the PV of the instruments\footnote{we will use $x_i$ to refer to both an instrument and its price} used to build the curve(s), and  $r_1\ldots r_N$ be the yields at the node points on the curve(s)\footnote{the distinction between one and many curves is not relevant here, so we will just talk above the yields $r_i$ which could refer to one or many curves}. The Jacobian is
\begin{equation}
\mathbf{J} = \left(
\begin{array}{ccc}
\frac{\partial x_{1}}{\partial r_1} & \ldots & \frac{\partial x_1}{\partial r_N}\\
\vdots&\ddots&\vdots\\
\frac{\partial x_N}{\partial r_1}&\ldots&\frac{\partial x_N}{\partial r_N}
\end{array}
\right)
\end{equation}
%
The sensitivity of the instruments to a parallel movement in the curve is given by summing each row. Mathematically this is
\[
\frac{\partial \mathbf{x}}{\partial r}|_{\text{parallel move}} = \mathbf{J}\mathbf{1} \quad \text{where} \quad \mathbf{1} = \left(
\begin{array}{c}
1\\
\vdots\\
1
\end{array}
\right)
\]
Sensitivities to other deformations of the curve can be found by replacing the vector $\mathbf{1}$ with one describing the appropriate deformation (e.g. steepening of the curve).
\subsection{Delta Hedging a Curve Sensitive Instrument}
Let $y$ be some other instrument whose price depends purely on the curve we have implied from the $N$ instruments $x_i$ above. A portfolio can be formed thus:
%
\begin{equation}
\Pi = y-\sum^N_{i=1}w_ix_i
\end{equation}
%
where the weights $w_i$ are the amounts held of $x_i$ (hold one unit of $y$). To be insensitive to any small movement of the curve requires
%
\begin{equation}
\frac{\partial \Pi}{\partial r_k}=0 \quad \Rightarrow \quad  \frac{\partial y}{\partial r_k} =\sum^N_{i=0}w_i\frac{\partial x_i}{\partial r_k} \quad \forall k
\end{equation}
%
Writing the weights $w_i$ and the curve sensitivity of $y$\footnote{The quantities $ \frac{\partial y}{\partial r_k}$ are calculated by the system exactly as the entries in the Jacobian are} in  vector form
\begin{equation}
\mathbf{w} = \left(
\begin{array}{c}
w_1\\
\vdots\\
w_N
\end{array}
\right)
\quad
\mathbf{y}' = \left(
\begin{array}{c}
\frac{\partial y}{\partial r_1 }\\
\vdots\\
\frac{\partial y}{\partial r_N}
\end{array}
\right)
\end{equation}
%
gives
\begin{equation}
\mathbf{y}'=\mathbf{J}^T\mathbf{w} \quad \Rightarrow \quad \mathbf{w} =\left( \mathbf{J}^{-1}\right)^T\mathbf{y}'
\end{equation}
%
In general every entry in $\mathbf{w}$ will have a non-zero value, implying some position will need to be taken in every instrument. Furthermore there is no guarantee that the weights will not contain large positive and negative values\footnote{this is often seen in mean-variance portfolio optimisation, where mathematically the equation for the weights is the same}, making the hedge extremely sensitive to non-negligible movements of the curve (i.e large gamma).
In practice one wants to pick a few hedging instrument that are sensitive to the curve and match the properties of $y$ (e.g. similar maturity, etc). Let the M instruments be $h_1\ldots h_M$, and again form a portfolio
%
\begin{equation}
\Pi = y-\sum^M_{i=1}w_ih_i
\end{equation}
%
With $M<N$ it is not possible to meet the condition $\frac{\partial \Pi}{\partial r_k}=0$ for all $k$. We define the sensitivity matrix $\mathbf{H}$ and the vector of small movements of the yields, $\mathbf{\Delta r}$ as
\begin{equation}
\mathbf{H} = \left(
\begin{array}{ccc}
\frac{\partial h_{1}}{\partial r_1} & \ldots & \frac{\partial h_1}{\partial r_N}\\
\vdots&\ddots&\vdots\\
\frac{\partial h_M}{\partial r_1}&\ldots&\frac{\partial h_M}{\partial r_N}
\end{array}
\right)
\quad
\mathbf{\Delta r} = \left(
\begin{array}{c}
\Delta r_1\\
\vdots\\
\Delta r_N
\end{array}
\right)
\end{equation}
%
so a small change in the value of the portfolio can be written as
\begin{equation}
\Delta \Pi \approx - \mathbf{\Delta r} \cdot (\mathbf{H}^T\mathbf{w}-\mathbf{y}')
\end{equation}
This is a random variable, which depends on the random vector $\mathbf{\Delta r}$. To keep things tractable we aim to minimise the expectation of the square of this value:
\begin{equation}
\begin{split}
\mathbb{E}[\Delta \Pi^2] = (\mathbf{H}^T\mathbf{w}-\mathbf{y}')^T \mathbf{\Theta} (\mathbf{H}^T\mathbf{w}-\mathbf{y}')\\
\text{where}  \quad\mathbf{\Theta}  = \mathbb{E}[ \mathbf{\Delta r} \mathbf{\Delta r}^T]
\end{split}
\end{equation}
So $\mathbf{\Theta}$ should be available from time series data of the changes in the yields - if $\mathbb{E}[\Delta \Pi]=0$ (i.e. there is no drift), then $\mathbf{\Theta}$ is the covariance matrix of changes in yields. Differentiating with respect to the weights we have
 %
\begin{equation}
\begin{split}
\frac{\partial \mathbb{E}[\Delta \Pi^2]}{\partial \mathbf{w}} = \mathbf{0} \Rightarrow\\
 \mathbf{H}^T\mathbf{\Theta} \mathbf{H} \mathbf{w}= \mathbf{H}^T\mathbf{\Theta}\mathbf{y}' \Rightarrow\\
 \mathbf{w}=( \mathbf{H}^T\mathbf{\Theta} \mathbf{H} )^{-1} \mathbf{H}^T\mathbf{\Theta}\mathbf{y}'
\end{split}
\end{equation}
This will in general give a more robust hedge to movements of the curve.

\section{Conclusion}
Any interest rate product with price sensitivity to a set (possibly only one) of yield curves only, can be used to imply these curves provided one can write down a function from yield curves to price.  If when attempting to imply several curves, there are no instruments that are sensitive to all the curves, then the problem decouples and the different curves can be found separately.


\subsection{The exterior (wedge) product}

The exterior product of two vectors is:

\(u\land v\)

\subsection{The exterior product is anticommutative}

This is anticommutative (alternating).

\(u\land v=-v\land u\)

This implies that:

\(u\land u=0\)

\subsection{The exterior product is distributive}

\((a+b)\land (c+d)=(a\land c)+(a\land d)+(b\land c)+(b\land d)\)

\subsection{Expanding the exterior product of two vectors}

We can look at the exterior product in component-basis terms.

Consider \(2\)-dimenional vector space with the following vectors:

\(u=ae_1+be_2\)

\(v=ce_1+de_2\)

The exterior product is:

\(u\land v=(ae_1+be_2)\land (ce_1+de_2)\)

\(u\land v=(ae_1\land ce_1)+(ae_1\land de_2)+(be_2\land ce_1)+(be_2\land de_2)\)

\(u\land v=ac(e_1\land e_1)+ad(e_1\land e_2)+bc(e_2\land e_1)+bd(e_2\land e_2)\)

\(u\land v=ad(e_1\land e_2)-bc(e_1\land e_2)\)

\(u\land v=(ad-bc)(e_1\land e_2)\)


\subsection{Von Neumann-Morgenstern utility theorem}

We can model any risk preference as:

\(U[L]=\sum_i p_iu(x_i)\)

If the agent is risk neutral we can use:

\(u(x_i)=x_i\)

If the agent is risk averse:

\(u(x_i)=\ln x_i\)

If the agent is risk loving we can use:

\(u(x_i)=x_i^2\)


\section{Fields}
\label{sec:fields}

\begin{outcome}
  \begin{enumerate}
  \item Solve systems of equations using scalars from a field other
    than the real numbers, such as $\Z_2$ or $\Z_5$.
  \end{enumerate}
\end{outcome}

So far in this chapter, we have worked with real numbers: all of the
scalars we used, for coefficients, constant terms, variables, and
parameters, were real numbers. But in fact, we have not used very many
properties of the real numbers, except for the fact that we can add,
subtract, multiply, and divide them. For example, we have never needed
to take a square root or to compute a trigonometric function.

In fact, most of linear algebra only requires addition, subtraction,
multiplication, and division. This opens the door to doing linear
algebra using other kinds of scalars besides the real numbers. For
example, we can do linear algebra over the rational numbers, complex
numbers, or even over some more exotic number systems that you will
learn about in this section. A system of scalars that one can do
linear algebra with is called a {\em field}.

\begin{definition}{Field}{field}
  \index{properties of addition!field}%
  \index{properties of multiplication!field}%
  A \textbf{field}%
  \index{field} is a set $K$, together with two operations called
  \textbf{addition}%
  \index{addition!in a field} and \textbf{multiplication}%
  \index{multiplication!in a field}, and together with two distinct
  elements $0$ and $1$, such that addition and multiplication satisfy
  the following properties:
  \begin{itemize}\setlength\itemsep{0em}
  \item[(A1)] {Commutative law of addition:} $a+b=b+a$;
    \index{commutative law!of addition}%
  \item[(A2)] {Associative law of addition:} $(a+b)+c = a+(b+c)$;
    \index{associative law!of addition}%
  \item[(A3)] {Unit law of addition:} $0+a = a$;
    \index{additive unit}%
  \item[(A4)] {Additive inverse:} for each $a\in K$, there exists an element $(-a)\in K$ such that $a+(-a)=0$;
    \index{additive inverse}%
    \index{inverse!additive}%
  \item[(M1)] {Commutative law of multiplication:} $ab=ba$;
    \index{commutative law!of multiplication}%
  \item[(M2)] {Associative law of multiplication:} $(ab)c=a(bc)$;
    \index{associative law!of multiplication}%
  \item[(M3)] {Unit law of multiplication:} $1a=a$;
    \index{multiplicative unit}%
    \index{unit!of multiplication|see{multiplicative unit}}%
  \item[(M4)] {Multiplicative inverse:} for each non-zero $a\in K$, there exists an element $a^{-1}\in K$ such that $aa^{-1}=1$;
    \index{multiplicative inverse}%
    \index{inverse!multiplicative}%
    \index{inverse!in a field}%
    \index{inverse!modulo p@modulo $p$}%
  \item[(D)] {Distributive law:} $a(b+c)=ab+ac$.
    \index{distributive law!of fields}%
  \end{itemize}
\end{definition}

Properties (A1)--(A4) are about addition, properties (M1)--(M4) are
about multiplication, and property (D) is about both addition and
multiplication. Here are some examples and non-examples of fields:

\begin{example}{Some fields and some non-fields}{fields-examples}
  \begin{enumialphparenastyle}
    \begin{enumerate}
    \item The set $\R$ of real numbers is a field.
    \item The set $\Q$ of rational numbers is a field.
    \item The set $\Z$ of integers satisfies all field
      properties except for (M4). It is therefore not a field.
    \item The set $\N=\set{0,1,2,\ldots}$ of natural numbers
      satisfies all field properties except (A4) and (M4). It is
      therefore not a field.
    \end{enumerate}
  \end{enumialphparenastyle}
\end{example}

A field doesn't have to be infinite. The following is an example of a
field with only two elements.

\begin{example}{The integers modulo 2}{z-mod-2}
  \index{modulo}%
  Consider the set of {\em bits}%
  \index{bit} (binary digits) $\set{0,1}$.  We can multiply them as
  usual, and add them almost as usual, subject to the alternative rule
  $1+1=0$ (instead of $1+1=2$). Here is a summary of the rules for
  addition and multiplication:
  \begin{equation*}
    \begin{array}{l|ll}
      + & 0 & 1 \\\hline
      0 & 0 & 1 \\
      1 & 1 & 0
    \end{array}
    \quad
    \begin{array}{l|ll}
      \cdot & 0 & 1 \\\hline
      0 & 0 & 0 \\
      1 & 0 & 1
    \end{array}
  \end{equation*}
  This particular alternative arithmetic is called ``arithmetic modulo
  2''.  In computer science, the addition is also called the ``logical
  exclusive or'' operation%
  \index{logical operation!exclusive or}, and multiplication is also
  called the ``logical and'' operation%
  \index{logical operation!and}. You can also think of $0$ as ``even''
  and $1$ as ``odd'', and not that odd plus odd makes even. For
  example, we can calculate like this:
  \begin{equation*}
    \begin{array}{lll}
      1\cdot((1+0)+1) + 1 &=& 1\cdot(1+1) + 1 \\
                          &=& 1\cdot 0 + 1 \\
                          &=& 0 + 1 \\
                          &=& 1.
    \end{array}
  \end{equation*}
  The binary digits form a field $\Z_2 = \set{0,1}$, also
  called \textbf{the field of integers modulo 2}%
  \index{field!integers modulo $p$}%
  \index{integers modulo $p$}.
\end{example}

You can convince yourself that the 9 properties of fields are all
satisfied by the integers modulo 2. This is a bit tedious, but it can
be checked by calculations.  For example, to verify (A1), we have to
check that $0+0=0+0$, $0+1=1+0$, $1+0=0+1$, and $1+1=1+1$. Perhaps the
most interesting properties are (A4) and (M4). For (A4), we can set
$(-0)=0$ and $(-1)=1$. It may be surprising that $(-1)=1$, but you can
check for yourself that $1+(-1)=1+1=0$ when calculating modulo
$2$. For (M4), we can set $1^{-1} = 1$.

When solving systems of linear equations, we only used addition,
subtraction, multiplication, and division. Therefore, we can solve
systems of equations using the elements of any field as the scalars,
instead of the real numbers.

\begin{example}{Solving a system of equations over $\Z_2$}{system-z2}
  Solve the following system in the integers modulo 2:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x + y     &=& 0 \\
      x     + z &=& 1 \\
          y + z &=& 1.
    \end{array}
  \end{equation*}
\end{example}

\begin{solution}
  As usual, we write the augmented matrix of the system of equations,
  then reduce it to {\rref} using elementary row operations. The only
  difference is that we will perform all arithmetic operations modulo
  2, rather than in the real numbers. The augmented matrix is:
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      1 & 1 & 0 & 0 \\
      1 & 0 & 1 & 1 \\
      0 & 1 & 1 & 1 \\
    \end{mymatrix}.
  \end{equation*}
  The first pivot entry is the $1$ in the upper left. We use a row
  operation to create a zero below it. Note that, because we are
  working modulo 2, adding $1$ and subtracting $1$ is the same thing,
  so $0-1=1$.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & 0 & 0 \\
      1 & 0 & 1 & 1 \\
      0 & 1 & 1 & 1 \\
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & 0 & 0 \\
      0 & 1 & 1 & 1 \\
      0 & 1 & 1 & 1 \\
    \end{mymatrix}.
  \end{equation*}
  The next pivot entry is in row 2 and column 2. We create a zero
  below it by subtracting row 2 from row 3, and a zero above it by
  subtracting row 2 from row 1:
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & 0 & 0 \\
      0 & \circled{1} & 1 & 1 \\
      0 & 1 & 1 & 1 \\
    \end{mymatrix}
    \stackrel{R_3\rowop R_3-R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 1 & 0 & 0 \\
      0 & \circled{1} & 1 & 1 \\
      0 & 0 & 0 & 0 \\
    \end{mymatrix}
    \stackrel{R_1\rowop R_1-R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 1 & 1 \\
      0 & \circled{1} & 1 & 1 \\
      0 & 0 & 0 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  The resulting system is in {\rref}. We can see that the system is
  consistent, because there is no row whose left-hand side is zero and
  whose right-hand side is non-zero. We also see that there are two
  pivot columns, and therefore two pivot variables, $x$ and $y$. On
  the other hand, $z$ is a free variable, so we set it equal to a
  parameter: $z=t$. Notice that this time, the parameter $t$ is not a
  real number, but an element of $\Z_2$. From the equation
  $x+z=1$, we get $x=1-z=1+t$. Can you guess why I have written $1+t$
  instead of $1-t$? This is because $(-1)=1$ in the integers modulo 2.
  So $1-t = 1+(-1)t = 1+1t = 1+t$. Similarly, from the equation
  $y+z=1$, we get that $y=1+t$. Therefore, the general solution to the
  system of equations is
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x &=& 1+t \\
      y &=& 1+t \\
      z &=& t,
    \end{array}
  \end{equation*}
  where $t\in\set{0,1}$ is an arbitrary parameter. Recall that this
  means that each time we plug in a particular value for $t$, we get a
  solution.

  There is one difference between solving equations in the real
  numbers and solving equations in $\Z_2$. In the real
  numbers, a system of equations has either no solution, a unique
  solution, or infinitely many solutions. This is because when there
  is a parameter, we automatically get infinitely many solutions. By
  contrast, in $\Z_2$, there are only two scalars, and
  therefore only two possible values for the parameter $t$, namely
  $t=0$ and $t=1$. For $t=0$ we get the solution $(x,y,z) = (1,1,0)$,
  and for $t=1$ we get the solution $(x,y,z) = (0,0,1)$.  Thus, when
  the general solution has one parameter in $\Z_2$, there are
  only two solutions, instead of infinitely many.
\end{solution}

\begin{example}{A game with buttons and lights}{button-game}
  Consider a game with 9 lights arranged in a square:
  \begin{center}
    \begin{tabular}{|c|c|c|}
      \hline
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lighton} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lightoff} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lighton} \\\hline
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lightoff} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lighton} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lightoff} \\\hline
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lightoff} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lightoff} &
      \rule{0ex}{8.5mm}\includegraphics[width=7mm]{figures/lighton} \\\hline
    \end{tabular}
  \end{center}
  Each light is also a button. When a button is pressed, its own
  light, and all the lights neighboring it (i.e., above, below, to the
  left and to the right) are toggled (i.e., any light that was off is
  turned on and vice versa). Figure out which buttons to press to
  turn off all the lights if the starting position is as shown above.
\end{example}

\begin{solution}
  We number the lights and buttons from top to bottom, left to right,
  like this:
  \begin{center}
    \begin{tabular}{|c|c|c|}
      \hline
      1 & 2 & 3\rule{0mm}{2.3ex}\\\hline
      4 & 5 & 6\rule{0mm}{2.3ex}\\\hline
      7 & 8 & 9\rule{0mm}{2.3ex}\\\hline
    \end{tabular}~.
  \end{center}
  Let $x_i$ be a variable in $\Z_2$, corresponding to the event
  ``button $i$ is pressed'' (or more precisely, ``button $i$ is
  pressed an odd number of times'', because pressing a button twice is
  the same as not pressing it at all. That is why we are working
  modulo 2). The light in position 1 is initially on. It is toggled
  each time buttons 1, 2, and 4 are pressed, i.e., it is toggled
  $x_1+x_2+x_4$ times. We want this light to be off in the end. So we
  must have $1+x_1+x_2+x_4=0$. Similarly, the light in position 2 is
  initially off. To ensure that it stays off, we must have
  $0+x_1+x_2+x_3+x_5=0$. In this way, we obtain 9 linear equations in
  9 variables:
  \begin{equation*}
    \begin{array}{lclcl}
      1 &+& x_1+x_2+x_4 &=& 0 \\
      0 &+& x_1+x_2+x_3+x_5 &=& 0 \\
      1 &+& x_2+x_3+x_6 &=& 0 \\
      0 &+& x_1+x_4+x_5+x_7 &=& 0 \\
      1 &+& x_2+x_4+x_5+x_6+x_8 &=& 0 \\
      0 &+& x_3+x_5+x_6+x_9 &=& 0 \\
      0 &+& x_4+x_7+x_8 &=& 0 \\
      0 &+& x_5+x_7+x_8+x_9 &=& 0 \\
      1 &+& x_6+x_8+x_9 &=& 0.
    \end{array}
  \end{equation*}
  If we write this system in standard form, we obtain the following
  augmented matrix:
  \begin{equation*}
    \begin{mymatrix}{rrrrrrrrr|r}
      1&1&0 & 1&0&0 & 0&0&0 & 1 \\
      1&1&1 & 0&1&0 & 0&0&0 & 0 \\
      0&1&1 & 0&0&1 & 0&0&0 & 1 \\

      1&0&0 & 1&1&0 & 1&0&0 & 0 \\
      0&1&0 & 1&1&1 & 0&1&0 & 1 \\
      0&0&1 & 0&1&1 & 0&0&1 & 0 \\

      0&0&0 & 1&0&0 & 1&1&0 & 0 \\
      0&0&0 & 0&1&0 & 1&1&1 & 0 \\
      0&0&0 & 0&0&1 & 0&1&1 & 1 \\
    \end{mymatrix}.
  \end{equation*}
  We solve this system of equations by doing Gauss-Jordan
  elimination with scalars in $\Z_2$.  The {\rref} is
  \begin{equation*}
    \begin{mymatrix}{rrrrrrrrr|r}
      1&0&0 & 0&0&0 & 0&0&0 & 0 \\
      0&1&0 & 0&0&0 & 0&0&0 & 0 \\
      0&0&1 & 0&0&0 & 0&0&0 & 1 \\

      0&0&0 & 1&0&0 & 0&0&0 & 1 \\
      0&0&0 & 0&1&0 & 0&0&0 & 1 \\
      0&0&0 & 0&0&1 & 0&0&0 & 0 \\

      0&0&0 & 0&0&0 & 1&0&0 & 0 \\
      0&0&0 & 0&0&0 & 0&1&0 & 1 \\
      0&0&0 & 0&0&0 & 0&0&1 & 0 \\
    \end{mymatrix}.
  \end{equation*}
  The unique solution is $(x_1,x_2,x_3,x_4,x_5,x_6,x_7,x_8,x_9) =
  (0,0,1,1,1,0,0,1,0)$. This means that we must press buttons 3, 4,
  5, and 8.
\end{solution}

\begin{example}{The integers modulo 5}{z-mod-5}
  Consider the set $\Z_5=\set{0,1,2,3,4}$, called the \textbf{integers
    modulo 5}%
  \index{integers modulo $p$}. We define their addition and
  multiplication by computing the usual addition and multiplication,
  then ``reducing'' the answer modulo 5. Here, ``reducing'' a number
  means repeatedly subtracting 5 until the answer is between $0$ and
  $4$. Imagine a clock showing 5 numbers instead of the usual 12:
  \begin{center}
    \includegraphics[width=2.5cm]{figures/clock}
  \end{center}
  If we want to calculate 3 o'clock plus 4 hours, we get 2 o'clock,
  because whenever the clock reaches 5, it resets to 0. This is how
  addition and multiplication modulo 5 are defined:
  \begin{equation*}
    \begin{array}{l|lllll}
      +&0&1&2&3&4 \\\hline
      0&0&1&2&3&4 \\
      1&1&2&3&4&0 \\
      2&2&3&4&0&1 \\
      3&3&4&0&1&2 \\
      4&4&0&1&2&3 \\
    \end{array}
    \quad
    \begin{array}{l|lllll}
      \cdot&0&1&2&3&4 \\\hline
      0&0&0&0&0&0 \\
      1&0&1&2&3&4 \\
      2&0&2&4&1&3 \\
      3&0&3&1&4&2 \\
      4&0&4&3&2&1 \\
    \end{array}
  \end{equation*}
\end{example}

We note that the integers modulo 5 form a field. Most of the
properties are tedious but easy to verify. Perhaps the most
interesting of the field properties is (M4). It says that for each
non-zero element $a$, there is another element $a^{-1}$ such that
$aa^{-1}=1$.  By looking at the multiplication table, we see that
$1\cdot 1=1$, $2\cdot 3=1$, $3\cdot 2=1$, and $4\cdot 4=1$. Therefore
we can set $1^{-1}=1$, $2^{-1}=3$, $3^{-1}=2$, and $4^{-1}=4$.

\begin{example}{Division in $\Z_5$}{division-z5}
  What is 2 divided by 3 in $\Z_5$?
\end{example}

\begin{solution}
  There are no fractions in $\Z_5$. The key to dividing is
  this: instead of dividing by $a$, multiply by $a^{-1}$. So we have:
  \begin{equation*}
    2/3 = 2\cdot 3^{-1} = 2\cdot 2 = 4.
  \end{equation*}
  So 2 divided by 3 equals 4. This makes sense, because 4 times 3
  equals 2, when calculating modulo 5.
\end{solution}

\begin{example}{Solving a system of equations over $\Z_5$}{system-z5}
  Solve the following system of linear equations over $\Z_5$:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      2x + z &=& 1 \\
      x+4y+z &=& 3 \\
      x+2y+3z &=& 2. \\
    \end{array}
  \end{equation*}
\end{example}

\begin{solution}
  We perform the usual Gauss-Jordan algorithm on the augmented
  matrix. The only thing to keep in mind is that, instead of dividing
  a row by $a$, we should multiply it by $a^{-1}$. And of course, we
  should reduce all intermediate results modulo $5$. For example, to
  change the first pivot entry from $2$ to $1$, we multiply by
  $2^{-1}=3$, instead of dividing by $2$.
  \begin{equation*}
    \begin{mymatrix}{rrr|r}
      \circled{2} & 0 & 1 & 1 \\
      1 & 4 & 1 & 3 \\
      1 & 2 & 3 & 2
    \end{mymatrix}
    \stackrel{R_1\rowop 3R_1}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 3 & 3 \\
      1 & 4 & 1 & 3 \\
      1 & 2 & 3 & 2
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-R_1}{\stackrel{R_3\rowop R_3-R_1}{\roweq}}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 3 & 3 \\
      0 & \circled{4} & 3 & 0 \\
      0 & 2 & 0 & 4
    \end{mymatrix}
    \stackrel{R_2\rowop 4R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 3 & 3 \\
      0 & \circled{1} & 2 & 0 \\
      0 & 2 & 0 & 4
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    {}
    \stackrel{R_3\rowop R_3-2R_2}{\roweq}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 3 & 3 \\
      0 & \circled{1} & 2 & 0 \\
      0 & 0 & \circled{1} & 4
    \end{mymatrix}
    \stackrel{R_1\rowop R_1-3R_3}{\stackrel{R_2\rowop R_2-2R_3}{\roweq}}
    \begin{mymatrix}{rrr|r}
      \circled{1} & 0 & 0 & 1 \\
      0 & \circled{1} & 0 & 2 \\
      0 & 0 & \circled{1} & 4
    \end{mymatrix}.
  \end{equation*}
  The final matrix is in {\rref}, and we see that the system has the
  unique solution $(x,y,z) = (1,2,4)$. Please double-check the
  solution with respect to the original equations.
\end{solution}

\begin{example}{The integers modulo 6 are not a field}{not-field-z6}
  Do the integers modulo 6 form a field?
\end{example}

\begin{solution}
  The integers modulo 6 are the set $\Z_6=\set{0,1,2,3,4,5}$, with the
  addition and multiplication modulo 6:
  \begin{equation*}
    \begin{array}{l|llllll}
      +&0&1&2&3&4&5 \\\hline
      0&0&1&2&3&4&5 \\
      1&1&2&3&4&5&0 \\
      2&2&3&4&5&0&1 \\
      3&3&4&5&0&1&2 \\
      4&4&5&0&1&2&3 \\
      5&5&0&1&2&3&4 \\
    \end{array}
    \quad
    \begin{array}{l|llllll}
      +&0&1&2&3&4&5 \\\hline
      0&0&0&0&0&0&0 \\
      1&0&1&2&3&4&5 \\
      2&0&2&4&0&2&4 \\
      3&0&3&0&3&0&3 \\
      4&0&4&2&0&4&2 \\
      5&0&5&4&3&2&1 \\
    \end{array}
  \end{equation*}
  Then $\Z_6$ satisfies all of the field axioms except
  (M3). To see why (M3) fails, let $a=2$, and note, by looking at the
  multiplication table, that there is no $b\in\Z_6$ such that
  $ab=1$. Therefore, $\Z_6$ is not a field.
\end{solution}

We conclude this section with a fact that we will not prove.

\begin{theorem}{The integers modulo a prime}{z-mod-p}
  Let $n$ be a positive integer. Then the set
  $\Z_n=\set{0,\ldots,n-1}$ of integers modulo $n$, with
  addition and multiplication modulo $n$, forms a field if and only if
  $n$ is prime. Thus, for example, $\Z_2$, $\Z_3$,
  $\Z_5$, $\Z_7$, and $\Z_{11}$ are fields,
  whereas $\Z_4$, $\Z_6$, and $\Z_9$ are not.
\end{theorem}

Another field that is very useful in mathematics and the natural
sciences is the field $\C$ of \textbf{complex numbers}%
\index{complex number}%
\index{field!of complex numbers}. You can read about the complex
numbers in Appendix~\ref{app:complex}.

\begin{example}{Solving a system of equations over $\C$}{system-c}
  Solve the following system of equations over the complex numbers:
  \begin{equation*}
    \begin{array}{c@{~}c@{~}c}
      x - y  + z      &=& -1+i, \\
      x + iy + 3z &=& ~~1+3i. \\
    \end{array}
  \end{equation*}
\end{example}

\begin{solution}
  We perform Gauss-Jordan elimination on the augmented matrix. When
  multiplying or dividing, we have to use complex number arithmetic.
  For example,
  \begin{equation*}
    \frac{2}{1+i} = \frac{2}{1+i}\cdot\frac{1-i}{1-i} = \frac{2-2i}{2} = 1-i.
  \end{equation*}
  The row operations are:
  \begin{equation*}
    \begin{mymatrix}{ccc|c}
      1 & -1 & 1 & -1+i \\
      1 &  i & 3 & 1+3i \\
    \end{mymatrix}
    \stackrel{R_2\rowop R_2-R_1}{\roweq}
    \begin{mymatrix}{ccc|c}
      1 &  -1 & 1 & -1+i \\
      0 & 1+i & 2 & 2+2i \\
    \end{mymatrix}
    \stackrel{R_2\rowop R_2/(1+i)}{\roweq}
    \begin{mymatrix}{ccc|c}
      1 & -1 & 1 & -1+i \\
      0 &  1 & 1-i & 2 \\
    \end{mymatrix}
  \end{equation*}
  \begin{equation*}
    \stackrel{R_1\rowop R_1+R_2}{\roweq}
    \begin{mymatrix}{ccc|c}
      1 &  0 & 2-i & 1+i \\
      0 &  1 & 1-i & 2 \\
    \end{mymatrix}.
  \end{equation*}
  Therefore the general solution is $z=t$, $y=2-(1-i)t$,
  $x=1+i-(2-i)t$, where $t\in\C$ is a parameter, i.e., $t$ is any
  complex number. In vector form, the general solution is
  \begin{equation*}
    \begin{mymatrix}{c} x \\ y \\ z \end{mymatrix}
    = \begin{mymatrix}{c} 1+i \\ 2 \\ 0 \end{mymatrix}
    + t\begin{mymatrix}{c} -2+i \\ -1+i \\ 1 \end{mymatrix}.
  \end{equation*}
\end{solution}


\subsection{Quotients of Riemann Surfaces}
\begin{definition}
    Let a group $G$ act by homeomorphism on a space $X$.
    The action is called properly discontinuous if every compact $K\subset X$ has $\{g\in G:g(K)\cap K\neq\varnothing\}$ finite.\\
    The action is free if for every $x\in X$ the stabiliser $\operatorname{Stab}_G(x)$ is trivial.
\end{definition}
\begin{example}
    If $\Lambda$ is a lattice in $\mathbb C$, then the action of $\Lambda$ on $\mathbb C$ by translation is properly discontinuous and free.
\end{example}
\begin{lemma}
    Let $G$ be a group acting properly discontinuously and freely on a Riemann surface $R$.
    The quotient $G\backslash R$ is Hausdorff and the quotient map $\pi:R\to G\backslash R$ is a regular covering map.
\end{lemma}
\begin{proof}
    For any $p,q\in R$ with $\pi(p)\neq\pi(q)$, as $R$ is Hausdorff and locally Euclidean, we can find open $U\ni p,V\ni q$ such that $\bar{U},\bar{V}$ are compact.
    Then $K=\bar{U}\cap\bar{V}$ is also compact, so $\{g\in G:g(K)\cap K\neq\varnothing\}\supset\{g\in G:g(\bar{U})\cap\bar{V}\neq\varnothing\}$ is finite.
    Say this set is $\{g_1,\ldots,g_n\}$.
    Then for each $i$ there are disjoint open neighbourhoods $U_i\ni x$ and $V_i\ni g_iy$.
    Now $U'=U\cap\bigcap_iU_i$ and $V'=V\cap\bigcap_ig_i^{-1}(V_i)$ are disjoint open neighbourhoods of $p,q$ with $GU'$ disjoint from $GV'$.
    Hence as $\pi$ is open, $\pi(U')$ and $\pi(V')$ are the required disjoint open neighbourhoods of $\pi(p),\pi(q)$.\\
    To see $\pi$ is a regular covering map, we use a similar argument.
    For any $p\in R$, again take $U\ni p$ with compact closure $K=\bar{U}$ and let $\{1,g_1,\ldots,g_n\}$ be the set $\{g\in G:g(K)\cap K\neq\varnothing\}$.
    As the action is free, $g_ix\neq x$ for each $i$, hence there exists disjoint open neighbourhoods $U_i\ni x$ and $V_i\ni g_ix$.
    But then $U'=U\cap\bigcap_i(U_i\cap g_i^{-1}V_i)$ is evenly covered via $\pi$.
    Hence $\pi$ is a regular covering map.
\end{proof}
\begin{proposition}
    Let $R$ be a Riemann surface and let $G$ be a group acting freely and properly discontinuously by conformal equivalences on $R$.
    Then the quotient $S=G\backslash R$ can be made a Riemann surface so that the quotient map $\pi:R\to S$ is analyic and a regular covering map.
\end{proposition}
\begin{proof}
    It is obvious that $S$ is connected since $R$ is.
    The preceding lemma shows that $S$ is Hausdorff.
    The conformal structure on $S$ can be constructed analogously to what we did for the complex torus.
\end{proof}
\begin{theorem}
    Let $R$ be a compact Riemann surface of genus $g_R\ge 2$ and suppose that a group $G$ acts freely and properly discontinuously on $R$ by conformal equivalences, then $G$ is finite and $|G|\le g_R-1$.
\end{theorem}
\begin{proof}
    $S=G\backslash R$ is a Riemann surface and the quotient map $\pi:R\to S$ is an analytic covering map.
    $\deg\pi=|G|$ by construction, so in particular $G$ is finite.
    \footnote{Alternatively, just take $K=R$ in the definition of properly discontinuous action.}
    Also $\pi$ is a covering map, so it has to be unramified, therefore Riemann-Hurwitz gives $g_R-1=|G|(g_S-1)$.
    As $g_R\ge 2$, $g_S-1\ge 1$, consequently $g_R-1\ge |G|$.
\end{proof}
Note that this fails when $g_R=1$.
\begin{example}[Non-example]
    The torus $\mathbb C/\Lambda$ is a group as well, and it acts on itself by conformal equivalences via left translation $(z+\Lambda)(z_0+\Lambda)\mapsto (z+z_0)+\Lambda$.
    Easily any finite subgroup of $\mathbb C/\Lambda$ acts on $\mathbb C/\Lambda$ properly continuously through the same action, but the size of these subgroups is unbounded.
\end{example}


\section{Partial correlations}

\subsection{Definition of a partial correlation}

\marginnote{
A partial correlation is a~measure of the degree of dependence between two random
variables while controlling for the~effect of other random variables:
\[
\pCorr(X,Y; Z) = \frac{\pCov(X,Y;Z)}{\sqrt{\pVar(X;Z)\pVar(Y;Z)}}
\]
where $\pVar(X;Z) = \Var(X - \alpha Z)$, $\alpha$ is such a~constant that $\Cov(X - \alpha Z, Z) = 0$,
and $\pCov(X,Y; Z) = \Cov(X - \alpha Z, Y - \beta Z)$, $\alpha$, $\beta$ are such
constants that $\Cov(X - \alpha Z, Z) = \Cov(Y - \beta Z, Z) = 0$.
}

A partial correlation can be defined in two ways.
We will provide both definitions and show their equivalence.

\begin{definition}
A partial correlation between random variables $X$ and $Y$ holding random variable $Z$
fixed is the~correlation coefficient between the~residuals in regression of $X$ onto
$Z$ and the~residuals in regression of $Y$ onto $Z$.
\end{definition}

Firstly, we project a random variable $X$ onto $Z$, which yields $\E(X \vert Z)$.
The residuals in this regression are $X - \E(X \vert Z)$ — a vector in $\Lin^{\perp}(Z)$.
We will call this variable `cleansed' and label it as $\widetilde X$.
Applying the same procedure for $Y$ yields the `cleansed' variable $\tilde Y = Y - \E(Y \vert Z) \in \Lin^{\perp}(Z)$.
The~angle between $\widetilde X$ and $\widetilde Y$ ($\varphi$ in Figure~\ref{fig:pcorr_def1})
is the~correlation coefficient between these `cleansed' random variables and
the~partial correlation between the~original ones.

\begin{figure}
\centering
\includegraphics[width=0.55\linewidth]{figures/03_partial_correlation_definition.pdf}
\caption{Partial correlation between $X$ and $Y$ while $Z$ is fixed.}
\label{fig:pcorr_def1}
\end{figure}

\marginnote{
Let us define $\widetilde X$ and $\widetilde Y$ as
\begin{align*}
\widetilde X &= \alpha_{XY} \widetilde Y + \tilde{u}_{XY}, \widetilde X \perp Z \\
\widetilde Y &= \alpha_{YX} \widetilde X + \tilde{u}_{YX}, \widetilde Y \perp Z
\end{align*}
Then assuming that the~error term $u_{XY}$ is uncorrelated with $\widetilde Y$,
we obtain:
\begin{align*}
&\Cov(\widetilde Y, \widetilde X - \alpha_{XY} \widetilde Y) = 0 \\
&\Cov(\widetilde Y, \widetilde X) - \alpha_{XY} \Cov(\widetilde Y, \widetilde Y) = 0 \\
&\alpha_{XY} = \frac{\Cov(\widetilde Y, \widetilde X)}{\Var(\widetilde Y)}
\end{align*}
In the same manner we get $\alpha_{YX}$:
\[
\alpha_{YX} = \frac{\Cov(\widetilde Y, \widetilde X)}{\Var(\widetilde X)}
\]
Note, that $\alpha_{XY}$ and $\alpha_{XY}$ are of the~same sign.
}

\begin{definition}
A partial correlation between random variables $X$ and $Y$ holding random variable $Z$
fixed is the~geometric mean between the coefficient $\beta_{XY}$ in regression
\[
X = \beta_{XY} Y + \beta_{XZ} Z + u_X
\]
and the~coefficinent $\beta_{YX}$ in regression
\[
Y = \beta_{YX} X + \beta_{YZ} Z + u_Y
\]
The partial correlation has the same sign as the~coefficients $\beta_{XY}$ and $\beta_{YX}$.
\end{definition}

Following the~definition, we need to start with regressing variable $X$ onto
$Y$ and $Z$. Then, the vector we obtain $\hat X = \E(X \vert Y, Z)$
can be broken up into the sum of $\beta_{XY} Y$ and $\beta_{XZ} Z$.
Projecting $\beta_{XY} Y$ onto $\Lin^{\perp}(Z)$ results in a~vector
$\alpha_{XY} \widetilde Y$ where $\widetilde Y = Y - \E(Y \vert Z)$
is the projection of $Y$ onto $\Lin^{\perp}(Z)$.

\marginnote{
Multiplying these coefficients, we get the final result:
\begin{multline*}
\alpha_{XY} \cdot \alpha_{XY} = \frac{\Cov^2(\widetilde Y, \widetilde X)}{\Var(\widetilde Y)\Var)\widetilde X} \\
= \Corr^2(\widetilde X, \widetilde Y) = \pCorr^2(X,Y; Z)
\end{multline*}
}

By the~properties of similar triangles
\[
\frac{\beta_{XY} Y}{Y} = \frac{\alpha_{XY} \widetilde Y}{\widetilde Y} \Leftrightarrow \beta_{XY} = \alpha_{XY}
\]

In the same way we perform a regression of $Y$ onto $X$ and $Z$ and repeat the
same steps for $X$. Finally, we get the whole picture:

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.55\linewidth]{figures/03_partial_correlation_regression_definition.pdf}
\label{fig:part_corr_alt}}
\subfigure[]{
\includegraphics[width=0.35\linewidth]{figures/03_partial_correlation_regression_definition_lin.pdf}
\label{fig:part_corr_alt_lin}}
\caption{\subref{fig:part_corr_alt}: Alternative definition of the partial correlation;
\subref{fig:part_corr_alt_lin}: $\Lin^{\perp}(Z)$.}
\end{center}
\end{figure}

Having plotted $\Lin^{\perp}(Z)$, now we can express $\cos \varphi$ in terms of $\beta_{XY}$
and $\beta_{YX}$. We use the fact that $\beta_{XY} \beta_{YX} > 0$:

\begin{equation*}%\label{eq:part_cor_cos}
\begin{split}
\cos \varphi &= \frac{\vert \beta_{XY} \widetilde Y \vert}{\vert \widetilde X \vert} \\
\cos \varphi &= \frac{\vert \beta_{YX} \widetilde X \vert}{\vert \widetilde Y \vert} \\
\cos^2 \varphi &= \vert \beta_{XY} \beta_{YX} \vert \stackrel{\beta_{XY} \beta_{YX} > 0}{=} \beta_{XY} \beta_{YX}
\end{split}
\end{equation*}

Recall that the angle $\varphi$ can be interpreted as the correlation
between $\widetilde X$ and $\widetilde Y$.
These random variables are constructed in such a way that both of them
are uncorrelated with $Z$. Thus, it follows that
\[
\cos^2 \varphi = \Corr^2(\widetilde X, \widetilde Y) = \pCorr^2(X,Y; Z) = \beta_{XY} \beta_{YX}
\]

\subsection{Partial correlation as correlation between residuals}

\marginnote{
Let us define cleansed $X$ and $Y$ first as
\begin{align*}
\widetilde{X} &= \alpha_1 \widetilde{Y} + \tilde{u}, \widetilde{X} \perp Z \\
\widetilde{Y} &= \beta_1 \widetilde{X} + \tilde{v}, \widetilde{Y} \perp Z
\end{align*}
Then
\begin{align*}
\alpha_1 &= \frac{\Cov(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{Y})} \\
\beta_1 &= \frac{\Cov(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X})}
\end{align*}
}

\begin{theorem}
The partial correlation between $X$ and $Y$ holding $Z$ fixed is the negative
correlation coefficient between the residuals $u$ in the regression model
\[
X = \alpha_1 Y + \alpha_2 Z + u
\]
and the residuals $v$ in the model
\[
Y = \beta_1 X + \beta_2 Z + v
\]
\end{theorem}

\begin{proof}
The first step is to find the residuals in the regressions.
For example, in order to get $u$ we regress $X$ onto $\Lin(Y,Z)$
which results in $\hat X = X - \E(X \vert Y, Z)$.
Then we take the difference $X - \hat X = u$
and project it and $X$ itself onto $\Lin^{\perp}(Z)$ as demonstrated
in Figure~\ref{fig:pcorr_t_x}.
We denote the result as $\tilde u$ and $\widetilde X$ respectively.

Figure~\ref{fig:pcorr_t_y} shows the same steps for obtaining $\tilde v$ and $\widetilde Y$.

\marginnote[-6\baselineskip]{
Substituting these into $\Cov(\tilde{u}, \tilde{v})$, we obtain:
\begin{align*}
&\Cov(\tilde{u}, \tilde{v}) = \\
&\Cov\left(\widetilde{X} - \frac{\Cov(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{Y})} \widetilde{Y},
\widetilde{Y} -  \frac{\Cov(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X})} \widetilde{X}\right) \\
&= \Cov(\widetilde{X}, \widetilde{Y}) - \Cov(\widetilde{X}, \widetilde{Y}) \\
&- \Cov(\widetilde{X}, \widetilde{Y}) + \frac{\Cov^3(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X})\Var(\widetilde{Y})} \\
&= - \Cov(\widetilde{X}, \widetilde{Y}) \left(1 - \frac{\Cov^2(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X})\Var(\widetilde{Y})} \right)
\end{align*}
Next, we deal with variances of $\tilde{u}$ and $\tilde{v}$:
\begin{align*}
\Var(\tilde{u}) &= \Var(\widetilde{X}) - \frac{\Cov^2(\widetilde{X}, \widetilde{Y})}{\Var^2(\widetilde{Y})} \Var(\widetilde{Y}) \\
&- 2 \Cov(\widetilde{X}, \widetilde{Y}) \frac{\Cov(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{Y})} \\
&= \Var(\widetilde{X}) \left(1 - \frac{\Cov^2(\widetilde{X}, \widetilde{Y}}{\Var(\widetilde{X}) \Var(\widetilde{Y})}\right) \\
&\Var(\tilde{v}) = \Var(\widetilde{Y}) \left(1 - \frac{\Cov^2(\widetilde{X}, \widetilde{Y}}{\Var(\widetilde{X}) \Var(\widetilde{Y})}\right)
\end{align*}
Now we can write out $\Corr(\tilde{u}, \tilde{v})$:
\begin{align*}
&\Corr(\tilde{u}, \tilde{v}) = \\
&= -\frac{\Cov(\widetilde{X}, \widetilde{Y}) \left(1 - \frac{\Cov^2(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X})\Var(\widetilde{Y})} \right)}{\sqrt{\Var(\widetilde{X})\Var(\widetilde{Y}) \left(1 - \frac{\Cov^2(\widetilde{X}, \widetilde{Y})}{\Var(\widetilde{X}) \Var(\widetilde{Y})}\right)^2 }} \\
&= -\Corr(\widetilde{X}, \widetilde{Y})
\end{align*}
}

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/03_partial_correlation_residuals_x.pdf}
\label{fig:pcorr_t_x}}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/03_partial_correlation_residuals_y.pdf}
\label{fig:pcorr_t_y}}
\caption{\subref{fig:pcorr_t_x}: $\hat u$ form regression of $X$ onto $Y$ and $Z$, $\hat u$ projected;
\subref{fig:pcorr_t_y}: $\hat v$ from regression of $Y$ onto $X$ and $Z$, $\hat v$ projected.}
\end{center}
\end{figure}

After putting these figures together, we need to measure the angle
between the $\tilde u$ and $\tilde v$.
Translating the $\tilde v$ vector to the orgin of $\tilde x$ as shown in Figure~\ref{fig:pcorr_t_lin},
we conclude that the desired angle is the bigger of the vertical angles.
Hence, we can derive it from the property of the quadrilateral
by substracting all the known angles from $360^\circ$.
Thus, the desired angle is $180^\circ - \varphi$.
\begin{align*}
\cos\varphi &= - \cos(180^\circ - \varphi) \\
\Corr(\widetilde X, \widetilde Y) &= -\Corr(\tilde u, \tilde v) \\
\pCorr(X,Y; Z) &= -\Corr(u, v)
\end{align*}

\begin{figure}[ht!]
\begin{center}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/03_partial_correlation_residuals_xy.pdf}
\label{fig:pcorr_t_xy}}
\subfigure[]{
\includegraphics[width=0.45\linewidth]{figures/03_partial_correlation_residuals_xy_lin.pdf}
\label{fig:pcorr_t_lin}}
\caption{\subref{fig:pcorr_t_x}: The residuals of both regressions;
\subref{fig:pcorr_t_y}: $\Lin^{\perp}(Z)$.}
\setfloatalignment{b}
\end{center}
\end{figure}
\end{proof}


\section{Bayesian inference for the Gaussian distribution}

\subsection{The univariate Gaussian}
\begin{align}
t &\sim \mathcal{N}(\mu,\sigma^2)\\
p(t|\mu, \sigma^2)&=\frac{1}{\sqrt{2\pi\sigma^2}}\exp\left( -\frac{1}{2}\left(\frac{t-\mu}{\sigma} \right)^2\right)
\end{align}
\begin{itemize}
\item
The Gaussian has \emph{mean} $\mu$ and \emph{variance} $\sigma^2$ and \emph{precision} $\beta=1/\sigma^2$
\end{itemize}

\begin{bbbox}{The univariate Gaussian}

\begin{flalign*}
	\mu = 0; \sigma^2 = 1, \\
	P(t) = \frac{1}{\sqrt{2\pi}} \mbox{e}^{-\frac{1}{2}t^2} \\
	\int_{-\infty}^{\infty} P(t)
		&= \frac{1}{\sqrt{2\pi}} \int_{-\infty}^{\infty} \mbox{e}^{-\frac{1}{2}t^2} dt\\
		&= \frac{1}{\sqrt{2\pi}} \sqrt{2\pi} = 1
\end{flalign*}

\begin{itemize}
\item  Q: What are the \emph{mode} and the \emph{median} of the Gaussian?\\
		  Mode and median of the Gaussian are equal to the mean. This can easily be seen graphically.

\end{itemize}
\end{bbbox}

\begin{bbbox}{Maximum Likelihood estimation of $\mu$ and $\beta$}

	  \begin{align*}
	  	D &= \{ t_1 , t_2 ,\cdots , t_n \} \\
	  	L(D|\mu,\beta) &= \prod_{n=1}^N \mathcal{N} \left( t_n,\mu,\sigma^2 \right) \\
	  				   &= \sqrt{\frac{\beta}{2\pi}}^N \mbox{e}^{-\frac{\beta}{2} \sum_{n=1}^N \left(t_n - \mu\right)^2} \\
		\log L &= \frac{N}{2} \log \left( \frac{\beta}{2\pi} \right) - \frac{\beta}{2} \sum_{n=1}^N \left(t_n - \mu\right)^2 \\
		\end{align*}

		\text{Maximum likelihood solution for the mean } $\mu$ \\
		\begin{align*}
		0 &= \frac{\partial \log L}{\partial \mu} = -\frac{\beta}{2} \sum_{n=1}^N 2 \left(t_n - \mu \right) (-1) \\
		0 &= \sum_{n=1}^N t_n - \sum_{n=1}^N \mu \\
		\mu &= \frac{1}{N} \sum_{n=1}^N t_n \\
		\end{align*}

		\text{Maximum likelihood solution for the variance } $\sigma^2 = \frac{1}{\beta}$ \\
		\begin{align*}
		0 &= \frac{\partial \log L}{\partial \beta} = \frac{N}{2\beta} - \frac{1}{2} \sum_{n=1}^N 2 \left(t_n - \mu \right)^2 \\
		\sigma^2 &= \frac{1}{\beta} = \frac{1}{N} \sum_{n=1}^N 2 \left(t_n - \hat{\mu} \right)^2 \\
	  \end{align*}
Q: How would you find the conjugate prior for the Gaussian? \\

\end{bbbox}

\textbf{(very important) aside: Products of Gaussian pdfs are (unnormalized) Gaussians pdfs}
\begin{itemize}
\item Suppose $p_1(x)=\mathcal{N}(x,\mu_1, \frac{1}{\beta_1})$ and $ p_2(x)=\mathcal{N}(x,\mu_2, \frac{1}{\beta_2})$, then
 \begin{align}
p_1(x) p_2(x) &\propto \mathcal{N}(x, \mu, 1/\beta)\\
\beta&=\beta_1+\beta_2\\
\mu&=\frac{1}{\beta}(\beta_1 \mu_1 +  \beta_2 \mu_2)
\end{align}

\begin{bbbox}{Products of Gaussians}
	Quadratic form for the Gaussian:
	\begin{align*}
		p(t|\mu, \sigma^2)
		&=\frac{1}{\sqrt{2\pi\sigma^2}}\exp\left( -\frac{1}{2}\left(\frac{t-\mu}{\sigma} \right)^2\right) \\
		&= \frac{1}{Z} \exp \left( \underbrace{a}_{-\frac{1}{2}\text{precision}}x^2 + \underbrace{b}_{\text{precision} \times \text{mean}}x \right) \\
	\end{align*}

	We show that the product of two Gaussians $p_1(x)$ and $p_2(x)$ is again Gaussian: \\
	\begin{align*}
	p_1(x) &= \frac{1}{Z_1} \exp \left( -\frac{\beta_1}{2} \left(x-\mu_1 \right)^2 \right) 	\\
	p_2(x) &= \frac{1}{Z_2} \exp \left( -\frac{\beta_2}{2} \left(x-\mu_2 \right)^2 \right) 	\\
	p_1(x) p_2(x) &= \frac{1}{Z_s} \exp \left( -\frac{\beta_1}{2} \left( x - \mu_1 \right)^2
					  -\frac{\beta_2}{2} \left( x - \mu_2 \right)^2	\right)\\
				  &= \frac{1}{Z_q} \exp \left( -\left( \frac{\beta_1}{2} + \frac{\beta_2}{2}\right) x^2
	                 + 2\left( \frac{\beta_1}{2} \mu_1 + \frac{\beta_2}{2} \mu_2 \right) x \right)\\
	\beta &= \beta_1 + \beta_2 \\
	\beta \mu &= = \beta_1 \mu_1 + \beta_2 \mu_2 \\
	\mu &= \frac{1}{\beta} \left( \beta_1 \mu_1 + \beta_2 \mu_2 \right)
	\end{align*}
	The quadratic form helps us to simply read out the parameters of our resulting Gaussian. We will see later that this observation comes in handy for Bayesian inference with Gaussians.
\end{bbbox}

In general:
\begin{align}
p_1(x) p_2(x) ... p_n(x) &\propto \mathcal{N}(x, \mu, 1/\beta)\\
\beta&=\sum_n \beta_n\\
\mu&=\frac{1}{\beta} \sum_n \mu_n \beta_n
\end{align}

This is also true for multivariate Gaussians!

\end{itemize}

\subsection{Bayesian inference for Gaussians}

\begin{itemize}
\item Suppose we are given data $D=\{x_1, \ldots, x_N\}$.
\item We assume that the data is Gaussian-distribution with known variance $\sigma^2$ and unknown mean $\mu$.
\item Our prior for $\mu$ is Gaussian: $\mu \sim \mathcal{N}(\mu_o, \sigma^2_o)$
\item Posterior distribution over $\mu$ given the data: [on board]
\end{itemize}

\begin{bbbox}{Posterior inference for the Gaussian}
	Here we want to derive the posterior  distribution for the mean of a Gaussian. For now we will assume that the variance $\sigma^2$ of our Gaussian is known and that the only parameter of interest is $\mu$. \\
	We define a prior over $\mu$:   $p(\mu) = \mathcal{N}(\mu_0, \sigma^2_0)$ which is again Gaussian. \\
	From before we know that our posterior distribution over $\mu$ is again Gaussian:
	\begin{align*}
		p(\mu|\sigma^2, \underbrace{\mu_0, \sigma_0^2}_{Prior},D)
		            &\propto p(D|\mu,\sigma^2,\mu_0, \sigma_0^2) p(\mu|\mu_0, \sigma_0^2) \\
		            &\propto \prod_{n=1}^N \mathcal{N}(x_n,\mu,\sigma^2) \mathcal{N}(\mu,\mu_0, \sigma_0^2) \\
		            &\propto \mathcal{N}(\mu,\mu_{post},\sigma_{post}^2) \\
		\frac{1}{\sigma_{post}^2} &= \sum_{n=1}^N \left(\frac{1}{\sigma^2} \right) + \frac{1}{\sigma_0^2} \\
		\sigma_{post}^2 &= \frac{1}{\frac{N}{\sigma^2} + \frac{1}{\sigma_0^2}} \\
		\mu_{post} &= \sigma_{post}^2 \left( \sum_{n=1}^N \left( x_n\frac{1}{\sigma^2} \right) + \mu_0 \frac{1}{\sigma_0^2} \right)
	\end{align*}

Behaviour for large $N$: [on board]
\begin{align*}
	\text{for large N: } \sigma_{post}^2 = \frac{\sigma^2}{N} \\
	\mu_{ML} &= \frac{1}{N} \sum_{n=1}^N x_n \\
	\mu_{post} &= \frac{\left( \sum_{n=1}^N \left( x_n\frac{1}{\sigma^2} \right) + \mu_0 \frac{1}{\sigma_0^2} \right)}{\frac{N}{\sigma^2} + \frac{1}{\sigma_0^2}} \\
	N \rightarrow \infty: &\mu_{post} = \mu_{ML} \\
	\sigma_0^2 \rightarrow 0: &\mu_{post} = \mu_{ML} \\
\end{align*}

\end{bbbox}

\begin{figure}
	\centering
	\includegraphics[width=.5\textwidth]{./lecture4/Figure212.pdf}
	\caption{Bishop Figure 2.12}
\end{figure}

\subsubsection{What if the variance is not given?}
\begin{itemize}
\item For simplicity, assume mean to be known.
\item More convenient to work with precision $\lambda=1/\sigma^2$.
\item Conjugate prior: Gamma distribution $\mbox{Gam}(\lambda|a,b)$
\begin{align}
p(\lambda|a, b)=\frac{1}{\Gamma(a)}b^a \lambda^{a-1} exp(-b\lambda)
\end{align}
\item Posterior is $\mbox{Gam}(\lambda|a_N,b_B)$
\begin{align}
a_N &= a + \frac{N}{2}\\
b_N &= b + \frac{1}{2} \sum_{n=1}^N (x_n -\mu)^2
\end{align}

\begin{figure}
\centering
\begin{subfigure}[b]{0.45\textwidth}
                \centering
                \includegraphics[width=\textwidth]{./lecture4/Figure213b.pdf}
    \end{subfigure}%
	~
	\begin{subfigure}[b]{0.45\textwidth}
                \centering
                \includegraphics[width=\textwidth]{./lecture4/Figure213c.pdf}
    \end{subfigure}%
   	\caption{Gamma distribution with different parameterizations. The Gamma distribution is a conjugate prior for the presision of a Gaussian. Figures taken from Bishop page 100.}

\end{figure}
\end{itemize}

\subsubsection{What if both the mean and the variance are unknown?}
\begin{figure}[h]
	\centering
		\includegraphics[width=0.7\textwidth]{./lecture4/Figure214.pdf}
		\caption{The Gaussian-Gamma distribution, a conjugate prior for mean and precision of a Gaussian. Bishop figure 2.14, page 102.}
\end{figure}
\begin{itemize}
\item Conjugate prior: Gaussian-Gamma distribution
\begin{align}
p(\mu,\lambda) & = \mathcal{N}\left(\mu |\mu_o (\beta\lambda)^{-1}\right) \mbox{Gam}(\lambda|a,b)
\end{align}
\end{itemize}


\subsubsection{Repeated Linear Factors}
If $Q(x)$ has repeated roots, it factors into
\begin{equation*}
	Q(x) = R(x)(x-a)^k\text{, }k \geq 2\text{ and }R(a) \neq 0.
\end{equation*}

\noindent
When making the common denominator for each repeated root of multiplicity $k$, we do
\begin{equation*}
	\frac{P(x)}{R(x)(x-a)^k} = \left(\text{Decomposition of }R(x)\right) + \frac{A_1}{x-a}+\ldots+\frac{A_k}{(x-a)^k}.
\end{equation*}

\noindent
You would then multiply each side by the denominator like in the linear factors case and solve for the coefficients.
The only additional difficulty is that you might have to use previous results or solve a system of linear equations to get some of the constants.

\begin{example}
	Find the partial fraction of the following expression:
	\begin{equation*}
		\frac{x^2+5x-6}{x^3-7x^2+16x-12}.
	\end{equation*}
\end{example}
\noindent
Factoring,
\begin{equation*}
	x^3-7x^2+16x-12 = (x-3)(x-2)^2.
\end{equation*}
So,
\begin{equation*}
	\frac{x^2+5x-6}{x^3-7x^2+16x-12} =  \frac{A_1}{x-3} + \frac{A_2}{x-2} + \frac{A_3}{(x-2)^2}.
\end{equation*}
Multiplying each side by the denominator,
\begin{equation*}
	x^2+5x-6 = A_1(x-2)^2 + A_2(x-2)(x-3) + A_3(x-3).
\end{equation*}
At $x=2$,
\begin{equation*}
	8 = A_3(2-3) \implies A_3 = -8.
\end{equation*}
At $x=3$,
\begin{equation*}
	18 = A_1(3-2)^2 \implies A_1 = 18.
\end{equation*}
Now we'll use our results for $A_1$ and $A_3$ to find $A_2$ using a value for $x$ that isn't 2 or 3 so the $A_2$ term doesn't become 0. A good choice is $x=0$.\\
At $x=0$,
\begin{equation*}
	-6 = 18(0-2)^2 + A_2(0-2)(0-3) + -8(0-3) \implies A_2 = -17.
\end{equation*}
So,
\begin{equation*}
	\frac{x^2+5x-6}{x^3-7x^2+16x-12} = \frac{18}{x-3} - \frac{17}{x-2} - \frac{8}{(x-2)^2}.
\end{equation*}