\documentclass[a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[margin=1in]{geometry}
\usepackage{setspace}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{graphicx}

\title{Chapter 5\\Vector Spaces}
\author{solutions by Hikari}
\date{August 2021}

\begin{document}

\newcommand{\br}[2]{\langle#1|#2\rangle}
\newcommand{\brr}[2]{\left\langle#1|#2\right\rangle}
\newcommand{\pdv}[2]{\frac{\partial#1}{\partial#2}}
\newcommand{\M}{\mathrm}
\newcommand{\V}{\mathbf}
\newcommand{\VE}{\mathbf{\hat{e}}}

\maketitle

\section*{5.1 Vectors in Function Spaces}

\paragraph{5.1.1}
If there are two expansions of $f(x)$, so
\[
f(x)=\sum_{n=0}^\infty a_n\varphi_n(x)=\sum_{n=0}^\infty b_n\varphi_n(x)
\]
then
\[
g(x)=\sum_{n=0}^\infty(a_n-b_n)\varphi_n(x)=0
\]
\[
\br{g(x)}{g(x)}=\sum_{n=0}^\infty(a_n-b_n)^2=0
\]
so $a_n=b_n$, and the expansion is unique.

\paragraph{5.1.2}
If
\[
f(x)=\sum_{i=1}^Nc_i\varphi_i(x)=\sum_{i=1}^Nc'_i\varphi_i(x)
\]
so
\[
\sum_{i=1}^N(c_i-c'_i)\varphi_i(x)
\]
By definition of linear independence, the linear combination of a linear independent set equals zero only if all the coefficient is zero. The set of $\varphi_i$ is linear independent, so $c_i-c'_i$ must be zero, which means $c_i=c'_i$, the components are unique.

\paragraph{5.1.3}
The mean square error $M$ is
\[
M=\int_0^1(f(x)-\sum_jc_jx^j)^2dx
\]
\[
=\int_0^1\left[f(x)^2+(\sum_jc_jx^j)(\sum_jc_jx^j)-2f(x)(\sum_jc_jx^j) \right]dx
\]

When $M$ is minimized, $\pdv{M}{c_i}=0$ for every $i$, so
\[
\pdv{M}{c_i}=\int_0^1\left[2x^i(\sum_jc_jx^j)-2f(x)x^i \right]dx=0
\]
so
\[
\sum_j\int_0^1x^{i+j}dx\cdot c_j=\int_0^1x^if(x)dx
\]
Let $\int_0^1x^{i+j}dx=A_{ij}$,\quad $\int_0^1x^if(x)dx=b_i$, then the equation becomes
\[
\sum_jA_{ij}c_j=b_i
\]
or $\M{A}\V{c}=\V{b}$ in matrix form.

\paragraph{5.1.4}
Let $F(x)=\varphi_i(x)$, then
\[
a_j=\delta_{ij}=\int_a^b\varphi_i(x)\varphi_{j}(x)w(x)dx
\]
so the basis are orthonormal.

When the mean square error is minimized,
\[
\pdv{}{c_n}\left(\int_a^b[F(x)-\sum_{k=o}^mc_k\varphi_k(x)]^2w(x)dx \right)=0
\]
so
\[
\int_a^b 2[F(x)-\sum_{k=o}^mc_k\varphi_k(x)](-\varphi_n(x))w(x)dx=0
\]
\[
-\int_a^bF(x)\varphi_n(x)w(x)dx+\sum_{k=0}^mc_k\delta_{kn}=0
\]
Note that the first term is $-a_n$, and the second term is $c_n$, so
\[
c_n=a_n
\]

\paragraph{5.1.5}
(a)
\[
\int_{-\pi}^\pi\frac{\sin(2n+1)x}{2n+1}\frac{\sin(2m+1)x}{2m+1}dx=\int_{-\pi}^\pi\frac{\cos(2(n-m)x)-\cos(2(n+m+1)x)}{2(2n+1)(2m+1)}
\]
\[=
\begin{cases}
0,\quad\quad\quad\;\;\textit{when $n\neq m$}\\
\frac{\pi}{(2n+1)^2},\quad\textit{when $n=m$}
\end{cases}
\]
so
\[
\int_{-\pi}^\pi\left[f(x)\right]^2dx=\int_{-\pi}^\pi\frac{4h^2}{\pi^2}(\sum_{n=0}^\infty\frac{\sin(2n+1)x}{2n+1})(\sum_{m=0}^\infty\frac{\sin(2m+1)x}{2m+1})dx
\]
\[
=\frac{4h^2}{\pi^2}\sum_{n=0}^\infty\frac{\pi}{(2n+1)^2}=\frac{4h^2}{\pi}\sum_{n=0}^\infty\frac{1}{(2n+1)^2}
\]

(b)
\[
\sum_{n=0}^\infty\frac{1}{(2n+1)^2}=\frac{1}{1^2}+\frac{1}{3^2}+\frac{1}{5^2}+\frac{1}{7^2}+\cdots
\]
\[
=(\frac{1}{1^2}+\frac{1}{2^2}+\frac{1}{3^2}+\frac{1}{4^2}+\frac{1}{5^2}+\frac{1}{6^2}+\frac{1}{7^2}+\cdots)-\frac{1}{2^2}(\frac{1}{1^2}+\frac{1}{2^2}+\frac{1}{3^2}+\cdots)
\]
\[
=\frac{3}{4}\zeta(2)=\frac{\pi^2}{8}
\]
so
\[
\frac{4h^2}{\pi}\sum_{n=0}^\infty\frac{1}{(2n+1)^2}=\frac{4h^2}{\pi}\frac{\pi^2}{8}=\frac{\pi}{2}h^2
\]

\paragraph{5.1.6}
\[
\left[\int_a^bf(x)g(x)dx \right]^2+\frac{1}{2}\int_a^bdx\int_a^bdy\left[f(x)g(y)-f(y)g(x) \right]^2=\int_a^b\left[f(x) \right]^2dx\int_a^b\left[g(x) \right]^2dx
\]
and note that $\frac{1}{2}\int_a^bdx\int_a^bdy\left[f(x)g(y)-f(y)g(x) \right]^2\geq0$, so
\[
\left[\int_a^bf(x)g(x)dx \right]^2\leq\int_a^b\left[f(x) \right]^2dx\int_a^b\left[g(x) \right]^2dx
\]
To prove the identity, note that
\[
\frac{1}{2}\int_a^bdx\int_a^bdy\left[f(x)g(y)-f(y)g(x) \right]^2=\frac{1}{2}\int_a^bdx\int_a^bdy\left[f(x)^2g(y)^2+f(y)^2g(x)^2-2f(x)g(x)f(y)g(y) \right]
\]
\[=
\frac{1}{2}\int_a^bdx\left[f(x)^2\int_a^b[g(y)]^2dy+(\int_a^b[f(y)]^2dy)\;g(x)^2-2f(x)g(x)\int_a^bf(y)g(y)dy \right]
\]
\[
=\frac{1}{2}\left[\int_a^b[f(x)]^2dx\int_a^b[g(y)]^2dy+\int_a^b[f(y)]^2dy\int_a^b[g(x)]^2dx-2\int_a^bf(x)g(x)dx\int_a^bf(y)g(y)dy \right]
\]
\[
=\int_a^b[f(x)]^2dx\int_a^b[g(x)]^2dx-\left[\int_a^bf(x)g(x)dx\right]^2
\]
Rearranging the terms we get the identity.

\paragraph{5.1.7}
The basis are orthonormal, so $\langle a_i\varphi_i|a_j\varphi_j \rangle$ is zero when $i\neq j$ and is $|a_i|^2$ when $i=j$. Let $f=\sum_ka_k\varphi_k$, $k$ can be infinite, and let $\sum_na_n\varphi_n$ be an incomplete expansion of $f$, then
\[
I=\left\langle f-\sum_na_n\varphi_n\Big|f-\sum_na_n\varphi_n \right\rangle
\]
\[
=\langle f|f \rangle-\left\langle \sum_na_n\varphi_n\Big|f \right\rangle-\left\langle f\Big|\sum_na_n\varphi_n \right\rangle+\left\langle \sum_na_n\varphi_n\Big|\sum_na_n\varphi_n \right\rangle
\]
\[
=\langle f|f \rangle-\left\langle \sum_na_n\varphi_n\Big|\sum_ka_k\varphi_k \right\rangle-\left\langle \sum_ka_k\varphi_k\Big|\sum_na_n\varphi_n \right\rangle+\left\langle \sum_na_n\varphi_n\Big|\sum_na_n\varphi_n \right\rangle
\]
\[
=\langle f|f \rangle-\sum_n|a_n|^2-\sum_n|a_n|^2+\sum_n|a_n|^2
\]
\[
=\langle f|f \rangle-\sum_n|a_n|^2\geq0
\]
so
\[
\langle f|f \rangle\geq\sum_n|a_n|^2
\]

\paragraph{5.1.8}
By integrating by parts, we have
\[
\int_0^1\sin\pi x\,dx=\frac{\pi}{2}
\]
\[
\int_0^1x\sin\pi x\,dx=\frac{1}{\pi}-\frac{1}{\pi}\int_0^1\cos\pi x\,dx=\frac{1}{\pi}
\]
\[
\int_0^1x^2\sin\pi x\,dx=\frac{1}{\pi}-\frac{2}{\pi^2}\int_0^1\sin\pi x\,dx=\frac{1}{\pi}-\frac{4}{\pi^3}
\]
\[
\int_0^1x^3\sin\pi x\,dx=\frac{1}{\pi}-\frac{6}{\pi^2}\int_0^1x\sin\pi x\,dx=\frac{1}{\pi}-\frac{6}{\pi^3}
\]
Let $\sin\pi x=\sum a_n\varphi_n$, then
\[
a_0=\frac{\langle\varphi_0|\sin\pi x\rangle}{\langle\varphi_0|\varphi_0\langle}=\frac{I_0}{1}=\frac{2}{\pi}
\]
\[
a_1=\frac{\langle\varphi_1|\sin\pi x\rangle}{\langle\varphi_1|\varphi_1\rangle}=\frac{2I_1-I_0}{\frac{1}{3}}=0
\]
\[
a_2=\frac{\langle\varphi_2|\sin\pi x\rangle}{\langle\varphi_2|\varphi_2\rangle}=\frac{6I_2-6I_1+I_0}{\frac{1}{5}}=\frac{10}{\pi}-\frac{120}{\pi^3}
\]
\[
a_3=\frac{\langle\varphi_3|\sin\pi x\rangle}{\langle\varphi_3|\varphi_3\rangle}=\frac{20I_3-30I_2+12I_1-I_0}{\frac{1}{7}}=0
\]
so
\[
\sin\pi x=\frac{2}{\pi}\varphi_0+(\frac{10}{\pi}-\frac{120}{\pi^3})\varphi_2+\cdots
\]

\paragraph{5.1.9}
Integrating by parts for $n$ times, we have
\[
\int_0^\infty x^ne^{-2x}dx=\frac{n!}{2^{n+1}}
\]
Let $e^{-x}=\sum a_nL_n(x)$, then
\[
a_0=\langle L_0|e^{-x}\rangle=\int_0^\infty e^{-2x}dx=\frac{1}{2}
\]
\[
a_1=\langle L_1|e^{-x}\rangle=\int_0^\infty e^{-2x}dx-\int_0^\infty xe^{-2x}dx=\frac{1}{4}
\]
\[
a_2=\langle L_2|e^{-x}\rangle=\int_0^\infty e^{-2x}dx-2\int_0^\infty xe^{-2x}dx+\frac{1}{2}\int_0^\infty x^2e^{-2x}dx=\frac{1}{8}
\]
\[
a_3=\langle L_3|e^{-x}\rangle=\int_0^\infty e^{-2x}dx-3\int_0^\infty xe^{-2x}dx+\frac{3}{2}\int_0^\infty x^2e^{-2x}dx-\frac{1}{6}\int_0^\infty x^3e^{-2x}dx=\frac{1}{16}
\]
so
\[
e^{-x}=\frac{1}{2}L_0+\frac{1}{4}L_1+\frac{1}{8}L_2+\frac{1}{16}L_3+\cdots
\]

\paragraph{5.1.10}
Expand $\varphi_n$ in the $\chi_n$ basis, we have
\[
\varphi_n=\sum_m\chi_m\langle\chi_m|\varphi_n\rangle
\]
so
\[
f=\sum_n\varphi_na_n=\sum_n\sum_m\chi_m\langle\chi_m|\varphi_n\rangle a_n
\]

\paragraph{5.1.11}
\[
\sum_j|\VE_j\rangle\br{\VE_j}{\V{a}}=\sum_j\VE_j(\VE_j\cdot\V{a})=\V{a}
\]

\paragraph{5.1.12}
\[
\br{\V{a}}{\V{a}}=a_1^2-2a_1a_2+ka_2^2=(a_1-a_2)^2+(k-1)a_2^2>0\quad\textit{(when $\V{a}\neq0$)}
\]
so $k-1>0$,\; $k>1$.
\[
\br{\V{a}}{\V{b}}^*=a_1b_1-a_1b_2-a_2b_1+ka_2b_2=b_1a_1-b_1a_2-b_2a_1+kb_2a_2=\br{\V{b}}{\V{a}}
\]
\[
\br{\V{a}}{\V{b}+\V{b'}}=a_1(b_1+b'_1)-a_1(b_2+b'_2)-a_2(b_1+b'_1)+ka_2(b_2+b'_2)\]
\[
=(a_1b_1-a_1b_2-a_2b_1+ka_2b_2)+(a_1b'_1-a_1b'_2-a_2b'_1+ka_2b'_2)=\br{\V{a}}{\V{b}}+\br{\V{a}}{\V{b'}}
\]
\[
\br{\V{a}}{x\V{b}}=a_1xb_1-a_1xb_2-a_2xb_1+ka_2xb_2=x(a_1b_1-a_1b_2-a_2b_1+ka_2b_2)=x\br{\V{a}}{\V{b}}
\]
so the condition for the scalar product to be valid is $k>1$.

\section*{5.2 Gram-Schmidt Orthogonalization}

\paragraph{5.2.1}
\[
P_0^*(x)=1
\]
\[
\psi_1(x)=x-1\frac{\br{1}{x}}{\br{1}{1}}=x-\frac{\int_0^1xdx}{\int_0^1dx}=x-\frac{1}{2}
\]
\[
P_1^*(x)=\frac{\psi_1(x)}{\psi_1(1)}=2x-1
\]
\[
\psi_2(x)=x^2-1\frac{\br{1}{x^2}}{\br{1}{1}}-(2x-1)\frac{\br{2x-1}{x^2}}{\br{2x-1}{2x-1}}=x^2-\frac{\int_0^1x^2dx}{\int_0^1dx}-(2x-1)\frac{\int_0^1(2x^3-x^2)dx}{\int_0^1(2x-1)^2dx}=x^2-x+\frac{1}{6}
\]
\[
P_2^*(x)=\frac{\psi_2(x)}{\psi_2(1)}=6x^2-6x+1
\]
\[
\psi_3(x)=x^3-1\frac{\br{1}{x^3}}{\br{1}{1}}-(2x-1)\frac{\br{2x-1}{x^3}}{\br{2x-1}{2x-1}}-(6x^2-6x+1)\frac{\br{6x^2-6x+1}{x^3}}{\br{6x^2-6x+1}{6x^2-6x+1}}=x^3-\frac{3}{2}x^2+\frac{3}{5}x-\frac{1}{20}
\]
\[
P_3^*(x)=\frac{\psi_3(x)}{\psi_3(1)}=20x^3-30x^2+12x-1
\]

\paragraph{5.2.2}
\[
\varphi_0=1
\]
\[
L_0=\frac{\varphi_0}{{\br{\varphi_0}{\varphi_0}}^{1/2}}=\frac{1}{(\int_0^\infty e^{-x}dx)^{1/2}}=\pm1\quad\textit{(choose $1$)}
\]
\[
\varphi_1=x-1\int_0^\infty1 xe^{-x}dx=x-1
\]
\[
L_1=\frac{\varphi_1}{{\br{\varphi_1}{\varphi_1}}^{1/2}}=\frac{x-1}{(\int_0^\infty(x-1)^2e^{-x}dx)^{1/2}}=\pm(x-1)\quad\textit{(choose $-x+1$)}
\]
\[
\varphi_2=x^2-1\int_0^\infty1x^2e^{-x}dx-(1-x)\int_0^\infty(1-x)x^2e^{-x}dx=x^2-4x+2
\]
\[
L_2=\frac{\varphi_2}{{\br{\varphi_2}{\varphi_2}}^{1/2}}=\frac{x^2-4x+2}{(\int_0^\infty (x^2-4x+2)^2e^{-x}dx)^{1/2}}=\pm\frac{x^2-4x+2}{2}\quad\textit{(choose $\frac{x^2-4x+2}{2}$)}
\]
(The choice of sign in the normalization is arbitrary and is so chosen to match the answer given in the text.)

\paragraph{5.2.3}
\[
\psi_0(x)=1
\]
\[
\varphi_0(x)=\frac{\psi_0}{\br{\psi_0}{\psi_0}^{1/2}}=\frac{1}{(\int_0^\infty xe^{-x}dx)^{1/2}}=1
\]
\[
\psi_1(x)=x-1\int_0^\infty1x\cdot xe^{-x}dx=x-2
\]
\[
\varphi_1(x)=\frac{\psi_1}{\br{\psi_1}{\psi_1}^{1/2}}=\frac{x-2}{(\int_0^\infty(x^2-4x+4)xe^{-x}dx)^{1/2}}=\frac{x-2}{\sqrt{2}}
\]
\[
\psi_2(x)=x^2-1\int_0^\infty1x^2\cdot xe^{-x}dx-\frac{x-1}{\sqrt{2}}\int_0^\infty\frac{x-2}{\sqrt{2}}x^2\cdot xe^{-x}dx=x^2-6x+6
\]
\[
\varphi_2(x)=\frac{\psi_2}{\br{\psi_2}{\psi_2}^{1/2}}=\frac{x^2-6x+6}{(\int_0^\infty(x^4-12x^3+48x^2-72x+36)xe^{-x}dx)^{1/2}}=\frac{x^2-6x+6}{2\sqrt{3}}
\]
(using the formula $\int_0^\infty x^ne^{-x}dx=n!$ can facilate the calculation.)

\paragraph{5.2.4}
To calculate the scalar product, we need the Gaussian integral $\int_{-\infty}^\infty x^ne^{-x^2}dx$. When $n$ is odd, $\int_{-\infty}^\infty x^ne^{-x^2}dx=0$ because $x^ne^{-x^2}$ is odd function. When $n$ is even, from Example 1.10.7 we have $\int_{-\infty}^\infty e^{-x^2}dx=\sqrt{\pi}$, and by substitution we have
\[
\int_{-\infty}^\infty e^{-ax^2}dx=\frac{\sqrt{\pi}}{\sqrt{a}}
\]
differentiate both sides regarding $a$,
\[
\frac{d}{da}\int_{-\infty}^\infty e^{-ax^2}dx=\int_{-\infty}^\infty (-x^2)e^{-ax^2}dx=-\frac{1}{2}\frac{\sqrt{\pi}}{a^{-\frac{3}{2}}}
\]
so
\[
\int_{-\infty}^\infty x^2e^{-ax^2}dx=\frac{\sqrt{\pi}}{2a^{-\frac{3}{2}}}
\]
differentiate again,
\[
\frac{d}{da}\int_{-\infty}^\infty x^2e^{-ax^2}dx=\int_{-\infty}^\infty (-x^4)e^{-ax^2}dx=-\frac{3}{4}\frac{\pi}{a^{-\frac{5}{2}}}
\]
so
\[
\int_{-\infty}^\infty x^4e^{-ax^2}dx=\frac{3\sqrt{\pi}}{4a^{-\frac{3}{2}}}
\]
\medskip

Let $\varphi_n$ be the non-scaled function, and $H_n=a_n\varphi_n$, so $\br{H_n}{H_n}=\int_{-\infty}^\infty a_n^2\varphi_n^2e^{-x^2}dx=2^nn!\sqrt{\pi}$. Then
\[
\varphi_0=1
\]
\[
\br{H_0}{H_0}=\int_{-\infty}^\infty a_0^21^2e^{-x^2}dx=a_0^2\sqrt{\pi}=\sqrt{\pi}
\]
so $a_0=1$, and $H_0=1$.
\[
\varphi_1=x-1\frac{\int_{-\infty}^\infty1\cdot xe^{-x^2}dx}{\int_{-\infty}^\infty1\cdot1e^{-x^2}dx}=x
\]
\[
\br{H_1}{H_1}=\int_{-\infty}^\infty a_1^2x^2e^{-x^2}dx=a_1^2\frac{\sqrt{\pi}}{2}=2\sqrt{\pi}
\]
so $a_1=2$, and $H_1=2x$.
\[
\varphi_2=x^2-1\frac{\int_{-\infty}^\infty1\cdot x^2e^{-x^2}dx}{\int_{-\infty}^\infty1\cdot1e^{-x^2}dx}-2x\frac{\int_{-\infty}^\infty2x\cdot x^2e^{-x^2}dx}{\int_{-\infty}^\infty2x\cdot2xe^{-x^2}dx}=x^2-\frac{1}{2}
\]
\[
\br{H_2}{H_2}=\int_{-\infty}^\infty a_2^2(x^2-\frac{1}{2})^2e^{-x^2}dx=a_2^2\frac{\sqrt{\pi}}{2}=8\sqrt{\pi}
\]
so $a_2=4$, and $H_2=4x^2-2$.

\paragraph{5.2.5}
\[
\int_{-1}^1\frac{x^{2n}}{\sqrt{1-x^2}}dx=
\begin{cases}
\pi,\quad &n=0\\
\pi\frac{(2n-1)!!}{(2n)!!},\quad &n=1,2,3\cdots
\end{cases}
\]
from Exercise 13.3.2, and $\int_{-1}^1\frac{x^{2n+1}}{\sqrt{1-x^2}}dx=0$ because $\frac{x^{2n+1}}{\sqrt{1-x^2}}$ is an odd function.
\medskip

Let $\varphi_n$ be the non-scaled function, and
$T_n=a_n\varphi_n$, so $\br{T_n}{T_n}=\int_{-1}^1a_n^2\varphi_n^2\frac{1}{\sqrt{1-x^2}}dx$. Then
\[
\varphi_0=1
\]
\[
\br{T_0}{T_0}=\int_{-1}^1a_0^2\frac{1}{\sqrt{1-x^2}}dx=a_0^2\pi=\pi
\]
so $a_0=1$, and $T_0=1$.
\[
\varphi_1=x-1\frac{\int_{-1}^11\cdot x\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^11\cdot1\frac{1}{\sqrt{1-x^2}}dx}=x
\]
\[
\br{T_1}{T_1}=\int_{-1}^1a_1^2x^2\frac{1}{\sqrt{1-x^2}}dx=a_1^2\frac{\pi}{2}=\frac{\pi}{2}
\]
so $a_1=1$, and $T_1=x$.
\[
\varphi_2=x^2-1\frac{\int_{-1}^11\cdot x^2\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^11\cdot1\frac{1}{\sqrt{1-x^2}}dx}-x\frac{\int_{-1}^1x\cdot x^2\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^1x\cdot x\frac{1}{\sqrt{1-x^2}}dx}=x^2-\frac{1}{2}
\]
\[
\br{T_2}{T_2}=\int_{-1}^1a_2^2(x^2-\frac{1}{2})^2\frac{1}{\sqrt{1-x^2}}dx=a_2^2\frac{\pi}{8}=\frac{\pi}{2}
\]
so $a_2=2$, and $T_2=2x^2-1$.
\[
\varphi_3=x^3-1\frac{\int_{-1}^11\cdot x^3\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^11\cdot1\frac{1}{\sqrt{1-x^2}}dx}-x\frac{\int_{-1}^1x\cdot x^3\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^1x\cdot x\frac{1}{\sqrt{1-x^2}}dx}-(2x^2-1)\frac{\int_{-1}^1(2x^2-1)x^3\frac{1}{\sqrt{1-x^2}}dx}{\int_{-1}^1(2x^2-1)^2\frac{1}{\sqrt{1-x^2}}dx}=x^3-\frac{3}{4}x
\]
\[
\br{T_3}{T_3}=\int_{-1}^1a_3^2(x^3-\frac{3}{4}x)^2\frac{1}{\sqrt{1-x^2}}dx=a_3^2\frac{\pi}{32}=\frac{\pi}{2}
\]
so $a_3=4$, and $T_3=4x^3-3x$.

\paragraph{5.2.6}
Note that
\[
\int_{-1}^1x^{2n+1}\sqrt{1-x^2}dx=0
\]
because $x^{2n+1}\sqrt{1-x^2}$ is an odd function.
\medskip

Let $\varphi_n$ be the non-scaled function, and $U_n=a_n\varphi_n$, so $\br{U_n}{U_n}=\int_{-1}^1a_n^2\varphi_n^2\sqrt{1-x^2}dx$. Then
\[
\varphi_0=1
\]
\[
\br{U_0}{U_0}=\int_{-1}^1a_0^2\sqrt{1-x^2}dx=a_0^2\frac{\pi}{2}=\frac{\pi}{2}
\]
so $a_0=1$, and $U_0=1$.
\[
\varphi_1=x-1\frac{\int_{-1}^11\cdot x\sqrt{1-x^2}dx}{\int_{-1}^11\cdot1\sqrt{1-x^2}dx}=x
\]
\[
\br{U_1}{U_1}=\int_{-1}^1a_1^2x^2\sqrt{1-x^2}dx=a_1^2\frac{\pi}{8}=\frac{\pi}{2}
\]
so $a_1=2$, and $U_1=2x$.
\[
\varphi_2=x^2-1\frac{\int_{-1}^11\cdot x^2\sqrt{1-x^2}dx}{\int_{-1}^11\cdot1\sqrt{1-x^2}dx}-2x\frac{\int_{-1}^12x\cdot x^2\sqrt{1-x^2}dx}{\int_{-1}^12x\cdot 2x\sqrt{1-x^2}dx}=x^2-\frac{1}{4}
\]
\[
\br{U_2}{U_2}=\int_{-1}^1a_2^2(x^2-\frac{1}{4})^2\sqrt{1-x^2}dx=a_2^2\frac{\pi}{32}=\frac{\pi}{2}
\]
so $a_2=4$, and $U_2=4x^2-1$.

\paragraph{5.2.7}
\[
\psi_0=1
\]
\[
\varphi_0=1
\]
\[
\psi_1=x-1\frac{\int_0^\infty1\cdot xe^{-x^2}dx}{\int_0^\infty1\cdot1e^{-x^2}dx}=x-\frac{1}{\sqrt{\pi}}
\]
\[
\varphi_1=x-\frac{1}{\sqrt{\pi}}
\]

\paragraph{5.2.8}
\[
\V{a_1'}=
\begin{pmatrix}1\\1\\1\end{pmatrix}
\]
\[
\V{a_1}=\frac{\V{a_1'}}{(\V{a_1'}\cdot\V{a_1'})^{1/2}}=\frac{1}{\sqrt{3}}\begin{pmatrix}1\\1\\1\end{pmatrix}
\]
\[
\V{a_2'}=\V{c_2}-\V{a_1}(\V{a_1}\cdot\V{c_2})=\frac{1}{3}\begin{pmatrix}-1\\-1\\2\end{pmatrix}
\]
\[
\V{a_2}=\frac{\V{a_2'}}{(\V{a_2'}\cdot\V{a_2'})^{1/2}}=\frac{1}{\sqrt{6}}\begin{pmatrix}-1\\-1\\2\end{pmatrix}
\]
\[
\V{a_3'}=\V{c_3}-\V{a_1}(\V{a_1}\cdot\V{c_3})-\V{a_2}(\V{a_2}\cdot\V{c_3})=\frac{1}{2}\begin{pmatrix}1\\-1\\0\end{pmatrix}
\]
\[
\V{a_3}=\frac{\V{a_3'}}{(\V{a_3'}\cdot\V{a_3'})^{1/2}}=\frac{1}{\sqrt{2}}\begin{pmatrix}1\\-1\\0\end{pmatrix}
\]

\section*{5.3 Operators}

\paragraph{5.3.1}
\[
\br{f}{Ag}=\br{A^{\dagger}f}{g}=\br{g}{A^\dagger f}^*=\br{(A^\dagger)^\dagger g}{f}^*=\br{f}{(A^\dagger)^\dagger g}
\]
so \[
\br{f}{(A-(A^\dagger)^\dagger)g}=0
\]
for any $f$ and $g$. If $A-(A^\dagger)^\dagger\neq0$, which means there are some $g$ such that $(A-(A^\dagger)^\dagger) g=\varphi\neq0$, then let $f=\varphi$, and $\br{f}{(A-(A^\dagger)^\dagger)g}=\br{\varphi}{\varphi}>0$, contradict. So $A-(A^\dagger)^\dagger$ must be zero, which means \[
(A^\dagger)^\dagger=A
\]

\paragraph{5.3.2}
\[
\br{f}{UVg}=\br{U^\dagger f}{Vg}=\br{V^\dagger U^\dagger f}{g}
\]
also
\[
\br{f}{UVg}=\br{(UV)^\dagger f}{g}
\]
so
\[
\br{((UV)^\dagger-V^\dagger U^\dagger)f}{g}=0
\]
for any $f$ and $g$, so $(UV)^\dagger-V^\dagger U^\dagger$ must be zero, which means
\[
(UV)^\dagger=V^\dagger U^\dagger
\]

\paragraph{5.3.3}
(a)
\[
(A_1)_{ij}=\br{\varphi_i|A_1}{\varphi_j}=\br{x_i|\sum_{k=1}^3x_k(\pdv{}{x_k})}{x_j}=\br{x_i}{x_j}=\delta_{ij}
\]
\[
(A_2)_{ij}=\br{\varphi_i|A_2}{\varphi_j}=\br{x_i|x_1(\pdv{}{x_2})-x_2(\pdv{}{x_1})}{x_j}=\br{x_i}{x_1\delta_{2j}-x_2\delta_{1j}}=\delta_{i1}\delta_{2j}-\delta_{i2}\delta_{1j}
\]
In matrix forms,
\[
\M{A_1}=
\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1
\end{pmatrix}
\quad\quad \M{A_2}
\begin{pmatrix}
0&1&0\\
-1&0&0\\
0&0&0
\end{pmatrix}
\]

(b)
\[
\psi_i=\br{\varphi_i}{\psi}=\br{x_i}{x_1-2x_2+3x_3}=\delta_{i1}-2\delta_{i2}+3\delta_{i3}
\]
In matrix form,
\[
\boldsymbol{\psi}=
\begin{pmatrix}
1\\-2\\3
\end{pmatrix}
\]

(c) From matrix equation,
\[
\boldsymbol{\chi}=\left[
\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1
\end{pmatrix}
-
\begin{pmatrix}
0&1&0\\
-1&0&0\\
0&0&0
\end{pmatrix}
\right]
\begin{pmatrix}
1\\-2\\3
\end{pmatrix}=
\begin{pmatrix}
3\\-1\\3
\end{pmatrix}
\]
which is $3x_1-x_2+3x_3$.

By direct application,
\[
\chi=(A_1-A_2)\psi=\sum_{i=1}^3x_i(\pdv{}{x_i})(x_1-2x_2+3x_3)-\left[x_1(\pdv{}{x_2})-x_2(\pdv{}{x_1}) \right](x_1-2x_2+3x_3)
\]
\[
=x_1-2x_2+3x_3-(-2x_1-x_2)=3x_1-x_2+3x_3
\]
which is the same with the results from matrix multiplication.

\paragraph{5.3.4}
(a)
\[
AP_0=x\frac{d}{dx}(\frac{1}{\sqrt{2}})=0
\]
\[
AP_1=x\frac{d}{dx}(\sqrt{\frac{3}{2}}x)=\sqrt{\frac{3}{2}}x=P_1
\]
\[
AP_2=x\frac{d}{dx}\left(\sqrt{\frac{5}{2}}(\frac{3}{2}x^2-\frac{1}{2})\right)=\sqrt{\frac{5}{2}}\,3x^2=2P_2+\sqrt{5}P_0
\]
\[
AP_3=x\frac{d}{dx}\left(\sqrt{\frac{7}{2}}(\frac{5}{2}x^3-\frac{3}{2}x)\right)=\sqrt{\frac{7}{2}}\frac{15}{2}x^3-\sqrt{\frac{7}{2}}\frac{3}{2}x=3P_3+\sqrt{21}P_1
\]
We can evaluate $A_{ij}$ by $A_{ij}=\br{P_i}{AP_j}$, and $\br{P_i}{P_j}=\delta_{ij}$. In matrix form,
\[
\M{A}=
\begin{pmatrix}
0&0&\sqrt{5}&0\\
0&1&0&\sqrt{21}\\
0&0&2&0\\
0&0&0&3
\end{pmatrix}
\]

(b)
\[
x^3=P_0\int_{-1}^1\frac{1}{\sqrt{2}}\,x^3dx+P_1\int_{-1}^1\sqrt{\frac{3}{2}}x\cdot x^3dx+P_2\int_{-1}^1\sqrt{\frac{5}{2}}(\frac{3}{2}x^2-\frac{1}{2})\cdot x^3dx+P_3\int_{-1}^1\sqrt{\frac{7}{2}}(\frac{5}{2}x^3-\frac{3}{2}x)x^3dx\]
\[=\frac{\sqrt{6}}{5}P_1+\frac{2\sqrt{14}}{35}P_3
\]

(c)
\renewcommand{\arraystretch}{1.5}
\[
\begin{pmatrix}
0&0&\sqrt{5}&0\\
0&1&0&\sqrt{21}\\
0&0&2&0\\
0&0&0&3
\end{pmatrix}
\begin{pmatrix}
0\\\frac{\sqrt{6}}{5}\\0\\\frac{2 \sqrt{14}}{35}
\end{pmatrix}=
\begin{pmatrix}
0\\\frac{3\sqrt{6}}{5}\\0\\\frac{6\sqrt{14}}{35}
\end{pmatrix}
\]
which is
\[
\frac{3\sqrt{6}}{5}(\sqrt{\frac{3}{2}}x)+\frac{6\sqrt{14}}{35}\sqrt{\frac{7}{2}}(\frac{5}{2}x^3-\frac{3}{2}x)=3x^3
\]
which is the same as $Ax^3=x\frac{d}{dx}(x^3)=3x^3$.

\section*{5.4 Self-Adjoint Operators}

\paragraph{5.4.1}
(a)
\[
\br{f}{(A+A^\dagger)g}=\br{(A+A^\dagger)^\dagger f}{g}=\br{(A^\dagger+A)f}{g}=\br{(A+A^\dagger)f}{g}
\]
\[
\br{f}{i(A-A^\dagger)g}=\br{-i(A-A^\dagger)^\dagger f}{g}=\br{-i(A^\dagger-A)f}{g}=\br{i(A-A^\dagger)f}{g}
\]

(b)
For every operator $A$,
\[
A=\frac{1}{2}(A+A^\dagger)-\frac{i}{2}i(A-A^\dagger)
\]
where both $A+A^\dagger$ and $i(A-A^\dagger)$ are Hermitian.

\paragraph{5.4.2}
Let $A,B$ be Hermitian.

If $AB$ is Hermitian, then $\br{f}{ABg}=\br{ABf}{g}$, but also
\[
\br{f}{ABg}=\br{Af}{Bg}=\br{BAf}{g}
\]
so $\br{(AB-BA)f}{g}=0$ for any $f,g$, which means $(AB-BA)$ must be zero, and therefore $AB=BA$,

If $AB=BA$, then
\[
\br{f}{ABg}=\br{Af}{Bg}=\br{BAf}{g}=\br{ABf}{g}
\]
so $AB$ is Hermitian.

\paragraph{5.4.3}
$A,B$ are Hermitian because they are quantum mechanical operators. $C=-i(AB-BA)$, so
\[
\br{f}{Cg}=\br{C^\dagger d}{g}=\br{i(B^\dagger A^\dagger-A^\dagger B^\dagger)f}{g}=\br{i(BA-AB)f}{g}=\br{-i(AB-BA)f}{g}=\br{Cf}{g}
\]
so $C$ is Hermitian.

\paragraph{5.4.4}
$\mathcal{L}$ is Hermitian, so
\[
\br{\psi|\mathcal{L}^2}{\psi}=\br{\psi|\mathcal{L}\mathcal{L}}{\psi}=\br{\mathcal{L}\psi}{\mathcal{L}\psi}\geq0
\]
by the definition of scalar product.

\paragraph{5.4.5}
(a) In spherical polar coordinate, $\varphi_1=C\sin\theta\cos\varphi$, $\varphi_2=C\sin\theta\sin\varphi$, $\varphi_3=C\cos\theta$. So
\[
\br{\varphi_1}{\varphi_1}=\int_0^\pi\int_0^{2\pi}(|C|^2\sin^2\theta\cos^2\varphi)\sin\theta\, d\theta\,d\varphi
\]
\[
=|C|^2\int_0^\pi\sin^3\theta\,d\theta\int_0^{2\pi}\cos^2\varphi\,d\varphi=|C|^2\frac{4\pi}{3}=1
\]
so
\[
C=\sqrt{\frac{3}{4\pi}}e^{i\theta}
\]
By symmetry this $C$ also made $\varphi_2$ and $\varphi_3$ normalized.

\[
\br{\varphi_1}{\varphi_2}=\int_0^\pi\int_0^{2\pi}|C|^2\sin^2\theta\cos^2\varphi\sin\theta\,d\theta\,d\varphi
\]
\[
=|C|^2\int_0^\pi\sin^3\theta\,d\theta\int_0^{2\pi}\sin\varphi\cos\varphi\,d\varphi=0
\]
By symmetry $\br{\varphi_2}{\varphi_3}$ and $\br{\varphi_3}{\varphi_1}$ are also zero.
\medskip

(b) Let $i,j,k$ be a cyclic permutation of $x,y,z$, then all the three operators have the form
\[
L_i=-i(x_j\pdv{}{x_k}-x_k\pdv{}{x_j})
\]
Note that $\pdv{}{x_k}(\frac{1}{r})=-\frac{x_k}{r^3}$, so
\[
L_i\varphi_b=-i\left[x_j\pdv{}{x_k}(\frac{Cx_b}{r})-x_k\pdv{}{x_j}(\frac{Cx_b}{r}) \right]
\]
\[
=-iC\left[x_j\pdv{x_b}{x_k}\frac{1}{r}-x_jx_b\frac{x_k}{r^3}-x_k\pdv{x_b}{x_j}\frac{1}{r}+x_kx_b\frac{x_j}{r^3} \right]
\]
\[
=-i\frac{C}{r}\left[x_j\delta_{kb}-x_k\delta_{jb} \right]=-i\left[\varphi_j\delta_{kb}-\varphi_k\delta_{jb} \right]
\]
so
\[
\br{\varphi_a|L_i}{\varphi_b}=-i\left[\br{\varphi_a}{\varphi_j}\delta_{kb}-\br{\varphi_a}{\varphi_k}\delta_{jb} \right]=-i\left(\delta_{aj}\delta_{kb}-\delta_{ak}\delta_{jb} \right)
\]
\[
=
\begin{cases}
-i & \textit{when $a=j,\;b=k$}\\
i & \textit{when $a=k,\;b=j$}
\end{cases}
\]
The components of $L_i$ are $(L_i)_{ab}=\br{\varphi_a|L_i}{\varphi_b}$, so in matrix form,
\[
L_x=
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}\quad
L_y=
\begin{pmatrix}
0&0&i\\
0&0&0\\
-i&0&0
\end{pmatrix}\quad
L_z=
\begin{pmatrix}
0&-i&0\\
i&0&0\\
0&0&0
\end{pmatrix}\quad
\]

(c)
\[
L_xL_y-L_yL_x=
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}
\begin{pmatrix}
0&0&i\\
0&0&0\\
-i&0&0
\end{pmatrix}-
\begin{pmatrix}
0&0&i\\
0&0&0\\
-i&0&0
\end{pmatrix}
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}=
\begin{pmatrix}
0&1&0\\
-1&0&0\\
0&0&0
\end{pmatrix}=iL_z
\]
\medskip

\noindent
(we can prove $[L_x,L_y]=iL_z$, $[L_y,L_z]=iL_x$, $[L_z,L_x]=iL_y$ together:
Note that
\[
\br{\varphi_a|L_i}{\varphi_b}=\begin{cases}
-i & \textit{when $a=j,\;b=k$}\\
i & \textit{when $a=k,\;b=j$}
\end{cases}
\]
is equivalent with $(L_i)_{ab}=-i\varepsilon_{iab}$.
Let $i,j,k$ be a cyclic permutation of $x,y,z$, then
\[
(L_iL_j)_{ab}=\sum_c(L_i)_{ac}(L_j)_{cb}=\sum_c-\varepsilon_{iac}\varepsilon_{jcb}=-\varepsilon_{iak}\varepsilon_{jkb}=\varepsilon_{iak}\varepsilon_{jbk}=\delta_{ij}\delta_{ab}-\delta_{ib}\delta_{aj}=-\delta_{ib}\delta_{aj}
\]
by Exercise 2.1.9. So
\[
(L_iL_j-L_jL_i)_{ab}=-\delta_{ib}\delta_{aj}+\delta_{jb}\delta_{ai}=\sum_l\varepsilon_{ijl}\varepsilon_{abl}=\varepsilon_{ijk}\varphi_{abk}=\varepsilon_{abk}=i(-i\varepsilon_{kab})=i(L_k)_{ab}
\]
which means $[L_i,L_j]=iL_k$ )

\section*{5.5 Unitary Operators}

\paragraph{5.5.1}
(There are mistakes in the matrix $\M{U}$ given in the text: $\M{U}_{33}$ should be $\frac{-i}{\sqrt{2}}$ and $\M{U}_{43}$ should be $\frac{i}{\sqrt{2}}$. It can be verified by checking $\chi_3=U_{33}\chi_3'+U_{43}\chi_4'$. The author probably forget to take the complex conjugate of $\chi_3'$ when calculating $\br{\chi_3'}{\chi_3}=\int_0^\pi\int_0^{2\pi}\sin\theta\,d\theta d\varphi(\chi_3')^*\chi_3$, as well as $\br{\chi_4'}{\chi_3}$. )
\medskip

(a)
\renewcommand{\arraystretch}{1}
\[
f(\theta,\varphi)=\V{c}=
\begin{pmatrix}
3\\2i\\-1\\0\\1
\end{pmatrix}
\]
\renewcommand{\arraystretch}{1.5}
\[
\V{c'}=\M{U}\V{c}=
\begin{pmatrix}
\frac{-1}{\sqrt{2}}&\frac{-i}{\sqrt{2}}&0&0&0\\
\frac{1}{\sqrt{2}}&\frac{-i}{\sqrt{2}}&0&0&0\\
0&0&\frac{-i}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0\\
0&0&\frac{i}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0\\
0&0&0&0&1
\end{pmatrix}
\begin{pmatrix}
3\\2i\\-1\\0\\1
\end{pmatrix}=
\begin{pmatrix}
\frac{-1}{\sqrt{2}}\\\frac{5}{\sqrt{2}}\\\frac{i}{\sqrt{2}}\\\frac{-i}{\sqrt{2}}\\1
\end{pmatrix}
\]
\[
\sum_ic_i'\chi_i'=\sqrt{\frac{15}{8\pi}}\sin\theta\cos\theta(\frac{1}{\sqrt{2}}e^{i\varphi}+\frac{5}{\sqrt{2}}e^{-i\varphi})+\sqrt{\frac{15}{32\pi}}\sin^2\theta(\frac{i}{\sqrt{2}}e^{2i\varphi}-\frac{i}{\sqrt{2}}e^{-2i\varphi})+\chi_5'
\]
\[
=3\sqrt{\frac{15}{4\pi}}\sin\theta\cos\theta\cos\varphi-2i\sqrt{\frac{15}{4\pi}}\sin\theta\cos\theta\sin\varphi-\sqrt{\frac{15}{4\pi}}\sin^2\theta\sin\varphi\cos\varphi+\chi_5
\]
\[
=3\chi_1-2i\chi_2-\chi_3+\chi_5=\sum_ic_i\chi_i=f(\theta,\varphi)
\]

(b)
\[
\M{U}^{-1}\M{U}=
\begin{pmatrix}
\frac{-1}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0&0&0\\
\frac{-i}{\sqrt{2}}&\frac{-i}{\sqrt{2}}&0&0&0\\
0&0&\frac{-i}{\sqrt{2}}&\frac{i}{\sqrt{2}}&0\\
0&0&\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0\\
0&0&0&0&1
\end{pmatrix}
\begin{pmatrix}
\frac{-1}{\sqrt{2}}&\frac{-i}{\sqrt{2}}&0&0&0\\
\frac{1}{\sqrt{2}}&\frac{-i}{\sqrt{2}}&0&0&0\\
0&0&\frac{-i}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0\\
0&0&\frac{i}{\sqrt{2}}&\frac{1}{\sqrt{2}}&0\\
0&0&0&0&1
\end{pmatrix}=
\begin{pmatrix}
1&0&0&0&0\\
0&1&0&0&0\\
0&0&1&0&0\\
0&0&0&1 &0\\
0&0&0&0&1
\end{pmatrix}
\]

\paragraph{5.5.2}
\renewcommand{\arraystretch}{1.0}
(a) The transformation is $x=z'$, $y=y'$, $z=-x'$. The new basis is defined as $\varphi_1'=x'$, $\varphi_2'=y'$, $\varphi_3'=z'$, so $\varphi_1=\varphi_3'$, $\varphi_2=\varphi_2'$, $\varphi_3=-\varphi_1'$, which in matrix representation becomes
\[
\begin{pmatrix}
0&0&-1\\
0&1&0\\
1&0&0\\
\end{pmatrix}
\]

(b) The transformation corresponds to rotating $\frac{\pi}{2}$ counterclockwise about $y$-axis, so the Euler angles are $\alpha=0$, $\beta=\frac{\pi}{2}$, $\gamma=0$. By Eq. 3.37, \[
S(\alpha,\beta,\gamma)=
\begin{pmatrix}
0&0&-1\\
0&1&0\\
1&0&0\\
\end{pmatrix}
\]
which is the same as (a).

(c)
\[
\begin{pmatrix}
0&0&-1\\
0&1&0\\
1&0&0\\
\end{pmatrix}
\begin{pmatrix}
2\\-3\\1
\end{pmatrix}=
\begin{pmatrix}
-1\\-3\\2
\end{pmatrix}
\]
so $f'=-x'-3y'+2z'=z-3y+2x=f$, consistent.

\paragraph{5.5.3}
$\varphi_1'=-\varphi_3$, $\varphi_2'=\varphi_2$, $\varphi_3'=\varphi_1$, so the inverse transformation matrix is
\[
\M{U'}=\begin{pmatrix}
0&0&1\\
0&1&0\\
-1&0&0\\
\end{pmatrix}
\]
\[
\M{U'}\M{U}=\begin{pmatrix}
0&0&1\\
0&1&0\\
-1&0&0\\
\end{pmatrix}
\begin{pmatrix}
0&0&-1\\
0&1&0\\
1&0&0\\
\end{pmatrix}
=\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1\\
\end{pmatrix}
\]
so the two matrix are matrix inverses of each other.

\paragraph{5.5.4}
(
The transformation matrix $V$ given is not unitary. A possible unitary $V$ is
\[
\begin{pmatrix}
1&0&0\\
0&\cos\theta&i\sin\theta\\
0&\sin\theta&-i\cos\theta
\end{pmatrix}
\]
which will be used to solve the problem.)

(a)
\[
\begin{pmatrix}
i\sin\theta&\cos\theta&0\\
-\cos\theta&i\sin\theta&0\\
0&0&1
\end{pmatrix}
\begin{pmatrix}
3\\-1\\-2
\end{pmatrix}=
\begin{pmatrix}
-\cos\theta+3i\sin\theta\\
-3\cos\theta-i\sin\theta\\
-2
\end{pmatrix}
\]
\[
\begin{pmatrix}
1&0&0\\
0&\cos\theta&i\sin\theta\\
0&\sin\theta&-i\cos\theta
\end{pmatrix}
\begin{pmatrix}
-\cos\theta+3i\sin\theta\\
-3\cos\theta-i\sin\theta\\
-2
\end{pmatrix}=
\begin{pmatrix}
-\cos\theta+3i\sin\theta\\
-3\cos^2\theta-i\sin\theta(\cos\theta+2)\\
-3\sin\theta\cos\theta+i(2\cos\theta-\sin^2\theta)
\end{pmatrix}
\]
so
\[
f(x)=(-\cos\theta+3i\sin\theta)\chi_1+(-3\cos^2\theta-i\sin\theta(\cos\theta+2))\chi_2+(-3\sin\theta\cos\theta+i(2\cos\theta-\sin^2\theta))\chi_3
\]

(b)
\[
(UV)=\begin{pmatrix}
i\sin\theta&\cos\theta&0\\
-\cos\theta&i\sin\theta&0\\
0&0&1
\end{pmatrix}
\begin{pmatrix}
1&0&0\\
0&\cos\theta&i\sin\theta\\
0&\sin\theta&-i\cos\theta
\end{pmatrix}=
\begin{pmatrix}
i\sin\theta&\cos^2\theta&i\sin\theta\cos\theta\\
-\cos\theta&i\sin\theta\cos\theta&-\sin^2\theta\\
0&\sin\theta&-i\cos\theta\\
\end{pmatrix}
\]
\[
\begin{pmatrix}
i\sin\theta&\cos^2\theta&i\sin\theta\cos\theta\\
-\cos\theta&i\sin\theta\cos\theta&-\sin^2\theta\\
0&\sin\theta&-i\cos\theta\\
\end{pmatrix}
\begin{pmatrix}
3\\-1\\-2
\end{pmatrix}=
\begin{pmatrix}
-\cos^2\theta+i\sin\theta(3-2\cos\theta)\\
-3\cos\theta+2\sin^2\theta-i\sin\theta\cos\theta\\
-\sin\theta+2i\cos\theta
\end{pmatrix}
\]
\[
(VU)=
\begin{pmatrix}
1&0&0\\
0&\cos\theta&i\sin\theta\\
0&\sin\theta&-i\cos\theta
\end{pmatrix}
\begin{pmatrix}
i\sin\theta&\cos\theta&0\\
-\cos\theta&i\sin\theta&0\\
0&0&1
\end{pmatrix}=
\begin{pmatrix}
i\sin\theta&\cos\theta&0\\
-\cos^2\theta&i\sin\theta\cos\theta&i\sin\theta\\
-\sin\theta\cos\theta&i\sin^2\theta&-i\cos\theta
\end{pmatrix}
\]
\[
\begin{pmatrix}
i\sin\theta&\cos\theta&0\\
-\cos^2\theta&i\sin\theta\cos\theta&i\sin\theta\\
-\sin\theta\cos\theta&i\sin^2\theta&-i\cos\theta
\end{pmatrix}
\begin{pmatrix}
3\\-1\\-2
\end{pmatrix}=
\begin{pmatrix}
-\cos\theta+3i\sin\theta\\
-3\cos^2\theta-i\sin\theta(\cos\theta+2)\\
-3\sin\theta\cos\theta+i(2\cos\theta-\sin^2\theta)
\end{pmatrix}
\]
So only applying $VU$ gives the same answer as (a), which is quite obvious because $U$ is applied first.

\paragraph{5.5.5}
\newcommand{\PP}[1]{\mathcal{P}_{#1}}
\newcommand{\FF}[1]{\mathcal{F}_{#1}}
\newcommand{\II}[1]{\int_{-1}^1#1\,dx}
(a) Let the normalized $\PP{n}=a_nP_n$ and $\FF{n}=b_nP_n$. Let the factors $a_n$ and $b_n$ be positive real numbers (every $a_ne^{i\theta}$ can also normalize the functions, as well as $b_ne^{i\theta}$)
\[
\II{|a_0|^2P_0^2}=|a_0|^22=1,\quad a_0=\frac{1}{\sqrt{2}},\quad\PP{0}=\frac{1}{\sqrt{2}}
\]
\[
\II{|a_1|^2P_1^2}=|a_1|^2\frac{2}{3}=1,\quad a_1=\sqrt{\frac{3}{2}},\quad\PP{1}=\sqrt{\frac{3}{2}}x
\]
\[
\II{|a_2|^2P_2^2}=|a_2|^2\frac{2}{5}=1,\quad a_2=\sqrt{\frac{5}{2}},\quad\PP{2}=\sqrt{\frac{5}{2}}(\frac{3}{2}x^2-\frac{1}{2})
\]

\[
\II{|b_0|^2F_0^2}=|b_0|^2\frac{2}{5}=1,\quad b_0=\sqrt{\frac{5}{2}},\quad\FF{1}=\sqrt{\frac{5}{2}}x^2
\]
\[
\II{|b_1|^2F_1^2}=|b_1|^2\frac{2}{3}=1,\quad b_1=\sqrt{\frac{3}{2}},\quad\FF{1}=\sqrt{\frac{3}{2}}x
\]
\[
\II{|b_2|^2F_2^2}=|b_2|^28=1,\quad b_2=\frac{1}{\sqrt{8}},\quad\FF{2}=\frac{1}{\sqrt{8}}(5x^2-3)
\]

(b) $U_{ij}=\II{F_i^*P_j}$. Note that except $U_{00},U_{02},U_{11},U_{20},U_{22}$, all the other $U_{ij}$ vanish because $F_i^*P_j$ are odd functions for these $i,j$.
\[
U_{00}=\II{\frac{\sqrt{5}}{2}x^2}=\frac{\sqrt{5}}{3}
\]
\[
U_{02}=\II{\frac{5}{2}(\frac{3}{2}x^4-\frac{1}{2}x^2)}=\frac{2}{3}
\]
\[
U_{11}=\II{\frac{3}{2}x^2}=1
\]
\[
U_{20}=\II{\frac{1}{4}(5x^2-3)}=-\frac{2}{3}
\]
\[
U_{22}=\II{\frac{\sqrt{5}}{4}(\frac{15}{2}x^4-7x^2+\frac{3}{2})}=\frac{\sqrt{5}}{3}
\]
so
\[
\M{U}=
\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&\frac{2}{3}\\
0&1&0\\
-\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}
\]

(c)$V_{ij}=\II{P_i^*F_j}$. Note that except $V_{00},V_{02},V_{11},V_{20},V_{22}$, all the other $V_{ij}$ vanish because $P_i^*F_j$ are odd functions for these $i,j$.
\[
V_{00}=\II{\frac{\sqrt{5}}{2}x^2}=\frac{\sqrt{5}}{3}
\]
\[
V_{02}=\II{\frac{1}{4}(5x^2-3)}=-\frac{2}{3}
\]
\[
V_{11}=\II{\frac{3}{2}x^2}=1
\]
\[
V_{20}=\II{\frac{5}{2}(\frac{3}{2}x^4-\frac{1}{2}x^2)}=\frac{2}{3}
\]
\[
V_{22}=\II{\frac{\sqrt{5}}{4}(\frac{15}{2}x^4-7x^2+\frac{3}{2})}=\frac{\sqrt{5}}{3}
\]
so
\[
\M{V}=
\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&-\frac{2}{3}\\
0&1&0\\
\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}
\]

(d)
\[
\M{U}^{-1}=\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&-\frac{2}{3}\\
0&1&0\\
\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}=\M{V}
\]
\[
\M{V}^{-1}=\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&\frac{2}{3}\\
0&1&0\\
-\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}=\M{U}
\]
\[
\M{U}\M{U}^{-1}=\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&\frac{2}{3}\\
0&1&0\\
-\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}
\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&-\frac{2}{3}\\
0&1&0\\
\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}=
\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1
\end{pmatrix}
\]
\[
\M{V}\M{V}^{-1}=\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&-\frac{2}{3}\\
0&1&0\\
\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}
\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&\frac{2}{3}\\
0&1&0\\
-\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}=
\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1
\end{pmatrix}
\]

(e)
\renewcommand{\arraystretch}{2}
\[
\V{f}_{P}=
\begin{pmatrix}
\II{(\frac{1}{\sqrt{2}})(5x^2-3x+1)}\\
\II{(\sqrt{\frac{3}{2}}x)(5x^2-3x+1)}\\
\II{\sqrt{\frac{5}{2}}(\frac{3}{2}x^2-\frac{1}{2})(5x^2-3x+1)}\\
\end{pmatrix}=
\begin{pmatrix}
\frac{8\sqrt{2}}{3}\\-\sqrt{6}\\\frac{2\sqrt{10}}{3}
\end{pmatrix}
\]
\[
\V{f}_{F}=
\begin{pmatrix}
\II{(\sqrt{\frac{5}{2}}x^2)(5x^2-3x+1)}\\
\II{(\sqrt{\frac{3}{2}}x)(5x^2-3x+1)}\\
\II{\frac{1}{\sqrt{8}}(5x^2-3)(5x^2-3x+1)}\\
\end{pmatrix}=
\begin{pmatrix}
\frac{4\sqrt{10}}{3}\\-\sqrt{6}\\-\frac{2\sqrt{2}}{3}
\end{pmatrix}
\]
\[
\M{U}\V{f}_{P}=\begin{pmatrix}
\frac{\sqrt{5}}{3}&0&\frac{2}{3}\\
0&1&0\\
-\frac{2}{3}&0&\frac{\sqrt{5}}{3}
\end{pmatrix}
\begin{pmatrix}
\frac{8\sqrt{2}}{3}\\-\sqrt{6}\\\frac{2\sqrt{10}}{3}
\end{pmatrix}=
\begin{pmatrix}
\frac{4\sqrt{10}}{3}\\-\sqrt{6}\\-\frac{2\sqrt{2}}{3}
\end{pmatrix}=\V{f}_F
\]

\section*{5.6 Transformations of Operators}

\paragraph{5.6.1}
\renewcommand{\arraystretch}{1.2}
(a)
\[
S_x=
\begin{pmatrix}
0&\frac{1}{2}\\
\frac{1}{2}&0
\end{pmatrix},\quad
S_y=
\begin{pmatrix}
0&\frac{-i}{2}\\
\frac{i}{2}&0
\end{pmatrix}
S_z=
\begin{pmatrix}
\frac{1}{2}&0\\
0&\frac{-1}{2}
\end{pmatrix}
\]

(b)
\[
\br{\varphi_1'}{\varphi_2'}=\br{C(\alpha+\beta)}{C(\alpha-\beta)}=|C|^2(\br{\alpha}{\alpha}-\br{\alpha}{\beta}+\br{\beta}{\alpha}-\br{\beta}{\beta})=|C|^2(1-1)=0
\]
\[
\br{\varphi_1'}{\varphi_1'}=\br{C(\alpha+\beta)}{C(\alpha+\beta)}=|C|^2(\br{\alpha}{\alpha}+\br{\alpha}{\beta}+\br{\beta}{\alpha}+\br{\beta}{\beta})=|C|^22=1
\]
so $C=\frac{1}{\sqrt{2}}$ can normalize $\varphi_1'$ and $\varphi_2'$. The transformation matrix $\M{U}$ is
\[
\M{U}=
\begin{pmatrix}
\br{\varphi_1'}{\varphi_1}&\br{\varphi_1'}{\varphi_2}\\
\br{\varphi_2'}{\varphi_1}&\br{\varphi_2'}{\varphi_2}
\end{pmatrix}=
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}
\]

(c)
\[
S_x'=\M{U}S_x\M{U}^{-1}=
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}
\begin{pmatrix}
0&\frac{1}{2}\\
\frac{1}{2}&0
\end{pmatrix}
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}=\begin{pmatrix}
\frac{1}{2}&0\\
0&\frac{-1}{2}
\end{pmatrix}
\]
\[
S_y'=\M{U}S_y\M{U}^{-1}=
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}
\begin{pmatrix}
0&\frac{-i}{2}\\
\frac{i}{2}&0
\end{pmatrix}
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}=\begin{pmatrix}
0&\frac{i}{2}\\
\frac{-i}{2}&0
\end{pmatrix}
\]
\[
S_z'=\M{U}S_z\M{U}^{-1}=
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}
\begin{pmatrix}
\frac{1}{2}&0\\
0&\frac{-1}{2}
\end{pmatrix}
\begin{pmatrix}
\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
\frac{1}{\sqrt{2}}&\frac{-1}{\sqrt{2}}
\end{pmatrix}=\begin{pmatrix}
0&\frac{1}{2}\\
\frac{1}{2}&0
\end{pmatrix}
\]

\paragraph{5.6.2}
(a) Note that $y\pdv{}{z}(e^{-r^2})-z\pdv{}{y}(e^{-r^2})=e^{-r^2}(-2r)(y\frac{z}{r}-z\frac{y}{r})=0$, so
\[
L_x\varphi_1=0
\]
\[
L_x\varphi_2=-iC(-ze^{-r^2})=i\varphi_3
\]
\[
L_x\varphi_3=-iC(ye^{-r^2})=-i\varphi_2
\]
so in matrix form,
\[
L_x=
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}
\]

(b)
\[
L_x'=\M{U}L_x\M{U}^{-1}=
\begin{pmatrix}
1&0&0\\
0&\frac{1}{\sqrt{2}}&\frac{-i}{\sqrt{2}}\\
0&\frac{1}{\sqrt{2}}&\frac{i}{\sqrt{2}}\\
\end{pmatrix}
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}
\begin{pmatrix}
1&0&0\\
0&\frac{1}{\sqrt{2}}&\frac{1}{\sqrt{2}}\\
0&\frac{i}{\sqrt{2}}&\frac{-i}{\sqrt{2}}\\
\end{pmatrix}=
\begin{pmatrix}
0&0&0\\
0&1&0\\
0&0&-1
\end{pmatrix}
\]

(c) $\varphi_i'$ is the $i^{th}$ column of $\M{U}^{-1}$ in $\varphi_i$ basis, so
\[
\varphi_1'=Cxe^{-r^2}\quad\varphi_2'=C\frac{y+iz}{\sqrt{2}}e^{-r^2}\quad\varphi_3'=C\frac{y-iz}{\sqrt{2}}e^{-r^2}
\]

$L_x\varphi_i'$ is the $i^{th}$ column of $L_x'$ in $\varphi_i'$ basis, so
\[
L_x\varphi_1'=0\quad L_x\varphi_2'=\varphi_2'=C\frac{y+iz}{\sqrt{2}}e^{-r^2}\quad L_x\varphi_3'=-\varphi_3'=-C\frac{y-iz}{\sqrt{2}}e^{-r^2}
\]

\paragraph{5.6.3}
Definition:
\[
\varphi_1=T_{11}\chi_1
\]
\[
\varphi_2=T_{12}\chi_1+T_{22}\chi_2
\]
\[
\varphi_3=T_{13}\chi_1+T_{23}\chi_2+T_{33}\chi_3
\]
\[
\M{T}=
\begin{pmatrix}
T_{11}&T_{12}&T_{13}\\
0&T_{22}&T_{23}\\
0&0&T_{33}
\end{pmatrix}
\]
Let $\psi_i$ be the orthogonalized but yet normalized functions, so
\[
\psi_1=\chi_1
\]
\[
\psi_2=\chi_2-\psi_1\frac{\br{\psi_1}{\chi_2}}{\br{\psi_1}{\psi_1}}
\]
\[
\psi_3=\chi_3-\psi_2\frac{\br{\psi_2}{\chi_3}}{\br{\psi_2}{\psi_2}}-\psi_1\frac{\br{\psi_1}{\chi_3}}{\br{\psi_1}{\psi_1}}
\]
By comparing the coefficients of $\chi_n$ in $\varphi_n$ and $\psi_n$, we have
\[
\varphi_n=T_{nn}\psi_n
\]
Let
\[
S_{ij}=\br{\chi_i}{\chi_j}
\]
\[
\M{S}=
\begin{pmatrix}
\br{\chi_1}{\chi_1}&\br{\chi_1}{\chi_2}&\br{\chi_1}{\chi_3}\\
\br{\chi_2}{\chi_1}&\br{\chi_2}{\chi_2}&\br{\chi_2}{\chi_3}\\
\br{\chi_3}{\chi_1}&\br{\chi_3}{\chi_2}&\br{\chi_3}{\chi_3}\\
\end{pmatrix}
\]
\[
D_1=\br{\chi_1}{\chi_1}
\]
\[
D_2=
\begin{vmatrix}
\br{\chi_1}{\chi_1}&\br{\chi_1}{\chi_2}\\
\br{\chi_2}{\chi_1}&\br{\chi_2}{\chi_2}\\
\end{vmatrix}
\]
\[
D_3=
\begin{vmatrix}
\br{\chi_1}{\chi_1}&\br{\chi_1}{\chi_2}&\br{\chi_1}{\chi_3}\\
\br{\chi_2}{\chi_1}&\br{\chi_2}{\chi_2}&\br{\chi_2}{\chi_3}\\
\br{\chi_3}{\chi_1}&\br{\chi_3}{\chi_2}&\br{\chi_3}{\chi_3}\\
\end{vmatrix}
\]
Note that $S_{ij}^*=\br{\chi_j}{\chi_1}$, $D_n^*=D_n$. In this problem when $|A|^2=k$, let $A=\sqrt{k}$, without the phase factor.
\medskip

From Eq. 5.79 we have
\[
\M{T}^\dagger\M{S}\M{T}=\M{I}
\]
note that from the derivation, the equation will hold for any dimension, so
\[
T_{11}^*\br{\chi_1}{\chi_1}T_{11}=1
\]
\[
T_{11}=\frac{1}{\sqrt{D_1}}
\]
\[
\begin{pmatrix}
T_{11}^*&0\\
T_{12}^*&T_{22}^*\\
\end{pmatrix}
\begin{pmatrix}
\br{\chi_1}{\chi_1}&\br{\chi_1}{\chi_2}\\
\br{\chi_2}{\chi_1}&\br{\chi_2}{\chi_2}\\
\end{pmatrix}
\begin{pmatrix}
T_{11}&T_{12}\\
0&T_{22}\\
\end{pmatrix}=
\begin{pmatrix}
1&0\\
0&1\\
\end{pmatrix}
\]
Take the determinant of both sides:
\[
T_{11}^*T_{22}^*D_2T_{11}T_{22}=1
\]
\[
|T_{22}|^2=\frac{1}{|T_{11}|^2D_2}=\frac{D_1}{D_2},\quad T_{22}=\sqrt{\frac{D_1}{D_2}}
\]
\[
\begin{pmatrix}
T_{11}^*&0&0\\
T_{12}^*&T_{22}^*&0\\
T_{13}^*&T_{23}^*&T_{33}^*
\end{pmatrix}
\begin{pmatrix}
\br{\chi_1}{\chi_1}&\br{\chi_1}{\chi_2}&\br{\chi_1}{\chi_3}\\
\br{\chi_2}{\chi_1}&\br{\chi_2}{\chi_2}&\br{\chi_2}{\chi_3}\\
\br{\chi_3}{\chi_1}&\br{\chi_3}{\chi_2}&\br{\chi_3}{\chi_3}\\
\end{pmatrix}
\begin{pmatrix}
T_{11}&T_{12}&T_{13}\\
0&T_{22}&T_{23}\\
0&0&T_{33}
\end{pmatrix}=
\begin{pmatrix}
1&0&0\\
0&1&0\\
0&0&1
\end{pmatrix}
\]
\[
T_{11}^*T_{22}^*T_{33}^*D_2T_{11}T_{22}T_{33}=1
\]
\[
|T_{33}|^2=\frac{1}{|T_{22}|^2|T_{11}|^2D_3}=\frac{D_2}{D_3},\quad T_{33}=\sqrt{\frac{D_2}{D_3}}
\]

\[
\psi_1=\chi_1
\]
\[
\varphi_1=T_{11}\chi_1=\frac{\chi_1}{\sqrt{D_1}}
\]
\[
\psi_2=\chi_2-\psi_1\frac{\br{\psi_1}{\chi_2}}{\br{\psi_1}{\psi_1}}=\chi_2-\chi_1\frac{S_{12}}{S_{11}}
\]
\[
\varphi_2=T_{22}\psi_2=\sqrt{\frac{D_1}{D_2}}(\chi_1\frac{-S_{12}}{S_{11}}+\chi_2)=\chi_1\frac{-S_{12}}{\sqrt{D_1D_2}}+\chi_2\sqrt{\frac{D_1}{D_2}}
\]
\[
\psi_3=\chi_3-\psi_2\frac{\br{\psi_2}{\chi_3}}{\br{\psi_2}{\psi_2}}-\psi_1\frac{\br{\psi_1}{\chi_3}}{\br{\psi_1}{\psi_1}}
\]
\[
=\chi_3-(\chi_2-\chi_1\frac{S_{12}}{S_{11}})\frac{S_{23}-S_{13}\frac{S_{21}}{S_{11}}}{S_{22}-\frac{S_{12}S_{21}}{S_{11}}}-\chi_1\frac{S_{13}}{S_{11}}
\]
\[
=\chi_3+\chi_2\frac{S_{13}S_{21}-S_{11}S_{23}}{S_{11}S_{22}-S_{12}S_{21}}+\chi_1\frac{S_{11}S_{12}S_{13}-S_{12}S_{21}S_{13}-S_{11}S_{22}S_{13}+S_{12}S_{21}S_{13}}{(S_{11}S_{22}-S_{12}S_{21})S_{11}}
\]
\[
=\chi_3+\chi_2\frac{S_{13}S_{21}-S_{11}S_{23}}{D_2}+\chi_1\frac{S_{11}S_{12}S_{13}-S_{12}S_{21}S_{13}-S_{11}S_{22}S_{13}+S_{12}S_{21}S_{13}}{D_2D_1}
\]
\[
\varphi_3=T_{33}\psi_3=\chi_3\sqrt{\frac{D_2}{D_3}}+\chi_2\frac{S_{13}S_{21}-S_{11}S_{23}}{\sqrt{D_2D_3}}+\chi_1\frac{S_{11}S_{12}S_{13}-S_{12}S_{21}S_{13}-S_{11}S_{22}S_{13}+S_{12}S_{21}S_{13}}{D_1\sqrt{D_2D_3}}
\]
so
\[\M{T}=
\begin{pmatrix}
T_{11}&T_{12}&T_{13}\\
0&T_{22}&T_{23}\\
0&0&T_{33}
\end{pmatrix}
\]
\[
T_{11}=\frac{1}{\sqrt{D_1}}\]
\[T_{12}=\frac{-S_{12}}{\sqrt{D_1D_2}}\] \[T_{22}=\sqrt{\frac{D_1}{D_2}}
\]
\[
T_{13}=\frac{S_{11}S_{12}S_{13}-S_{12}S_{21}S_{13}-S_{11}S_{22}S_{13}+S_{12}S_{21}S_{13}}{D_1\sqrt{D_2D_3}}
\]
\[
T_{23}=\frac{S_{13}S_{21}-S_{11}S_{23}}{\sqrt{D_2D_3}}
\]
\[
T_{33}=\sqrt{\frac{D_2}{D_3}}
\]

\section*{5.7 Invariants}

\paragraph{5.7.1}
In matrix representation,
\[
\M{X}\M{P}-\M{P}\M{X}=i\M{I}
\]
\[
\M{U}\M{X}\M{P}\M{U}^{-1}-\M{U}\M{P}\M{X}\M{U}^{-1}=i\M{U}\M{I}\M{U}^{-1}=i\M{I}
\]
\[
\M{U}\M{X}\M{U}^{-1}\M{U}\M{P}\M{U}^{-1}-\M{U}\M{P}\M{U}^{-1}\M{U}\M{X}\M{U}^{-1}=i\M{I}
\]
\[
\M{X}'\M{P}'-\M{P}'\M{X}'=i\M{I}
\]
so $[x,p]=i$ is invariant under unitary transformation.

\paragraph{5.7.2}
\[
\boldsymbol{\sigma}_1'=
\begin{pmatrix}
\cos\theta&\sin\theta\\
-\sin\theta&\cos\theta
\end{pmatrix}
\begin{pmatrix}
0&1\\
1&0
\end{pmatrix}
\begin{pmatrix}
\cos\theta&-\sin\theta\\
\sin\theta&\cos\theta
\end{pmatrix}=
\begin{pmatrix}
\sin2\theta&\cos2\theta\\
\cos2\theta&-\sin2\theta
\end{pmatrix}
\]
\[
\boldsymbol{\sigma}_2'=
\begin{pmatrix}
\cos\theta&\sin\theta\\
-\sin\theta&\cos\theta
\end{pmatrix}
\begin{pmatrix}
0&-i\\
i&0
\end{pmatrix}
\begin{pmatrix}
\cos\theta&-\sin\theta\\
\sin\theta&\cos\theta
\end{pmatrix}=
\begin{pmatrix}
0&-i\\
i&0
\end{pmatrix}
\]
\[
\boldsymbol{\sigma}_3'=
\begin{pmatrix}
\cos\theta&\sin\theta\\
-\sin\theta&\cos\theta
\end{pmatrix}
\begin{pmatrix}
1&0\\
0&-1
\end{pmatrix}
\begin{pmatrix}
\cos\theta&-\sin\theta\\
\sin\theta&\cos\theta
\end{pmatrix}=
\begin{pmatrix}
\cos2\theta&-\sin2\theta\\
-\sin2\theta&-\cos2\theta
\end{pmatrix}
\]
\[
\boldsymbol{\sigma}_1'\boldsymbol{\sigma}_2'-\boldsymbol{\sigma}_2'\boldsymbol{\sigma}_1'=
\begin{pmatrix}
i\cos2\theta&-i\sin2\theta\\
-i\sin2\theta&-i\cos2\theta
\end{pmatrix}-
\begin{pmatrix}
-i\cos2\theta&i\sin2\theta\\
i\sin2\theta&i\cos2\theta
\end{pmatrix}
=\begin{pmatrix}
2i\cos2\theta&-2i\sin2\theta\\
-2i\sin2\theta&-2i\cos2\theta
\end{pmatrix}=2i\boldsymbol{\sigma}_3'
\]
so $[\boldsymbol{\sigma}_1',\boldsymbol{\sigma}_2']=2i\boldsymbol{\sigma}_3'$ is still valid under transformation.

\paragraph{5.7.3}
(a) From Exercise 5.6.2(a),
\[
L_x\varphi_1=0
\]
\[
L_x\varphi_2=-iC(-ze^{-r^2})=i\varphi_3
\]
\[
L_x\varphi_3=-iC(ye^{-r^2})=-i\varphi_2
\]
so in matrix form,
\[
L_x=
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}
\]

(b)
\[
L_x\left[(x+iy)e^{-r^2} \right]=L_x\frac{\varphi_1+i\varphi_2}{C}=\frac{i(i\varphi_3)}{C}=-\frac{\varphi_3}{C}=-ze^{-r^2}
\]

(c)
In matrix form, the above equation becomes
\[
\begin{pmatrix}
0\\0\\-1
\end{pmatrix}=
\begin{pmatrix}
0&0&0\\
0&0&-i\\
0&i&0
\end{pmatrix}
\begin{pmatrix}
1\\i\\0
\end{pmatrix}
\]
after transformation,
\[
\begin{pmatrix}
0\\\frac{i}{\sqrt{2}}\\\frac{-i}{\sqrt{2}}
\end{pmatrix}=
\begin{pmatrix}
0&0&0\\
0&1&0\\
0&0&-1
\end{pmatrix}
\begin{pmatrix}
1\\\frac{i}{\sqrt{2}}\\\frac{i}{\sqrt{2}}
\end{pmatrix}
\]
where the transformed $L_x$ has been obtained in Exercise 5.6.2(b).

(d) From Exercise 5.6.2(c),
\[
\varphi_1'=Cxe^{-r^2}\quad\varphi_2'=C\frac{y+iz}{\sqrt{2}}e^{-r^2}\quad\varphi_3'=C\frac{y-iz}{\sqrt{2}}e^{-r^2}
\]

(e)
\[
\varphi_2'\frac{i}{\sqrt{2}}+\varphi_3'\frac{-i}{\sqrt{2}}=-Cze^{-r^2}
\]
\[
\varphi_1'+\varphi_2'\frac{i}{\sqrt{2}}+\varphi_3'\frac{i}{\sqrt{2}}=C(x+iy)e^{-r^2}
\]
\[
L_x\varphi_1'=L_x(Cxe^{-r^2})=0
\]
\[L_x\varphi_2'=L_x(C\frac{y+iz}{\sqrt{2}}e^{-r^2})=C\frac{y+iz}{\sqrt{2}}e^{-r^2}=\varphi_2'
\]
\[
L_x\varphi_3'=L_x(C\frac{y-iz}{\sqrt{2}}e^{-r^2})
=-C\frac{y-iz}{\sqrt{2}}e^{-r^2}=-\varphi_3'
\]
so the vectors and operators transform correctly by $\M{U}$.

\end{document}

Mathematical Methods for Physicists/Chapter 05/main.tex


\chapter{Topos} \label{topos}
In this section we will examine the categorical interpretation of finite sets.
In particular, we will prove that decidable Kuratowski finite types form a
\(\Pi\)-pretopos.
A lot of the work for this proof has been done already: in
Theorem~\ref{cardinal-kuratowski} we saw that discrete Kuratowski finite types
were equivalent to Cardinally finite types.
We will use the latter definition implementation-wise from now on, as it is
slightly easier to work with: CuTT's transport means we can do this without loss
of generality.

There are two reasons we're interested in the categorical and topos-theoretic
interpretation of finite sets: first, it's an important theoretical grounding
for finite sets, which allows us to understand them in the context of other
set-like constructions.
Secondly, and more practically, the language of a topos is (or in our case the
\(\Pi\)-pretopos) is a common standard framework for doing mathematics
generally.
This makes it a good basis for an API for building QuickCheck-like generators,
for example.
\section{Categories in HoTT}
At first glance, HoTT seems like a perfect setting for category theory: the
univalence axiom identifies isomorphisms with equality, a useful tool for
category theory missing from MLTT.
While this initial impression is broadly true, the construction of categories in
HoTT is unfortunately quite complex and involved (much of the following is a
summary of \citet[chapter 9]{hottbook}).

\todo{references here are tricky, need to disentangle the contributions quite
  precisely}
Much of this section is simply a summary of parts of \citet[chapter
9]{hottbook}.
The formal proofs we provide are part translation of those proofs in that
chapter, part from \cite{iversenFredefoxCat2018}
\cite{huProofrelevantCategoryTheory2020}, and part our own.

First, we need to think about the type of objects and arrows.
We cannot, unfortunately, leave them unrestricted: because of the potential for
higher homotopy in HoTT types \todo{This sentence is a tongue twister}, we have
to restrict the type of arrows to just the sets.
This notion: that of a category with all the usual laws such that arrows are a
set, is called a \emph{precategory}.
\begin{agdalisting}
  \ExecuteMetaData[agda/Categories.tex]{precategory}
\end{agdalisting}
We will use long arrows to refer to morphisms within a category:
\begin{agdalisting}
  \ExecuteMetaData[agda/Categories.tex]{morph-arrow}
\end{agdalisting}

From here, we can define a notion of isomorphisms.
\begin{agdalisting}
  \ExecuteMetaData[agda/Categories.tex]{isomorphism}
\end{agdalisting}
It's a condition on this type which separates the precategories from the
categories: if it satisfies a form of univalence, it the precategory is a full
category.
\begin{agdalisting}
  \ExecuteMetaData[agda/Categories.tex]{cat-univalence}
\end{agdalisting}
\section{The Category of Sets}
Next we'll look at how to construct the category of sets (in the HoTT sense).
Much of this work comes directly from \citet[chapter 10]{hottbook} and
\citet{rijkeSetsHomotopyType2015}.
The formalisation, however, is novel, as far as we know.

The objects are represented by a \(\Sigma\):
\begin{agdalisting}
  \ExecuteMetaData[agda/Snippets/Category.tex]{hset}
\end{agdalisting}
This will be quite similar to our objects for finite sets.

Since sets in HoTT don't form a topos, there are quite a few smaller lemmas we
need to prove to get as close as we can (a \(\Pi W\)-pretopos): we won't include
them here, other than the closure proofs in the following section.
\section{Closure}
The two most involved proofs for showing that discrete Kuratowski sets form a
\(\Pi\)-pretopos are those proofs that show closure under \(\Pi\) and
\(\Sigma\).
We will describe them here.
\subsection{Closure of the Ordered Predicates}
First, we will show that split enumerability (and, by extension, manifest
enumerability) are closed under \(\Pi\) and \(\Sigma\).
This is the first stepping stone on our way to prove that cardinal finiteness is
closed under the same.

Practically speaking, these proofs also open up a wide number of other closure
proofs to us.
By proving that dependent products and sums are finite, we get the non-dependent
cases for free.

\begin{lemma} \label{split-enum-sigma} \todo{Convert to Agda}
  Split enumerability is closed under \(\Sigma\).
  \begin{agdalisting}
    \ExecuteMetaData[agda/Cardinality/Finite/SplitEnumerable.tex]{split-enum-sigma}
  \end{agdalisting}
\end{lemma}
\begin{proof}
  Our task is to construct the two components of the output pair: the support
  list, and the cover proof.
  We'll start with the support list: this is constructed by taking the Cartesian
  product of the input support lists.
  \begin{agdalisting}
    \ExecuteMetaData[agda/Cardinality/Finite/SplitEnumerable.tex]{sup-sigma}
  \end{agdalisting}
  We use do notation here because we're working the list monad: this applies the
  latter function (\(ys\)) to every element of the list \(xs\), and concatenates
  the results.

  To show that this does indeed cover every element of the target type is a
  little intricate, but not necessarily difficult. \todo{Should a proof of this
    be included?}
\end{proof}

Next we'll look at closure under \(\Pi\).
In MLTT, this is of course not provable: since all of the finiteness predicates
we have seen so far imply decidable equality, and since we don't have any kind
of decidable equality on functions in MLTT, we know that we won't be able to
show that any kind of function is finite; even one like \(\AgdaDatatype{Bool}
\rightarrow \AgdaDatatype{Bool}\).

CuTT is not so restricted.
Since we have things like function extensionality and transport, we can indeed
prove the finiteness of function types.
Our proof here makes use directly of the univalence axiom, and makes use
furthermore of all the previous closure proofs.
\begin{theorem} \label{split-enum-pi}
  Split enumerability is closed under dependent functions
  (\(\Pi\)-types).
  \begin{agdalisting}
    \ExecuteMetaData[agda/Cardinality/Finite/ManifestBishop.tex]{pi-clos}
  \end{agdalisting}
\end{theorem}
\begin{proof}
  Let \(A\) be a split enumerable type, and \(U\) be a type family from \(A\),
  which is split enumerable over all points of \(A\).

  As \(A\) is split enumerable, we know that it is also manifestly Bishop finite
  (lemma~\ref{split-enum-to-manifest-bishop}), and consequently we know \(A
  \simeq \AgdaDatatype{Fin}\;n\), for some \(n\) (lemma~\ref{bishop-equiv}).
  We can therefore replace all occurrences of \(A\) with \(\AgdaDatatype{Fin}\;n\),
  changing our goal to:
  \begin{equation}
    \frac{
      \AgdaDatatype{\ensuremath{\mathcal{E}!}}\;(\AgdaDatatype{Fin}\;n) \; \; \; \left((x : \AgdaDatatype{Fin}\;n) \rightarrow \AgdaDatatype{\ensuremath{\mathcal{E}!}}\;\left( U\;x \right)\right)
    }{
      \AgdaDatatype{\ensuremath{\mathcal{E}!}}\left((x : \AgdaDatatype{Fin}\;n) \rightarrow U\;x\right)
    }
  \end{equation}

  We then define the type of \(n\)-tuples over some type family.
  \begin{agdalisting}
    \ExecuteMetaData[agda/Data/Tuple/UniverseMonomorphic.tex]{tuple-def}
  \end{agdalisting}
  We can show that this type is equivalent to functions (proven in our formalisation):
  \begin{agdalisting}
    \ExecuteMetaData[agda/Data/Tuple/UniverseMonomorphic.tex]{tuple-iso}
  \end{agdalisting}
  And therefore we can simplify again our goal to the following:
  \begin{equation}
    \frac{
      \AgdaDatatype{\ensuremath{\mathcal{E}!}}\;(\AgdaDatatype{Fin}\;n) \; \; \; ((x : \AgdaDatatype{Fin}\;n) \rightarrow \AgdaDatatype{\ensuremath{\mathcal{E}!}}\left( U\;x \right))
    }{
      \AgdaFunction{\ensuremath{\mathcal{E}!}}\;\left(\AgdaFunction{Tuple}\;n\;U\right)
    }
  \end{equation}

  We can prove this goal by showing that \(\AgdaFunction{Tuple}\;n\;U\) is split
  enumerable: it is made up of finitely many products of points of \(U\), which
  are themselves split enumerable, and \agdatop, which is also split enumerable.
  Lemma~\ref{split-enum-sigma} shows us that the product of finitely many split
  enumerable types is itself split enumerable, proving our goal.
\end{proof}
\subsection{Closure on Cardinal Finiteness}
Since we don't have a function of type \(\mathcal{C}(A) \rightarrow
\AgdaDatatype{\ensuremath{\mathcal{B}}}\;A\), closure proofs on \(\AgdaDatatype{\ensuremath{\mathcal{B}}}\) do not transfer over to
\(\mathcal{C}\) trivially (unlike with \(\AgdaDatatype{\ensuremath{\mathcal{E}!}}\) and \(\AgdaDatatype{\ensuremath{\mathcal{B}}}\)).
The cases for \(\bot\), \(\top\), and \(\AgdaDatatype{Bool}\) are simple to adapt: we
can just propositionally truncate their Bishop finiteness proof.

Non-dependent operators like \(\times\), \(\uplus\), and \(\rightarrow\) are
also relatively straightforward: since \(\AgdaDatatype{\ensuremath{\lVert\_\rVert}}\) forms a monad, we
can apply \(n\)-ary functions to values inside it, combining them together.
\begin{agdalisting}
  \ExecuteMetaData[agda/Cardinality/Finite/ManifestBishop.tex]{times-clos-sig}
\end{agdalisting}
Into a truncated context:
\begin{agdalisting}
  \ExecuteMetaData[agda/Cardinality/Finite/Cardinal.tex]{times-clos-impl}
\end{agdalisting}

Unfortunately, for the dependent type formers like \(\Sigma\) and \(\Pi\), the
same trick does not work.
We have closure proofs like:
\begin{equation}
  \frac{
    \AgdaDatatype{\ensuremath{\mathcal{B}}}\;A \; \; \; ((x : A) \rightarrow \AgdaDatatype{\ensuremath{\mathcal{B}}}\;(U\;x))
  }{
    \AgdaDatatype{\ensuremath{\mathcal{B}}}\;((x : A) \rightarrow U\;x)
  }
\end{equation}
If we apply the monadic truncation trick we can derive closure proofs like the
following:
\begin{equation}
  \frac{
    \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;A \;\AgdaDatatype{\ensuremath{\rVert}} \; \; \; \AgdaDatatype{\ensuremath{\lVert}}\; ((x : A) \rightarrow \AgdaDatatype{\ensuremath{\mathcal{B}}}\;(U\;x)) \;\AgdaDatatype{\ensuremath{\rVert}}
  }{
    \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;((x : A) \rightarrow U\;x) \;\AgdaDatatype{\ensuremath{\rVert}}
  }
\end{equation}
However our \emph{desired} closure proof is the following:
\begin{equation}
  \frac{
    \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;A \;\AgdaDatatype{\ensuremath{\rVert}} \; \; \; ((x : A) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;(U\;x) \;\AgdaDatatype{\ensuremath{\rVert}})
  }{
    \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;((x : A) \rightarrow U\;x) \;\AgdaDatatype{\ensuremath{\rVert}}
  }
\end{equation}
They don't match!

The solution would be to find a function of the following type:
\begin{equation}
  ((x : A) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaDatatype{\ensuremath{\mathcal{B}}}\;(U\;x) \;\AgdaDatatype{\ensuremath{\rVert}}) \rightarrow
  \AgdaDatatype{\ensuremath{\lVert}}\; (x : A) \rightarrow \AgdaDatatype{\ensuremath{\mathcal{B}}}\;(U\;x) \;\AgdaDatatype{\ensuremath{\rVert}}
\end{equation}
However we might be disheartened at realising that this is a required goal: the
above equation is \emph{extremely} similar to the axiom of choice!
\begin{definition}[Axiom of Choice]
  In HoTT, the axiom of choice is commonly defined as follows \cite[lemma
  3.8.2]{hottbook}.
  For any set \(A\), and a type family \(U\) which is a set at all the points
  of \(A\), the following function exists:
  \begin{equation}
    \left( (x : A) \rightarrow  \AgdaDatatype{\ensuremath{\lVert}}\; U(x) \;\AgdaDatatype{\ensuremath{\rVert}} \right) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; (x : A) \rightarrow U(x) \;\AgdaDatatype{\ensuremath{\rVert}}
  \end{equation}
\end{definition}
Luckily the axiom of choice \emph{does} hold for cardinally finite types,
allowing us to prove the following:
\begin{lemma}
  \begin{equation}
    \agdacal{C}\;A \rightarrow ((x : A) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; U(x) \;\AgdaDatatype{\ensuremath{\rVert}}) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; (x : A) \rightarrow U(x) \;\AgdaDatatype{\ensuremath{\rVert}}
  \end{equation}
\end{lemma}
\begin{proof}
  Let \(A\) be a cardinally finite type, \(U\) be a type family on \(A\), and
  \(f\) be a dependent function of type \(\Pi(x : A) , \AgdaDatatype{\ensuremath{\lVert}}\; U(x) \;\AgdaDatatype{\ensuremath{\rVert}}\).

  First, since our goal is itself propositionally truncated, we have access to
  values under truncations: put another way, in the context of proving our goal,
  we can rely on the fact that \(A\) is manifestly Bishop finite.
  Using the same technique as we did in lemma~\ref{split-enum-pi}, we can switch
  from working with dependent functions from \(A\) to \(n\)-tuples, where \(n\)
  is the cardinality of \(A\).
  This changes our goal to the following:
  \begin{equation}
    \AgdaFunction{Tuple}\;n\;(\AgdaDatatype{\ensuremath{\lVert\_\rVert}}\;\AgdaFunction{\ensuremath{\circ}}\; U) \rightarrow \AgdaDatatype{\ensuremath{\lVert}}\; \AgdaFunction{Tuple}\;n\;U\;\AgdaDatatype{\ensuremath{\rVert}}
  \end{equation}
  Since \(\AgdaDatatype{\ensuremath{\lVert\_ \rVert}}\) is closed under finite products, this function
  exists (in fact, using the fact that \(\AgdaDatatype{\ensuremath{\lVert\_ \rVert}}\) forms a monad, we
  can recognise this function as \verb+sequenceA+ from the \verb+Traversable+
  class in Haskell).
\end{proof}

This gets us all of the necessary closure proofs on \(\mathcal{C}\).
\section{The Absence of the Subobject Classifier}
\begin{agdalisting} \label{filter-subobject}
  \ExecuteMetaData[agda/Cardinality/Finite/SplitEnumerable.tex]{subobject}
\end{agdalisting}

\section{Closure}
For the first three closure proofs, we only consider split enumerability:
as it is the strongest of the finiteness predicates, we can derive the other
closure proofs from it.

\section{The Category of Finite Sets}
HoTT and CuTT seem to be especially suitable settings for formalisations of
category theory.
The univalence axiom in particular allows us to treat categorical isomorphisms
as equalities, saving us from the dreaded ``setoid hell''.

We follow \cite[chapter 9]{hottbook} in its treatment of
categories in HoTT, and in its proof that sets do indeed form a category.
We will first briefly go through the construction of the category
\(\mathit{Set}\), as it differs slightly from the usual method in type theory.

First, the type of objects and arrows:
\begin{alignat}{3}
  &\text{Obj}_\mathit{Set}      &&\coloneqq \Sigma(x : \mathbf{Type}) , \text{isSet}(x) \\
  &\text{Hom}_\mathit{Set}(x , y) &&\coloneqq  \text{fst}(x) \rightarrow \text{fst}(y)
\end{alignat}
As the type of objects makes clear, we have already departed slightly from the
simpler \(\text{Obj}_\mathit{Set} \coloneqq \mathbf{Type}\) way of doing things:
of course we have to, as HoTT allows non-set types.
Furthermore, after proving the usual associativity and identity laws for
composition (which are definitionally true in this case), we must further show
\(\text{isSet}(\text{Hom}_\mathit{Set}(x,y))\); even then we only have a
precategory.

To show that \(\mathit{Set}\) is a category, we must show that categorical
isomorphisms are equivalent to equivalences.
In a sense, we must give a univalence rule for the category we are working in.

We have provided formal proofs that \(\mathit{Set}\) does indeed form a
category, and the following:
\begin{theorem}[The Category of Finite Sets]
  Finite sets form a category in HoTT when defined like so:
  \begin{equation}
    \begin{alignedat}{3}
      &\text{Obj}_\mathit{FinSet}      &&\coloneqq \Sigma(x : \mathbf{Type}) , \mathcal{C}(x) \\
      &\text{Hom}_\mathit{FinSet}(x , y) &&\coloneqq  \text{fst}(x) \rightarrow \text{fst}(y)
    \end{alignedat}
  \end{equation}
\end{theorem}
\section{The \(\Pi\)-pretopos of Finite Sets}
For this proof, we follow again the proof that \(\mathit{Set}\) forms a \(\Pi
W\)-pretopos from \cite[chapter 10]{hottbook} and
\cite{rijkeSetsHomotopyType2015}.
The difference here is that clearly we do not have access to \(W\)-types, as
they would permit infinitary structures.

We first must show that \(\mathit{Set}\) has an initial object and finite,
disjoint sums, which are stable under pullback.
We also must show that \(\mathit{Set}\) is a regular category with effective
quotients.
We now have a pretopos: the presence of \(\Pi\) types make it a
\(\Pi\)-pretopos.

We have proven the above statements for both \(\mathit{Set}\) and
\(\mathit{FinSet}\).
As far as we know, this is the first formalisation of either.
\begin{theorem} \label{finite-topos}
  The category of finite sets, \(\mathit{FinSet}\), forms a \(\Pi\)-pretopos.
\end{theorem}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../paper"
%%% End:


\ifpdf
    \graphicspath{{Chapter3/Figs/Vector/}{Chapter3/Figs/PDF/}{Chapter3/Figs/}}
\else
    \graphicspath{{Chapter3/Figs/Raster/}{Chapter3/Figs/}}
\fi

\chapter{Linear-Time Gaussian Processes on Hyperspheres}
\label{chapter:vish}

Stationary kernels, i.e. translation invariant covariances, are ubiquitous in machine learning when the input space is Euclidean. When working on the hypersphere, their spherical counterpart are dot-product kernels, which are invariant to rotations. They are the main object studied in this chapter. In \cref{sec:rkhs-dotproduct-kernels}, we first show how we can construct the Reproducing Kernel Hilbert Space (RKHS) of dot-product kernel on the hypersphere. \Cref{sec:vish} uses the RKHS to construct an efficient variational interdomain inducing variable for Gaussian processes on the hypersphere. We will then show how to expand the GP onto the complete domain $\Reals^d$ and conclude with a series of experiments, showing the speed and accuracy of the proposed approach.

\section{Mercer Representation of Dot-Product Kernels}
\label{sec:rkhs-dotproduct-kernels}

Consider the unit sphere in $\Reals^d$  as the input domain
\begin{equation}
    \mathcal{X} = \dsphere = \{x \in \Reals^d: \norm{x}_2 = 1\}
\end{equation}
and $\nu$ to be the Lebesgue measure on $\dsphere$, so that
\begin{equation}
    \darea = \int_{\dsphere} \calcd{\nu(x)} = \frac{2 \pi ^ {d/2}}{\Gamma(d/2)}.
\end{equation}
is the surface area of $\dsphere$. We adopt the usual $L_2$ inner product for functions $f: \dsphere \rightarrow \Reals$ and $g: \dsphere \rightarrow \Reals$ restricted to the sphere
\begin{equation}
     \langle f, g\rangle_{L_{2}(\dsphere)} = \frac{1}{\darea} \int_{\dsphere} f(x)\,g(x) \, \calcd{\nu(x)}.
\end{equation}

We define a dot-product kernel, also known as a zonal kernel (we will use both terms interchangeably), as a p.d. kernel of the form
\begin{equation}
    k(x, x') = \kappa(x\transpose x'),
\end{equation}
where $\kappa: [-1, 1] \rightarrow \Reals$ is a continuous function and referred to as the \emph{shape function}. In other words, dot-product kernels only depend on the distance on the great circle between two inputs, rather than their location. They are the counterpart of stationary kernels $k(x, x') = \kappa(x - x')$, who are functions of the difference only. For instance, where stationary kernels are translation invariant, dot-product kernel are rotationally invariant.

We can associate a kernel operator $\mathcal{K}$ to a zonal kernel, as explained in \cref{section:theory:spectral-formulation}, which exhibits the form
\begin{equation}
    \label{eq:kernel-operator-zonal}
  \mathcal{K} f = \int_{\dsphere} \kappa(x\transpose\cdot) f(x) \calcd{\nu(x)}.
\end{equation}
To construct a Mercer representation of a zonal kernel's RKHS we require the eigensystem of the kernel operator $\mathcal{K}$, or, equivalently, the set of eigenfunctions $\{\phi_n\}$ and associated eigenvalues $\{\lambda_n\}$ for which
\begin{equation}
    \mathcal{K} \phi_n = \lambda_n \phi_n\qquad\text{and}\qquad \langle \phi_n, \phi_m \rangle_{L_2(\dsphere)} = \delta_{nm}.
\end{equation}

To obtain the eigensystem of $\mathcal{K}$, we will show that $\mathcal{K}$ commutes with the Laplace-Beltrami operator $\LaplaceBeltrami$, henceforth referred to as simply the Laplacian or the Laplace operator. We want to remind the reader that commuting operators share the same eigenfunctions, but not necessarily the same eigenvalues. Therefore, to find the eigenfunctions of the kernel operator $\mathcal{K}$, it suffice to find the eigenfunctions of the Laplacian. Fortunately, the diagonalisation of the Laplace operator on $\dsphere$ is a well-studied problem. In the following paragraph we first proof the commutativity of the operators before covering the definition of the RKHS.

\paragraph{Commutativity of $\mathcal{K}$ and $\LaplaceBeltrami$.}
We now proof that the Laplacian and the kernel operator of zonal kernels commute. For this, we first show that for zonal kernels and $x,s \in \dsphere$ the following property holds
\begin{align}
    \LaplaceBeltrami_x k(x, x')
     &= \nabla_x \cdot \nabla_x \kappa(x\transpose x') && \text{Definition $\LaplaceBeltrami$ and zonal kernels}\\
     &= \nabla_x \cdot (x' \kappa'(x\transpose x'))  && \text{Chainrule} \\
     &= \norm{x'}^2 \kappa''(x\transpose x') = \kappa''(x \transpose x') && \text{Because } \norm{x'} = 1.
\end{align}
Similarly, for $\LaplaceBeltrami_s k(x, x')$ we obtain $k''(x \transpose x')$, and as a result
\begin{equation}
    \label{eq:LaplaceBeltrami-x-s}
\LaplaceBeltrami_x k(x, x') = \LaplaceBeltrami_{x'} k(x, x').
\end{equation}
Relying on integration by parts and the previous result, we obtain
\begin{align}
    \mathcal{K} \left[\LaplaceBeltrami f\right] &= \int_{\dsphere} k(x, x') \left[\LaplaceBeltrami_x f(x)\right] \calcd{\nu(x)} && \text{Definition $\mathcal{K}$, \cref{eq:kernel-operator-zonal}}\\
    &= \int_{\dsphere} f(x) \LaplaceBeltrami_x k(x, x')  \calcd{\nu(x)} && 2\,\times\,\text{Integration by parts}\\
    &= \int_{\dsphere} f(x) \LaplaceBeltrami_{x'} k(x, x')  \calcd{\nu(x)}  && \text{\cref{eq:LaplaceBeltrami-x-s}} \\
    &= \LaplaceBeltrami \left[ \mathcal{K} f \right]
\end{align}
which shows that the two operators commute and in turn implies that they share the same eigenfunctions. This result is of particular relevance to us since there is a huge body of literature on diagonalisation of the Laplace-Beltrami operator on $\dsphere$, and that it is well known that its eigenfunctions are given by the spherical harmonics. The spherical harmonics $\phi_{n,j}$ form an orthonormal basis for the square-integrable functions on the hypersphere. They are indexed with a level $n$ and an index within each level $j \in \{1, \dots, \dnumharmonicsforlevel\}$. See \cref{appendix:spherical-harmonics} for a comprehensive introduction to spherical harmonics, as well as a practical algorithm to compute them for relatively large $d$. \Cref{fig:harmonics} shows the first 4 levels of spherical harmonics in $\Reals^3$.

The above reasoning about the commutativity of the Laplace-Beltrami and kernel operator and the diagonalisation of the Laplace-Beltrami operator by the spherical harmonics, can be summarised by the following theorem:
\begin{theorem}[Mercer decomposition]
    \label{theorem:mercer-zonal}
Any zonal kernel $k(x, x') = \kappa(x\transpose x')$ on the hypersphere can be decomposed as
\begin{equation}
\label{eq:kernel-form}
    k(x, x') = \sum_{n=0}^{\infty} \sum_{j=1}^{\dnumharmonicsforlevel} \lambda_{n} \phi_{n,j}(x) \phi_{n,j}(x'),
\end{equation}
where $x,x' \in \dsphere$ and $\lambda_{n}$ are positive coefficients, $\phi_{n,j}$ denote the elements of the spherical harmonic basis in $\dsphere$, and $\dnumharmonicsforlevel$ corresponds to the number of spherical harmonics for a given level $n$. As a result of the Funk-Hecke theorem, the associated eigenvalues only depending on the level $n$
\begin{equation}
    \label{eq:compute-eigenvalues}
        \lambda_{n} = \frac{\omega_{d}}{C_n^{(\alpha)}(1)} \int_{-1}^1 \kappa(t)\,C_n^{(\alpha)}(t)\,(1 - t^2)^{\frac{d-3}{2}} \calcd{t},
\end{equation}
where $C_n^{(\alpha)}$ is the Gegenbauer polynomial of degree $n$, $\alpha = \frac{d-2}{2}$, $\omega_d$ is a constant that depends on the surface area of the hypersphere. Analytic expressions are given in \cref{appendix:spherical-harmonics}.
\end{theorem}
Although it is typically stated without a proof, this theorem is already known in some communities (see \citet{wendland2005} for a functional analysis exposition, or \citet{peacock1999cosmological} for its use in cosmology). Given the Mercer decomposition, we can equivalently, define the associated RKHS as
\begin{equation}
    \label{eq:rkhs-sphere}
    \rkhs = \left\{
    f =
    \sum_{n=0}^\infty \sum_{j=1}^{\dnumharmonicsforlevel} {f}_{n,j} \sh_{n, j}:
    \norm{f}_{\rkhs} < \infty
    \right\},
    \quad\text{where}\quad
    \langle g, h \rangle_{\rkhs} =
    \sum_{n,j}
            \frac{{g}_{n, j} {h}_{n, j}}{\lambda_{n}}
\end{equation}
is the \emph{reproducing} inner product between two functions $g,h \in \rkhs$. The reproducing property of the RKHS implies that for $f \in \rkhs$: $f(x) = \langle f, k(x, \cdot) \rangle_\rkhs$, which will enable the construction of spherical harmonic inducing features of discussing in the following section.

%%%%%%%%%%%%%%%%%%%%%%
\section{Variational Spherical Harmonic Gaussian Processes}
\label{sec:vish}

We can now build on the results from the previous section, combined with interdomain inducing variables (see \cref{section:interdomain-inducing-variables}), to propose a novel sparse variational GP model. Specifically, we are using interdomain inducing variables that are defined through the zonal kernel's RKHS inner product. As a result we will obtain expressive features $\kux$ that exhibit non-local influence, and efficient inducing variables that induce diagonal structure in $\Kuu$. We achieve this by defining the inducing variables $u_m$ to be the inner product between the GP and spherical harmonics:\footnote{Note that in the context of inducing variables, we switch to a single integer $m$ to index the spherical harmonics and order them first by increasing level $n$, and then by increasing $j$ within a level.}
\begin{equation}
  u_m = \langle f, \sh_m\rangle_{\rkhs}\quad\text{for}\quad m \in \{1, \ldots, M\}.
\end{equation}
To leverage these new inducing variables we need to compute two quantities: 1) the covariance between $u_m$ and $f$ for $\kux$, and 2) the covariance between the inducing variables themselves for the $\Kuu$ matrix. Firstly, we compute the covariance of the inducing variables and the GP
\begin{align}
   \left[\kux \right]_m
    &= \ExpSymb[f(\cdot)\,u_m] && \text{Definition covariance} \\
    &= \ExpSymb[f(\cdot)\,\langle f, \sh_m\rangle_{\rkhs}] && \text{Definition $u_m$} \\
    &= \langle k(\cdot, \cdot), \sh_m \rangle_{\rkhs}   && {k(\cdot, \cdot) = \ExpSymb[f(\cdot) f]} \\
    &= \phi_{m}(\cdot) && \text{Reproducing property.}
\end{align}
where we relied on the linearity of the expectation and the inner product and the reproducing property of $\rkhs$. Interestingly, using this construction we obtain the RKHS basis functions $\{\phi_m\}$ as the features for the sparse approximation. Secondly, the covariance between the inducing variables is given by
\begin{equation}
    \left[\Kuu \right]_{m, m'} = \ExpSymb \left[u_m\,u_{m'} \right]
    % = \ExpSymb \left[ \langle f, Y_{m} \rangle  \langle f, Y_{m'} \rangle  \right]
    = \langle \sh_{m}, \sh_{m'} \rangle_{\rkhs}
    = {\delta_{mm'} \over \lambda_m},
\end{equation}
where $\delta_{mm'}$ is the Kronecker delta. Crucially, this means that $\Kuu$ is a diagonal matrix $\textrm{diag}(\lambda_1, \lambda_2, \ldots, \lambda_M)^{-1}$ containing the inverse eigenvalues on its diagonal. Finally, substituting $\Kuu$ and $\kux$ into the sparse variational approximation (\cref{eq:qf}) leads to
\begin{equation}
\begin{aligned}
    q(f) =
    \GP\Big(\tilde{\bm{\phi}}^\top(x)\,\vm;\ k(x, x') + &\tilde{\bm{\phi}}^\top(x)(\MS - \Kuu)\tilde{\bm{\phi}}(\vx') \Big),\\
    &\text{with}\quad\tilde{\bm{\phi}}(x) = [\lambda_m \phi_m(x)]_{m=1}^M \in \Reals^M.
\end{aligned}
\end{equation}
Through this construction we created a SVGP model with 1) spherical harmonic basis functions, and 2) a diagonal $\Kuu$ matrix, which means that we do not incur the cubic cost of matrix inversion in the evaluation of the model or the variational lower bound. The sperical harmonic basis functions have as advantage that they have global support, so that even a few of them are able to capture the rough shape of the function. This is similar to the Fourier series approximation, in which the first few terms are already able to roughly approximate the function. In the experiments we will discuss this phenomenon in more detail.

Note that while the inducing variables $\{u_m\}$ are independent under the prior (i.e. $p(\vu) = \NormDist{\bm{0}, \Kuu}$ with $\Kuu$ a diagonal matrix), that does not imply that the posterior over the inducing variables is diagonal as well. As a result, we still parameterise $q(\vu) = \NormDist{\vm, \MS}$, where $\MS$ is a full-rank matrix which captures all the pairwise covariances.

\section{Homogeneous Extension to $\Reals^d$}

\begin{figure}[t]
\centering
\begin{minipage}{.48\textwidth}
  \centering
  \includegraphics[width=\linewidth]{harmonics}
  \captionof{figure}{Spherical Harmonics on $\sphere^2$}
  \label{fig:harmonics}
\end{minipage}\hfill
\begin{minipage}{.48\textwidth}
  \centering
  \includegraphics[width=\linewidth,trim=1.5cm 1.6cm 1.cm 1.55cm,clip=true]{mapping}
  \captionof{figure}{Example Homogeneous Extension}
  \label{fig:mapping}
\end{minipage}
\end{figure}

So far, the domain of interest has been the hypersphere $\dsphere$ . However, most regression problems in machine learning have inputs defined on the complete Euclidean space. We can address this problem as follows. Consider $x \in \Reals^{d-1}$ to be the input of the dataset and $y \in \Reals$ the corresponding output.
\begin{enumerate}
    \item We start by concatenating the datapoint $x \in \Reals^{d-1}$ with a bias term $b \in \Reals$ to obtain $x = [x, b] \in \Reals^d$. This operation corresponds to embedding the $(d\!-\!1)$-dimensional datapoint on a plane in $\Reals^d$ where $x_d = b$.
    \item Subsequently, the data is linearily projected to the hypersphere through normalisation: $x = {x \over \norm{x}} \in \dsphere$ and $y = {y \over \norm{x}}$.
    \item Based on the projected data we learn $f \sim \GP$ on $\dsphere$ using the approach outlined in the previous section.
    \item Finally, the function on the sphere can be extended to the whole domain by a homogeneous extension thereof: $g: \Reals^d \rightarrow \Reals$ with $g(x) = \norm{x} f({x \over \norm{x}})$. In other words, by linearily extrapolating the values on the hypersphere we obtain the values of the function in the data plane.
\end{enumerate}
This procedure is explained in \cref{fig:mapping}, which gives an illustration of the mapping between a 2D dataset (grey dots) embedded into a 3D space and its projection (orange dots) onto the unit half-circle using a linear mapping.

\subsection{First-order Arc Cosine kernel}
\label{sec:arccosine}
Although this setup may seem very arbitrary, it is inspired by the important works on the limits of neural networks as Gaussian processes. The first order Arc Cosine kernel \citep{cho2009kernel} is one of the most well-known examples, and mimics the computation of infinitely wide fully connected layers with ReLU activations. Let $\sigma(t) = \max(0, t)$, then the covariance between function values of $f(x) = \sigma(w\transpose x)$ for $w \sim \NormDist{0, d^{-1/2} \Eye_d}$ and $w \in \Reals^d$ is given by
\begin{equation}
\label{eq:arccosine}
    k(x, x') = \Exp{w}{\sigma(w\transpose x)\, \sigma(w\transpose x')} = \underbrace{\norm{x} \norm{x'}}_{\text{radial}}\ \underbrace{\frac{1}{\pi}\big( \sqrt{1 - t^2} + t\, (\pi - \arccos t) \big)}_{\text{angular (shape function) } \kappa(t)},
\end{equation}
where $t = \frac{\vx^\top \vx'}{\norm{\vx}\norm{\vx'}}$. Indeed, we observe how the first order Arc Cosine kernel factorises in a radial and an angular component. The radial component simply linearly extrapolates the values on the hypersphere, while the angular component encodes the covariance on the hypersphere and is only a function of the geodesic distance. This factorisation exactly matches our setup. We can generalise the zonal kernels of the previous section and their corresponding RKHS to the complete domain $\Reals^d$ using the following definition
\begin{equation}
    k(x, x') = \norm{x} \norm{x'} \kappa\left( \frac{x^\top x'}{\norm{x}\norm{x'}} \right), % \quad \text{and} \quad f(x) = \norm{x} f\left({x \over \norm{x}}\right).
\end{equation}
which leads to an RKHS consisting of functions defined on $\Reals^d$ of the form $g(x) = \norm{x}\,f(\frac{x}{\norm{x}})$, where $f \in \rkhs$ (from \cref{eq:rkhs-sphere}) is defined on the unit hypersphere but fully determines the function on $\Reals^d$. Straightforwardly we can extend the Mercer decomposition of \cref{theorem:mercer-zonal} to
\begin{equation}
    k(x, x') = \sum_{n=0}^{\infty} \sum_{j=1}^{\dnumharmonicsforlevel} \lambda_{n} \norm{x} \phi_{n,j}(x) \norm{x'} \phi_{n,j}(x'),
\end{equation}
where the eigenvalues $\lambda_n$ and reproducing inner-product remain unchanged and are given by \cref{eq:compute-eigenvalues} and \cref{eq:rkhs-sphere}, respectively.

\section{Relation to Variational Fourier Features}
\label{sec:vff-vs-vish}

The most closely related method to the presented method, which going forward we are going to refer to as Variational Inducing Spherical Harmonics (VISH), is the Variational Fourier Feature (VFF) approach proposed by \citet{hensman2017variational}. VFF is an interdomain method where the inducing variables are given by a Mat\'ern RKHS inner product between the GP and elements of the Fourier basis:
\begin{equation}
u_m = \langle f, \psi_m \rangle_\rkhs,
\end{equation}
where $\psi_0 = 1$, $\psi_{2m}=\cos(m x)$ and $\psi_{2m+1}=\sin(m x)$ if the input space is $[0, 2 \pi]$. This leads to
\begin{equation}
    \Kuu = \left[\langle \psi_i , \psi_{j} \rangle_\rkhs^{} \right]_{i, j = 0}^{M-1}\qquad\text{and}\qquad\kux = \left[ \psi_i(x)\right]_{i = 0}^{M-1}\, .
\end{equation}
This results in several advantages. First, the features $\kux$ are exactly the elements of the Fourier basis, which are independent of the kernel parameters and can be precomputed. Second, the matrix $\Kuu$ is the sum of a diagonal matrix plus low rank matrices. This structure can be used to drastically reduce the computational complexity. Finally, the variance of the inducing variables typically decays quickly with increasing frequencies, which means that by selecting the first $M$ elements of the Fourier basis we pick the features that carry the most signal.

The main flaw of VFF comes from the way it extends to multidimensional input spaces. The approach in~\citet{hensman2017variational} for getting a set of $d$-dimensional inducing functions consists of taking the outer product of $d$ univariate basis and to consider separable kernels so that the elements of $\Kuu$ are given by the product of the inner products in each dimension. For example, in dimension 2, a set of $M^2$ inducing functions is given by $\{(x_1, x_2) \mapsto \psi_i(x_1)\psi_j(x_2)\}_{0 \leq i, j \leq M-1}$, and entries on $\Kuu$ are $\langle \psi_i\psi_j, \psi_k\psi_l \rangle_{\mathcal{H}_{k_1 k_2}^{}}^{} = \langle \psi_i, \psi_k \rangle_{\mathcal{H}_{k_1}^{}}^{} \langle \psi_j, \psi_l \rangle_{\mathcal{H}_{k_2}^{}}^{}$. This construction scales poorly with the dimension: for example choosing a univariate basis as simple as $\{1, \cos, \sin\}$ for an eight-dimensional problem already results in more that 6,500 inducing functions. Additionally, this construction is very inefficient in terms of captured variance, as we illustrate in  \Cref{fig:decay-vff} for a 2D input space. The figure shows that the prior variance associated with the inducing function $\psi_i(x_1)\psi_j(x_2)$ vanishes quickly when both $i$ and $j$ increase. This means that most of the inducing functions on which the variational posterior is built are irrelevant, whereas some important ones such as $\psi_i(x_1)\psi_0(x_2)$ or $\psi_0(x_1)\psi_j(x_2)$ for $i, j \geq \sqrt{M}$ are important but ignored. Although we used a 2D example to illustrate this poor behaviour, it is important to bear in mind that the issue gets exacerbated for higher dimensional input spaces.

On the contrary for VISH, given that we ordered the spherical harmonic by increasing level $n$, choosing the first $M$ elements means we will select first inducing functions with low angular frequency (see \cref{fig:harmonics}). Provided that the kernel spectral density is a decreasing function (this will be true for classic covariances, but not for quasi-periodic ones), this means that the selected inducing variables correspond to the ones carrying the most signal according to the prior. In other words, the decomposition of the kernel can be compared to an `infinite dimensional principal component analysis', and our choice of the inducing function is optimal since we pick the ones with the largest variance. This is illustrated in \cref{fig:decay-vish}, which shows the analogue of \cref{fig:decay-vff} for spherical harmonic inducing functions.

\begin{figure}[t!]
  \centering
\begin{subfigure}{0.49\textwidth}
  \includegraphics[width=\textwidth]{VFF_v2}
  \caption{VFF}
  \label{fig:decay-vff}
\end{subfigure}\hfil % <-- added
\begin{subfigure}{0.49\textwidth}
  \includegraphics[width=\textwidth]{VISH_v2}
  \caption{VISH}
  \label{fig:decay-vish}
\end{subfigure}\hfil % <-- added
\caption{Illustration of the variance of the prior and the inducing variables when using VFF or VISH with a two dimensional input space. Each square corresponds to an inducing function $\psi_{i,j}$ (VFF) or $\phi_{n,j}$ (VISH) and the area of each square is proportional to the variance of the associated inducing variable. For a given number of inducing variables as highlighted with the blue line, VFF does not select the inducing variables that contribute the most to the prior. The selection in VISH on the other hand is optimal.}
\label{fig:decay-vff-vs-vish}
\end{figure}

% especially the \emph{arc-sine} kernel \citep{williams1998computation} and the \emph{arc-cosine} kernels \citep{cho2009kernel}. An arc-cosine kernel corresponds to the infinitely-wide limit of a single-layer ReLU-activated network with Gaussian weights. Let $\vx, \vx' \in \Reals^{d}$, such that $x_d = x'_d = b$, be an augmented input vector whose last entry corresponds to the bias. Then the arc-cosine kernel can be written as
% \begin{multline*}
%     k_{ac}(\vx, \vx') = \norm{\vx}\,\norm{\vx'}\,\underbrace{\frac{1}{\pi} \big( \sin \theta + (\pi - \theta) \cos \theta \big)}_{=J(\theta)}, \\
%     \text{where}\quad\theta = \cos^{-1}\left(\frac{\vx\transpose\vx'}{\norm{\vx}\norm{\vx'}}\right).
% \end{multline*}
% The kernel is a function of the norm of the inputs $\norm{\vx}\,\norm{\vx'}$ and a factor $J(\theta)$ that only depends on the geodestic distance $\theta$ (or the great-circle distance) between the projection of $\vx$ and $\vx'$ on the unit hypersphere.

\section{Experiments}

We evaluate our method Variational Inference with Spherical Harmonics (VISH) on a regression and a classification problem to demonstrate that VISH performs competitively in terms of accuracy and uncertainty quantification, while also being extremely fast (modelling a dataset of 6 million 8D datapoints in less than 2 minutes on a standard desktop).

\subsection{Toy Experiment: Banana Classification}

The banana dataset is a 2D binary classification problem \citep{hensman2015scalable}. In \cref{fig:banana} we show three different fits of VISH with $M\in \{9,\ 225,\ 784\}$ spherical harmonics, which correspond respectively to maximum levels of $2$, $14$, and $27$ for our inducing functions. Since the variational framework provides a guarantee that more inducing variables must be monotonically better \citep{titsias2009}, we expect that increasing the number of inducing functions will provide improved approximations. This is indeed the case as we show in the rightmost panel: with increasing $M$ the ELBO converges and the fit becomes tighter.

% While this is expected behaviour for SVGP methods, it is not guaranteed by VFF. Given the Kronecker-structure used by VFF for this 2D experiment \citet{hensman2017variational} report that using a full rank covariance matrix for the variational distribution was intolerably slow. They also show that enforcing the Kronecker structure on the posterior results in an ELBO that {\em decreased} as frequencies were added, and they finally propose a sum-of-two-Kroneckers structure, but provide no guarantee that this would converge to the exact posterior in the limit of larger $M$. In VISH we do not need to impose any structure on the approximate covariance matrix $\MS$, so we retain the guarantee that adding more basis functions will move us closer to the posterior process. The method remains fast despite optimising over full covariance matrices: fitting the models displayed in \cref{fig:banana} only takes a few seconds on a standard desktop.

\begin{figure}[tbh]
    \centering
    \includegraphics[width=\textwidth]{banana}
    \caption{Classification of the 2D banana dataset with growing number of spherical harmonic basis functions. The right plot shows the convergence of the ELBO with respect to increasing numbers of basis functions.\label{fig:banana}}
\end{figure}

\subsection{Large-Scale Regression on Airline Delay}

\begin{table}[tb]
\centering
\resizebox{\textwidth}{!}{\input{Chapter3/Tables/airline.tex} }
\caption{Predictive mean squared errors (MSEs), negative log predictive densities (NLPDs) and wall-clock time in seconds with one standard deviation based on 10 random splits on the airline arrival delays experiment. Total dataset size is given by $N$ and in each split we randomly select 2/3 and 1/3 for training and testing.}
\label{tab:airline}
\end{table}

This experiment illustrates three core capabilities of VISH: 1) it can deal with large datasets and 2) it is computationally and time efficient 3) the model improves performance in terms of NLPD.

We use the 2008 U.S. airline delay dataset to asses these capabilities. The goal of this problem is to predict the amount of delay $y$ given eight characteristics $x$ of a flight, such as the age of the aircraft (number of years since deployment), route distance, airtime, etc. We follow the exact same experiment setup as \citet{hensman2017variational}\footnote{\url{https://github.com/jameshensman/VFF}} and evaluate the performance on 4 datasets of size 10,000, 100,000, 1,000,000, and 5,929,413 (complete dataset), created by subsampling. For each dataset we use two thirds of the data for training and one third for testing. Every split is repeated 10 times and we report the mean and one standard deviation of the MSE and NLPD. For every run the outputs are normalized to be a centered unit Gaussian and the inputs are scaled to be within $[-1, 1]$.
%for VFF and SVGP. For VISH we normalize the inputs so that each column falls within $[-v_d, v_d]$. The hyperparameter $v_d$ corresponds to the prior variance of the weights of an infinite-width fully-connected neural net layer (see \citet{cho2009kernel}). We can optimise for this weight-variance by back-propagation through $k_u(x)$ w.r.t. the ELBO. This is similar to the lengthscale hyperparameters of stationary kernels.

\Cref{tab:airline} shows the outcome of the experiment. The results for VFF and SVGP are from \citet{hensman2017variational}. We observe that VISH improves on the other methods in terms of NLPD and is within error bars in terms of MSE. Given the variability in the data the GP models improve when more data is available during training.

Given the dimensionality of the dataset, a full-VFF model is completely infeasible. As an example, using just four frequencies per dimension would already lead to $M = 4^8 = 65,536$ inducing variables. So VFF has to resort to an additive model with a prior covariance structure given as a sum of Mat\'ern-3/2 kernels for each input dimension. Each of the functions $f_d$ is approximated using 30 frequencies.
We report two variants of VISH: one using all spherical harmonics up to degree 3 ($M$=210) and another up to degree 4 ($M$=660). As expected, the more inducing variables, the better the fit.

We also report the wall clock time for the experiments (training and evaluation) for $N=10,000$ and $N=5,929,413$. All these experiments were ran on a single consumer-grade GPU (Nvidia GTX 1070). On the complete dataset of almost 6 million records, VISH took $41\pm0.81$ seconds on average. A-VFF required $75.61\pm0.75$ seconds and the SVGP method needed approximately 15 minutes to fit and predict. This shows that VISH is roughly two orders of magnitude faster than SVGP. A-VFF comes close to VISH but has to impose additive structure to keep its computational advantage.

\section{Conclusion}

In this chapter we introduced the RKHS associated with dot-product kernels on the hypersphere. Using the Mercer representation of this RKHS we designed an efficient spherical harmonic inducing function for sparse variational Gaussian processes. Our general setup is closely related to VFF, and we inherit several of its advantages such as a considerable speed-up compared to classic sparse GP models as a result of sparsity in the $\Kuu$ matrix, and having fixed features $\kux$ .with a global influence on the approximation. However, by projecting the data onto the hypersphere and using dedicated GP models on this manifold our approach succeeds where other sparse GPs methods fail. First, VISH provides good scaling properties as the dimension of the input space increases.  Second, we showed that under some relatively weak hypothesis we are able to select the optimal features to include in the approximation. This is due to the intricate link between ``stationary'' covariances on the sphere and the Laplace-Beltrami operator. Third, the Mercer representation of the kernel means that the matrices to be inverted at training time are exactly diagonal ---resulting in a very cheap-to-compute sparse approximate GP.

In the next chapter, we are going to employ the same RKHS. However, instead of using the eigenfunctions of the kernel operator as the inducing functions, we are going to rely on the reproducing property to construct inducing functions that closely mimic the activation functions in neural networks.


\subsubsection{Differentiation according to Gestures} \label{differentiation_according_to_gestures}
Also interesting are the different results depending on the gesture. For this purpose the focus is again on the best
performing simulation without filter. Figure \ref{fig:gesture_result} shows the $Precision$ and $Recall$ values for
every gesture over all records. The best recognized gesture measured under $F_{1}score$ for the best performing
simulation is gesture $GesA$.

\begin{figure}[H]
    \begin{center}
        \resizebox {\textwidth} {!} {
            \begin{tabular}{cc}
                \resizebox {!} {\height} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0.5,
                            xmax=1,
                            ymin=0.5,
                            ymax=1,
                            width=\axisdefaultwidth,
                            height=\axisdefaultwidth,
                            xlabel=$Precision$,
                            ylabel=$Recall$,
                            samples=100]
                            \addplot+[
                                blue,
                                only marks,
                                nodes near coords,
                                every node near coord/.style={at={(0.475,0.245)}, color=black},
                                point meta=explicit symbolic] table[x=x, y=y, meta=label] {../data/fig/gesture_result/gesture.dat};
                            \addplot[gray, domain=0.5:1] {(0.4 * x) / (2 * x - 0.4)};
                            \addplot[gray, domain=0.5:1] {(0.5 * x) / (2 * x - 0.5)};
                            \addplot[gray, domain=0.5:1] {(0.6 * x) / (2 * x - 0.6)};
                            \addplot[gray, domain=0.5:1] {(0.7 * x) / (2 * x - 0.7)};
                            \addplot[gray, domain=0.5:1] {(0.8 * x) / (2 * x - 0.8)};
                            \addplot[gray, domain=0.5:1] {(0.9 * x) / (2 * x - 0.9)};
                        \end{axis}
                    \end{tikzpicture}
                } &
                \resizebox {!} {\height} {
                    \begin{tikzpicture}
                        \begin{axis}[
                            xmin=0,
                            xmax=1,
                            ymin=0,
                            ymax=1,
                            width=\axisdefaultwidth,
                            height=\axisdefaultwidth,
                            xlabel=$Precision$,
                            ylabel=$Recall$,
                            samples=100]
                            \addplot+[
                                blue,
                                only marks,
                                nodes near coords,
                                every node near coord/.style={at={(-0.05,0)}, color=black},
                                point meta=explicit symbolic] table[x=x, y=y, meta=label] {../data/fig/gesture_result/gesture.dat};
                            \addplot[gray, domain=0.051:1] {(0.1 * x) / (2 * x - 0.1)};
                            \addplot[gray, domain=0.11:1] {(0.2 * x) / (2 * x - 0.2)};
                            \addplot[gray, domain=0.16:1] {(0.3 * x) / (2 * x - 0.3)};
                            \addplot[gray, domain=0.21:1] {(0.4 * x) / (2 * x - 0.4)};
                            \addplot[gray, domain=0.26:1] {(0.5 * x) / (2 * x - 0.5)};
                            \addplot[gray, domain=0.31:1] {(0.6 * x) / (2 * x - 0.6)};
                            \addplot[gray, domain=0.36:1] {(0.7 * x) / (2 * x - 0.7)};
                            \addplot[gray, domain=0.41:1] {(0.8 * x) / (2 * x - 0.8)};
                            \addplot[gray, domain=0.46:1] {(0.9 * x) / (2 * x - 0.9)};
                        \end{axis}
                    \end{tikzpicture}
                }
            \end{tabular}
        }
    \end{center}
    \caption{$Precision$ and $Recall$ of all gestures over all records for the simulation that uses $\eta$DTW with a
    Sakoe-Chiba band of size 36\% depending on the input time series length, \textit{HAveD} as threshold determination
    and \textit{Mid} as window size determination without any filter. The left plot is a zoomed version of the
    right plot. Gray lines are illustrating the distribution of $F_{1}score$ in $\frac{1}{10}$ steps.}
    \label{fig:gesture_result}
\end{figure}