\newpage
\section{CNN for Darcy equation}
\begin{breakablealgorithm}
	\caption{$u={\rm DarcyCNN}(\kappa; f_d, coarse\_grid\_size,)$}
	\label{alg:mgnet}
	\begin{algorithmic}
		\State Initialization:  $\kappa$
		%		\State Initialization $u^{1,0}$
%		\For{$ = 1:N_{epoch}$}
%		\For{$i = 1:\nu_$}
		\State Encode
		\begin{equation}
		u^{} = K^{3}  \ast_2 \sigma \circ K^{ 2}\ast \sigma (K^{ 1}\ast \kappa).
		\end{equation}

%		\EndFor
		\State Restriction to the coarse grid
		\begin{equation}
		u^{} = \sigma \circ \theta^{. 1} \circ u^{}
		\end{equation}
		\State
		Convolution on the coarse grid
		\begin{equation}
		u^{} =  \sigma \circ K^{ 5}\ast \sigma (K^{ 4}\ast u^{}).
		\end{equation}
		\State Decode
		\begin{equation}
		u^{} =  \sigma \circ \theta^{ 3}\circ \sigma (\theta^{ 2}\circ u^{}).
		\end{equation}
%		\EndFor
	\end{algorithmic}
\end{breakablealgorithm}

	where $K^{3}  \ast_2$ is the average pooling with kernel size $2  \times 2$ and stride 2, $K^{1}$, $K^{2} $, $K^{4} $ and $K^{5}  $ are multi-channel kernels with kernel size $3  \times 3$ and the size of these kernel matrices are $(1 \times f_d/2)$, $ (f_d/2 \times f_d)$, $(1,f_d/2)$ and $(f_d/2, f_d)$ respectively.

The loss function is
$$
L = \frac{1}{N} \sum_{i=1}^{N} \frac{\| u_{pred} - u_{test}\|_{L^2}^2}{\| u_{test}\|_{L^2}^2}
$$

For this problem, we have a comparison for the results with and without nonlinear activation functon.
\begin{figure}[H]
	\centering
	\includegraphics[width=0.65\textwidth]{figures/Darcy_CNN/Darcy_CNN_2000.png}
	\includegraphics[width=0.65\textwidth]{figures/Darcy_CNN/Darcy_CNN_2000_nonlinear.png}
	\caption{Comparison of the solutions obtained by CNN with and without activation function by SGD (Top: without activation, Bottom:  with activation, 8,2 million parameters).}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=0.65\textwidth]{figures/Darcy_CNN/Darcy_CNN_Adam_2000.png}
	\includegraphics[width=0.65\textwidth]{figures/Darcy_CNN/Darcy_CNN_2000__Adam_nonlinear.png}
	\caption{Comparison of the solutions obtained by CNN with and without activation function by Adam method (Top: without activation, Bottom:  with activation).}
\end{figure}

\begin{figure}[!htbp]
	\centering
	\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/darcy_loss_2000_adam.png}
	\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/darcy_loss_2000_adam_nonlinear.png}
	\caption{Training curves  with and without activation function by Adam method (Left: without activation, Right:  with activation).}
\end{figure}

	\begin{figure}[!htbp]
	\centering
	\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/darcy_loss_2000_SGD.png}
	\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/darcy_loss_2000_SGD_nonlinear.png}
	\caption{Training curves  with and without activation function by SGD method (Left: without activation, Right:  with activation).}
\end{figure}

Below we consider the decoding process as the transpose of the encoding process.
\begin{breakablealgorithm}
	\caption{$u={\rm DarcyCNN_{transpose}}(\kappa; f_d, coarse\_grid\_size,)$}
	\label{alg:mgnet}
	\begin{algorithmic}
		\State Initialization:  $\kappa$
		%		\State Initialization $u^{1,0}$
		%		\For{$ = 1:N_{epoch}$}
		%		\For{$i = 1:\nu_$}
		\State Encode
		\begin{equation}
		u^{} = K^{3}  \ast_2   K^{ 2}\ast  (K^{ 1}\ast \kappa).
		\end{equation}

		%		\EndFor
		\State Restriction to the coarse grid
		\begin{equation}
		u^{} =   W u^{}
		\end{equation}
		\State
		Convolution on the coarse grid
		\begin{equation}
		u^{} =   K^{ 5}\ast K^{ 4}\ast u^{}.
		\end{equation}
		\State Decode
		\begin{equation}
		u^{} =  K^{ 1,T} \ast K^{ 2,T}\ast K^{3}  \ast_2^T (W^{ T}  K^{ 4,T}\ast K^{ 5,T}\ast u^{}).
		\end{equation}
		%		\EndFor
	\end{algorithmic}
\end{breakablealgorithm}
Where $K^T$ is the transpose of kernel $K$. For a kernel with size of $3 \times 3$ without padding, its transpose is the central symmetry of itself with 2 paddings.\\

In this network, the parameters are 13 million and 6.5 million respectively.

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/Darcy_transpose_value_mean.png}
		\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/Darcy_transpose_novalue_mean.png}
		\caption{Comparison of solutions obationed by FEM and neural network including transpose(Left: Transposed parameters untrained, Right: Transposed parameters trained) with zero-mean input.}
	\end{figure}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/Darcy_transpose_value.png}
		\includegraphics[width=0.45\textwidth]{figures/Darcy_CNN/Darcy_transpose_novalue.png}
		\caption{Comparison of solutions obationed by FEM and neural network including transpose(Left:  Transposed parameters untrained, Right: Transposed parameters trained) with original input.}
	\end{figure}

The problem we feel confused is that unlike the process in PCA method, what we need to do is to solve a darcy equation, where the input data and output data are in two different fields. Therefore, whether it makes sense that using the encode transpose-encode process to solve this problem.

\newpage
\subsection{PCA and POD}
For the differnence between PCA and POD, here we consider following linear problem
$$
A_i x_i = b_i, \quad i = 1,\cdots,N
$$
Consider the solutions $x_i $ are in a hyperplane $y_i = W\tilde{x}_i+\mu$ where $W \in \mathbb{R}^{d\times d'}, d'\ll d $ and $y_i$ is an approximation of $x_i$. To approximate $x_i$ as closely as possible, i.e. minimize loss function
\begin{equation}\label{lossPOD}
L = \sum_{i}^{N}=\| y_i - x_i\|^2 =  \sum_{i}^{N}=\| W\tilde{x}_i +\mu - x_i\|^2
\end{equation}
Then, the best choice of $\tilde{x}_i $ and $\mu$ is
$$
\mu = \frac{1}{N}\sum_{i}^{N}x_i = \bar{x}, \quad
\tilde{x}_i = W^T(x_i-\bar{x})
$$
Then \ref{lossPOD} can rewritten as
$$
L =  \sum_{i}^{N}=\| WW^T(x_i-\bar{x}) +\bar{x} - x_i\|^2 .
$$
The choice of $W$ is the same as PCA method.\\

For the linear problem $Ax=b$, we consider $x = W\tilde{x}+\bar{x}$ and premultiplicate $W^T$ on the eqauation, we get
$$
W^TAW\tilde{x} = W^T(b-A\bar{x}).
$$
Solve the equation above and with the decoder, we can get the approximation of the solution as
$$
x \approx W\tilde{x} + \bar{x}.
$$
In the POD, we consier the solution in the specific hyperplane $y_i = W\tilde{x}_i$, then the low-dimension solution $\tilde{x}$ satisfies
$$
W^TAW\tilde{x} = W^Tb
$$
Apply the decoded to get the approximation solution as
$$
x = W\tilde{x}
$$
For the construction of CNN to solve this problem, the matrix A is associated with $\kappa$ and can be regarded as a linear mapping of $\kappa$, and the right side term $b$ is fixed.

%Here for a input $\kappa$, I wonder the encoded is processed on $\kappa$, or on a linear  mapping of $\kappa$ which approximate the solution $x$. As we all know, the traditional encode-decode process is all about $x$. However, here we need to apply the encoder on $\kappa$ and have some extra computation to solve the equation in a lower dimension. Finally we apply the decoder to get the approximation of the solution.
\subsection{Mgnet-encoder}
In order to make the distance between the approximate solution and true solution  as small as possible, i.e. minimize loss function
$$
L = \sum_{i}^{N} \| g  \circ f(x_i) - x_i \|
$$
If we fix the decoder as a linear function, i.e.
$$
g(y) = Wy + b
$$
Hdre, we use the following MgNet as the encoder
\begin{breakablealgorithm}%[!htb]
	\caption{$\mu^J = {\text{MgNet}}(f; J,\nu_1, \cdots, \nu_J)$}
	\label{alg:L-Slash11d}
	\begin{algorithmic}
		\State Set up
		$$
		f^1 = {\color{red} \theta\ast} f, \quad \mu^{1}=0.
		$$
		\State Smoothing and restriction from fine to coarse level (nested)
		\For{$\ell = 1:J$}
		\For{$i = 1:\nu_\ell$}
		\State
		\begin{equation}\label{eq:smoothing}
		\mu^{\ell} \leftarrow \mu^{\ell} + S^\ell \ast (f^\ell - A_\ell \ast \mu^{\ell}).
		\end{equation}
		\EndFor
		\State Form restricted residual and set initial guess:
		$$
		\mu^{\ell+1} \leftarrow \Pi_{\ell}^{\ell+1}\mu^\ell , \quad
		f^{\ell+1} \leftarrow R^\ell \ast_2 (f^\ell -  A_\ell \ast
		\mu^{\ell}) + A_{\ell+1}\ast 	\mu^{\ell+1} ,    %A_{\ell+1} = R       \ast_2 A_\ell \ast (R\ast_2^\top).
		$$
		\EndFor
		\State
		Finally, we ultilize the Adaptive average pooling to let the number of element in every channel of $\nu^J$ be 1.
		$$\nu^J = Avg \ast \nu^J$$
	\end{algorithmic}
\end{breakablealgorithm}
Here, $\mu_J$ is the reduced vector. For the decoder, we denote $(u^l,f^l)$ as a new variable.
Then we can induce that
$$\mu^{l+1}=\phi^{l+1}\mu^l + \xi^{l+1}f^{l+1}$$
$$f^{l+1} = R^l f^l + M^l \mu^l $$
where $\phi^{l+1}= (N^{l+1})^{v_{l+1}}\Pi_l^{l+1}, N^{l+1}= I - S^{l+1}A^{l+1} $, $\xi^{l+1}=\big(\sum_{i=1}^{v_{l+1}-1}(N^{l+1})^i\big)S^{l+1}$ and $M^l = A^{l+1}\Pi_l^{l+1} - R^lA^l$.

We can also rewritten the equation above as
\[
\left( \begin{array}{l}
f^{l+1} \\
\mu^{l+1}
\end{array} \right) = \left( {\begin{array}{*{20}{c}}
	R^l & M^l\\
	\xi^{l+1}R^l&\phi^{l+1}+\xi^{l+1}M^l
	\end{array}} \right)\left( \begin{array}{l}
f^{l}\\
\mu^{l}
\end{array} \right), \quad
\left( \begin{array}{l}
f^{1} \\
\mu^{1}
\end{array} \right) = \left( {\begin{array}{{c}}
	\theta \\
	\xi^{1}\theta
	\end{array}} \right)\left( \begin{array}{l}
f\\
\end{array} \right)
\]
We can then define the mgnet-decoder as
\begin{breakablealgorithm}%[!htb]
	\caption{$f = {{\rm MgNet_{transpose}}}(u^J; J,\nu_1, \cdots, \nu_J)$}
	\begin{algorithmic}
		\State Transpose of Adapative Average pooling
		$$\mu^J = Avg^T \ast \nu^J $$
		\State Prolongation back to the fine level
		\State
		$$
		f^{J-1} = R^{J-1, T}\xi^{J, T}\mu^J,  \qquad \mu^{J-1} = (M^{J-1,T}\xi^{J, T} + \phi^{J-1, T})\mu^{J}
		$$
		\For{$\ell = J-2:1$}
		\State
		\begin{equation}
		\mu^{\ell}= M^{\ell,T}f^{\ell+1} + (M^{\ell,T}\xi^{\ell+1, T} + \phi^{\ell, T})\mu^{\ell+1}.
		\end{equation}
		\State
		\begin{equation}
		f^{\ell} = R^{\ell, T}f^{\ell+1} + R^{\ell, T}\xi^{\ell+1, T}\mu^{\ell+1}
		\end{equation}
		\EndFor
		\State Transpose of pre-convolution:
		$$
		f = \theta^T \ast f^1 + \theta^T \ast \xi^{1,T} \mu^1
		$$
	\end{algorithmic}
\end{breakablealgorithm}

\subsection{Numerical results}
Then we have the some results for the following test.

\begin{itemize}
	\item Model equations
	\begin{align*}
	- \nabla \cdot (\kappa \nabla p) & = 0  \qquad \text{in}~~ \Omega \\
	\kappa \nabla p \cdot n &= 0 \qquad \text{on} ~~ \Gamma_\text{N} \\
	p  &= 0 \qquad \text{on} ~~\Gamma_{\text{D}1} \\
	p  &= 1 \qquad \text{on}~~ \Gamma_{\text{D}2} \\
	\end{align*}
	\begin{center}
		\vskip -20pt
		\includegraphics[height=5cm]{figures/Darcy_CNN/Darcy_domain.png}
	\end{center}
\end{itemize}
The loss fucntion is
$$
L = \sum_{i}^{N} \| g  \circ f(x_i) - x_i \|
$$
where $x_i$ is the FEM solution given the permeability $\kappa_i$, $f$ is the Mgnet encoder and $g$ is the Mgnet-Transpose decoder.

\begin{itemize}
	\item Uniformly generate 2000 training samples: $\kappa_1 \sim \mathcal{U}(2.5,7,5), \kappa_2 \sim \mathcal{U}(1,3)$ and $\kappa_3 \sim \mathcal{U}(0.5,1.5)$
	\item Grid size: $33*33 ~ (d=1089)$
	\item Number of training samples: 2000
	\item Number of epochs: 2000
	\item Number of test samples: 500
	\item Mgnet learning rate: $10^{-3}$
	\item DNN learning rate: 0.1
\end{itemize}
Here, we give the results of DNN and MgNet:

\subsubsection{Case 1: Fix J, change $\nu_i$}
Let $J=4,$, number of $\mu$'s channel equals to 10, number of $f$'s channel equals to 1, we test the cases when $\nu_i=1,2,3,4$.
\begin{table}
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& $\nu_i$=1 & $\nu_i$=2 & $\nu_i$=3& $\nu_i$=4 \\
			\hline
			Train loss & 1.1861 $\times 10^{-2}$ & 1.197 $\times 10^{-2}$& 1.0793 $\times 10^{-2}$ & 1.0795 $\times 10^{-2}$\\
			\hline
			Mean rel. errors & 6.358 $\times 10^{-3}$ &6.199 $\times 10^{-3}$ &5.947 $\times 10^{-3}$ &6.010 $\times 10^{-3}$ \\
			\hline
		\end{tabular}\caption{Training errors and test errors(3456 parameters).}
	\end{center}
\end{table}

\subsubsection{Case 2: Fix $\nu_i$, change J}
Let $\nu_i$=2, number of $\mu$'s channel equals to 10, number of $f$'s channel equals to 1, we test the cases when $J=3,4,5,6$
.\begin{table}
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& $J=3$ & $J=4$ & $J=5$& $J=6$ \\
			\hline
			Train loss & 1.3235 $\times 10^{-2}$ & 1.1197 $\times 10^{-2}$& 1.0760 $\times 10^{-2}$ & 1.0663 $\times 10^{-2}$\\
			\hline
			Mean rel. errors & 6.613 $\times 10^{-3}$ &6.199 $\times 10^{-3}$ &6.022 $\times 10^{-3}$ &5.956 $\times 10^{-3}$ \\
			\hline
			Number of Para & 2367& 3456& 4545 & 5634\\
			\hline
		\end{tabular}\caption{Training errors and test errors for various J.}
	\end{center}
\end{table}
\begin{figure}[H]
	\centering
	\includegraphics[width=0.6\textwidth]{figures/Darcy_CNN/loss_J.png}
	\caption{Training curves for various J=3,4,5,6.}
\end{figure}
\subsubsection{Case 3: Change number of $u$'s channel}
Let $\nu_i$=2, number of $f$'s channel equals to 1, $J=4$, we test the cases when the number of $\mu$'s channel equals to $10,20,30,40$
.\begin{table}
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& $n_u=10$ & $n_u=20$ & $n_u=30$& $n_u=40$ \\
			\hline
			Train loss & 1.1197 $\times 10^{-2}$ & 1.0772 $\times 10^{-2}$& 0.9634 $\times 10^{-2}$ & 0.8550 $\times 10^{-2}$\\
			\hline
			Mean rel. errors & 6.199$\times 10^{-3}$ & 6.091 $\times 10^{-3}$ &5.796 $\times 10^{-3}$ &5.574$\times 10^{-3}$ \\
			\hline
			Number of Para & 3456& 12276& 26496 & 71136\\
			\hline
		\end{tabular}\caption{Training errors and test errors for various number of $u$'s channel.}
	\end{center}
\end{table}

\begin{figure}[H]
	\centering
	\includegraphics[width=0.6\textwidth]{figures/Darcy_CNN/loss_u_channel.png}
	\caption{Training curves for various number of $u$'s channel.}
\end{figure}

\subsubsection{Case 4: DNN results}
For the one-layer DNN, we test the cases when the number of reduced dimension is $10, 20, 30 ,40$.
.\begin{table}[H]
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& 10 & 20 & 30& 40 \\
			\hline
			Train loss & 1.0584 $\times 10^{-2}$ & 0.8902 $\times 10^{-2}$& 0.7296$\times 10^{-2}$ & 0.6819 $\times 10^{-2}$\\
			\hline
			Mean rel. errors & 6.110$\times 10^{-3}$ & 5.852 $\times 10^{-3}$ &5.571$\times 10^{-3}$ &5.470$\times 10^{-3}$ \\
			\hline
			Number of Para & 10890& 21780& 32670 & 54450\\
			\hline
		\end{tabular}\caption{Training errors and test errors for various reduced dimension.}
	\end{center}
\end{table}
\textbf{In this test, we noticed that the weight obtained by DNN almost satisfies the orthogonality, and the weight obtained by MgNet also has the trendency to satisfy the orthogonality where the elements on the diagnol close to 1 and the non-diagol elements close to 0.}

\subsubsection{Case 5: POD results}
For the POD method, we test the cases when the number of reduced basis is $10, 20, 30 ,40$
.\begin{table}
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& 10 & 20 & 30& 40 \\
			\hline
			Mean rel. errors & 6.345$\times 10^{-3}$ & 6.111 $\times 10^{-3}$ &5.918$\times 10^{-3}$ &5.788$\times 10^{-3}$ \\
			\hline
		\end{tabular}\caption{Test results for various reduced basis number.}
	\end{center}
\end{table}

However, there exists a problem for this numerical test because the variance of every entry  of the solution is actually verly small ($10^-5$). From the results above, we can only infer that CNN can use less paramters to obtain the weight than DNN does. To illustrate this conclusion, we consider further numerical tests.

\subsection{Extend tests}
Here we let the source term of the equation be 1, i.e.,
\begin{itemize}
	\item Model equations
	\begin{align*}
		- \nabla \cdot (\kappa \nabla p) & = 1  \qquad \text{in}~~ \Omega \\
		\kappa \nabla p \cdot n &= 0 \qquad \text{on} ~~ \Gamma_\text{N} \\
		p  &= 0 \qquad \text{on} ~~\Gamma_{\text{D}1} \\
		p  &= 1 \qquad \text{on}~~ \Gamma_{\text{D}2} \\
	\end{align*}
	\begin{center}
		\vskip -20pt
		\includegraphics[height=5cm]{figures/Darcy_CNN/Darcy_domain.png}
	\end{center}
\end{itemize}

Then the variance of every entry is about 0.11, which is lager than the previous case. Fix the number of $u$'s channel as 1, we have the following results.

\subsubsection{Case 1: Change $\nu_i$}
\begin{table}[H]
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			$\nu_i$& 1& 2 & 3& 4 \\
			\hline
			Train loss & 53.4168 & 55.3550 & 53.3935 & 53.8467\\
			\hline
			%			Mean rel. errors & 6.110$\times 10^{-3}$ & 5.852 $\times 10^{-3}$ &5.571$\times 10^{-3}$ &5.470$\times 10^{-3}$ \\
			%			\hline
			Number of Para & 135 & 135 & 135 & 135\\
			\hline
		\end{tabular}\caption{Training errors for various reduced dimension(lr = $10^{-2}$).}
	\end{center}
\end{table}
\subsubsection{Case 2: Change $J$}
\begin{table}[H]
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			J& 3& 4 & 5& 6 \\
			\hline
			Train loss & 44.7566 & 55.3550 & 66.2886& 78.9738\\
			\hline
			%			Mean rel. errors & 6.110$\times 10^{-3}$ & 5.852 $\times 10^{-3}$ &5.571$\times 10^{-3}$ &5.470$\times 10^{-3}$ \\
			%			\hline
			Number of Para & 99 & 135 & 171 & 207\\
			\hline
		\end{tabular}\caption{Training errors for various reduced dimension(lr = $10^{-2}$).}
	\end{center}
\end{table}
\subsubsection{Case 3: DNN results}
.\begin{table}[H]
	\begin{center}
		\begin{tabular}{|c|c|c|c|c|}
			\hline
			& 10 & 20 & 30& 40 \\
			\hline
			Train loss & 60.4035  & 50.5609 & 45.3879 & 42.3671 \\
			\hline
%			Mean rel. errors & 6.110$\times 10^{-3}$ & 5.852 $\times 10^{-3}$ &5.571$\times 10^{-3}$ &5.470$\times 10^{-3}$ \\
%			\hline
			Number of Para & 10890& 21780& 32670 & 54450\\
			\hline
		\end{tabular}\caption{Training errors for various reduced dimension(lr = $10^{-5}$).}
	\end{center}
\end{table}
\subsection{Some disscusion about the orthogonality}
Review that the loss function is
$$
L = \Sigma_{i=1}^N \| WW^Tx_i  -x_i \| ^2
$$
where $W\in \mathbb{R}^{d\times k}, k \ll d$. Here the matrix $A=WW^T$ is a symmetric positive semi-definite matrix, which have the following decomposition
$$
Q^TAQ  = \Lambda
$$
where Q is a orthogonal matrix and $\Lambda = diag(x_k, 0)$ is a diagnoal matrix, $x_k$ ia a strictly positive  $k$ element vector.

Then the loss function can be written as
$$
\begin{aligned}
L&= \sum x_i^TA^TAx_i - 2x_i^TAx_i +x_i^Tx_i^T\\
&= \sum x_i^TQ \Lambda^2 Q^Tx_i - 2x_i^TQ\Lambda Q^Tx_i
\end{aligned}
$$
To minimize the loss function, for each element of $x_k=(\lambda_i)$, we need to minimize the following problem for each $\lambda_i, i = 1,\cdots, k$
$$
M \lambda_i^2 - 2M \lambda_i
$$
where $M=M(Q,x_i)$ is  positive and depends on $x_i$ and Q. Therefore, $\lambda_i$ should be 1 for the purpose.

Then the original problem is equivalent to maximize the following problem
$$
\begin{aligned}
L & = \sum_{i=1}^{N}x_i^TQ \Lambda Q^T x_i\\
& = \sum_{i=1}^N \sum_{j=1}^k x_i q_j q_j^T x_i\\
& = \sum_{j=1}^k q_j^TXX^T q_j
\end{aligned}
$$
where $X = (x_1,\cdots,x_N)$, $\{q_j\}, j=1,\cdots k$ is the first $k$ columns of orthogonal matrix $Q$.

Then the problem is equivalent to the PCA method which requires the orthogonality, i.e., the orthogonality is a necessary condition to unique the minimizer.

On the other hand, if we consider $A = WW^T$. From above we know that the eigenvalues of $A$ is 1 and 0, and $A$ has a decomposition as $A = WW^T$ where $W$ is not unique.

Without the constaint of orthogonality, this weight $W$ is not unique. Therefore, CNN can generate one of the weight matrix $W_{CNN}$, if this weight matrix is orthogonal, then CNN has no difference from POD method. If not, CNN may be better than POD method.

\subsection{Othogonality is necessary and sufficient}
In the following part, I will prove that the weight matrix must satisfy the orthogonality.

Firstly, for the matrix $A = WW^T$ where $A\in \mathbb{R}^{n\times n}, W \in \mathbb{R}^{n\times r}, r\ll n$. There exists an orthogonal matrix $Q$ and a diagnoal matrix $\Lambda$ such that
$$
A = Q^T \Lambda Q
$$
Then we have $\Lambda = QWW^TQ^T$. Consider the decomposition of $\Lambda$, since the eigenvalues of $A$ is 1 and 0, $\Lambda$ can be written as $diag(e_r, 0)$ where $e_r$ is a r-dim vector of all ones. We have
$$
\Lambda= \left( \begin{array}{c}
	A_r \\
	A_{n-r}
	\end{array}\right) (A_r^T \ A_{n-r}^T) = \left(\begin{array}{c c}
	A_rA_r^T & A_rA_{n-r}^T \\
	A_{n-r}A_r^T & A_{n-r}A_{n-r}^T
\end{array}\right)
= \left(\begin{array}{c c}
	I_r & 0 \\
	0 & 0
\end{array}\right)
$$
Since $ A_{n-r}A_{n-r}^T$ is a zero matrix, we can induce that $A_{n-r}$ is a zero matrix and $A_r$ is an orthogonal matrix due to $A_rA_r^T= I_r$. Using these results leads to
$$
\left( \begin{array}{c}
	Q_r \\
	0
\end{array}\right) = QW
$$
i.e.,
$$
W = Q^T\left( \begin{array}{c}
	Q_r \\
	0
\end{array}\right)
$$
where $Q_r$ is an arbitary orthogonal matrix. Then we have
$$
W^TW = (Q_r^T \ 0) QQ^T \left(\begin{aligned}
	Q_r \\
	0
\end{aligned}\right) = (Q_r^T \ 0) \left(\begin{aligned}
Q_r \\
0
\end{aligned}\right) = I_r
$$
Therefore, orthogonality is a sufficient and necessary conditon for the weight matrix. For any two weight matrices $W_1$ and $W_2$, there exists a unique orthgonal matrix $Q_r$ such that $W_1 = W_2Q_r$.


\section{The effect of pixelization on point-source models}

Since \climax\ constructs its models in the image plane, point source
models are necessarily pixelized with a resolution of $dx$.  Here we
consider the effect this has on the minimum \math{\chisq}\ achievable in
\climax.

Consider a source at a true location $x$.  In general, the best-fit
pixelized model will be for a source located at $x + \delx$, where
$\delx < \mdx/2$, with Fourier transform

\begin{equation}
V_j = Se^{-2\pi i u_j\delx} = S(\cos{2\pi u_j\delx} - \sin{2\pi u_j\delx})
\end{equation}

The best-fit \math{\chisq}\ will then be:

\begin{eqnarray}\nonumber
  \chisq = {1\over{2N}}\{&\sum_j&{(R_j - S\cos{2\pi u_j(x + \delx)})^2/\sigma_j^2} + \\\nonumber
                         &\sum_j&{(I_j + S\sin{2\pi u_j(x + \delx)})^2/\sigma_j^2}\}
\end{eqnarray}

Now $R_j = N^R_j + S\cos{2\pi u_j x}$ and $I_j = N^I_j - S\sin{2\pi u_j x}$ so that:

\begin{eqnarray}\nonumber
  \chisq = {1\over{2N\sigma^2}}\{&\sum_j&{(N^R_j + S\cos{2\pi u_j x} - S\cos{2\pi u_j(x + \delx)})^2} + \\\nonumber
                                 &\sum_j&{(N^I_j - S\sin{2\pi u_j x} + S\sin{2\pi u_j(x + \delx)})^2}\}
\end{eqnarray}

if we take the weights to be equal $\sigma_j \equiv \sigma$.  Ignoring
cross terms with $N^R_j$ and $N^I_j$, which will average to zero, and
writing $\cos(x + \delx) = \cos{x}\cos{\delx} - \sin{x}\sin{\delx}$ and
$\sin(x + \delx) = \sin{x}\cos{\delx} + \cos{x}\sin{\delx}$, we have:

\begin{eqnarray}\nonumber
  \chisq = 1 + {S^2\over{2N\sigma^2}}\{&\sum_j&{( cx - cx\,c\delx + sx\,s\delx})^2 + \\\nonumber
                                     &\sum_j&{(-sx + sx\,c\delx + cx\,s\delx)^2}
\end{eqnarray}

or

\begin{eqnarray}\nonumber
  \chisq = 1 + {\snr^2\over{2N}}\{&\sum_j&{( cx^2 - 2cx^2\,c\delx + 2cx\,sx\,s\delx + cx^2\,c\delx^2 - 2cx\,sx\,c\delx\,s\delx  + sx^2\,s\delx^2}) + \\\nonumber
                                  &\sum_j&{(sx^2 - 2sx^2\,c\delx - 2sx\,cx\,s\delx + sx^2\,c\delx^2 + 2cx\,sx\,c\delx\,s\delx + cx^2\,s\delx^2)}
\end{eqnarray}

or

\begin{equation}
  \chisq = 1 + {\snr^2\over{N}}\sum_j{(1 - \cos{2\pi u_j\delx})}.
\end{equation}

Clearly, if $\delx\sim 0$, the sum converges to 1.  In general though, we have in the continuous limit:

\begin{eqnarray}\nonumber
  \sum_j{(\cos{2\pi u_j\delx})} &\sim& {1\over{\Delta u^\prime}}\int^{u^\prime_{max}}_{u^\prime_{min}}{\cos{u^\prime}du^\prime} \\\nonumber
  &=& {1\over{\Delta u^\prime}}\{\sin{u^\prime_{max}} - \sin{u^\prime_{min}}\} \\\nonumber
  &=& {1\over{2\pi\Delta u\delx}}\{\sin{2\pi\delx u_{max}} - \sin{2\pi\delx u_{min}}\}.
\end{eqnarray}

In the worst-case scenario,

\begin{eqnarray}\nonumber
\delx \sim {{\Delta x}\over{2}} = {1\over{4u_{max}}}\\\nonumber
\Delta u = {1\over{N\Delta x}} = {{2u_{max}}\over{N}}
\end{eqnarray}

and we have

\begin{eqnarray}\nonumber
  {1\over{2\pi\Delta u\delx}}\{\sin{2\pi\delx u_{max}} - \sin{2\pi\delx u_{min}}\}
  &=&  {N\over{\pi}}\{1 - \sin{{\pi\over{2}}{r}}\}
\end{eqnarray}

where $r = u_{min}/u_{max}$, and

\begin{eqnarray}\nonumber
  \chisq &=& 1 + {\snr^2\over{N}}\{N - {N\over{\pi}}\{1 - \sin{{\pi\over{2}}{r}}\}\} \\\nonumber
         &=& 1 + \snr^2\{1 - {1\over{\pi}}\{1 - \sin{{\pi\over{2}}{r}}\}\},
\end{eqnarray}

which ranges between $\chisq = 1 + \snr^2$ and $\chisq \simeq 1 + {2\over{3}}\snr^2$, depending on the dynamic range $r$ of the data.

Let's take a typical SZA observation: say we have a 6 hour track,
split into 40-second integrations, for $n_{vis} = 28 \times 6 \times
3600/40 \sim 15000$ band-averaged visibilities per frequency channel.
If the visibility data are binned into a $128\times128 = 16384$ cell
array, then we have on order one visibility per uv cell.  If the map
noise is $0.2$~mJy/beam, then the noise on each cell is of order
$\sigma \sim 0.2 \sqrt{16384} = 26~$mJy.  This means that for a
$1$~mJy source, the contribution to $\chisq$ will be $(1/26)^2 \sim 1.5e-3$.


%!TEX root = ../thesis.tex
% ******************************* Thesis Appendix A ********************************

\chapter{Probability Density}
 The probability density function is the function that gives the probability of a particular value occurring. $P(x)=f(x)$

And the equivalent form that all probabilities sum to 1 is the following equation:
\begin{equation}
    \int_{-\infty}^{\infty}f(x)dx = 1
\end{equation}

The expected or mean value of x is
\begin{equation}
   \bar{x}= \int_{-\infty}^{\infty}xf(x)dx
\end{equation}

In 2D when we analyse a point distribution with a constant probability of a point occurring, the point density $k$, the probability density function is the following;

\begin{equation}
  f(r)=k
\end{equation}

When we integrate this in radial coordinates over a circular area with radius $R$;
\begin{equation}
  \int_0^\infty k.2\pi r.dr = \int_0^R k.2\pi r.dr = 1
\end{equation}

\begin{equation}
k. \pi R^2 = 1
\end{equation}

\begin{equation}
    k=\frac{1}{\pi R^2}
\end{equation}
The probability $k$ of finding a point at radius, $r$, is $\frac{1}{\pi R^2}$.
The probability of finding $N$ points is $\frac{N}{\pi R^2}$

Therefore the probability of finding a point between $r$ and $r + dr$ is
\begin{equation}
    $\frac{2 \pi}{\pi R^2} dr$
\end{equation}


\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\begin{document}

\section{singular 1/r}
We want to compute by quadrature:
\begin{equation}
  \int_a^b f(r) \,\mathrm{d}r \quad \mathrm{with} \quad f(r) = \frac{1}{r} \quad .
\end{equation}
For $a=10^{-3}$ and $b=1.0$, the analytical result is $\mathrm{log}(b)-\mathrm{log}(a) \approx 6.91$.
Standard trapezoidal quadrature on 100 points leads to
\begin{equation}
  \sum_{i=0}^{99} \frac{1/x_i + 1/x_{i+q}}{2} \Delta x \approx 10.1
\end{equation}
where
\begin{equation}
  x_i = a+\frac{i}{100-1}\Delta x ~ \mathrm{and} ~ \Delta x = \frac{b-a}{100-1} \quad .
\end{equation}

This specific quadrature can be made much more accurate by a substitution that leads to
\begin{equation}
  \int_{s(a)}^{s(b)} \frac{1}{r} r\,\mathrm{d}s = \left[ s \right]_{s(a)}^{s(b)} \quad ,
\end{equation}
for which we need
\begin{equation}
  \mathrm{d}r = r \, \mathrm{d}s \quad .
\end{equation}
Rephrase:
\begin{equation}
  \frac{\mathrm{d}r}{r} = \mathrm{d}s \Leftrightarrow \int \frac{\mathrm{d}r}{r} = \int \mathrm{d}s \Leftrightarrow
  \mathrm{log}(r) = s \quad .
\end{equation}
The quadrature for the substituded integral is then:
\begin{equation}
  \sum_{i=0}^{99} ...
\end{equation}

\section{nearly-singular 1/(r+eps)}
We want to compute:
\begin{equation}
  \int_a^b f(r) \,\mathrm{d}r ~ \mathrm{where} ~ f(r) \propto \frac{1}{\sqrt{r^2 + \epsilon^2}} \quad .
\end{equation}
If we take $f$ to be the pure kernel function, an analytical solution to this can be found:
\begin{equation}
  \int_a^b \frac{1}{\sqrt{r^2 + \epsilon^2}} \,\mathrm{d}r = \mathrm{log}\left( b + \sqrt{b^2 + \epsilon^2} \right) - \mathrm{log}\left( a + \sqrt{a^2 + \epsilon^2} \right) \quad .
\end{equation}
Again do the substitution trick:
\begin{equation}
  \int_a^b f(r) \,\mathrm{d}r = \int_{s(a)}^{s(b)} f(r) \left(\sqrt{r^2 + \epsilon^2}\right) \,\mathrm{d}s \quad ,
\end{equation}
where we used
\begin{equation}
  \mathrm{d}r = \left(\sqrt{r^2 + \epsilon^2}\right) \,\mathrm{d}s \quad .
\end{equation}
The substitution is therefore
\begin{equation}
  s = \mathrm{log}\left(r + \sqrt{r^2 + \epsilon^2}\right) \quad .
\end{equation}

\end{document}


\subsection{General linear groups \(GL(n, F)\)}

The general linear group, \(GL(n, F)\), contains all \(n\odot n\) invertible matrices \(M\) over field \(F\).

The binary operation is multiplication.


\section{Intermediate Value Theorem}
\begin{theorem}[Intermediate Value Theorem (IVT)]
	If $f$ is continuous on the closed interval $[a,b]$, then for all $c \in [f(a), f(b)]$, there exists $x \in [a,b]$ such that $f(x) = c$.
\end{theorem}

That is, if $f$ is continuous on $[a,b]$, then $f$ must take on every value between $f(a)$ and $f(b)$.
This encapsulates the idea that if a function is continuous on some interval, then it is "connected" on that interval.

\begin{example}
	Use the IVT to show that $e^{-x} = x$ has at least one solution.
\end{example}
\begin{answer}
	Let $f(x) = e^{-x} - x$.
	We are looking for $x$ where $f(x) = 0$.
	$f(0) = 1$ and $f(1) = \frac{1}{e} - 1$.
	Since $f$ is continuous on the closed interval $[0,1]$, it must take on every value between $1$ and $\frac{1}{e} - 1$.
	Since $1$ is positive and $\frac{1}{e} - 1$ is negative, 0 is between these two values.
	Thus, by the IVT, there must exist a solution between $x=0$ and $x=1$.
\end{answer}


% based on
% https://github.com/sympy/sympy/wiki/Quick-examples

\documentclass[12pt]{pylatex}
\usepackage{examples}

\begin{document}

\section*{Visible Python code}

This is the first of two-part example. In this first part the Python code and tags are made visible using {\tt\small\verb|\PySetup{action=show}|} (this is the default). In the second part the Python code will be hidden using {\tt\small\verb|\PySetup{action=hide}|}. In both cases the Python output is identical (apart from the different tag names).

\vspace{12pt}

\PySetup{action=show}

\begin{python}
   from sympy import *

   x, y = symbols('x y')

   ans = limit((sin(x)-x)/x**3, x, 0)               # py(ans.101,ans)
   ans = (1/cos(x)).series(x, 0, 6)                 # py(ans.102,ans)
   ans = diff(cos(x**2)**2 / (1+x), x)              # py(ans.103,ans)
   ans = integrate(x**2 * cos(x), x)                # py(ans.104,ans)
   ans = integrate(x**2 * cos(x), (x, 0, pi/2))     # py(ans.105,ans)

   f = Function('f')

   eqn = Eq(Derivative(f(x),x,x) + 9*f(x), 1)       # py(ans.106,eqn)
   sol = dsolve(eqn, f(x))                          # py(ans.107,sol)

   eqn = Eq(f(x).diff(x, x) + 9*f(x), 1)            # py(ans.108,eqn)
   sol = dsolve(eqn, f(x))                          # py(ans.109,sol)

   my_int = Integral(cos(x),(x,0,2))                # py(ans.110,my_int)
   my_ans = my_int.doit()                           # py(ans.111,my_ans)
\end{python}

\clearpage

\section*{Visible Python code output}

Here is the output from the above Python code. This line of text is here for no other reason than to fill out the line so that we have two lines of text before the Python output. This makes it easier to compare the vertical spacing between this and the next example.
% ----------------------------------------------------------------------------------
% Note
% ----------------------------------------------------------------------------------
% For perfect alignment of the py(ans.101) and py(ans.201) tags in the pdf file, the
% following \begin{align*} and \begin{python} environments must be in vertical mode.
% The easiest way to ensure this is to precede each environment by a blank line.
% ----------------------------------------------------------------------------------

\vspace{-10pt}

\begin{align*}
   &\py*{ans.101}\\
   &\py*{ans.102}\\
   &\py*{ans.103}\\
   &\py*{ans.104}\\
   &\py*{ans.105}\\
   &\py*{ans.106}\\
   &\py*{ans.107}\\
   &\py*{ans.108}\\
   &\py*{ans.109}\\
   &\py*{ans.110}\\
   &\py*{ans.111}
\end{align*}

\clearpage

\section*{Hidden Python code and output}

\PySetup{action=hide}

In this example the Python code is hidden and consumes no vertical space on the page. Compare the gap between this pair of lines and the following output text against that seen on the previous page.

\begin{python}
   from sympy import *

   x, y = symbols('x y')

   ans = limit((sin(x)-x)/x**3, x, 0)               # py(ans.201,ans)
   ans = (1/cos(x)).series(x, 0, 6)                 # py(ans.202,ans)
   ans = diff(cos(x**2)**2 / (1+x), x)              # py(ans.203,ans)
   ans = integrate(x**2 * cos(x), x)                # py(ans.204,ans)
   ans = integrate(x**2 * cos(x), (x, 0, pi/2))     # py(ans.205,ans)

   f = Function('f')

   eqn = Eq(Derivative(f(x),x,x) + 9*f(x), 1)       # py(ans.206,eqn)
   sol = dsolve(eqn, f(x))                          # py(ans.207,sol)

   eqn = Eq(f(x).diff(x, x) + 9*f(x), 1)            # py(ans.208,eqn)
   sol = dsolve(eqn, f(x))                          # py(ans.209,sol)

   my_int = Integral(cos(x),(x,0,2))                # py(ans.210,my_int)
   my_ans = my_int.doit()                           # py(ans.211,my_ans)
\end{python}

\vspace{-10pt}

\begin{align*}
   &\py*{ans.201}\\
   &\py*{ans.202}\\
   &\py*{ans.203}\\
   &\py*{ans.204}\\
   &\py*{ans.205}\\
   &\py*{ans.206}\\
   &\py*{ans.207}\\
   &\py*{ans.208}\\
   &\py*{ans.209}\\
   &\py*{ans.210}\\
   &\py*{ans.211}
\end{align*}

\clearpage

\section*{Visible Python markup}

There is nothing special in this particular example. Its purpose is to provide a contrast with the following example where the tags have been hidden from view.

\vspace{5pt}

\PySetup{action=show}

\begin{python}
   from sympy import *

   x = Symbol('x')

   ans = diff(cos(x)**2, x)                         # py(ans.301,ans)
   ans = integrate(2*sin(x)*exp(-x), x)             # py(ans.302,ans)
\end{python}

\begin{align*}
   &\py*{ans.301}\\
   &\py*{ans.302}
\end{align*}

\section*{Hidden Python markup}
This example creates the impression that there are no tags in the following block. But looks can be deceptive, the truth is revealed on the following page.

\vspace{5pt}

\PySetup{action=hide}

\begin{python}
   from sympy import *

   x = Symbol('x')

   ans = diff(cos(x)**2, x)                         # py(ans.401,ans)
   ans = integrate(2*sin(x)*exp(-x), x)             # py(ans.402,ans)
\end{python}

\PySetup{action=verbatim}

\begin{python}
   from sympy import *

   x = Symbol('x')

   ans = diff(cos(x)**2, x)
   ans = integrate(sin(x)*exp(-x), x)
\end{python}

\begin{align*}
   &\py*{ans.401}\\
   &\py*{ans.402}
\end{align*}

\clearpage

\section*{How to hide the tags}
The easiest way to hide the tags from view is to include two blocks, one containing the tags but hidden from view using {\tt\small\verb|\PySetup{action=hide}|}, the other block is a copy of the first but with all tags stripped out. That second block is made visible using {\tt\small\verb|\PySetup{action=verbatim}|}. This is a bit cumbersome and prone to errors (the second block must be a faithful copy of the first block). Better solutions could be devised but they all appear to require significant changes to the current preprocessors -- so much so that the this cloning trick was deemed a reasonable compromise.

Here is the LaTeX code from the previous example.

\vspace{-5pt}

\begin{minipage}[t]{0.65\textwidth}
\bgcolour{white}
\latexstyle
\begin{latex}
*  \PySetup{action=hide}
*
*  \begin{python}
\end{latex}
\pythonstyle
\begin{python}
*     from sympy import *
*
*     x = Symbol('x')
*
*     ans = diff(cos(x)**2, x)                         # py(ans.401,ans)
*     ans = integrate(2*sin(x)*exp(-x), x)             # py(ans.402,ans)
\end{python}
\latexstyle
\begin{latex}
*  \end{python}
*
*  \PySetup{action=verbatim}
*
*  \begin{python}
\end{latex}
\pythonstyle
\begin{python}
*     from sympy import *
*
*     x = Symbol('x')
*
*     ans = diff(cos(x)**2, x)
*     ans = integrate(sin(x)*exp(-x), x)
\end{python}
\latexstyle
\begin{latex}
*  \end{python}
*
*  \begin{align*}
*     &\py*{ans.401}\\
*     &\py*{ans.402}
*  \end{align*}
\end{latex}
\end{minipage}

\end{document}


\subsection{Rings}\label{subsec:rings}

\begin{definition}\label{def:semiring}
  A \term{semiring} \( (R, +, \cdot) \) is an algebraic structure with two binary operations:
  \begin{thmenum}[series=def:semiring]
    \thmitem{def:semiring/addition} The \term{addition} \( + \) is \hyperref[def:magma/associative]{associative}, \hyperref[def:unital_magma]{unital} and \hyperref[def:magma/commutative]{commutative}, i.e. \( (R, +) \) is a commutative \hyperref[def:unital_magma/monoid]{monoid}. The additive identity is usually denoted by \( 0 \). We denote this monoid by \( R^+ \).

    \thmitem{def:semiring/multiplication} The \term{multiplication} \( \cdot \) (usually written using juxtaposition) is \hyperref[def:magma/associative]{associative}, i.e. \( (R, \cdot) \) a \hyperref[def:magma/associative]{semigroup}. We denote this semigroup by \( R^\times \).

    \thmitem{def:semiring/distributivity} We require that \( \cdot \) \term{distributes} over \( + \), i.e.
    \begin{align}\label{eq:def:semiring/distributivity}
      (x + y)z = xz + yz
      &&
      x(y + z) = xy + xz
    \end{align}
  \end{thmenum}

  The \term{trivial semiring} consists only of the additive identity \( \{ 0 \} \).

  We say that \( x \) is \term{nilpotent} if \( x^n = 0 \) for some nonnegative integer \( n \).

  The following are special kinds of semirings:
  \begin{thmenum}[resume=def:semiring]
    \thmitem{def:semiring/dioid} \term{Dioids} are unital semirings, that is, both \( (R, +) \) and \( (R, \cdot) \) are monoids. The term \enquote{semiring} is sometimes reserved for dioids while \enquote{hemiring} is used for what we call semirings.

    We define integer exponentiation as a shorthand for iterated multiplication. That is, if multiplication is invertible, for \( r \in R \) and \( n \in \BbbZ \) we define
    \begin{equation*}
      r^n \coloneqq \begin{cases}
        1,                      & n = 0  \\
        r^{n - 1} \cdot r,      & n > 0  \\
        r^{n + 1} \cdot r^{-1}, & n < 0.
      \end{cases}
    \end{equation*}

    If multiplication is not invertible, we skip defining negative exponents.

    \thmitem{def:semiring/no_zero_divisor} If \( xy = 0 \) whenever \( x \) and \( y \) are nonzero, we say that the semiring has \term{no zero divisors}.

    \thmitem{def:semiring/ring} \term{Rings} are semirings with invertible addition, i.e. \( (R, +) \) forms an \hyperref[def:abelian_group]{abelian group}. The category of rings is denoted by \( \cat{Ring} \).

    \thmitem{def:semiring/unital_ring} \term{Unital rings} are \hyperref[def:semiring/ring]{rings} in which multiplication is unital, i.e. \( R, \cdot \) is a \hyperref[def:unital_magma/monoid]{monoid} with identity \( 1 \). Some authors define all rings to be unital. Invertible elements under multiplication are called \term{units} (see \fullref{rem:units_in_rings_etymology}) and the operation itself is called \term{division}.

    We may additionally require that the ring is nontrivial, so that \( 0 \neq 1 \) (see \fullref{thm:def:semiring/properties/identities_are_equal_iff_trivial_ring}).

    \thmitem{def:semiring/commutative_ring} \term{Commutative rings} are \hyperref[def:semiring/ring]{rings} in which multiplication is \hyperref[def:magma/commutative]{commutative}, i.e. \( R, \cdot \) is a commutative semigroup.

    \thmitem{def:semiring/commutative_unital_ring} \term{Commutative unital rings} are (obviously) both commutative and unital and as usually assumed to be nontrivial. Despite being ubiquitous, they do not have an established one-word name.

    We are mostly interested in these rings because if \( R \) is a commutative unital ring, so is its polynomial \hyperref[def:algebra_of_polynomials]{ring} \( R[X] \) and, by induction, multivariate polynomial \hyperref[def:multivariate_polynomial]{rings} (\( R[X, Y] = R[X][Y] \)). So we are interested in properties of \( R \) that are preserves by \( R[X] \). This is the reason, so many types of commutative rings and ideals are studied. We are interested in integral domains when we speak of divisibility and factorization and since zero is absorbing, it does not interact nicely with factorization. Refer to \fullref{sec:commutative_algebra}.

    \thmitem{def:semiring/integral_domain} \term{Integral domains} are nontrivial commutative unital \hyperref[def:semiring/commutative_unital_ring]{rings} with no zero \hyperref[def:commutative_ring_division]{divisors}. This implies that \( 1 \neq 0 \) because otherwise \( 1 \cdot 1 = 0 \) and \( 1 \) would be a zero divisor. This in turn implies that \fullref{thm:def:semiring/properties/identities_are_equal_iff_trivial_ring}.

    \Fullref{thm:def:semiring/properties/cancellable_iff_not_zero_divisor} shows that a commutative unital ring \( R \) is an integral domain if and only if its multiplication is cancellable.

    \thmitem{def:semiring/unique_factorization_domain} \term{Unique factorization domains} are integral domains in which every element has unique \hyperref[def:factorization_in_ring]{factorization} exists.

    \thmitem{def:semiring/principal_ideal_domain} \term{Principal ideal domains} are integral \hyperref[def:semiring/integral_domain]{domains} in which every \hyperref[def:semiring_ideal]{ideal} is \hyperref[def:principal_ideal]{principal}.

    By \fullref{thm:pid_is_ufd}, every principal ideal domain is a unique factorization domain.

    \thmitem{def:semiring/euclidean_domain} \term{Euclidean domains} are integral \hyperref[def:semiring/integral_domain]{domains} which allow division with remainders (see \fullref{def:euclidean_domain}).

    By \fullref{thm:euclidean_domain_is_pid}, every Euclidean domain is a principal ideal domain.

    \thmitem{def:semiring/division_ring} \term{Division rings} are nontrivial unital \hyperref[def:semiring/unital_ring]{rings} in which all nonzero elements are units, i.e. \( (F \setminus \{ 0 \}, \cdot) \) is a \hyperref[def:group]{group}. The nontriviality is a requirement because we want \( 1 \) not to be a zero divisor (see discussion in \fullref{def:semiring/integral_domain}).

    The multiplicative inverse of an element in a division ring is called its \term{reciprocal}.

    In order to fit multiplicative invertibility as an axiom, we can use the following formula:
    \begin{equation*}
      \forall \xi ((\xi \doteq 0) \lor \exists \eta (\xi \cdot \eta \doteq 1))
    \end{equation*}
    or add an additional operation \( (\cdot)^{-1} \) that inverts all nonzero elements and fixes zero, that is,
    \begin{equation*}
      \forall \xi (\xi \cdot \xi^{-1} \doteq 1),
    \end{equation*}
    where we define \( 0^{-1} = 0 \). This is only a formalism since \( 0 \) is not actually \enquote{invertible}, but it is required if we wish to avoid existential quantifiers.

    \thmitem{def:semiring/field} \term{Fields} are \hyperref[def:magma/commutative]{commutative} division \hyperref[def:semiring/division_ring]{rings}, i.e. \( (F \setminus \{ 0 \}, \cdot) \) is an \hyperref[def:abelian_group]{abelian group}. The category of fields is denoted by \( \cat{Field} \).
  \end{thmenum}
\end{definition}

\begin{example}\label{ex:semirings}
  We give examples and counterexamples of semirings. Note that the order of definitions in \fullref{def:semiring} is not preserved.

  \begin{refenum}
    \refitem{def:semiring/euclidean_domain} The base building block for the examples will be the ring \( (\BbbZ, +, \cdot) \) of \hyperref[def:set_of_integers]{integers}, which itself is an Euclidean domain (see \fullref{def:set_of_integers}).

    Another example of Euclidean domains are the polynomial rings over a field (see \fullref{thm:polynomials_over_field_are_euclidean_domain}).

    \refitem{def:semiring/dioid} By removing additive inverses from the integers, we obtain the dioid \( (\BbbN, +, \cdot) \) of \hyperref[def:set_of_natural_numbers]{natural numbers}.

    Take only the commutative monoid \( (\BbbN, +) \) of \hyperref[def:set_of_natural_numbers]{natural numbers} with addition. The endomorphism semiring
    \begin{equation*}
      \End(\BbbN)
    \end{equation*}
    is a noncommutative dioid.

    Tropical \hyperref[def:tropical_semiring]{semirings} are another example noncommutative dioids.

    \refitem{def:semiring} Simple examples of semirings (but not rings) without unity are proper semiring ideals. For example, the ideal \( 2\BbbN \) of positive even numbers is a non-unital semiring.

    Another example can be given by taking a subsemiring, but not a unital subsemiring of the endomorphism dioid \( \End(\BbbN, \oplus) \), for example the functions
    \begin{balign*}
       & f_n: \BbbN \to \BbbN        \\
       & f_n(x) \coloneqq x + n,
    \end{balign*}
    where \( n > 0 \). They are closed under composition and thus form a semiring themselves, but they do not contain the identity function, so the semiring is not unital.

    \refitem{def:semiring/unital_ring} The endomorphism rings \( \End(G) \) for any abelian group \( G \) are unital, but non-commutative rings. This includes the matrix space \( R^{n \times n} \) (see \fullref{thm:finite_dimensional_operators_are_isomorphic_to_matrices}).

    \refitem{def:semiring/ring} Consider the Banach space \( C_0(\BbbC) \) of complex functions vanishing at \hyperref[def:function_spaces/c0]{infinity}. If we take addition to be pointwise addition and multiplication to be composition, then \( C_0(\co) \) becomes a non-commutative ring with no multiplicative identity because \( C_0(\co) \) does not contain the identity function.

    \refitem{def:semiring/commutative_ring} For an example of a commutative ring without unit, consider again the Banach space \( C_0(\co) \), however define multiplication as pointwise function multiplication rather than by composition. The constant function \( f(x) = 1 \) does not vanish at infinity, hence \( (C_0(\co), +, \cdot) \) is a commutative, but not unital ring.

    \refitem{def:semiring/commutative_unital_ring} We are mostly interested in different types of commutative unital rings since the polynomials over well-behaved commutative rings preserve this behavior. This is important because multivariate \hyperref[def:multivariate_polynomial]{polynomials} are defined inductively as polynomials over polynomial rings. See \fullref{thm:geometric_nullstellensatz} for an application.

    An example of a nontrivial commutative unital ring that has zero divisors is the matrix algebra \( \BbbZ^{n \times n} \) over the integers. It is a ring under addition and matrix multiplication. We have
    \begin{equation*}
      \begin{pmatrix}
        1 & 0 \\
        1 & 0
      \end{pmatrix}
      \begin{pmatrix}
        0 & 0 \\
        0 & 1
      \end{pmatrix}
      =
      \begin{pmatrix}
        0 & 0 \\
        0 & 0
      \end{pmatrix},
    \end{equation*}
    thus there are zero divisors in \( \BbbZ^{n \times n} \).

    \refitem{def:semiring/integral_domain}\cite[388]{Knapp2016BasicAlgebra} The integral domain \( \BbbZ[\sqrt{-5}] \) is not a unique factorization domain because
    \begin{equation*}
      6 = (1 + \sqrt{-5}) (1 - \sqrt{-5}) = 2 \cdot 3.
    \end{equation*}

    Note that \( \BbbZ[\sqrt{-5}] \) is an integral domain by \fullref{thm:polynomials_over_integral_domain_are_integral_domain}.

    \refitem{def:semiring/unique_factorization_domain}\cite{ProofWiki:polynomials_in_integers_is_not_principal_ideal_domain} The unique factorization domain \( \BbbZ[X] \) is not a principal ideal domain.

    Note that \( \BbbZ[X] \) is a unique factorization domain by \fullref{thm:polynomials_over_integral_domain_are_integral_domain}.

    Consider the ideal \( I \) of polynomials with an even constant term.

    Assume that \( I \) is generated by the polynomial \( p(X) \in \BbbZ[X] \). Since \( 2 \in I \), then \( p(X) \) divides \( 2 \), so \( p(X) \in \{ -2, -1, 1, 2 \} \), that is \( p(X) \) is a unit of \( \BbbZ[X] \). But then \( I = \braket{p(X)} = \BbbZ[X] \), which contradicts the definition of \( I \).

    The obtained contradiction proves that \( \BbbZ[X] \) is not a principal ideal domain.

    \refitem{def:semiring/principal_ideal_domain} Principal ideal domains are not Euclidean domains in general. Such domains are discussed in \cite{Anderson1988}.

    \refitem{def:semiring/division_ring} The \hyperref[def:ring_localization]{localization} of a noncommutative ring over its nonzero cancellative elements (characterized by \fullref{thm:ring_localization_universal_property}), if it exists, forms a division ring.

    \refitem{def:semiring/field} The canonical examples of fields include the rational \hyperref[def:set_of_rational_numbers]{numbers} \( \BbbQ \), the \hyperref[def:set_of_real_numbers]{real numbers} \( \BbbR \) and the \hyperref[def:set_of_complex_numbers]{complex numbers} \( \BbbC \).

    More generally, any nontrivial commutative unital can be embedded in a field by \fullref{def:field_of_fractions}.
  \end{refenum}
\end{example}

\begin{proposition}\label{thm:def:semiring/properties}
  Any semiring \( R \) has the following basic properties:
  \begin{thmenum}
    \thmitem{thm:def:semiring/properties/zero_is_absorbing} Multiplication by \( 0 \) is \term{absorbing}, that is, \( x0 = 0x = 0 \) for any \( x \in R \).
    \thmitem{thm:def:semiring/properties/one_exponents} For any integer \( n \), \( 1^n = 1 \).
    \thmitem{thm:def:semiring/properties/identities_are_equal_iff_trivial_ring} In a unital \hyperref[def:semiring/unital_ring]{ring}, the additive and multiplicative identities are equal if and only if the ring is trivial.
    \thmitem{thm:def:semiring/properties/cancellable_iff_not_zero_divisor} An element \( x \in R \) of a commutative ring is a zero divisor if and only if it is cancellable (with respect to multiplication).
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:def:semiring/properties/zero_is_absorbing} Follows from \fullref{thm:def:left_module/properties/ring_zero_is_absorbing} and \fullref{thm:def:left_module/properties/module_zero_is_absorbing}.
  \SubProofOf{thm:def:semiring/properties/one_exponents} By definition, \( r \cdot 1 = r \) for any \( r \in R \). Hence, for \( 1 \cdot 1 = 1 \). Proceeding by induction, we can show that for \( 1^n = 1 \) for positive \( n \). For negative \( n \), since \( 1 \) is its own inverse, we also have \( 1^n = 1 \).

  \SubProofOf{thm:def:semiring/properties/identities_are_equal_iff_trivial_ring}
  Let \( 0 = 1 \) in a unital ring \( R \). Let \( r \in R \). Then, by \fullref{thm:def:semiring/properties/zero_is_absorbing},
  \begin{equation*}
    r = 1r = 0r = 0.
  \end{equation*}

  Thus, \( r = 0 \). Since \( r \) was arbitrary, we conclude that \( R = \{ 0 \} \) is the trivial ring.

  \SubProofOf{thm:def:semiring/properties/zero_is_absorbing} Follows from \fullref{thm:def:left_module/properties/ring_zero_is_absorbing} and \fullref{thm:def:left_module/properties/module_zero_is_absorbing}.

  \SubProofOf{thm:def:semiring/properties/one_exponents} By definition, \( r \cdot 1 = r \) for any \( r \in R \). Hence, for \( 1 \cdot 1 = 1 \). Proceeding by induction, we can show that for \( 1^n = 1 \) for positive \( n \). For negative \( n \), since \( 1 \) is its own inverse, we also have \( 1^n = 1 \).

  \SubProofOf{thm:def:semiring/properties/identities_are_equal_iff_trivial_ring} The trivial ring only has one element, hence \( 0 = 1 \).

  \SufficiencySubProofOf{thm:def:semiring/properties/cancellable_iff_not_zero_divisor}
  Suppose that \( x \in R \) is not cancellable. Then there exist \( y \neq z \) for which \( xy = xz \). We have \( y - z \neq 0 \) and
  \begin{equation*}
    x(y - z) = xy - xz = 0.
  \end{equation*}

  Thus, \( x \) is a zero divisor.

  \NecessitySubProofOf{thm:def:semiring/properties/cancellable_iff_not_zero_divisor} If \( x \) is a zero divisor, fix \( y \in R \) such that \( xy = 0 \). For any \( z \in R \) we have
  \begin{equation*}
    xy = 0 = x(yz)
  \end{equation*}
 , but \( y \neq yz \) in general.

  Thus, \( x \) is not cancellable.
\end{proof}

\begin{definition}\label{def:semiring_characteristic}
  Fix a nontrivial unital semiring \( R \) (the nontrivial condition is essential because of \fullref{thm:embedding_preserves_characteristic}). We define its \term{characteristic} \( \op{char}(R) \) via any of the equivalent definitions:
  \begin{thmenum}
    \thmitem{def:semiring_characteristic/direct} \( \op{char}(R) \) is the smallest number of times \( 1_R \) must be added to itself in order to obtain \( 0_R \) and zero if \( 0_R \) cannot be obtained in this way.

    \thmitem{def:semiring_characteristic/homomorphism} The semiring \( \BbbZ_{\geq 0} \) of nonnegative integers can be embedded into \( R \) via the unique homomorphism
    \begin{balign*}
       & \iota: \BbbZ_{\geq 0} \to R                     \\
       & \iota(k) \coloneqq \begin{cases}
        0_R,                & k = 0 \\
        \iota(k - 1) + 1_R, & k > 1
      \end{cases}
    \end{balign*}
    that adds the identity \( 1_R \) to itself \( n \) times. This allows us to use the positive integers in any semiring. Note that if addition in \( R \) is invertible (that is, if \( R \) is a ring), we can embed all integers by defining \( \iota(n) \coloneqq -\iota(-n) \) for \( n < 0 \).

    We define \( \op{char}(R) \) as the positive integer \( n \) such that
    \begin{equation*}
      n\BbbZ_{\geq 0} \cong \ker\iota.
    \end{equation*}
  \end{thmenum}
\end{definition}
\begin{proof}
  We will show that \( \iota \) is unique. Let \( \varphi: \BbbZ_{\geq 0} \to R \) be another embedding. Obviously
  \begin{equation*}
    \iota(1_{\BbbZ}) = 1_R = \varphi(1_{\BbbZ}).
  \end{equation*}

  Then by induction on \( k \) we can show that
  \begin{equation*}
    \iota(k) = \underbrace{1_R + \cdots + 1_R}_{k \text{ times }} = \varphi(k).
  \end{equation*}

  Hence, \( \iota = \varphi \).
\end{proof}

\begin{proposition}\label{thm:embedding_preserves_characteristic}
  If \( R \) is a nontrivial unital semiring and \( T \) is a unital supersemiring of \( R \), then
  \begin{equation*}
    \op{char}(T) = \op{char}(R).
  \end{equation*}
\end{proposition}
\begin{proof}
  Note that if \( R \) was the trivial unital semiring, its characteristic would be \( 1 \) and yet the characteristic of \( T \) could be nonzero. So the theorem would not hold if \( R \) was trivial.

  Now assume that \( R \) is nontrivial and let \( \iota: \BbbZ_{\geq 0} \to R \) be the embedding defining \( \op{char}(R) \). Then it is obviously also an embedding of \( \BbbZ_{\geq 0} \) into \( T \). Since \( 0_R = 0_T \), then \( \ker \iota = \ker \varphi = n\BbbZ_{\geq 0} \), where \( n = \op{char}(R) \).
\end{proof}

\begin{example}\label{ex:semiring_characteristic}
  We find examples of semiring characteristics via the embedding \( \iota \) defined in \fullref{def:semiring_characteristic}:

  \begin{thmenum}
    \thmitem{ex:semiring_characteristic/nonnegative_integers} The zero-based \hyperref[def:set_of_natural_numbers]{natural numbers} \( \BbbN \) have characteristic \( \op{char}(\BbbN) = 0 \) because \( \iota \) is an isomorphism. Consequently, any supersemiring of \( \BbbN \) has characteristic zero, most notably the integers \( \BbbZ \) and the fields \( \BbbQ \), \( \BbbR \), \( \BbbC \).

    \thmitem{ex:semiring_characteristic/integers_modulo} The integers modulo \( n \) (see \fullref{def:ring_of_integers_modulo}) have characteristic \( \op{char}(\BbbZ_n) = n \) because of \fullref{thm:integers_modulo_isomorphic_to_quotient_group}.

    \thmitem{ex:semiring_characteristic/polynomial_ring} An \hyperref[def:algebra_over_ring]{algebra} \( A \) over a nontrivial commutative unital ring \( R \) has characteristic \( \op{char}(A) = R \) because of the canonical embedding of \( R \) in \( A \). In particular, polynomial \hyperref[def:algebra_of_polynomials]{rings} \( R[X] \) have the same characteristic as their ring.

    \thmitem{ex:semiring_characteristic/galois_fields} The \hyperref[thm:galois_field_existence]{Galois field} \( \BbbF_{p^n} \) has characteristic \( p \) because it is a field extension of \( \BbbF_p \).
  \end{thmenum}
\end{example}

\begin{definition}\label{def:semiring_kernel}
  The \term{kernel} \( \ker(f) \) of a semiring homomorphism \( f: R \to S \) is the \hyperref[def:zero_locus]{zero locus} of \( f \), that is, \hyperref[thm:def:function/properties/preimage]{preimage} \( f^{-1}(0_S) \).

  It is an instance of \fullref{def:zero_morphisms/kernel}.
\end{definition}

\begin{definition}\label{def:quotient_semiring}
  Let \( R \) be a ring and \( I \) be an ideal of \( M \). Define the \term{quotient ring} to be the quotient \hyperref[def:quotient_left_module]{module} when considering \( R \) as a module over itself.
\end{definition}

\begin{theorem}\label{thm:homomorphism_theorem_for_rings}
  Let \( \varphi: R \to T \) be a homomorphism of rings. We have the isomorphism
  \begin{equation*}
    R / \ker \varphi \cong \img \varphi.
  \end{equation*}
\end{theorem}
\begin{proof}
  Special case of \fullref{thm:homomorphism_theorem_for_left_modules}.
\end{proof}

\begin{proposition}\label{thm:ring_homomorphism_simpler_conditions}
  A function \( f: R \to S \) between the rings \( R \) and \( S \) is a homomorphism in the sense of \fullref{def:first_order_homomorphism} if and only if for any \( x, y \in R \) it satisfies
  \begin{equation}\label{thm:ring_homomorphism_simpler_conditions/condition}
    \begin{dcases}
      f(x + y) & = f(x) + f(y), \\
      f(xy)    & = f(x) f(y),   \\
      f(1_R)   & = 1_S.
    \end{dcases}
  \end{equation}

  Note that the last condition is only for unital rings.

  In other words, if a function satisfies \fullref{thm:ring_homomorphism_simpler_conditions/condition}, the following are automatically satisfied:
  \begin{itemize}
    \item \( f(0_R) = 0_S \)
    \item for all \( x \in R \), we have \( f(-x) = -f(x) \)
    \item for all units \( x \in R \), we have \( f(x^{-1}) = f(x)^{-1} \)
  \end{itemize}
\end{proposition}
\begin{proof}
  Since \( (R, +) \) and \( (S, +) \) are groups, the first two equalities from \fullref{thm:group_homomorphism_single_condition}.

  The proof of \( f(x^{-1}) = f(x)^{-1} \) is analogous to \fullref{thm:group_homomorphism_single_condition}.
\end{proof}

\begin{definition}\label{def:ring_of_integers_modulo}
  The \hyperref[def:set_of_integers]{integers} \( \BbbZ \) form a ring under addition and multiplication. Fix a positive integer \( n > 1 \). We extend the group \( \BbbZ_n \) of integers modulo \( n \) (see \fullref{def:group_of_integers_modulo}) with the operation
  \begin{equation*}
    x \odot y \coloneqq \rem(xy, n).
  \end{equation*}

  The ring \( \BbbZ_n \) is called the \term{ring of integers modulo} \( n \).
\end{definition}
\begin{proof}
  Note that
  \begin{balign*}
     & \phantom{\cong}\thickspace \rem(x, n) \rem(y, n)
     & \pmod n \cong                           \\ &\cong
    (x - n \quot(x, n)) (y - n \quot(y, n))
     & \pmod n \cong                           \\ &\cong
    xy - n \quot(x, n) - n \quot(y, n) + n^2 \quot(x, n) \quot(y, n)
     & \pmod n \cong                           \\ &\cong
    xy
     & \pmod n. \phantom{\cong}
  \end{balign*}

  The proof that multiplication in \( \BbbZ_n \) is associative, unital and commutative becomes trivial.

  We will prove that multiplication distributes over addition. Fix \( x, y, z \in \BbbZ_n \). We have
  \begin{balign*}
    (x \oplus y) \odot z
     & =
    \rem((x \oplus y) z, n)
    =    \\ &=
    \rem(\rem(x + y, n) z, n)
    =    \\ &=
    \rem((x + y - n \quot(x + y, n)) z, n)
    =    \\ &=
    \rem((x + y)z, n).
  \end{balign*}
  and
  \begin{balign*}
    (x \odot z) \oplus (y \odot z)
     & =
    \rem([(x \odot z) + (y \odot z)], n)
    =    \\ &=
    \rem([xz - n \quot(xz, n) + yz - n \quot(yz, n)], n)
    =    \\ &=
    \rem(xz + yz, n)
    =    \\ &=
    \rem((x + y)z, n).
  \end{balign*}

  Hence,
  \begin{equation*}
    (x \oplus y) \odot z = (x \odot z) \oplus (y \odot z).
  \end{equation*}
\end{proof}

\begin{definition}\label{def:tropical_semiring}\mcite{nLab:tropical_semiring}
  Fix a partially \hyperref[def:partially_ordered_set]{ordered} \hyperref[def:abelian_group]{abelian group} \( (M, +, \leq) \). Let \( \infty \) be a sentinel symbol not in \( M \). Define
  \begin{equation*}
    T \coloneqq M \cup \{ \infty \}
  \end{equation*}
  with operations
  \begin{balign*}
     & \oplus: T \times T \to T                        \\
     & x \oplus y \coloneqq \begin{cases}
      \min \{ x, y \}, & x \neq \infty \text{ and } y \neq \infty \text{ and they are comparable}, \\
      \infty,          & x = \infty \text{ or } y = \infty
    \end{cases} \\
    \\
     & \odot: T \times T \to T                         \\
     & x \odot y \begin{cases}
      x + y,  & x \neq \infty \text{ and } y \neq \infty, \\
      \infty, & x = \infty \text{ or } y = \infty
    \end{cases}
  \end{balign*}

  This makes \( (T, \oplus, \odot) \) into a \hyperref[def:semiring/dioid]{dioid} with additive identity \( \infty \) and multiplicative identity \( 0 \). We call \( (T, \oplus, \odot) \) the \( \min \)-\term{tropical semiring} or simply the \term{tropical semiring} over \( M \). We define the \( \max \)-\term{tropical semiring} analogously by simply replacing \( \min \) with \( \max \).
\end{definition}

\begin{definition}\label{def:semiring_direct_product}
  Let \( \{ X_k \}_{k \in \mscrK} \) be a nonempty family of rings.

  Analogously to \fullref{def:group_direct_product}, we define their \term{direct product} as the ring \( \prod_{k \in \mscrK} X_k \), the operations defined componentwise as
  \begin{balign*}
     & \{ x_k \}_{k \in \mscrK} + \{ y_k \}_{k \in \mscrK}
    \coloneqq
    \{ x_k + y_k \}_{k \in \mscrK},                         \\
     & \{ x_k \}_{k \in \mscrK} \cdot \{ y_k \}_{k \in \mscrK}
    \coloneqq
    \{ x_k \cdot y_k \}_{k \in \mscrK}.
  \end{balign*}

  We define their \term{direct sum} as the subring of \( \prod_{k \in \mscrK} X_k \) (see \fullref{def:semiring_direct_product}) where only finitely many components of any ring element are different from zero.
\end{definition}

\begin{proposition}\label{thm:ring_categorical_limits}
  We are interested in \hyperref[def:category_of_cones/limit]{categorical limits} and \hyperref[def:category_of_cones/colimit]{colimits} in \( \cat{Ring} \). Fix an indexed family  \( \{ X_k \}_{k \in \mscrK} \) of rings.
  \begin{thmenum}
    \thmitem{thm:ring_categorical_limits/product} Their \hyperref[def:discrete_category_limits]{categorical product} is their direct \hyperref[def:semiring_direct_product]{product} \( \prod_{k \in \mscrK} X_k \), the projection morphisms being inherited from \fullref{thm:discrete_category_limits_in_set}.
  \end{thmenum}
\end{proposition}

\begin{definition}\label{def:opposite_ring}\mcite[555]{Knapp2016BasicAlgebra}
  The opposite ring \( R^{-1} \) of \( R \) is defined as the same abelian group with the order of multiplication reversed. They are obviously isomorphic for commutative rings.
\end{definition}

\begin{definition}\label{def:ring_commutator}
  Let \( R \) be a ring. The commutator of \( x, y \in R \) is defined as
  \begin{equation*}
    [x, y] \coloneqq xy - yx.
  \end{equation*}

  The commutator ideal of \( R \) is the ideal \hyperref[def:generated_ring_ideal]{generated} by all the commutators in \( G \).
\end{definition}

\begin{proposition}\label{thm:quotient_by_commutator_ideal}
  The quotient \( R / I \) of any unital \hyperref[def:semiring/unital_ring]{ring} \( R \) by its commutator ideal \( I \) is \hyperref[def:semiring/commutative_ring]{commutative}.
\end{proposition}

\begin{definition}\label{def:endomorphism_dioid}
  Let \( (X, +) \) be an monoid and let \( \End(X) \) be set of endomorphism over \( X \). We define two operations:
  \begin{itemize}
    \item Pointwise addition \( [f + g](x) \coloneqq f(x) + g(x) \).
    \item Multiplication by composition \( [fg](x) \coloneqq f(g(x)) \).
  \end{itemize}

  These operations make \( \End(X) \) into a dioid. If \( X \) is a group, then \( \End(X) \) is a ring.

  If \( X \) is a dioid, we define \( \End(X) \) to be a set of dioid endomorphisms (that is, we want the additive group homomorphisms to preserve multiplication and units). Then \( \End(X) \) is again a dioid and, if \( X \) is a unital ring, so is \( \End(X) \).
\end{definition}

\begin{definition}\label{def:function_support}
  The \term{support} of a function \( f: S \to R \) from a set \( S \) to a semiring \( R \) is the set
  \begin{equation*}
    \supp(f) \coloneqq \{ x \in S \colon f(x) \neq 0_R \}.
  \end{equation*}
\end{definition}

\begin{definition}\label{def:functions_vanish_nowhere}
  Let \( \mathcal{F} \) be a family of functions from a set \( S \) to a ring \( R \). We say that \( \mathcal{F} \) \term{vanishes nowhere} if for every \( x \in S \) there exists a function \( f \in \mathcal{F} \) such that \( f(x) \neq 0_R \).
\end{definition}

\begin{definition}\label{def:ordered_semiring}
  Extending \fullref{def:preordered_magma} to (semi)rings, we define a \term{preordered semiring} to be a semiring \( \BbbR \) with a magma preorder \( \leq \) that additionally satisfies
  \begin{equation}\label{eq:def:ordered_semiring/nonnegativity}
    0 \leq y \T{and} 0 \leq y \T{implies} 0 \leq xy.
  \end{equation}
\end{definition}


% !TEX root = index.tex
\section{Rank \& Nullity}
  We saw that matrices encode linear transformations between Euclidean spaces.
  A matrix of size $\ell \times k$ ( having $\ell$ rows and $k$ columns ) can multiply a vector of size $k$ to get a vector of size $\ell$.

  \begin{align*}
    A : \bbr^k &\longrightarrow \bbr^\ell \\
    \vec{v} &\longmapsto A \vec{v}
  \end{align*}

  \begin{align}
    \label{equation:matrixMult}
    \begin{bmatrix}
      A_{11} & A_{12} & \cdots & A_{1k} \\
      A_{21} & A_{22} & \cdots & A_{2k} \\\\
      \vdots & \vdots & \ddots & \vdots \\\\
      A_{\ell 1} & A_{\ell 2} & \cdots & A_{\ell k}
    \end{bmatrix}
    \begin{bmatrix} v_1 \\ v_2 \\ \vdots \\ v_k \end{bmatrix}
      &=
      v_1 \begin{bmatrix}
        A_{11} \\
        A_{21} \\\\
        \vdots \\\\
        A_{\ell 1}
      \end{bmatrix}
      + v_2 \begin{bmatrix}
        A_{12} \\
        A_{22} \\\\
        \vdots \\\\
        A_{\ell 2}
      \end{bmatrix}
      + \dots +
      v_k \begin{bmatrix}
        A_{1k} \\
        A_{2k} \\\\
        \vdots \\\\
        A_{\ell k}
      \end{bmatrix}
      \\\nonumber\\\nonumber
      &=
      \begin{bmatrix}
        A_{11} v_1  + A_{12} v_2 + \cdots + A_{1k} v_k \\
        A_{21} v_1 + A_{22} v_2 + \cdots + A_{2k} v_k \\\\
        \vdots \\\\
        A_{\ell 1} v_1 + A_{\ell 2} v_2+ \cdots + A_{\ell k} v_k
      \end{bmatrix}\\\nonumber\\
      \nonumber
      \left[\mbox{size }\ell \times k \right]
      \cdot \left[\mbox{size } k \right]
      &=
      \left[\mbox{size } \ell \right] \\\nonumber
  \end{align}

  % Another way to visualize the same formula is by taking the ``dot'' product of the $i^{th}$ column of the matrix $A$ with the vector $\vec{v}$.
  % \begin{equation}\label{eq:appendrow}
  %   \left[ \begin{array}{cccc}
  %         A_{11} & A_{12} & \cdots & A_{1n} \\
  %         \colorbox{green}{$A_{21}$} & \colorbox{green}{$A_{22}$} & \cdots & \colorbox{green}{$A_{2n}$} \\
  %         \vdots & \vdots & \ddots & \vdots \\
  %         A_{m1} & A_{m2} & \cdots & A_{mn}
  %       \end{array}\right]
  %   \left[  \begin{array}{c}
  %             \colorbox{green}{$v_1$} \\
  %             \colorbox{green}{$v_2$} \\\\
  %             \vdots \\\\
  %             \colorbox{green}{$v_n$}
  %           \end{array} \right]
  %         =
  %   \left[      \begin{array}{c}
  %               A_{11} v_1 + A_{12} v_2 + \dots + A_{1n} v_n \\
  %               \colorbox{green}{$A_{21} v_1 + A_{22} v_2 + \dots + A_{2n} v_n$} \\
  %               \vdots \\
  %               A_{m1} v_1 + A_{m2} v_2 + \dots + A_{mn} v_n
  %             \end{array} \right]
  % \end{equation}

\begin{qbox}
  Using equation \eqref{equation:matrixMult}, check that the $i^{th}$ column of the matrix $A$ is exactly the vector $A\vec{e}_i$.
\end{qbox}

\begin{qbox}[Practice problems]
  Compute $A \begin{bmatrix} x \\ y \\ z \end{bmatrix}$ for each of the following matrices.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 1 & 1 \\ 1 & 1 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 1 & 1 & 0\end{bmatrix}$
      \item $\begin{bmatrix} 1 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \\ 1 & 1 & 1\end{bmatrix}$
    \end{enumerate}
  \end{multicols}
\end{qbox}

\subsection{Linear systems}
A \emph{linear system} (of $\ell$ equations in $k$ variables) is a collection of linear\footnote{Linear = degree 1 = only addition and scalar multiplication.} equations.
\begin{align}
  \label{equation:generalLinearSystem}
  \begin{split}
    A_{11} x_1  + A_{12} x_2 + \cdots + A_{1k} x_k &= \alpha_1 \\
    A_{21} x_1  + A_{22} x_2 + \cdots + A_{2k} x_k &= \alpha_2 \\
    \vdots \\
    A_{\ell 1} x_1 + A_{\ell 2} x_2+ \cdots + A_{\ell k} x_k &= \alpha_\ell
  \end{split}
\end{align}
where $A_{ij}$'s and $\alpha_i$'s are real numbers, and $x_i$'s are the variables we are solving for.
By equation \eqref{equation:matrixMult} this is the same as
\begin{align*}
  \begin{bmatrix}
    A_{11} & A_{12} & \cdots & A_{1k} \\
    A_{21} & A_{22} & \cdots & A_{2k} \\\\
    \vdots & \vdots & \ddots & \vdots \\\\
    A_{\ell 1} & A_{\ell 2} & \cdots & A_{\ell k}
  \end{bmatrix}
  \begin{bmatrix} x_1 \\ x_2 \\ \vdots \\ x_k \end{bmatrix}
    &=
  \begin{bmatrix} \alpha_1 \\ \alpha_2 \\\\ \vdots \\\\ \alpha_\ell \end{bmatrix}
    \\\\
  A \vec{x}
  &= \vec{\alpha}
\end{align*}

Thus matrices encode not only the linear transformations but also linear systems. This observation allows us to study linear systems using techniques from linear algebra.

\subsubsection{Image}
\begin{definition}
  Let $A$ be a matrix of size $\ell \times k$.
  The \emph{image} of a matrix $A$ is simply the image of the corresponding linear transformation $A:\bbr^k \rightarrow \bbr^\ell$ i.e. it is the set of all vectors $\vec{y} \in \bbr^\ell$ such that $A \vec{x} = \vec{y} $ for some $\vec{x} \in \bbr^k$.
  \begin{align*}
    \im(A) = \{ A \vec{x} \mid \vec{x} \in \bbr^k \}
  \end{align*}
\end{definition}

\begin{qbox}
  Show that the $\im(A)$ is a subspace of $\bbr^\ell$.
\end{qbox}

\begin{definition}
  The dimension of $\im(A)$ is called the \emph{rank} of $A$.
  \begin{align*}
    \rank A := \dim \im (A)
  \end{align*}
\end{definition}

\begin{qbox}
  Check that equation \eqref{equation:generalLinearSystem} having a solution is the same as saying that $\vec{\alpha}$ is in the image of $A$.
\end{qbox}

\begin{qbox}
  Using \eqref{equation:matrixMult}, show that $\im(A)$ is precisely the span of the column vectors of $A$.
\end{qbox}

Combining all the above statements we get,
\begin{theorem}
  The equation \eqref{equation:generalLinearSystem} has a solution if and only if the vector $\vec{\alpha}$ is in the vector space spanned by the column vectors of $A$.
\end{theorem}

Finding the image i.e. the span of column vectors in general is non-trivial. However, there is a very fast algorithm called Gaussian elimination for computing it.

\begin{qbox}
  \label{q:rankComputation1}
  Find rank of each of the following matrices.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 1 & 1 \\ 1 & 1 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 1 & 1 & 0\end{bmatrix}$
      \item $\begin{bmatrix} 1 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \\ 1 & 1 & 1\end{bmatrix}$
    \end{enumerate}
  \end{multicols}
\end{qbox}

\begin{qbox}(Do this problem if you have done the section on linear transformations.)
  \label{q:rankComputation2}
  Using geometry, find rank of each of the following matrices.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $[\id_{\bbr^2}]$
      \item $[\rot]$
      \item $[\refl]$
      \item $[\proj]$
    \end{enumerate}
  \end{multicols}
\end{qbox}

\subsubsection{Kernel}
Once we know a solution exists, we can ask - what is the number of solutions? In general, there can be infinitely many solutions, but the dimension of the solution space is finite. This dimension is called the nullity.
\begin{definition}
  The \emph{kernel} of a matrix $A$ is the set of vector $\vec{x} \in \bbr^k$ such that $A \vec{x} = 0$.
  \begin{align*}
    \ker A := \{ \vec{x} \mid A \vec{x} = \vec{0} \} \subseteq \bbr^k
  \end{align*}
\end{definition}

\begin{qbox}
  Prove that $\ker A$ is a subspace of $\bbr^k$.
\end{qbox}

\begin{definition}
  The \emph{nullity} of $A$ is the dimension of $\ker A$.
  \begin{align*}
    \nul A := \dim \ker (A)
  \end{align*}
\end{definition}

\begin{qbox}
  \label{q:nullityComputation1}
  Find nullity of each of the following matrices.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 1 & 1 \\ 1 & 1 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 1 & 1 & 0\end{bmatrix}$
      \item $\begin{bmatrix} 1 & 0 & 1 \end{bmatrix}$
      \item $ \begin{bmatrix} 1 & 0 & 0 \\ 0 & 1 & 0 \\ 0 & 0 & 1 \\ 1 & 1 & 1\end{bmatrix}$
    \end{enumerate}
  \end{multicols}
\end{qbox}

\begin{qbox}(Do this problem if you have done the section on linear transformations.)
  \label{q:nullityComputation2}
  Using geometry, find nullity of each of the following matrices.
  \begin{multicols}{2}
    \begin{enumerate}
      \item $[\id_{\bbr^2}]$
      \item $[\rot]$
      \item $[\refl]$
      \item $[\proj]$
    \end{enumerate}
  \end{multicols}
\end{qbox}

\begin{qbox}
  Consider a matrix $A$ of size $\ell \times k$ and a vector $\vec{\alpha} \in \bbr^\ell$.
  \begin{enumerate}
    \item Show that if $\vec{x}_1$, $\vec{x}_2$ are solutions of $A \vec{x} = \vec{\alpha}$ then $\vec{x}_1 - \vec{x}_2$ is a solution of $A \vec{x} = \vec{0}$.
    \item Show that if $\vec{x}_1$ is a solution of $A \vec{x} = \vec{\alpha}$ and $\vec{x}_0$ is a solution of $A \vec{x} = \vec{0}$ then $\vec{x}_1 + \vec{x}_0$ is also a solution of $A \vec{x} = \vec{\alpha}$.
  \end{enumerate}
\end{qbox}

Combining everything above we have the following fundamental theorem about the solution space of the system of equations.
\begin{theorem}
  \label{theorem:dimensionSolutionSpace}
  Let $A$ be a matrix of size $\ell \times k$ and let $\vec{\alpha}$ be a vector in $\bbr^\ell$.
  \begin{enumerate}
    \item The linear system $A \vec{x} = \vec{\alpha}$ has a solution if an only if $\vec{\alpha}$ is in $\im A$.
    \item If a solution exists, then the space of solutions has dimension\footnote{This is not strictly correct as the space of solutions of $A \vec{x} = \vec{\alpha}$ is not a vector space if $\alpha \neq 0$. Instead, we need to shift the space of solutions to the origin to make it a vector space. This is analogous to the fact that a line not passing through the origin is not a vector space, but we can shift it to the origin to make it one. Once shifted, the space of solutions becomes $\ker A$.} $\nul A$.
  \end{enumerate}
\end{theorem}

\begin{corollary}
  If $\nul A = 0$ then $A \vec{x} = \vec{\alpha}$ has either a unique solution or no solution.
  If $\nul A > 0$ then $A \vec{x} = \vec{\alpha}$ has either infinitely many solutions or no solution.
\end{corollary}

Once we know that we are looking for vector spaces we can use standard algorithms in linear algebra to find the solutions.

\begin{qbox}
  For your computations in Questions \ref{q:rankComputation1}, \ref{q:rankComputation2}, \ref{q:nullityComputation1}, and \ref{q:nullityComputation2} what is $\rank A + \nul A$?
  \begin{align*}
    \rank A + \nul A = ?
  \end{align*}
\end{qbox}


\chapter{DT Fourier Series}

Recall the complex exponential $z^n$ is the Eigenfunction of DT LTI systems. If we can decompose an input into a (possibly infinite) sum of such signals, we can easily determine the output using the superposition principle. In this section we consider the decomposition when the input is periodic, called the DT \emph{Fourier Series} (DTFS). The DTFS is similar, but not identical to the CTFS. Notably, the approximation requires only a finite number of terms, there are no convergence issues, and the resulting spectrum is a periodic function.

Recall a DT signal $x[n]$ is periodic, with fundamental frequency $\omega_0 = \frac{2\pi}{N}$ rad/sec, if $x[n] = x[n+kN]$ for integer multiple $k$ and fundamental period $N \in \mathbb{Z}$. As we shall see, in this case the complex base of the Eigenfunction becomes $z_k = e^{jk\omega_0}$, and the decomposition is a finite sum. This gives the input-output relationship for a stable DT LTI system as
\[
x[n] = \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n} \;\longrightarrow\;  y[n] = \sum\limits_{k = N_0}^{N_0 + N-1} H\left(e^{j k\omega_0}\right) a_k e^{jk\omega_0 n}
\]
where $H\left(e^{j k\omega_0}\right)$ are the Eigenvalues or DT frequency response. We now turn to how to find the coefficients $a_k$.

\section{Synthesis and Analysis Equation}

Similar to the CTFS we wish to show that any periodic DT signal can be represented by the sum of complex exponentials whose frequencies are harmonics of the fundamental. This differs from the CTFS in that there are only $N$ distinct harmonics, so that the sum is over a finite range
\[
\boxed{x[n] = \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n} }
\]
where $N$ is the period and $N_0$ is any starting index for the sum. Note the course text defines $<N> = \{ N_0, N_0 + 1,  \cdots (N_0 + N-1) \}$. This is called the \emph{synthesis equation} of the DT Fourier series.

One approach to find the coefficients $a_k$ is to note that there are a finite number of terms in the summation and the signal has a finite number of values over one period. This gives a system of $N$ linear equations in $N$ unknowns (the $a_k$'s)
\begin{align*}
  x[N_0] &= \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0N_0}\\
  x[N_0+1] &= \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0(N_0+1)}\\
  \vdots\hspace{2em} &= \hspace{2em}\vdots\\
  x[N_0+N-1] &= \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0(N_0+N-1)}\\
\end{align*}
which can be solved to find the coefficients using linear algebra.

\begin{example}
  Consider the periodic DT signal $x[n] = \cdots -1, 1, -1, 1, -1, 1, \cdots$ where $x[0] = 1$. The period is $N=2$ so that $\omega_0 = \pi$. If we let $N_0 = 0$, the system of equations is
  \begin{align*}
    x[0] &= \sum\limits_{k = 0}^{1} a_k = a_0 + a_1 = 1\\
    x[1] &= \sum\limits_{k = 0}^{1} a_k e^{jk\pi} = a_0 - a_1 = -1
  \end{align*}
  which has the solution $a_0 = 0$ and $a_1 = 1$ and $x[n] = e^{j\pi n}$.\\
  $\blacksquare$
\end{example}

Another approach is similar to that taken when deriving the CT Fourier Series. Beginning with the synthesis equation
\[
x[n] = \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n}
\]
we multiply both sides by $e^{-j\frac{2\pi r}{N} n}$ for $r \in \mathbb{Z}$ and sum over $N$ terms
\[
\sum\limits_{n = N_0}^{N_0 + N-1} x[n]e^{-j\frac{2\pi r}{N} n} = \sum\limits_{n = N_0}^{N_0 + N-1} \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n}e^{-j\frac{2\pi r}{N} n}
\]
We then interchange to order of summation on the right-hand-side
\[
\sum\limits_{n = N_0}^{N_0 + N-1} x[n]e^{-j\frac{2\pi r}{N} n} =  \sum\limits_{k = N_0}^{N_0 + N-1} a_k \underbrace{\sum\limits_{n = N_0}^{N_0 + N-1} e^{jk\omega_0n}e^{-j\frac{2\pi r}{N} n}}
\]
Since $\omega_0 = \frac{2\pi}{N}$, the bracketed term is
\[
\sum\limits_{n = N_0}^{N_0 + N-1} e^{j(k-r)\frac{2\pi}{N} n} = \begin{cases}
  N & \mbox{ if } k-r = 0, \pm N, \pm 2N, \cdots\\
  0 & \mbox{ else }
\end{cases}
= N\delta[(k-r) + mN] \mbox{ for arbitrary } m \in \mathbb{Z}
\]
and the right-hand side is
\[
\sum\limits_{k = N_0}^{N_0 + N-1} a_k N \delta[(k-r) + mN] = Na_r
\]
since $N_0 < mN < N_0 + 1$ for some $m$. Solving for $a_r$ gives the \emph{analysis equation} of the DT Fourier series:
\[
\boxed{a_r = \frac{1}{N} \sum\limits_{n = N_0}^{N_0+N-1} x[n]e^{-j\frac{2\pi}{N} r n }}
\]
where the summation can be over any interval of length $N$ and the symbol for the subscript (integer $r$) is arbitrary. The DT Fourier Series coefficients are also called the \emph{spectrum} of the signal. In general the $a_k$ are complex. \emph{Note the spectrum is periodic in $N$}. The function of $k$, $|a_k|$ is called the \emph{amplitude spectrum}. The function of $k$, $\angle a_k$ is called the \emph{phase spectrum}. When plotting the coefficients it is common to plot the amplitude and phase spectrum together over a single interval of length $N$ (since it is periodic).

\begin{example}
  A simple way to construct a DT periodic signal is to use the modulus $\%$ operator. For example,
  \[
  x[n] = \gamma^{n \% N} \mbox{ for any } \gamma \in \mathbb{C}
  \]
  is periodic in $N$, e.g. $x[n] = \left(\frac{1}{2}\right)^{n \% 4}$
  \begin{center}
    \includegraphics[scale=1]{graphics/dtfs_example1.pdf}
  \end{center}

  The synthesis equation is given by
  \[
  x[n] = \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n}
  \]
  Where the coefficients are found using the analysis equation. Let $N_0 = 0$ arbitrarily, then
  \begin{align*}
    a_k &= \frac{1}{N} \sum\limits_{n = 0}^{N-1} x[n]e^{-j\frac{2\pi}{N} k n }\\
    &= \frac{1}{N} \sum\limits_{n = 0}^{N-1} \gamma^n e^{-j\frac{2\pi}{N} k n }\\
    &= \frac{1}{N} \sum\limits_{n = 0}^{N-1} \left(\gamma e^{-j\frac{2\pi}{N} k}\right)^n\\
    a_k &= \frac{1}{N} \frac{1-\left(\gamma e^{-j\frac{2\pi}{N} k}\right)^N}{1-\left(\gamma e^{-j\frac{2\pi}{N} k}\right)}
  \end{align*}

We can plot the spectrum of this signal (using for example Matlab)
\begin{verbatim}
gamma = 0.5;
N = 4;
k = -10:10;
a = (1-(gamma*exp(-j*2*pi*k/N)).^N)./(N*(1-gamma*exp(-j*2*pi*k/N)));

h0 = subplot(2,1,1);
h1 = stem(k, abs(a));
h2 = xlabel('k');
h3 = ylabel('|a(k)|');
h4 = title('Amplitude Spectrum');
h5 = subplot(2,1,2);
h6 = stem(k, angle(a));
h7 = xlabel('k');
h8 = ylabel('Angle a(k)');
h9 = title('Phase Spectrum');
\end{verbatim}
Giving the amplitude and phase spectrum plot
\begin{center}
\includegraphics[scale=0.7]{graphics/dtfs_example1.png}
\end{center}
$\blacksquare$
\end{example}

\begin{example}
  Find the DTFS of $x[n] = \cos[\tfrac{\pi}{4}n]$. Note $N=8$ and $\omega_0 = \tfrac{\pi}{4}$. Using Euler's formula
  \[
  x[n] = \frac{1}{2}e^{j\frac{\pi}{4}n} + \frac{1}{2}e^{-j\frac{\pi}{4}n}
  \]
  The synthesis equation is
  \[
  x[n] = \sum\limits_{k = 0}^{N-1} a_k e^{jk\omega_0n} = a_0 + a_1e^{j\frac{\pi}{4}n}+ a_2e^{j\frac{2\pi}{4}n} + \cdots + a_7e^{j\frac{7\pi}{4}n}
  \]
  Comparing to the expansion above and noting that $e^{-j\frac{\pi}{4}n} = e^{j\frac{7\pi}{4}n}$ we see that
  \[
  a_k = \begin{cases}
    \frac{1}{2} & k=1\\
    \frac{1}{2} & k=7\\
    0 & \mbox{else}
  \end{cases}
  \]
  for $k \in [0,7]$ and $a_k = a_{k\%8}$ for all $k$.\\
  $\blacksquare$
\end{example}

\section{Properties of the DT Fourier Series}

Given two signals $x[n]$ and $y[n]$ periodic in $N$ with $\omega_0 = \frac{2\pi}{N}$, having DT Fourier coefficients
$a_k$ and $b_k$ respectively.

\begin{itemize}
\item Linearity. The coefficients of the signal
  \[
  z[n] = Ax[n] + By[n] \mbox{ for constants } A,B
  \]
  are $Aa_k + Bb_k$
\item Index Shifting. The coefficients of
  \[
  z[n] = x[n-n_0] \mbox{ are } e^{-jk\omega_0 n_0}a_k
  \]
  that is, it adds a phase shift.
\item Frequency Shift. The coefficients of
  \[
  z[n] = x[n]e^{jm\omega_0n} \mbox{ are } a_{k-m}
  \]
\item Index Reversal. The coefficients of
  \[
  z[n] = x[-n] \mbox{ are } a_{-k}
  \]
\item Multiplication. The coefficients of
  \[
  z[n] = x[n] \cdot y[n] \mbox{ are } \sum\limits_{m = N_0}^{N_0 + N -1} a_m\cdot b_{k-m}
  \]
  the discrete convolution of the individual signals' coefficients.
\item Convolution. The coefficients of
  \[
  z[n] = x[n] * y[n] \mbox{ are } N a_k b_k
  \]
\item Conjugate Symmetry. The coefficients of
  \[
  z[n] = x^*[n] = \Re{x[n]} - j\Im{x[n]} \mbox{ are } a_{-k}^*
  \]
  A consequence of this property is that real, even signals have real, even $a_k$; and real, odd signals have purely imaginary, odd  $a_k$. Thus if $x[n]$ is real  $|a_k|$ is an even periodic function of $k$ and $\angle a_k$ is an odd periodic function of $k$.
\item Parseval's Relation. The power of the signal with Fourier series coefficients is
  \[
  \frac{1}{N} \sum\limits_{n = N_0}^{N_0 + N -1} |x[n]|^2\;dt = \sum\limits_{k = N_0}^{N_0+N-1} |a_k|^2
  \]
\end{itemize}

\section{Comparison of CT and DT Fourier Series}
A summary of the CT and DT Fourier Series is as follows.\\

In CT, a periodic signal $x(t)$ can be decomposed as a countably infinite combination of complex sinusoids at harmonic frequencies of the fundamental. The Fourier series coefficients are a discrete signal that is a-periodic.

\[
x(t) \approx \sum\limits_{k = -\infty}^{\infty} a_k \, e^{j k\omega_0 t}
\hspace{2em}
a_k = \frac{1}{T_0} \int\limits_{0}^{T_0} x(t)e^{-jk\omega_0 t} \; dt
\]

In DT, a periodic signal $x[n]$ can be decomposed as a finite combination of complex sinusoids at harmonic frequencies of the fundamental. The Fourier series coefficients are a discrete signal that is periodic.

\[
x[n] = \sum\limits_{k = N_0}^{N_0 + N-1} a_k e^{jk\omega_0n}
\hspace{2em}
a_k = \frac{1}{N} \sum\limits_{n = N_0}^{N_0+N-1} x[n]e^{-j\frac{2\pi}{N} k n }
\]


\documentclass[./main.tex]{subfiles}

\usepackage{amsmath}
\usepackage{mathtools}

\begin{document}

\chapter{Algebraic solution to the multilateration problem}

\section{Solution based on three reference point}
Given are the three reference points $P_1(x_1, y_1, z_1)$, $P_2(x_2, y_2, z_2)$, $P_3(x_3, y_3, z_3)$ and the range measurements $s_1$, $s_2$, $s_3$ to point $N(x,y,z)$, the location $(x,y,z)$ of $N$ is the solutions of the following system of quadratic equations:

\begin{equation}
    \begin{dcases*}
        (x-x_1)^2 + (y-y_1)^2 + (z-z_1)^2 = s_1^2 \\
        (x-x_2)^2 + (y-y_2)^2 + (z-z_2)^2 = s_2^2 \\
        (x-x_3)^2 + (y-y_3)^2 + (z-z_3)^2 = s_3^2
    \end{dcases*}
    \label{eqn:three_ref_point_localization}
\end{equation}

\ref{eqn:three_ref_point_localization} can be arranged as:

\begin{equation}
    \begin{dcases*}
        (x^2 + y^2 + z^2) -2x_1x - 2y_1y - 2z_1z = s_1^2 - x_1^2 - y_1^2 - z_1^2 \\
        (x^2 + y^2 + z^2) -2x_2x - 2y_2y - 2z_2z = s_2^2 - x_2^2 - y_2^2 - z_2^2 \\
        (x^2 + y^2 + z^2) -2x_3x - 2y_3y - 2z_3z = s_1^2 - x_3^2 - y_3^2 - z_3^2
    \end{dcases*}
    \label{eqn:three_ref_point_localization}
\end{equation}
Or in the matrix representation:

\begin{equation}
    \begin{bmatrix}
        1 & -2x_1 & -2y_1 & -2z_1\\
        1 & -2x_2 & -2y_2 & -2z_2\\
        1 & -2x_3 & -2y_3 & -2z_3
    \end{bmatrix}
    \begin{bmatrix}
        x^2 + y^2 + z^2\\
        x\\
        y\\
        z\\
    \end{bmatrix}
    =
    \begin{bmatrix}
        s_1^2 - x_1^2 - y_1^2 - z_1^2\\
        s_2^2 - x_2^2 - y_2^2 - z_2^2\\
        s_1^2 - x_3^2 - y_3^2 - z_3^2\\
    \end{bmatrix}
    \label{eqn:three_ref_point_localization_matrix}
\end{equation}
Thus, \ref{eqn:three_ref_point_localization_matrix} is represented in the known form:
\begin{equation}
    \mathbf{A}_0 \mathbf{x}=\mathbf{b}_0
    \label{eqn:three_ref_point_localization_general_form}
\end{equation}
With the constraint: $\mathbf{x} \in \mathbf{E}$\\
Where $\mathbf{E}=\{(x_0, x_1, x_2, x_3)^T / x_0 = x_1^2 + x_2^2 + x_3^2 \}$
\subsection{Solution of equation \ref{eqn:three_ref_point_localization_general_form}}

\end{document}