% chapter included in vclmanual.tex
\documentclass[vcl_manual.tex]{subfiles}
\begin{document}

\chapter{Mathematical functions}\label{chap:MathematicalFunctions}
\flushleft

\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & exponent \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & extracts the exponent part of a floating point number. The result is an integer vector.\newline
exponent(a) = floor(log2(abs(a))).\newline
The value for a = 0 is implementation dependent \\ \hline
 \bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(1.0f, 2.0f, 3.0f, 4.0f);
Vec4i b = exponent(a);  // b = (0, 1, 1, 2)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & fraction \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & extracts the fraction part of a floating point number.\newline
a = pow(2,exponent(a)) * fraction(a) \newline
The value for a = 0, INF, or NAN is implementation dependent \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(2.0f, 3.0f, 4.0f, 5.0f);
Vec4f b = fraction(a);  // b = (1.00f, 1.50f, 1.00f, 1.25f)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & exp2 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & calculates integer powers of 2. The input is an integer vector, the output is a floating point vector. Overflow gives +INF, underflow gives zero. This function will never produce subnormals, and never raise exceptions \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4i a(-1, 0, 1, 2);
Vec4f b = exp2(a);     // b = (0.5f, 1.0f, 2.0f, 4.0f)
\end{lstlisting}
\vspacebig

\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & mul\_add \newline
nmul\_add \newline
mul\_sub \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & mul\_add(a,b,c) = a*b+c \newline
nmul\_add(a,b,c) = -a*b+c \newline
mul\_sub(a,b,c) = a*b-c \newline
These functions use fused multiply-and-add (FMA) instructions if available. Some compilers use FMA instructions automatically for expressions like a*b+c. Use these functions for optimal performance on all compilers or to specify calculation order, etc. \\ \hline
\bfseries Precision & The intermediate product a*b is calculated with infinite precision if the FMA instruction set is enabled. \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\vspacesmall

\section{Floating point categorization functions}\label{FloatingPointCategorizationFunctions}

\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & is\_finite \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns a boolean vector with true for elements that are normal, subnormal or zero, false for INF and NAN \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a( 0.0f, 1.0f, 2.0f, 3.0f);
Vec4f  b(-1.0f, 0.0f, 1.0f, 2.0f);
Vec4f  c = a / b;
Vec4fb d = is_finite(c);  // d = (true, false, true, true)
\end{lstlisting}
\vspacebig

\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & is\_inf \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns a boolean vector with true for elements that are +INF or -INF, false for all other values, including NAN \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a( 0.0f, 1.0f, 2.0f, 3.0f);
Vec4f  b(-1.0f, 0.0f, 1.0f, 2.0f);
Vec4f  c = a / b;
Vec4fb d = is_inf(c);  // d = (false, true, false, false)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & is\_nan \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns a boolean vector with true for all types of NAN, false for all other values, including INF \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a(-1.0f, 0.0f, 1.0f, 2.0f);
Vec4f  b = sqrt(a);
Vec4fb c = is_nan(b);  // c = (true, false, false, false)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & is\_subnormal \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns a boolean vector with true for subnormal (denormal) vector elements, false for normal numbers, INF and NAN \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a(1.0f, 1.0E-10f, 1.0E-20f, 1.0E-30f);
Vec4f  b = a * a;           // b = (1.0f, 1.E-20f, 1.E-40f, 0.f)
Vec4fb c = is_subnormal(b); // c = (false,false,true,false)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & is\_zero\_or\_subnormal \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns a boolean vector with true for zero and subnormal (denormal) vector elements, false for nonzero normal numbers, INF and NAN \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a(1.0f, 1.0E-10f, 1.0E-20f, 1.0E-30f);
Vec4f  b = a * a;           // b = (1.0f, 1.E-20f, 1.E-40f, 0.f)
Vec4fb c = is_zero_or_subnormal(b); // c = (false,false,true,true)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & infinite4f, infinite8f, infinite16f, \newline
infinite2d, infinite4d, infinite8d \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns positive infinity \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f  a = infinite4f(); // a = (INF, INF, INF, INF)
\end{lstlisting}
\vspacebig

\label{nan4f}
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & nan4f(unsigned int n) \newline
nan8f(unsigned int n) \newline
nan16f(unsigned int n) \newline
nan2d(unsigned int n) \newline
nan4d(unsigned int n) \newline
nan8d(unsigned int n) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Description & returns not-a-number (NAN). \newline
The optional parameter n may be used for error tracing. \newline
The maximum value of n is 0x003FFFFF.
This parameter can be retrieved later by the function nan\_code (page \pageref{nanCode}). \\ \hline
\bfseries Efficiency & good \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a = nan4f(); // a = (NAN, NAN, NAN, NAN)
\end{lstlisting}
\vspacesmall

\section{Floating point control word manipulation functions}\label{FPControlWordManipulationFunctions}

MXCSR is a control word that controls floating point exceptions, rounding mode and subnormal numbers. There is one MXCSR for each thread.

The MXCSR has the following bits:
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Bit index & \bfseries meaning \\ \hline
0 & Invalid Operation Flag \\ \hline
1 & Denormal (subnormal) Flag \\ \hline
2 & Divide-by-Zero Flag \\ \hline
3 & Overflow Flag \\ \hline
4 & Underflow Flag \\ \hline
5 & Precision Flag \\ \hline
6 & Denormals (subnormals) Are Zeros \\ \hline
7 & Invalid Operation Mask \\ \hline
8 & Denormal (subnormal) Operation Mask \\ \hline
9 & Divide-by-Zero Mask \\ \hline
10 & Overflow Mask \\ \hline
11 & Underflow Mask \\ \hline
12 & Precision Mask \\ \hline
13-14 & Rounding control: \newline
00: round to nearest or even \newline
01: round down towards -infinity \newline
10: round up towards +infinity \newline
11: round towards zero (truncate) \newline
If the rounding mode is temporarily changed then it must be set back to 00 for the vector class library to work correctly. \\ \hline
15 & Flush to Zero \\ \hline
\end{tabular}

Please see programming manuals from Intel or AMD for further explanation.

\vspacebig
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & get\_control\_word \\ \hline
\bfseries Description & reads the MXCSR control word \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
int m = get_control_word();  // default value m = 0x1F80
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & set\_control\_word(n) \\ \hline
\bfseries Description & writes the MXCSR control word \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
// Enable overflow and divide by zero exceptions:
set_control_word(0x1980);
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & reset\_control\_word \\ \hline
\bfseries Description & sets the MXCSR control word to the default value \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
reset_control_word();
\end{lstlisting}

\label{noSubnormals}
\vspacesmall
\begin{tabular}{|p{25mm}|p{100mm}|}
\hline
\bfseries Function & no\_subnormals \\ \hline
\bfseries Description & Disables the use of subnormal (denormal) values. \newline
Floating point numbers with an absolute value below  \newline
1.18E-38 for single precision or 2.22E-308 for double precision are represented by subnormal numbers. The handling of subnormal numbers is extremely time-consuming on many CPUs. The no\_subnormals function sets the "denormals are zeros" and "flush to zero" mode to avoid the use of subnormal numbers. It is recommended to call this function at the beginning of each thread in order to improve the speed of mathematical calculations if very low numbers are likely to occur. \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
no_subnormals();
\end{lstlisting}

\section{Floating point mathematical functions}\label{FPMathematicalFunctions}

Mathematical functions such as logarithms, exponential functions, power, trigonometric functions, etc. are available either as inline code or through external function libraries. These functions all take vectors as input and produce vectors as output.
\vspacesmall

The use of vector math functions is straightforward:
\begin{example}
\label{exampleSinFunction}
\end{example} % frame disappears if I put this after end lstlisting
\begin{lstlisting}[frame=single]
#include <stdio.h>
#include "vectorclass.h"
#include "vectormath_trig.h"    // trigonometric functions

int main() {
    Vec4f a(0.0f, 0.5f, 1.0f, 1.5f);// define vector
    Vec4f b = sin(a);               // sine function
    // b = (0.0000f, 0.4794f, 0.8415f, 0.9975f)

    // output results:
    for (int i = 0; i < b.size(); i++) {
        printf("%6.4f ", b[i]);
    }
    printf("\n");
    return 0;
}

\end{lstlisting}
\vspacesmall

\section{Inline mathematical functions}\label{InlineMathematicalFunctions}

The inline mathematical functions are available by including the appropriate header file, e. g. vectormath\_exp.h for powers, logarithms and exponential functions, and vectormath\_trig.h for trigonometric functions. An advantage of the inline version is that the compiler can optimize the code across function calls, eliminate common sub-expressions, etc. The disadvantage is that you may get multiple instances of the same function taking up space in the code cache.
\vspacesmall

The speed of the inline functions is similar to or better than external vector function libraries in most cases and many times faster than standard (scalar) math function libraries.
Unlike many other mathematical function libraries, this library does not use lookup tables. This gives better parallelism, a very small load on the data cache, and a more predictable execution time.
\vspacesmall

The precision is good. The calculation error is typically below 2 ULP (Unit in the Last Place = least significant bit) on the output. (The relative value of one ULP is $2^{-52}$ for double precision and $2^{-23}$ for single precision). Cases where the error can exceed 3 ULP are mentioned under the specific function.
\vspacesmall

The functions do not generate exceptions or set \codei{errno} when an input is out of range. This would be inefficient and it would be problematic for the error handler to detect which vector element caused the error. Instead, the functions return INF (infinity) or NAN (not a number) in case of error. Generally, an overflow will produce INF. A negative overflow produces -INF. An underflow towards zero returns 0. Other errors produce NAN. An efficient way of detecting errors is to let the INF and NAN codes propagate through the calculations and detect the error at the end of a series of calculations as explained on page \pageref{FloatingPointErrors}. It is possible to include an error code in a NAN and detect it with the function nan\_code on page \pageref{nanCode}.
\vspacesmall

Note that many of the inline math functions do not support subnormal numbers. Subnormal numbers may be treated as zero by the logarithm, exponential, power and root functions. It is recommended to set the “denormals are zero” and “flush to zero” flags by calling the function \codei{no\_subnormals()} first (see page \pageref{noSubnormals}). This may speed up some calculations and give more consistent results.
\vspacesmall

A description of each mathematical function is given below.
\vspacesmall

\section{Using an external library for mathematical functions}\label{ExternalMathLibrary}

As an alternative to the inline mathematical functions, you can use an external function library named Intel SVML (Short Vector Math Library).
\vspacesmall

You can use the Intel SVML library with any compiler. The SVML library is available for all platforms relevant to the vector class library. SVML is included with Intel C++ compilers, but it can be used with other compilers as well.
The performance of SVML will usually be good on both Intel and non-Intel processors, as long as you are not using the Intel compiler.
After installing an Intel compiler, you can look for the file name: svml\_dispmt.lib for Windows or libsvml.a for Linux. You also need the library libircmt.lib or libirc.a for a CPU dispatcher called by the svml library.
You also need the library svmlpatch.lib if compiling for 64-bit Windows. This library is provided in the folder svml\_patch.
\vspacesmall

Link the necessary library files into your C++ project. Be sure to choose the 32-bit version or 64-bit version of each library according to the platform you are compiling for.
\vspacesmall

Include the header file vectormath\_lib.h instead of vectormath\_exp.h, vectormath\_trig.h, or vectormath\_hyp.h if you want to use the SVML library.
It is not possible to mix the two kinds of mathematical functions (inline and library) in the same C++ file.
\vspacesmall

The available vector math functions are listed below. The efficiency is listed as poor because mathematical functions take more time to execute than most other functions, but they are still much faster than scalar alternatives. The details listed apply to the inline version. Details for the SVML library versions may be sought in the documentation for the specific library.
\vspacesmall

\section{Powers, exponential functions and logarithms}\label{ExpLogFunctions}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & pow(vector, vector), pow(vector, scalar) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & pow(a,b) = $a^b$ \newline
See also faster alternatives below for integer and rational powers. \\ \hline
\bfseries Range & Subnormal numbers are treated as zero. The result is NAN if a is negative and b is not an integer. NAN's are always propagated by the inline version of pow, even in cases where the IEEE 754 standard specifies otherwise. The library version may fail to propagate NANs in the cases pow(NAN,0) and pow(1,NAN). \\ \hline
\bfseries Precision & better than (0.8*abs(b)+2) ULP \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a( 1.0f,  2.0f, 3.0f, 4.0f);
Vec4f b( 0.0f, -1.0f, 0.5f, 2.0f);
Vec4f c = pow(a, b);
// c = (1.0000, 0.5000, 1.7321, 16.0000)
Vec4f d = pow(a, 2.4f);
// d = (1.0000, 5.2780, 13.9666, 27.8576)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & pow(vector, int) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & no extra header file required \\ \hline
\bfseries Library version & not available \\ \hline
\bfseries Description & see page \pageref{powVectorInt} \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(0.0f, 1.0f, 2.0f, 3.0f);
int   b = 3;
Vec4f c = pow(a, b);  // c = (0.0f, 1.0f, 8.0f, 27.0f)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & pow\_const(vector, const int) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & no extra header file required \\ \hline
\bfseries Library version & not available \\ \hline
\bfseries Description & see page \pageref{powConstVectorInt} \\ \hline
\bfseries Efficiency & medium, often better than pow(vector, int) \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(0.0f, 1.0f, 2.0f, 3.0f);
Vec4f c = pow_const(a, 3);  // c = (0.0f, 1.0f, 8.0f, 27.0f)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & pow\_ratio(vector x, const int a, const int b) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & not available \\ \hline
\bfseries Description & Raises all elements of x to the rational power a/b.\newline
a and b must be compile-time constant integers. \\ \hline
\bfseries Range & x may be zero only if a and b are positive. x may be negative only if b is odd.\newline
The range is the same as for cbrt (page \pageref{cbrt}) if b is 3.
 The result when x is infinite may be NAN in some cases.
Subnormal numbers are treated as zero in some cases.  \\ \hline
\bfseries Precision & slightly imprecise for extreme values of \codei{a} due to accumulating rounding errors.
The precision is similar to the cbrt function when b is 3 or 6. \\ \hline
\bfseries Efficiency & Quite good for b = 1, 2, 4, or 8. Reasonable for b = 3 or 6. No better than pow for other values of b. \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(1.0f, 2.0f, 3.0f, 4.0f);
// Reciprocal square root
Vec4f b = pow_ratio(a, -1, 2);  // c = (1.0, 0.707, 0.577, 0.500)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & exp \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & exponential function $e^x$ \\ \hline
\bfseries Range & double: abs(x) \textless 708.39. float: abs(x) \textless 87.3 \\ \hline
\bfseries Efficiency & Poor. The performance of the inline version for single precision vectors (Vec16f etc.) is better when the instruction set AVX512ER is supported. The performance can be improved further, at a slight loss of precision, when VCL\_FASTEXP is defined in addition to AVX512ER. \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
#include "vectormath_exp.h"
Vec16f a, b;
b = exp(a);
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & expm1 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & $e^x-1$. Useful to avoid loss of precision if x is close to 0 \\ \hline
\bfseries Range & double: abs(x) \textless{} 708.39. float: abs(x) \textless{} 87.3 \\ \hline
\bfseries Efficiency & Poor. (not improved with AVX512ER) \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & exp2 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & $2^x$ \\ \hline
\bfseries Range & double: abs(x) \textless{} 1020. float: abs(x) \textless{} 27. \\ \hline
\bfseries Efficiency & The performance of the inline version is good for single precision vectors if instruction set AVX512ER is supported. (VCL\_FASTEXP is not needed). \newline
Use pow or pow\_const instead if x is an integer. \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & exp10 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & $10^x$ \\ \hline
\bfseries Range & double: abs(x) \textless{} 307.65. float: abs(x) \textless{} 37.9. \\ \hline
\bfseries Efficiency & Poor. The performance of the inline version for single precision vectors (Vec16f etc.) is better when the instruction set AVX512ER is supported. The performance can be improved further, at a slight loss of precision, when VCL\_FASTEXP is defined in addition to AVX512ER. \newline
Use pow or pow\_const instead if x is an integer. \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
#define VCL_FASTEXP
#include "vectormath_exp.h"
Vec16f a, b;
b = exp10(a);
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & log \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & natural logarithm \\ \hline
\bfseries Range & The input must be a normal number. Subnormal numbers are treated as zero. \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & log1p \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & log(1+x) \newline
Useful to avoid loss of precision if x is close to 0 \\ \hline
\bfseries Range & x \textgreater{} -1 \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & log2 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & logarithm base 2  \\ \hline
\bfseries Range & The input must be a normal number. Subnormal numbers are treated as zero. \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & log10 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & logarithm base 10  \\ \hline
\bfseries Range & The input must be a normal number. Subnormal numbers are treated as zero. \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\label{cbrt}
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & cbrt \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & cube root  \\ \hline
\bfseries Range & float: 0, $\pm 10^{-28} ..{}  10^{28}$ \newline
                  double: 0, $\pm 10^{-200} ..{}  10^{200}$ \newline
                  The return value is 0 if abs(x) is too small \\ \hline
\bfseries Precision & 5 ULP \\ \hline
\bfseries Efficiency & Faster than pow \\ \hline
\end{tabular}

\vspacebig

\section{Trigonometric functions and inverse trigonometric functions}
All angles are in radians.

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & sin \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & sine function \\ \hline
\bfseries Range & Defined for abs(x) \textless{} $10^8$. 0 for big x \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(0.0f, 0.5f, 1.0f, 1.5f);// define vector
Vec4f b = sin(a);               // sine function
// b = (0.0000f, 0.4794f, 0.8415f, 0.9975f)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & cos \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & cosine function \\ \hline
\bfseries Range & Defined for abs(x) \textless{} $10^8$. 1 for big x \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & sincos \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h (some platforms only) \\ \hline
\bfseries Description & sine and cosine computed simultaneously.\\ \hline
\bfseries Range & Defined for abs(x) \textless{} $10^8$. 0 and 1 for big x \\ \hline
\bfseries Efficiency & faster than computing sin and cos separately \\ \hline
\end{tabular}
\begin{lstlisting}[frame=none]
// Example:
Vec4f a(0.0f, 0.5f, 1.0f, 1.5f);
Vec4f s, c;
s = sincos(&c, a);
// s = (0.0000, 0.4794, 0.8415, 0.9975)
// c = (1.0000, 0.8776, 0.5403, 0.0707)
\end{lstlisting}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & tan \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & tangent function \\ \hline
\bfseries Range & Defined for abs(x) \textless{} $10^7$. 0 for big x \newline
tan($\pi/2$) will not produce infinity because the value of $\pi/2$ cannot be represented exactly as a floating point number. The output will be big, though, when the input is as close to $\pi/2$ as possible.\\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & asin \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse sine function \\ \hline
\bfseries Range & -1 $\leq$ x $\leq$ 1 \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & acos \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse cosine function \\ \hline
\bfseries Range & -1 $\leq$ x $\leq$ 1 \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & atan \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & Inverse tangent \\ \hline
\bfseries Range & Results between $-\pi/2$ and $\pi/2$ \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & atan2 \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_trig.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & Inverse tangent with two parameters, x and y, gives the angle to a point in the (x,y) plane \\ \hline
\bfseries Range & Results between $-\pi$ and $\pi$ \newline
The result of atan2(0,0) is 0 by convention\\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacesmall

\section{Hyperbolic functions and inverse hyperbolic functions}\label{HyperbolicFunctions}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & sinh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & hyperbolic sine \\ \hline
\bfseries Range & double: abs(x) \textless{} 709. float: abs(x) \textless{} 88. \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & cosh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & hyperbolic cosine \\ \hline
\bfseries Range & double: abs(x) \textless{} 709. float: abs(x) \textless{} 88. \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & tanh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & hyperbolic tangent \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & asinh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse hyperbolic sine \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & acosh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse hyperbolic cosine \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & atanh \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_hyp.h \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse hyperbolic tangent \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacesmall

\section{Other mathematical functions}\label{OtherMathematicalFunctions}

\vspacesmall
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & erf \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & not available \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & error function \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & erfc \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & not available \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & error function complement \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & erfinv \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & not available \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse error function \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & cdfnorm \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & not available \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & cumulative normal distribution function \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & cdfnorminv \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & not available \\ \hline
\bfseries Library version & vectormath\_lib.h \\ \hline
\bfseries Description & inverse cumulative normal distribution function \\ \hline
\bfseries Efficiency & poor \\ \hline
\end{tabular}
\vspacebig

\label{nanCode}
\begin{tabular}{|p{30mm}|p{120mm}|}
\hline
\bfseries Function & Vec4ui nan\_code(Vec4f) \newline
Vec8ui nan\_code(Vec8f)\newline
Vec16ui nan\_code(Vec16f)\newline
Vec2uq nan\_code(Vec2d)\newline
Vec4uq nan\_code(Vec4d)\newline
Vec8uq nan\_code(Vec8d) \\ \hline
\bfseries Defined for & all floating point vector classes \\ \hline
\bfseries Inline version & vectormath\_exp.h \\ \hline
\bfseries Library version & not available \\ \hline
\bfseries Description & Extracts an error code hidden as payload in a NAN. This code can be generated with the functions nan4f etc. (page \pageref{nan4f}) and propagated through a series of calculations. When two NANs are combined (e.g. NAN1+NAN2), current processors propagate the first one. NANs produced by CPU instructions, such as 0./0. or sqrt(-1.) have a code of zero. NANs cannot propagate through integers and booleans.\newline
The return value is (0x00400000 + code) for a quiet NAN.
The sign bit is ignored.\newline
The return value is 0 for inputs that are not NAN. \\ \hline
\bfseries Efficiency & medium \\ \hline
\end{tabular}
\vspacesmall

\end{document}


\chapter{Graphics} \label{Ch:Graphics}

\section{Ground Track Plotting}

A ground track plot displays spherical latitude and longitude over a Poincare (unwrapped cylinder) projection of a body's surface geography. The algormithm used to generate a ground track plot ensures that line wrapping at the plot boundaries is handled correctly, and that when wrapping occurs, the plot is interpolated to the plot boundary.

Longitude is computed by converting position and velocity of the object to the body fixed system of the ground track plot central body.  Define $x$, $y$, and $z$ as the body fixed coordinates of the object.   The longitude is calculated using
%
\begin{equation}
     \lambda =  atan2(y,x)
\end{equation}
%
and the latitude is calculate using
%
\begin{equation}
     \phi = asin{\frac{z}{r}}
\end{equation}
%
where
%
\begin{equation}
    r = \sqrt{x^2 + y^2 + z^2}
\end{equation}

There are three special cases to consider when plotting a new point on a ground track: (1) the new point wraps off the right-hand side of the plot, (2) the new point wraps off the left-had side of the plot, and (3) the new point does not wrap off either plot boundary.  When wrapping oocurs as in cases (1) and (2), the plot algorithm must perform  "Pen Up" and "Pen Down" commands to avoid connecting points on opposite ends of the plot with a spurious straight line.  Secondly, when wrapping occurs, the system must interpolate the line segments to plot boundaries.

The test to determine the case depends  upon whether the object is moving clockwise or counterclockwise in the body fixed system of the ground track plot central body.    The orbit direction is determined by evaluating the z component of orbit angular momentum expressed in the body fixed system.  Define a variable $d$ that is positive for clockwise motion in the body fixed system (the spacecraft is moving to the right on a ground track plot), and negative for counterclockwise motion. (spacecraft is moving to the left on  a ground track plot), and 0 for no motion.   The variable $d$ is computed as follows
%
\begin{equation}
   d = sign(x \dot{y} - \dot{x}y)
\end{equation}
%
%
\begin{equation}
    m = \frac{\phi_i -\phi_{i-1}}{\lambda_i - \lambda_{i-1}}
\end{equation}

\begin{center}
\begin{minipage}{6 in}
\begin{small}
\begin{algorithm}[H]

    \KwIn{$\lambda_i,\lambda_{i-1},\phi_i,\phi_{i-1},d_i,d_{i-1}$}
    %
    \KwOut{Updated Ground Track Plot}
    $m\lambda_i^+ = mod(\lambda_i,2 \pi)$\;
    $m\lambda_{i-1}^+ = mod(\lambda_{i-1},2 \pi)$\;
    $m\lambda_i^- = mod(\lambda_i,-2 \pi)$\;
    $m\lambda_{i-1}^- = mod(\lambda_{i-1},-2 \pi)$\;
\% New point wraps off RHS border\;
  \uIf{ $ d_i = d_{i-1} = 1 $ \mbox{And} $m\lambda_{i-1}^+ < \pi $ \mbox{And} $m\lambda_i^+ > \pi$}
          {

           $m = \displaystyle\frac{\phi_i - \phi_{i-1}}{m\lambda_i^+ - m\lambda_{i-1}^+}$\;
           $\phi_b = m (\pi - m\lambda_i^+) + \phi_i$\;
           Plot line segment from ($\lambda_{i-1},\phi_{i-1}$) to ($\pi,\phi_b$)\;
           Plot line segment from ($-\pi,\phi_b$) to ($\lambda_i,\phi_i$)\;
          }{
          \% New point wraps off LHS border\;
  \uElseIf {$ d_i = d_{i-1} = -1 $ \mbox{And} $m\lambda_i^- < -\pi $ \mbox{And} $m\lambda_{i-1}^- > -\pi$}
              {

           $m = \displaystyle\frac{\phi_i - \phi_{i-1}}{m\lambda_i^- - m\lambda_{i-1}^- }$\;
           $\phi_b = m (-\pi - m\lambda_i^-) + \phi_i$\;
           Plot line segment from ($\lambda_{i-1},\phi_{i-1}$) to ($-\pi,\phi_b$)\;
           Plot line segment from ($\pi,\phi_b$) to ($\lambda_i,\phi_i$)\;
               }
               \% New does not wrap off plot border \;
  \Else
           {
                Plot line segment from ($\lambda_{i-1},\phi_{i-1}$) to ($\lambda_i,\phi_i$)\;
            }
    }
    \hspace{.2 in}
    %
    \label{alg:GroundTrackAlgorithm}\caption{Algorithm for Updating A Ground Track Plot}
    %
\end{algorithm}
\end{small}
\end{minipage}
\end{center}

\section{Footprint and Limb Computation}

\subsection{Overview}

Computing an instrument footprint or the Earth Limb as viewed from a spacecraft involves essentially three related problems: (1) Determining the intersection of a given line with a given ellipsoid, (2) determination of a line tangent to a known ellipsoid, and (3) determining how to select the points along the footprint or limb curve to provide a smooth plot in the graphics.    These problems are related and discussed in order below starting with the problem of computing the intersection of a line and an ellipsoid.

\subsection{Intersection of Line and Ellipsoid}

Begin by defining a ray $\boldsymbol{\ell}$ such that
%
\begin{equation}
    \boldsymbol{\ell} = \mathbf{p} + \alpha\hat{\mathbf{d}} \label{Eq:RayParametrization}
\end{equation}
%
where $\mathbf{p}$ (in this context the usually location of a spacecraft) is the the starting location of ray $\boldsymbol{\ell}$, $\hat{\mathbf{d}}$ is the unit vector in the direction of the ray, and $\alpha$ is the distance from coordinates $\mathbf{p}$ in the direction.  The equation for a tri-axial ellipoid is defined as
%
\begin{equation}
  \frac{x^2}{R_x^2} + \frac{y^2}{R_y^2} + \frac{z^2}{R_z^2} - 1 = 0 \label{Eq:TriAxialEllipsoid}
\end{equation}
%
where $R_x$, $R_y$, and $R_z$ are the ellipsoid radii in the $x$, $y$, and $z$ directions respectively.
For simplicity of notation, we'll assume all coordinates such $x$, $y$, and $z$, and ray $\boldsymbol\ell$ are
expressed in the body coordinates of the ellipsoid.
%
Subtituting Eq.~(\ref{Eq:RayParametrization}) into Eq.~(\ref{Eq:TriAxialEllipsoid}) results in
%
\begin{equation}
    \frac{\boldsymbol{\ell}^T \hat{\mathbf{i}}}{R_x^2} + \frac{\boldsymbol{\ell}^T \hat{\mathbf{j}}}{R_y^2}
    + \frac{\boldsymbol{\ell}^T \hat{\mathbf{k}}}{R_z^2} - 1 = 0 \label{Eq:EllipseSimltaneous}
\end{equation}
%
where $\hat{\mathbf{i}}$, $\hat{\mathbf{i}}$, and $\hat{\mathbf{i}}$ are unit vectors in the $x$, $y$, and $z$ directions respectively.  Define $\alpha^*$ as the value of $\alpha$ that simulteoulsy satisfies both the equation for ray $\boldsymbol{\ell}$ and the equation for the triaxial ellipoid.  Expanding Eq.~(\ref{Eq:EllipseSimltaneous}) and grouping terms by powers of $\alpha^*$ yields
%
\begin{equation}
    \alpha^{*2}\left( \frac{d_x^2}{R_x^2} + \frac{d_y^2}{R_y^2} + \frac{d_z^2}{R_z^2} \right) +
    2\alpha^{*}\left( \frac{d_x p_x}{R_x^2} + \frac{d_y p_y}{R_y^2} + \frac{d_z p_z}{R_z^2} \right)+
    \left( \frac{p_x^2}{R_x^2} + \frac{p_y^2}{R_y^2} + \frac{p_z^2}{R_z^2} - 1\right) = 0
\end{equation}
%
This is a quadratic equation and the solution is
%
\begin{equation}
    \alpha^* = \frac{-B \pm \sqrt{B^2 - 4 A C}}{2A}\label{Eq:Quadratic}
\end{equation}
%
where
%
\begin{eqnarray}
    A &=& \left( \frac{d_x^2}{R_x^2} + \frac{d_y^2}{R_y^2} + \frac{d_z^2}{R_z^2} \right)\\
    B &=& 2\left( \frac{d_x p_x}{R_x^2} + \frac{d_y p_y}{R_y^2} + \frac{d_z p_z}{R_z^2} \right)\\
    C &=& \left( \frac{p_x^2}{R_x^2} + \frac{p_y^2}{R_y^2} + \frac{p_z^2}{R_z^2} - 1\right)
\end{eqnarray}
%
There are three physical types of solutions for $\alpha^*$ given by Eq.~(\ref{Eq:Quadratic}).  If $(B^2 - 4 A C) < 0$ then ray $\boldsymbol\ell$ does not intersect
the ellipoid.  If $(B^2 - 4 A C) = 0$ then ray $\boldsymbol\ell$ is tangent to the ellipsoid (we'll revisit this relation when computing the limb curve). In this case,
%
\begin{equation}
    \boldsymbol{\ell}^* = \mathbf{p} -\frac{B}{2A}\hat{\mathbf{d}}
\end{equation}
%
Finally, if $(B^2 - 4 A C) > 0$ then ray $\boldsymbol\ell$ intersects the ellipsoid
at two locations.  For graphics purposes, we require the smaller value of $\alpha^*$ given by Eq.~(\ref{Eq:Quadratic}). If we define $\boldsymbol\alpha^*$ as a vector containing the two solutions for $\alpha$ corresponding to the two intersection points, then, the equation for the nearest intersection point, $\boldsymbol{\ell}^*$ is given by
%
\begin{equation}
    \boldsymbol{\ell}^* = \mathbf{p} + min(\boldsymbol\alpha^*)\hat{\mathbf{d}}
\end{equation}

\subsection{Determining the Limb Region}

\subsection{Selecting Points for Accurate Graphics}


\subsection{Estimators}

When we take statistics we are often concerned with inferring properties of the underlying probability function.

As the properties of the probability distribution function affect the chance of observing the sample, we can analyse samples to infer properties of the underlying distribution.

There are many properties would could be interested in. This includes moments and parameters of a specific probability distribution function.

An estimator is a statistic which is our estimate of one of these values.

Emphasise that statistics and estimators are different things. A statistic may be terrible estimator, but be useful for other purposes.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Written By Michael Brodskiy
% Class: Analytic Geometry & Calculus III (Math-292)
% Professor: V. Cherkassky
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\documentclass[12pt]{article}
\usepackage{alphalph}
\usepackage[utf8]{inputenc}
\usepackage[russian,english]{babel}
\usepackage{titling}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{enumitem}
\usepackage{amssymb}
\usepackage[super]{nth}
\usepackage{everysel}
\usepackage{ragged2e}
\usepackage{geometry}
\usepackage{fancyhdr}
\geometry{top=1.0in,bottom=1.0in,left=1.0in,right=1.0in}
\newcommand{\subtitle}[1]{%
  \posttitle{%
    \par\end{center}
    \begin{center}\large#1\end{center}
    \vskip0.5em}%

}
\usepackage{hyperref}
\hypersetup{
colorlinks=true,
linkcolor=blue,
filecolor=magenta,
urlcolor=blue,
citecolor=blue,
}

\urlstyle{same}

\title{Lecture V Notes}
\date{\today}
\author{Michael Brodskiy\\ \small Professor: V. Cherkassky}

% Mathematical Operations:

% Sum: $$\sum_{n=a}^{b} f(x) $$
% Integral: $$\int_{lower}^{upper} f(x) dx$$
% Limit: $$\lim_{x\to\infty} f(x)$$

\begin{document}

\maketitle

\section{Vectors in Space $-$ 13.1}

A vector function: $\overrightarrow{r}(t) = \langle f(t), g(t), h(t) \rangle = f(t)\hat{\bold{i}} + g(t)\hat{\bold{j}} + h(t)\hat{\bold{k}}$\\

Example: $$\overrightarrow{r}(t)=\langle t^2, ln(2-t), \sqrt{t} \rangle$$
The domains of each component function must be considered to find the domain of the vector function, $0\leq x < 2$\\

For a vector function: $$\lim_{t\to a} \overrightarrow{r}(t)=\langle \lim_{t\to a} f(t), \lim_{t\to a} g(t), \lim_{t\to a} h(t) \rangle$$\\

Example: Find the limit
$$\lim_{t\to 0} (e^{-3t}\hat{\bold{i}} + \frac{t^2}{\sin^2{t}}\hat{\bold{j}} + \cos(2t)\hat{\bold{k}})=\hat{\bold{i}}+\hat{\bold{j}}+\hat{\bold{k}}$$

\section{Derivatives and Integrals of Vector Functions $-$ 13.2}

$$\overrightarrow{r}'(t)=\lim_{h\to 0} \frac{\overrightarrow{r}(t+h)-\overrightarrow{r}(t)}{h}$$

$\overrightarrow{r}'(t)$ is a vector tangent to $\overrightarrow{r}(t)$, where $\overrightarrow{r}'(t)=\langle f'(t), g'(t), h'(t) \rangle$

The unit tangent vector, $\overrightarrow{T}(t)$, may found using the formula: $\overrightarrow{T}(t)=\frac{\overrightarrow{r}'(t)}{|\overrightarrow{r}'(t)|}$

The integral of the vector function $\overrightarrow{r}(t)$ is: $$\int_{a}^{b} \overrightarrow{r}(t) dt = \left(\int_{a}^{b} f(t) dt\right)\hat{\bold{i}} + \left(\int_{a}^{b} g(t) dt\right)\hat{\bold{j}} + \left(\int_{a}^{b} h(t) dt\right)\hat{\bold{k}}$$

\end{document}


%\section{Exercise 7. Metrizability \& number theory}
\noindent
\textit{
Let be X the vector space of all complex functions on the unit interval
$[0, 1]$, topologized by the family of seminorms
%
  \begin{align}
    \mathit{p_{x}(f\,)=|f(x)| \quad\quad (0\leq x\leq 1).\nonumber}
  \end{align}
%
This topology is called the topology of pointwise convergence.
Justify this terminology.
Show that there is a sequence $\{f_n\}$ in X such that (a) $\{f_n\}$ converges
to $0$ as $n \to\infty$, but (b) if $\{γ_n\}$ is any sequence of scalars such
that $γ_n\to\infty$ then $\{γ_nf_n\}$ does not converge to $0$.
(Use the fact that the collection of all complex sequences converging to $0$
has the same cardinality as $[0, 1]$.)
This shows that metrizability cannot be omited in (b) of Theorem 1.28.
}
\begin{proof}
%Our justification consists in proving that $\tau$-convergence and pointwise
%convergence are the same one.
%
%To do so, remark first that
The family of the seminorms $p_{\varit{x}}$ is separating:
By \citeresultFA{1.37}, the collection $\mathscr{B}$ of all finite intersections
of the sets
%
  \begin{align}
    %\tensor*[_x]{V}{_k}
    V(\varit{x},\varit{k})
      \Def
    \singleton{p_\varit{x} < 2^{\,\minus \varit{k}}}
      \quad\quad
    (\varit{\varit{x}} \in [0, 1], \varit{k}=1, 2, 3, \dots)
  \end{align}
%
is therefore a local base for a topology $\tau$ on $X$.
So,
%
  \begin{align}
    %
    \label{Inequality boolean series}
    %
    \sum_{n=1}^\infty \boolean{f_n \notin \cap_{i=1}^m U_i} \leq
    \sum_{n=1}^\infty \sum_{i=1}^m \boolean{f_n \notin U_{i}} =
    \sum_{i=1}^m \sum_{n=1}^\infty \boolean{f_n \notin U_i} %\off{U_{i}
    \quad\quad (f_n \in X, U_i \in \tau).
  \end{align}%
%
Now assume that $\{f_n\}$ $\tau$-converges to some $f$, \ie
%
  \begin{align}
    %
    \sum_{n=1}^\infty \boolean{f_n \notin f + W} < \infty \quad\quad (W \in \mathscr{B}).
  \end{align}
The special case %
%
$W = V(\varit{x}, \varit{k})$ %
%
means that %
%
  $\magnitude{f_n(\varit{\varit{x}}) - f(\varit{x})} < 2^{\,\minus \varit{k}}$ %
%
for almost all $n$,  \ie
  $\singleton{f_n(\varit{x})}$ converges to $f(\varit{x})$.
Conversely, assume that $\singleton{f_n}$ does not $\tau$-converges in $X$, \ie
%
  \begin{align}
    %
    \label{Divergence}
    %
    \forall f \in X, \exists W \in \localbase{B}:
      \sum_{n=1}^\infty\boolean{f_n \notin f +  W} = \infty.
    %
  \end{align}
%
$W$ is now the intersection of finitely many %
%
$V(\varit{\varit{x}}, \varit{k})$, %
%
say
%
  $ V(x_1, k_1), \dots, V(x_m, k_m)$. Thus,  %
%
  \begin{align}
    \sum_{i=1}^m \sum_{n=1}^\infty \boolean{f_n \notin f + V(x_i, k_i)}
      %
        \citegeq{\ref{Inequality boolean series}}
      %
    \sum_{n=1}^\infty \boolean{f_n \notin f + W}
    %
      \citeq{\ref{Divergence}}
    %
    \infty .
  \end{align}
%
We can now conclude that, for some index $\varit{i}$,
%
  \begin{align}
    \sum_{n=1}^\infty \boolean{
      f_n \notin f + V(x_i, k_i)
    } = \infty .
  \end{align}
In other word, %
%
$\singleton{f_n(x_i)}$ %
%
fails to converge to
$f(x_i)$.
We have so proved that
%
  $\tau$-convergence is a rewording of pointwise convergence.
%
% SECOND PART
We now establish the second part. \\%
\\\noindent
To do so, we split $\varit{x}$ into two variables: %
$\varit{r}$ if $\varit{x}$ is rational, $\alpha$ otherwise.
%
The proof is based on the following well-known result: Each $\alpha$ %
has a {\it unique} binary expansion. %
More precisely, there exists a bijection %
%
$b:[0, 1] \setminus \Q \to %
\set{\beta \in \{0, 1\}^{\N_+}}{\beta \text{ is not eventually periodic}}$ %
%
where %
$b(\alpha) = (\beta_1, \beta_2, \dots)$ %
is the only bit stream such that %
%
  \begin{align}
    \label{definition of alpha}
    \alpha = \sum_{k=1}^\infty \beta_k \cdot 2^{\,\minus k}.
  \end{align}
%
Remark that %
%
$b(\alpha)_1 + \dots + b(\alpha)_n \longrightarrow \infty$, %
%
since %
%
$b(\alpha)$
%
has infinite support, then fix %
%
  \begin{equation}
    \label{definition of f_n(alpha)}
    f_n(\alpha)\Def %
    \frac{1}{b(\alpha)_1+\cdots+ b(\alpha)_n} \tendsto{n}{\infty} 0.
  \end{equation}
%
The actual values $f_n(r)$ are of no interest, %
as long as every sequence $\set{f_n(r)}{\counting{n}}$ converges to $0$. %
For example, put $f_n(r) = r/n$, or just $f_n(r) = 0$. %
We also take %
%
  $\gamma_n \longrightarrow \infty$, \ie %
%
given any counting number $\varit{p}$, %
$\gamma_{n}$ is greater than $\varit{p}$ for almost all $\varit{n}$.
%
Next, we choose $n_{p}$ among those \textit{almost all} $\mathit{n}$ that are
large enough to satisfy
%
  \begin{align}
    \label{definition of n_p}
    n_p - n_{p-1} > p
  \end{align}
%
(start with $n_0=0$).
%
So, every list %
%
  $n_{p}, n_{p'}, n_{p''}, \dots $ %
%
that satisfies %
%
  $n_{p'}- n_{p} = n_{p''} - n_{p'} = \dots$ %
%
is finite %
(otherwise, %
    $n_{p'}-n_{p} \geq n_{p+1} - n_{p}
    > p
    \rightarrow  \infty %
    $
would hold from; see (\ref{definition of n_p})).
In other words, {\it the distribution of %
%
$\mathit{n_1, n_2, \dots}$ %
%
displays no periodic pattern}. %
As a consequence, the {\it characteristic function} %
  $\chi: k\mapsto \boolean{k \in \singleton{n_1, n_2, \dots}}$ %
is not eventually periodic. %
Combined with (\ref{definition of alpha}), this establishes that
%
\def\xgamma{\alpha_{\gamma}}
%
  \begin{align}
    \xgamma
      \Def
    \sum_{k=1}^\infty \chi_k 2^{\minus \varit{k}}
      %\notin \Q
  \end{align}
%
is irrational. Conversely, still with (\ref{definition of alpha}),  %
%
  \begin{align}
    b(\xgamma)_{k} = \chi_{k}.
  \end{align}
%
Now remark that %
%
  \begin{alignat}{2}
    \chi_1 + \cdots + \chi_{n_{1}} & &&=1 \\
    \chi_1 + \cdots + \chi_{n_{1}} & + \cdots + \chi_{n_{2}} &&=2 \\
    \nonumber \vdots&  &&\\
    \chi_1 + \cdots + \chi_{n_{1}} & + \cdots + \chi_{n_{2}} + \cdots +
    \chi_{n_{p}}&&=p.
  \end{alignat}
%
Combined with (\ref{definition of f_n(alpha)}), this yields %
%
  \begin{align}
    \gamma_{n_p} f_{n_{p}}(\xgamma) = \frac{\gamma_{n_p}}{p} > 1.
  \end{align}
%
There so exists a subsequence $\singleton{\gamma_{n_p}}$ such that %
%
  $\singleton{\gamma_{n_p} f_{\gamma_{n_p}}}$ %
%
fails to converge pointwise to $0$. %
In other words, (b) holds, which is in violent contrast with %
\citeresultFA{1.28}: $X$ is therefore not metrizable. So ends the proof.
\end{proof}
% END


\subsection{Noetherian rings}\label{subsec:noetherian_rings}

\begin{definition}\label{def:noetherian_module}\mcite[prop. 8.30]{Knapp2016BasicAlgebra}
  A module \( M \) over \( R \) is called \term{Noetherian} if it satisfies any of the following conditions:
  \begin{thmenum}
    \thmitem{def:noetherian_module/ascending_chain} Every strict chain of submodules of \( M \)
    \begin{equation*}
      M_1 \subsetneq M_2 \subsetneq \ldots
    \end{equation*}
    is finite.

    \thmitem{def:noetherian_module/finite_basis} Every submodule of \( M \) is finitely \hyperref[def:free_left_module]{generated}.
  \end{thmenum}
\end{definition}
\begin{proof}
  \ImplicationSubProof{def:noetherian_module/ascending_chain}{def:noetherian_module/finite_basis} We can construct a basis as follows: choose any element \( x_1 \) of \( M \). Next, choose an element \( x_2 \in M \setminus \braket {x_1} \), then \( x_3 \in M \setminus \braket {x_1, x_2} \) and, so on.

  This process must stop after finitely many steps because we obtain the strict chain
  \begin{equation*}
    \braket {x_1} \subsetneq \braket {x_1, x_2} \cdots
  \end{equation*}
  of submodules.

  \ImplicationSubProof{def:noetherian_module/finite_basis}{def:noetherian_module/ascending_chain} Suppose that all submodules of \( M \) are finitely generated. Let
  \begin{equation*}
    M_1 \subsetneq M_2 \subsetneq \ldots
  \end{equation*}
  be a strictly ascending chain of submodules.

  Then the submodule \( N \coloneqq \bigcup_{i=1}^\infty M_i \) is also finitely generated. There exists a member \( M_N \) of the chain containing all the generators of \( N \). Then no further strict inclusion of modules is possible. We conclude that the chain
  \begin{equation*}
    M_1 \subsetneq M_2 \subsetneq \ldots
  \end{equation*}
  is finite.
\end{proof}

\begin{definition}\label{def:noetherian_ring}
  A \term{Noetherian ring} is an Noetherian submodule over itself, i.e. it satisfies the conditions in \fullref{def:noetherian_module} on its ideals.
\end{definition}

\begin{theorem}[Hilbert basis theorem]\label{thm:hilberts_basis_theorem}\mcite[418]{Knapp2016BasicAlgebra}
  If \( R \) is Noetherian, so is \( R[X] \).
\end{theorem}

\begin{theorem}\label{thm:noetherian_rings_closed_under}
  The \hyperref[def:set]{classes} of Noetherian \hyperref[def:noetherian_module]{modules} and Noetherian \hyperref[def:noetherian_ring]{rings} are closed with respect to many operations, including the following:

  \begin{thmenum}
    \thmitem{thm:noetherian_rings_closed_under/localization}\cite[corollary 8.48]{Knapp2016BasicAlgebra} The \hyperref[def:ring_localization]{localizations} \( S^{-1} R \) of a Noetherian ring \( R \) are also Noetherian.

    \thmitem{thm:noetherian_rings_closed_under/modules}\cite[proposition 8.34]{Knapp2016BasicAlgebra} Any \hyperref[def:free_left_module]{finitely generated module} over a Noetherina ring is also \hyperref[def:noetherian_module]{Noetherian}.

    \thmitem{thm:noetherian_rings_closed_under/submodules}\cite[proposition 6.3(a)]{КоцевСидеров2016} Every submodule of a Noetherian module is Noetherian.

    \thmitem{thm:noetherian_rings_closed_under/quotients}\cite[proposition 6.3(a)]{КоцевСидеров2016} Every quotient of a Noetherian module is Noetherian.

    \thmitem{thm:noetherian_rings_closed_under/restoration}\cite[proposition 6.3(b)]{КоцевСидеров2016} If the module \( M \) has a Noetherian submodule \( N \) such that \( M / N \) is also Noetherian, then \( M \) itself is Noetherian.

    \thmitem{thm:noetherian_rings_closed_under/polynomial_ring} The polynomial ring \( R[X] \) over a Noetherian ring \( R \) is also Noetherian. See \fullref{thm:hilberts_basis_theorem}.
  \end{thmenum}
\end{theorem}

\begin{theorem}[Primary decomposition]\label{thm:primary_decomposition}\mcite[thm. 10.6]{КоцевСидеров2016}
  In a Noetherian ring \( R \), every ideal \( I \) can be represented as an intersection
  \begin{equation*}
    I = \bigcap_{i=1}^n P_i
  \end{equation*}
  of finitely many primary \hyperref[def:primary_ring_ideal]{ideals}.
\end{theorem}


\documentclass{article}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathtools}
\usepackage{parskip}
\counterwithin*{equation}{subsubsection}
\setcounter{secnumdepth}{4}

\begin{document}
    \section{Derive the normal equation}
	Given a data \textbf{X},\textbf{y} and assuming a probabilistic model given by $y=\beta^{T}x+\epsilon$ where $\epsilon\sim Normal(0,\sigma^2)$, show that the $\beta$ that maximizes the probability of obtaining the data is given by: $\beta=(X^{T}X)^{-}X^{T}y$.

	\subsection{Solution}
	\subsubsection{Feature space}
	Assume $m$ training examples $(x^{(1)},y^{(1)}),(x^{(2)},y^{(2)}),...,(x^{(m)},y^{(m)})$ with $n$ features $x_1, x_2,...,x_n$. At index 0, let $x^{(1)}_0, x^{(2)}_0,..., x^{(i)}_0,...,x^{(n)}_0$ all equal 1. Therefore, if there are $n$ features and a 0th index, there will be $n+1$ feature vectors.

	Let $X$ be the design matrix of $n+1$ feature vectors $x^{(i)}_{n+1}$ where $x^{(i)}$ denotes the $i$-th $n+1$-dimensional feature vector contained within $X$.

	Thus, each row of the matrix $X$ is filled by $(x^{(1\:\text{to}\:m)})^{T}$, making $X$ an $m\times(n+1)$-dimensional matrix of all the features of the training data:

	\begin{equation*}
	x^{(i)} = \begin{bmatrix}
	x^{(i)}_0\\x^{(i)}_1\\x^{(i)}_2\\.\\x^{(i)}_n
	\end{bmatrix}\in\rm I\!R^{n+1}\qquad
	X=\begin{bmatrix}
	x^{(1)}_0&x^{(1)}_1&x^{(1)}_2&.&.&.&x^{(1)}_n\\
	x^{(2)}_0&x^{(2)}_1&x^{(2)}_2&.&.&.&x^{(2)}_n\\
	.&.&.&.&.&.&.\\
	x^{(i)}_0&x^{(i)}_1&x^{(i)}_2&.&.&.&x^{(i)}_n\\
	.&.&.&.&.&.&.\\
	x^{(m)}_0&x^{(m)}_1&x^{(m)}_2&.&.&.&x^{(m)}_n
	\end{bmatrix}
	\end{equation*}

	Let $y$ be the vector of true values of the above described training examples.

	\begin{equation*}
	y=\begin{bmatrix}
	y_1\\y_2\\.\\.\\y_m
	\end{bmatrix}
	\end{equation*}

	\subsubsection{Hypothesis function}
	Given the hypothesis function:
	$$
	h_\beta(x)=\beta_0x_0+\beta_1x_1+...+\beta_nx_n
	$$
	(Recall that $x_0$ = 1.)

	The above hypothesis function can be represented using matrix notation. The regression coefficients of hypothesis function $h_
	\beta(x)$ can be represented as an $n+1$-dimensional vector:
	\begin{equation*}
	\beta=\begin{bmatrix}
	\beta_0\\\beta_1\\\beta_2\\.\\.\\.\\\beta_n
	\end{bmatrix}\in\rm I\!R^{n+1}
	\end{equation*}

	Similarly, each of the $m$ training examples is an $n+1$-dimensional vector $\Bigg[\begin{smallmatrix}
	x^{(i)}_0\\x^{(i)}_1\\.\\x^{(i)}_n
	\end{smallmatrix}\Bigg]$ with $x^{(i)}_0=1$ to allow for a convenient vector multiplication.

	Thus, the hypothesis function for each $x_i$, $h_\beta(x_i)$, can be written as:
	\begin{equation*}
	h_\beta(x_i)=\beta^{T}x+\epsilon
	\end{equation*}
	where $\beta$ and $x_i$ are $n+1$-dimensional vectors, and $\epsilon$ is the normally distributed error for each observation.

	\subsubsection{Training error}
	The training error for the above generalized example could be expanded algebraically as:
	\begin{equation*}
	Error\:(\epsilon)=\begin{bmatrix}
	y_1 - (\beta_0x^{(1)}_0+\beta_1x^{(1)}_1+\beta_2x^{(1)}_2+...+\beta_nx^{(1)}_n)\\
	y_2 - (\beta_0x^{(2)}_0+\beta_1x^{(2)}_1+\beta_2x^{(2)}_2+...+\beta_nx^{(2)}_n)\\
	y_3 - (\beta_0x^{(3)}_0+\beta_1x^{(3)}_1+\beta_2x^{(3)}_2+...+\beta_nx^{(3)}_n)\\
	...\\
	...\\
	y_m - (\beta_0x^{(m)}_0+\beta_1x^{(m)}_1+\beta_2x^{(m)}_2+...+\beta_nx^{(m)}_n)
	\end{bmatrix}=
	\begin{bmatrix}
	\epsilon_1\\
	\epsilon_2\\
	\epsilon_3\\
	.\\
	.\\
	\epsilon_m
	\end{bmatrix}\in\rm I\!R^{m}
	\end{equation*}

	This can be simplified using matrix notation and the matrices that were defined above. $y_{1\:\text{to}\:m}$ in the matrix above correspond to the matrix $y$ of true values. The $\beta_{0\:\text{to}\:n}$ correspond to the matrix $\beta$ of regression coefficients. The $x_{0\:\text{to}\:n}$ correspond to the design matrix $X$ of the $m\times(n+1)$-dimensional feature space.

	Using matrix addition and multiplication, the above matrix simplifies to:
	\begin{equation*}
	Error\:(\epsilon) =y_{m\times1}- X_{m\times n+1}\beta_{n+1}\in\rm I\!R^{m}
	\end{equation*}
	(The subscripts denote the dimensions of these matrices for convenience.)

	\subsubsection{Deriving the cost function}
	The likelihood of obtaining the model parameters from the data is given by:
	\begin{equation*}
	\mathcal{L}(\beta|y,X)=\Pr(X|\beta)
	\end{equation*}
	where $\mathcal{L}$ is the likelihood, $X$ is the design matrix (i.e., the data), and $\beta$ is the vector of model parameters.

	The probability of the data $X$ given the model parameters $\beta$ is the joint probability of each individual data point:
	\begin{equation*}
	\Pr(X|\beta)=\prod_{i=1}^{m}\Pr(y_i|x_i,\beta)
	\end{equation*}

	It is given that $\epsilon\sim Normal(0,\sigma)$. As the noise $\epsilon$ is additive, the linearity condition implies that $\Pr(y_i|x_i,\beta)\sim Normal(\beta^{T}x_i,\sigma_\epsilon^2)$.

	The goal is to find a set of model parameters $\beta$ that maximize the likelihood. Taking the logarithm of both sides helps simplify the equation. Since the logarithm is a monotonic function, the maximum of the log-likelihood occurs at the same value of $\beta$ as the maximum of the likelihood. Thus, taking the $\ln$ of both sides:
	\begin{align*}
	\ln\mathcal{L}(\beta)&= \ln \prod_{i=1}^{m}\Pr(y_i|x_i,\beta)\\
	&=\sum_{i=1}^{m}\left[\ln \Pr(y_i|x_i,\beta)\right]\\
	&=\sum_{i=1}^{m}\left[-\frac{1}{2\sigma_\epsilon^2}(y_i-\beta^Tx_i)^2-\ln \left(\sqrt{2\pi\sigma_\epsilon^2}\right)\right]
	\end{align*}
	The last step follows because $\Pr(y_i|x_i\beta)$ is a Gaussian probability density as noted above.

	As the goal is to maximize the above likelihood (or more precisely, log-likelihood) in terms of the model parameters, the above terms that do not depend on $\beta$ (i.e., $-\frac{1}{2\sigma_\epsilon^2}$, $-\ln \sqrt{2\pi\sigma_\epsilon^2}$) can be ignored. Thus, the optimization problem can be written as:
	\begin{equation*}
	\ln\mathcal{L}(\beta)=\sum_{i=1}^{m}(y_i-\beta^Tx_i)^2
	\end{equation*}
	This is the sum of least squares!

	\subsubsection{Rewriting the cost function using matrices}
	The goal is to minimize the least-squares cost function:
	\begin{equation*}
	J(\beta_{0...n})= \frac{1}{2m}\sum_{i=1}^{m}(y_i - h_\beta(x^{(i)}))^2
	\end{equation*}
	where, as above, $x^{(i)}$ is the $i$-th sample from a set of $m$ samples and $y^{(i)}$ is the $i$-th true value.

	Because the term $h_\beta(x^{(i)})-x^{(i)}=\epsilon^{(i)}$, i.e.,

	\begin{equation*}
	J(\beta_{0...n})= \frac{1}{2m}\sum_{i=1}^{m}(y_i - h_\beta(x^{(i)}))^2=\frac{1}{2m}\sum_{i=1}^{m}\epsilon_i^2
	\end{equation*}

	another way of stating this problem is minimizing the sum of the squared errors in the $Error$ vector $\epsilon$, i.e., $\epsilon^{T}\times\epsilon$. Concretely,

	\begin{equation*}
	\sum_{i=1}^{m}\epsilon_i^2 = \begin{bmatrix}
	\epsilon_1&\epsilon_2&\epsilon_3&.&.&\epsilon_m
	\end{bmatrix}
	*
	\begin{bmatrix}
	\epsilon_1\\
	\epsilon_2\\
	\epsilon_3\\
	.\\
	.\\
	\epsilon_m
	\end{bmatrix}=\epsilon_1^2+\epsilon_2^2+\epsilon_3^2+...+\epsilon_m^2
	\end{equation*}

	As above, $\epsilon=y-X\beta$. Thus,
	\begin{equation*}
	J(\beta_{0...n})=\frac{1}{2m}(y-X\beta)^{T}(y-X\beta)
	\end{equation*}

	Ignoring the constant $\frac{1}{2m}$,
	\begin{align*}
	J(\beta_{0...n})&=(y^T-(X\beta^T))(y-X\beta)\\
	&=y^Ty-y^TX\beta-(X\beta)^Ty+(X\beta)^TX\beta
	\end{align*}
	Take the transpose of the second term in the above equation $(y^TX\beta)^T=(X\beta)^Ty$. Thus,
	\begin{align*}
	J(\beta_{0...n})&=y^Ty-(X\beta)^Ty-(X\beta)^Ty+(X\beta)^TX\beta\\
	&=y^Ty-2(X\beta)^Ty+(X\beta)^TX\beta
	\end{align*}
	Distribute the transpose in the last term and the final equation for $J(\beta_{0...n})$ is:
	\begin{equation*}
	J(\beta_{0...n})= y^Ty-2(X\beta)^Ty+\beta^{T}X^TX\beta
	\end{equation*}

	\subsubsection{Minimizing the cost function}
	In order to find the minimum of the cost function, the derivative of $J(\beta_{0...n})$ must be taken and then set to zero:
	\begin{equation*}
	\frac{\partial{J}}{\partial{\beta}}=0
	\end{equation*}

	To simplify the operations, the derivative of each term of $J(\beta_{0...n})$ will be taken separately. $y^{T}y$ will be ignored given it has no $\beta$ terms and the derivative of a constant is 0.
	\begin{equation*}
	J(\beta_{0...n}) = P(\beta_{0...n})+Q(\beta_{0...n})+y^Ty
	\end{equation*}
	\begin{equation}
	P(\beta_{0...n})=\beta^{T}X^TX\beta
	\end{equation}
	\begin{equation}
	Q(\beta_{0...n})=-2(X\beta)^Ty
	\end{equation}

	\paragraph{Differentiate $P(\beta_{0...n})$}~\\
	\begin{equation*}
	P(\beta_{0...n})=\beta^{T}X^TX\beta
	\end{equation*}

	Importantly, the product $X^T_{n+1\times m}X_{m\times n+1}$ is a square, symmetrical $n+1 \times n+1$-dimensional matrix. For convenience $Z$ will be substituted for $X^TX$. Therefore, $P(\beta_{0...n})$ can be rewritten as: $\beta^TZ\beta$ where $Z$ is the square, symmetrical matrix defined above.

	For the case where a scalar $\alpha$ is given by
	\begin{equation*}
	\alpha=x^TAx
	\end{equation*}
	where x is $n\times1$, $A$ is $n\times n$, and $A$ does not depend on $x$:
	\begin{align*}
	\alpha &= \sum_{j=1}^{n}\sum_{i=1}^{n}a_{ij}x_ix_j\\
	\frac{\partial\alpha}{\partial x_k}&=\sum_{j=1}^{n}a_{kj}x_j+\sum_{i=1}^{n}a_{ik}x_i\qquad\text{for the $k$th element of $x$}\\
	\frac{\partial\alpha}{\partial x}&=x^TA^T+x^TA\hspace{53pt}\text{for all $k=1,2,...n$}\\
	&=x^T(A^T+A)
	\end{align*}

	For the special case where $A$ is a symmetrical matrix, $A^T=A$, therefore $(A^T+A)=2A$ and $\frac{\partial\alpha}{\partial x}=2x^TA$.

	Back to the problem at hand, $Z$ was noted to be a square, symmetrical matrix. Therefore,
	\begin{align*}
	P(\beta_{0...n})&=\beta^TZ\beta\\
	\frac{\partial P}{\partial\beta}&=2\beta^TZ\\
	&=2\beta^TX^TX\qquad\text{substituting $X^TX$ for $Z$}\\
	&=2(\beta^TX^TX)^T\qquad\text{take the transpose}\\
	&=2X^TX\beta
	\end{align*}

	\paragraph{Differentiate $Q(\beta_{0...n})$}~\\
	\begin{align*}
	Q(\beta_{0...n})&=-2(X\beta)^Ty\\
	&=-2\left( \begin{bmatrix}
	x^{(1)}_0&x^{(1)}_1&x^{(1)}_2&.&.&.&x^{(1)}_n\\
	x^{(2)}_0&x^{(2)}_1&x^{(2)}_2&.&.&.&x^{(2)}_n\\
	.&.&.&.&.&.&.\\
	x^{(i)}_0&x^{(i)}_1&x^{(i)}_2&.&.&.&x^{(i)}_n\\
	.&.&.&.&.&.&.\\
	x^{(m)}_0&x^{(m)}_1&x^{(m)}_2&.&.&.&x^{(m)}_n
	\end{bmatrix}\begin{bmatrix}
	\beta_0\\\beta_1\\\beta_2\\.\\.\\.\\\beta_n
	\end{bmatrix}\right)^T\begin{bmatrix}
	y_1\\y_2\\.\\.\\y_m
	\end{bmatrix}\\
	&=-2\left(\begin{bmatrix}
	\beta_0x^{(1)}_0+\beta_1x^{(1)}_1+\beta_2x^{(1)}_2+...+\beta_nx^{(1)}_n\\
	\beta_0x^{(2)}_0+\beta_1x^{(2)}_1+\beta_2x^{(2)}_2+...+\beta_nx^{(2)}_n\\
	...\\
	\beta_0x^{(i)}_0+\beta_1x^{(i)}_1+\beta_2x^{(i)}_2+...+\beta_nx^{(i)}_n\\
	...\\
	\beta_0x^{(m)}_0+\beta_1x^{(m)}_1+\beta_2x^{(m)}_2+...+\beta_nx^{(m)}_n
	\end{bmatrix}\right)^T\begin{bmatrix}
	y_1\\y_2\\.\\.\\y_m
	\end{bmatrix}
	\end{align*}

	\begin{multline*}
	Q(\beta_{0...n})=-2[\:y_1(\beta_0x^{(1)}_0+...+\beta_nx^{(1)}_n)\\
	+y_2(\beta_0x^{(2)}_0+...+\beta_nx^{(2)}_n)+...+y_m(\beta_0x^{(m)}_0+...+\beta_nx^{(m)}_n)\:]
	\end{multline*}

	Rearranging the above using sums:
	\begin{align}
	Q(\beta_{0...n})&=-2\sum_{r=1}^{m}y_r(\beta_0x_0^{(r)}+...+\beta_nx^{(r)}_n)\\
	&=-2\sum_{r=1}^{m}y_r\sum_{s=1}^{n}\beta_sx_s^{(r)}
	\end{align}

	Using equation (3) above to differentiate:
	\begin{equation*}
	\frac{\partial Q}{\partial \beta}=-2\sum_{r=1}^{m}y_r(\beta_0x_0^{(r)}+...+\beta_nx^{(r)}_n) \partial \beta
	\end{equation*}

	This can be rewritten as a series of partial derivatives:
	\begin{align*}
	\frac{\partial Q}{\partial \beta_0}&=-2(x_0^{(1)}y_1+x_1^{(1)}y_1+...+x_n^{(1)}y_m)\\
	\frac{\partial Q}{\partial \beta_1}&=-2(x_0^{(2)}y_1+x_1^{(2)}y_1+...+x_n^{(2)}y_m)\\
	\frac{\partial Q}{\partial \beta_2}&=-2(x_0^{(3)}y_1+x_1^{(3)}y_1+...+x_n^{(3)}y_m)\\
	...\\
	\frac{\partial Q}{\partial \beta_n}&=-2(x_0^{(m)}y_1+x_1^{(m)}y_1+...+x_n^{(m)}y_m)
	\end{align*}

	This can be collapsed as a vector of partial derivatives:
	\begin{equation*}
	\begin{bmatrix}
		\frac{\partial Q}{\partial \beta_0}\\
		\frac{\partial Q}{\partial \beta_1}\\
		\frac{\partial Q}{\partial \beta_2}\\
		.\\
		.\\
		\frac{\partial Q}{\partial \beta_n}
	\end{bmatrix}=-2\left(\begin{bmatrix}
	x_0^{(1)}&x_1^{(1)}&x_2^{(1)}&.&.&x_n^{(1)}\\
	x_0^{(2)}&x_1^{(2)}&x_2^{(2)}&.&.&x_n^{(2)}\\
	.&.&.&.&.&.\\
	.&.&.&.&.&.\\
	x_0^{(m)}&x_1^{(m)}&x_2^{(m)}&.&.&x_n^{(m)}
	\end{bmatrix}\right)^T\begin{bmatrix}
	y_1\\
	y_2\\
	.\\
	.\\
	y_m
	\end{bmatrix}
	\end{equation*}

	In other words,
	\begin{equation*}
	\frac{\partial Q}{\partial \beta_0}=-2\frac{\partial (X\beta)^Ty}{\partial \beta_0}=-2X^Ty
	\end{equation*}

	Putting this all together,
	\begin{align*}
	J(\beta_{0...n}) &= P(\beta_{0...n})+Q(\beta_{0...n})\\
	\frac{\partial J}{\partial\beta}&=\frac{\partial P}{\partial \beta}+\frac{\partial Q}{\partial \beta_0}\\
	&=2X^TX\beta-2X^Ty=0\qquad\text{and solve for $\beta$}\\
	2X^TX\beta&=2X^Ty\\
	X^TX\beta&=X^Ty\qquad\text{multiply both sides by $(X^TX)^-$}\\
	\beta&=(X^TX)^-X^Ty\qquad\blacksquare
	\end{align*}

	\section{Show that Regularized Linear Regression has a Bayesian interpretation}
	Given data \textbf{X},\textbf{y} and assuming a linear model $y=\beta^Tx$ with a prior distribution over $\beta$ given by a normal distribution with mean 0, show that the $\beta$ that maximizes the probability of having obtained the data is given by:
	\begin{equation*}
	\beta=(X^TX+\lambda I)^- X^Ty
	\end{equation*}
	where $\lambda$ depends on the variance of the prior distribution.

	\subsection{Solution}
	\subsubsection{Using Bayes' theorem to rephrase maximum likelihood estimation}
	The maximum likelihood estimator discussed \textbf{2.1.4} can be related to the most probable Bayes estimator given a uniform prior distribution. The maximum \textit{a posteriori} estimate is the vector of parameters $\beta$ that maximize the probability of $\beta$ given the data. Using Bayes' theorem to write this:
	\begin{equation*}
	\Pr(\beta|x_1, x_2,...,x_n)=\frac{h(x_1, x_2,...,x_n|\beta)\Pr(\beta)}{\Pr(x_1, x_2,...,x_n)}
	\end{equation*}
	where $\Pr(\beta)$ is the prior distribution for the parameters $\beta$ and $\Pr(x_1, x_2,...,x_n)$ is the probability of obtaining the data. The denominator is independent of $\beta$, so the Bayesian estimator is obtained by maximizing $h(x_1, x_2,...,x_n|\beta)\Pr(\beta)$ with respect to $\beta$.

	In the derivation of the cost function in \textbf{2.1.4}, the Bayesian estimator could be considered to correspond to the maximum likelihood estimator for a uniform prior distribution of $\beta$s given by $\Pr(\beta)\sim Uniform(0,\beta)$. In other words, it is solving for $\Pr(data|\beta)$, i.e., the probability of obtaining the data given the parameters $\beta$.

	In contrast to finding $\Pr(data|\beta)$, here the problem is written as finding $\Pr(\beta|data)$, which is derived using Bayes' theorem and \textbf{prior} knowledge (i.e., a prior) of the distribution of $\beta$.

	In \textbf{Exercise 3}, the prior distribution of the $\beta$ vector is given as $\Pr(\beta)\sim Normal(0,\beta)$. Therefore, the Bayesian estimator can be rewritten as:
	\begin{align*}
	\Pr(\beta|data)&=\frac{\Pr(data|\beta)*prior}{\Pr(data)}\\
	\Pr(\beta|x_1, x_2,...,x_n)&=h(x_1, x_2,...,x_n|\beta)\Pr(\beta)
	\end{align*}
	again ignoring the denominator because it is independent of $\beta$.

	\subsubsection{Maximum \textit{a posteriori} estimator}
	The maximum \textit{a posteriori} extimator is then given by:
	\begin{equation*}
	\mathcal{L}(\beta|X)=\Pr(X|\beta)\Pr(\beta)
	\end{equation*}
	where $\mathcal{L}$ is the likelihood, $X$ is the design matrix (i.e., the data), $\beta$ is the vector of model parameters, and $\Pr(\beta)$ is given by the normal distribution as described above.

	The likelihood of the model parameters given the data is the joint probability of each individual data point multiplied by the prior:
	\begin{equation*}
	\mathcal{L}(\beta|X)=\Pr(\beta)\prod_{i=1}^{m}\Pr(y_i|x_i,\beta)
	\end{equation*}

	As before, the log-likelihood is easier to work with:
	\begin{align}
	\ln\mathcal{L}(\beta|X)&=\ln\left[\Pr(\beta)\prod_{i=1}^{m}\Pr(y_i|x_i,\beta)\right]\\
	&=\ln\Pr(\beta)+\sum_{i=1}^{m}\ln\left[\Pr(y_i|x_i,\beta)\right]
	\end{align}

	From \textbf{2.1.4} above, the second term $\sum_{i=1}^{m}\ln\left[\Pr(y_i|x_i,\beta)\right]$ is the sum of squared residuals $\sum_{i=1}^{m}(y_i-\beta^Tx_i)^2$.

	To gain an intuition of how the first term can be written as a sum, assume the parameters $\beta$ are distributed normally and independently around the origin with variance $\sigma^2_\beta$, as given:
	\begin{align*}
	\Pr(\beta)&=\prod_{i=0}^{n}\Pr(\beta_i)\\
	&=\frac{1}{2\pi\sigma_\beta^2}\exp\left(-\frac{\sum_{i=0}^{n}\beta_i^2}{2\sigma_\beta^2}\right)\\
	&=\frac{1}{2\pi\sigma_\beta^2}\exp\left(-\frac{\beta^T\beta}{2\sigma_\beta^2}\right)\qquad\text{written as a vector}\\
	\ln\Pr(\beta)&=-\frac{1}{2\sigma_\beta^2}\beta^T\beta\qquad\text{taking the $\ln()$ of both sides}
	\end{align*}

	Plugging this back into equation (2) above and rewriting the sum of least squares in matrix form (see \textbf{2.1.5}), we obtain the below. Recall that the objective can be multiplied by any scalar without affecting the optimum:
	\begin{align*}
	\ln\mathcal{L}(\beta|X)&=-\frac{1}{2\sigma_\beta^2}\beta^T\beta-\left(\frac{1}{2\sigma_\epsilon^2}(y-X\beta)^{T}(y-X\beta)\right)\\
	&=-\frac{\sigma_\epsilon^2}{\sigma_\beta^2}\beta^T\beta-(y-X\beta)^{T}(y-X\beta)
	\end{align*}

	Rather than maximizing the above function, the signs can be reversed and the function minimized:
	\begin{align*}
	\ln\mathcal{L}(\beta|X)&=\frac{\sigma_\epsilon^2}{\sigma_\beta^2}\beta^T\beta+(y-X\beta)^{T}(y-X\beta)\qquad\text{set $\frac{\sigma_\epsilon^2}{\sigma_\beta^2}=\lambda$}\\
	&=\beta^{T}X^TX\beta+\lambda\beta^T\beta-2\beta^TX^Ty\\
	\end{align*}

	The first and third terms' partial derivatives with respect to $\beta$  were proven above in \textbf{Exercise 2}:
	\begin{equation*}
	2X^TX\beta-2X^Ty
	\end{equation*}

	The second term's partial derivative with respect to $\beta$ is:
	\begin{equation*}
	2\lambda\beta
	\end{equation*}

	Putting this all together:
	\begin{equation*}
	\frac{\partial\mathcal{L}}{\partial\beta}=2X^TX\beta-2X^Ty+2\lambda\beta
	\end{equation*}
	Set the derivative equal to 0 to minimize, and then solve for $\beta$. The $\lambda$ term, recall, depends on the variance of the prior distribution:
	\begin{align*}
	0&=X^TX\beta-X^Ty+\lambda\beta\\
	X^Ty&=X^TX\beta+\lambda\beta\\
	X^Ty&=(X^TX+\lambda I)\beta\\
	\beta&=(X^TX+\lambda I)^-X^Ty
	\qquad\blacksquare
	\end{align*}
\end{document}

tex files/Derivation of the Normal Equation.tex


\subsection{Linear Approximations with the Gradient}
\noindent
The 4th gradient property can be generalized a bit further. Suppose we have $f(x,y,z)$, $\vec{r}(u,v) = \langle x(u,v), y(u,v), z(u,v) \rangle$, and $g(u,v) = f\circ\vec{r}(u,v)$.
\begin{equation*}
	\frac{\partial g}{\partial u} = \frac{\partial f}{\partial x}\frac{\partial x}{\partial u} + \frac{\partial f}{\partial y}\frac{\partial y}{\partial u} + \frac{\partial f}{\partial z}\frac{\partial z}{\partial u}
\end{equation*}
\noindent
We can rewrite our linear approximation as $\Delta z\approx\nabla f\cdot\langle \Delta x, \Delta y\rangle$.\\
In fact, we how have a way to find derivatives of $f\circ\vec{r}(t)$.
\begin{equation*}
	\frac{\mathrm{d}f}{\mathrm{d}t} = \frac{\partial f}{\partial x}\frac{\mathrm{d}x}{\mathrm{d}t} + \frac{\partial f}{\partial y}\frac{\mathrm{d}y}{\mathrm{d}t} = \nabla f \cdot \langle x^{\prime}(t), y^{\prime}(t) \rangle
\end{equation*}


Mathematical models used to simulate water distribution systems are subject to uncertainty.
Effective characterization of uncertainty is critical for reliable analysis with simulation-based studies.
Particularly for contamination incidents, uncertainty quantification is needed to effectively
use simulation tools that provide insights into response actions.
Hydraulic parameters that might cause uncertainty include:
(1) customer demands at each node and time, (2) operational controls (e.g., valve settings, pump curves), (3)
infrastructure topography and characteristics (e.g., missing pipes or junctions, effective pipe diameters) and (4) initial conditions (e.g., tank levels, pump statuses).
Water quality parameters that might cause uncertainty include:
(1)  initial water quality, (2) contaminant species, (3) contaminant reaction dynamics, (4) the amount
of contaminant injected, (5) injection location, (6) injection time and (7) injection duration.

The \code{uq} subcommand examines the effect of hydraulic and water quality uncertainty on the extent of contamination in terms of the identification of the contamination source.
This subcommand can be used to quantify uncertainty after running source identification using the Bayesian probability based formulation (Section \ref{sec.bayesian_algorithm}). Given a particular confidence level, nodes can be categorized according to their probability of contamination. Nodes whose contamination probability, $\gamma_n$, that are above a
threshold are labeled with respect to their contamination state as LY for ``likely yes,'' LN for``likely no'' and UN for ``unknown.''
For example, for a 95\% confidence level:

\[ \left\{ \begin{array}{ll}
         \gamma_n \geq 0.975 & \mbox{LY},\\
        0.025\leq \gamma_n \leq 0.975 & \mbox{UN}, \\
        \gamma_n \leq 0.025 & \mbox{LN} \end{array} \right.
\]

With the \code{uq} subcommand
the effects of uncertainty in customer demand, isolation valve status, bulk reaction rate coefficient and
contaminant injection location, start time, duration and rate can be studied on the size and location of the contamination incident.

A flowchart representation of the \code{uq} subcommand is shown in Figure \ref{fig:uq_flowchart}. Given a list of EPANET 2.00.12 compatible network models (INP format)
coupled with a list of injection scenarios (TSG format), the \code{uq} subcommand
runs Monte Carlo simulations to estimate the probability that each node in the network is contaminated.

\begin{figure}[h]
	\centering
	\includegraphics[scale=0.30]{graphics/uq_flowchart.png}
	\caption{Uncertainty quantification flowchart.}
	\label{fig:uq_flowchart}
\end{figure}

\section{Uncertainty Quantification Method}\label{uqn_algorithms}
The \code{uq} subcommand runs an ensemble of scenarios where each scenario is defined using an INP and TSG file. The INP and TSG files
contain the hydraulic and water quality parameters for the scenario.
The results are used to compute the probability that each node is contaminated, using the following equation:
\begin{equation}
\gamma_n = \sum_{s \in S} \delta_{s,n}\beta_s
\label{nodeprob}
\end{equation}
where $\gamma_n $ is the probability that node $n$ is contaminated, $\beta_s$ is the probability of scenario $s$ and $\delta_{s,n}$ is a binary parameter that is $1$ if scenario $s$ contaminates node $n$, and $0$ otherwise. The values of $\delta_{s,n}$ are determined from the simulations over the full potential scenario set, and the values for $\beta_s$ are determined using equation~\eqref{source_probability} based on previous measurements provided in the measurements file. If no measurements file is provided, then the values for $\beta_s$ are assumed to be uniform. A threshold value is used to decide whether a node is contaminated or not in a single scenario.

\section{\code{uq} Subcommand}\label{sec.uq_subcommand}

The \code{uq} subcommand is executed using the following command line:
\begin{unknownListing}
wst uq <configfile>
\end{unknownListing}
where \code{configfile} is a WST configuration file in the YAML format.

The \code{---help} option prints information about this subcommand,
such as usage, arguments and a brief description:
\begin{unknownListing}
wst uq --help
\end{unknownListing}

\subsection{Configuration File}

The \code{uq} subcommand generates a template configuration
file using the following command line:

\begin{unknownListing}
wst uq --template <configfile>
\end{unknownListing}

The \code{uq} template configuration file is shown in
Figure \ref{fig:uq_template}. Brief descriptions of the
options are included in the template after the \# sign.

\begin{figure}[h]
  \unknownInputListing{examples/uq_config.yml}{}{1}{17}
  \caption{The \code{uq} configuration template file.}
  \label{fig:uq_template}
\end{figure}

\subsection{Configuration Options}\label{sec.uq_subcommand.config_options}

Full descriptions of the WST configuration options used by the \code{uq} subcommand are listed below.
\input{examples/uq_config}

\subsection{Subcommand Output}
The \code{uq} subcommand creates two YAML files called <output prefix>\_uq\_scenarios.yml and <output prefix>\_uq\_nodes.yml that contain
a list of probabilities for the scenarios and nodes, respectively.
The log file named <output prefix>uq\_output.log contains basic debugging information.

\section{Uncertainty Quantification Example}

A list of EPANET 2.00.12 INP and TSG files are required to run the \code{uq} subcommand. The configuration file for this example, uq\_ex1.yml, is shown in Figure \ref{fig:uq_confex1}.

\begin{figure}[h]
  \unknownInputListing{../../examples/uq_ex1.yml}{}{1}{17}
  \caption{The \code{uq} configuration file for example 1.}
  \label{fig:uq_confex1}
\end{figure}

The file with the list of scenarios for this example is shown in Figure \ref{fig:uq_scenarios}.

\begin{figure}[h]
  \unknownInputListing{../../examples/Net3/uq/list_scenarios.dat}{}{1}{3}
  \caption{List of scenarios.}
  \label{fig:uq_scenarios}
\end{figure}

Summary information is printed to the screen, as shown in Figure \ref{fig:uq_out1}.

\begin{figure}[h]
  \unknownInputListing{examples/uq/uq_ex1_screen_output.txt}{}{1}{31}
  \caption{Screen output for example 1.}
  \label{fig:uq_out1}
\end{figure}

The results from the \code{uq} subcommand can be represented in probability maps using the \code{visualization} subcommand (Chapter 11).


\appendix

\section{A Python prototype}

Observation in last paragraph allows us to code a bunch of Python functions,
lying on top of SymPy module, that produce
$\mathcal{C}_{\equiv 2}^{(\alpha+1)}$ consuming $\mathcal{C}_{\equiv 2}^{(\alpha)}$:
the construction is strictly \emph{blockwise}, according theorems stated
in previous section. Here's the implementation:

\begin{adjustwidth}{-1cm}{0cm}
    \inputminted{python}{../../PhD/projects/recurrences-unfolding/sympy-notebook/colouring.py}
\end{adjustwidth}

The following is a session that builds $\mathcal{C}_{\equiv 2}^{(7)}$:
\begin{minted}{python}
    catalan_matrix = Matrix([
        [1,0,0,0,0,0],
        [1,1,0,0,0,0],
        [2,2,1,0,0,0],
        [5,5,3,1,0,0],
        [14,14,9,4,1,0],
        [42,42,28,14,5,1]])
    alpha = 2
    bound = 2**alpha
    pc = catalan_matrix[:bound, :bound]
    pc = pc.applyfunc(lambda c: c % 2)
    # _ = colour_matrix(pc)
    pc = build_modular_catalan(pc)
    # _ = colour_matrix(pc)
    pc = build_modular_catalan(pc)
    # _ = colour_matrix(pc)
    pc = build_modular_catalan(pc)
    # _ = colour_matrix(pc)
    pc = build_modular_catalan(pc)
    # _ = colour_matrix(pc)
    pc = build_modular_catalan(pc)
    # _ = colour_matrix(pc)
\end{minted}

Function \mintinline{python}|build_modular_catalan| implements given theorems to avoid
computing convolutions and series expansion: the starting building block in the
above session is $\mathcal{C}_{\equiv 2}^{(2)}$, hence it is very quick.
If desired, it is possible to display coloured triangles as svg images:
just uncomment \mintinline{python}|# _ = colour_matrix(pc)| lines.


\section{Clustering}

% ===
\emph{\textit{k}-means:}\enskip
$\arg\min_\theta \sum_{i\leq n} \norm{\bm x_i - \theta_{c(\bm x_i)}}^2$

% ===
\subsection{Mixture Models}

\textit{Assume:}\enskip
$\bm x \sim p(\bm x\mid \pi_{1\ldots k}, \theta_{1\ldots k}) = \sum_{c\leq k} \pi_c p(\bm x \mid \bm\theta_c)$

\textbf{Find:}
$\highlight{\hat\theta = \arg\max_\theta  p(\mathcal X\mid \pi,\theta)}
\color{gray} = \prod_{\bm x} p(\bm x\mid \pi,\theta)$

\emph{Gaussian Mixtures:}\enskip
$\to$\enskip
$p(\bm x \mid \theta_c) = p(\bm x \mid \bm\mu, \bm\Sigma)$

Introduce \textit{latent indicator variables} for mode assignments $M_{\bm x c} \in \{0,1\}$. Then, the \textbf{log-likelihood:}

$L(\mathcal X, \bm M \mid \bm\theta) = \sum_{\bm x} \sum_{c\leq k} M_{\bm x c} \log (\pi_c p(\bm x \mid \bm\theta_c))$

% ===
\subsubsection{EM-Algorithm \normalfont\sffamily for Gaussian Mixtures}

\begin{highlightbox}
    \textbf{E-step:}\enskip
    Calculate

    $Q(\bm\theta; \bm\theta^{(t)}) = \E[\bm M \mid \mathcal X, \bm\theta^{(t)}]{L(\mathcal X, \bm M \mid \bm\theta)} = \ldots$ \\
    $= \sum\limits_{\bm x} \sum\limits_{c\leq k} \paren[\big]{
    \begingroup
        \color{OrangeRed} \underbracket[.7pt][2pt]{\the\everymath \E*[\bm M \mid \mathcal X, \bm\theta^{(t)}]{M_{\bm x c}}}%_{\gamma_{\bm x c}}
    \endgroup
    \cdot \log \pi_c p(\bm x\mid \bm\theta_c) }$

    where \highlight*{$\begingroup\color{OrangeRed} \gamma_{\bm x c} \endgroup = \frac{p(\bm x \mid c, \bm\theta^{(t)}) \: p(c\mid \bm\theta^{(t)})}{p(\bm x \mid \bm\theta^{(t)})}$}, \enskip
    $\sum_{c\leq k} \gamma_{\bm x c} = 1$

    \textbf{M-step:}\enskip
    $\bm\theta^{(t+1)} \in \arg\max_{\bm\theta} Q(\bm\theta; \bm\theta^{(t)})$

    s.t. $\sum_c \pi_c = 1$. \enskip
    Solve via Lagrangian, yields

    \begin{highlightbox*}
        $\pi_c {=} \frac{1}{\abs{\mathcal X}} \sum\limits_{\bm x} \gamma_{\bm x c}$,\hfill
        $\bm\mu_c {=} \frac{\sum_{\bm x} \gamma_{\bm x c} \bm x}{\sum_{\bm x} \gamma_{\bm x c}}$,\hfill
        $\sigma^2_c {=} \frac{\sum_{\bm x} \gamma_{\bm x c} (\bm x - \bm\mu_c)^2}{\sum_{\bm x} \gamma_{\bm x c}}$
    \end{highlightbox*}
\end{highlightbox}

% ===
\subsection{Non-parametric Bayesian Methods}

$\mathrm{Dir}(\bm x \mid \bm\alpha) = \frac{1}{B(\alpha)} \prod_{k=1}^n x_k^{\alpha_k - 1}$,\enskip
$B(\bm\alpha) = \frac{ \prod_{k=1}^n \Gamma(\alpha_k)}{\Gamma(\sum_{k=1}^n \alpha_k)}$

Rewrite \textbf{Finite mixture models:}

$p(x) = \sum_{k=1}^K \pi_k p(x\mid \theta_k)
= \int p(x\mid\theta) G(\theta) \diff\theta$

where $G(\theta) = \sum_{k=1}^\infty \pi_k \delta_{\theta_k}\!(\theta)$
$\color{gray} \leftarrow \text{ discrete distr.}$

\emph{Stick-breaking process:}

Draw $\theta_k \sim H$ and $\beta_k \sim \mathrm{Beta}(1,\alpha)$ for $k{=}1,2,\ldots$

$\pi_k = \beta_k (1 - \sum_{k=1}^{k-1} \pi_i)$
$\implies \highlight{\!\pi {=} \{ \pi_k \}_{k=1}^\infty \sim \mathrm{GEM}(\alpha)\!\!}$

\vspace{-2pt}$\implies
\begingroup \color{gray}
    \sum_{k=1}^\infty \pi_k \delta_{\theta_k}\!(\theta) =
\endgroup
\highlight{G(\theta) \sim \mathrm{DP}(\alpha, H)}$

Sample $\theta^{(1)}, \theta^{(2)}, \ldots$ from $G$.\enskip
Denote $\theta^{(i)} {=} \theta_{k_i}$.

$\implies \theta^{(i)}, \theta^{(j)}$ with $k_i {=} k_j$ belong to same ``cluster''

\emph{Chinese Rest. Process:}
\vspace{-5pt}

$P(\text{cust}_{n+1} \text{ joins table } \tau \mid \mathcal P) =
\begin{cases}
    \frac{\abs{\tau}}{\alpha + n} & \text{if } \tau \in \mathcal P, \\
    \frac{\alpha}{\alpha + n} & \text{new table}
\end{cases}$

$P(\text{partition } \mathcal P)
= \frac{\alpha^{\abs{\mathcal P}}}{\alpha^{(n)}} \prod_{\tau\in\mathcal P} (\abs{\tau} - 1)!$

\textbf{expec. \#clusters:}\enskip
$\E{\bm 1} = \sum_{i\leq N} \frac{\alpha}{\alpha + i} \sim \mathcal O(\alpha \log N)$

\emph{De Finetti:}
$(X_1, \ldots, X_n)$ are inftly \textbf{exchangable} RVs \textit{if}
$P(X_1, \ldots, X_n) = \int \paren*{\prod_{i=1}^n p(X_i \mid G)} \diff P(G)$

% Equivalent formulations (?)
\iffalse
    $P(X_1, \ldots X_n) = \int P(G) \prod_{i=1}^n p(X_i \mid G) \diff G$

    $P(X_1, \ldots X_n) = \int P(\theta) \prod_{i=1}^n p(X_i \mid \theta) \diff\theta$
\fi

\subsubsection{Finite GMM}
\begin{enumerate}
    \item Cluster centers:\enskip
        $\mu_k \sim \Gauss{\mu_0, \sigma_0}$
    \item Prob's of clusters:\enskip
        $\pi_{1\ldots K} \sim \mathrm{Dir}(\alpha_{1\ldots K})$
    \item Cluster assignments:\enskip
        $z_i \sim \mathrm{Categorical}(\pi_{1\ldots K})$
    \item Coordinates of data:\enskip
        $x_i \sim \Gauss{\mu_{z_i}, \sigma_{z_i}}$
\end{enumerate}

\subsubsection{DP Mixture Model (DP-GMM)}
\begin{enumerate}
    \item Cluster centers:\enskip
        $\mu_k \sim \Gauss{\mu_0, \sigma_0}$,\enskip
        $\color{gray} k {=} 1, 2, \ldots$
    \item Prob's of clusters:\enskip
        $\pi = (\pi_1, \pi_2, \ldots) \sim \mathrm{GEM}(\alpha)$
    \item Cluster assignments:\enskip
        $z_i \sim \mathrm{Categorical}(\pi)$
    \item Coordinates of data:\enskip
        $x_i \sim \Gauss{\mu_{z_i}, \sigma}$,\enskip
        $\color{gray} i {=} 1 \ldots N$
\end{enumerate}

\emph{Fitting a DP-MM:}\enskip
\textbf{Collapsed Gibbs sampler}

$p(z_i {=} k \mid \bm z_{-i}, \bm x, \alpha, \bm\mu) \propto
\begingroup \color{OrangeRed} \text{Prior} \endgroup \times
\begingroup \color{Green} \text{Likelihood} \endgroup$
\\\hfill
$\propto \begin{cases}
    \begingroup \color{OrangeRed}
        \frac{\abs{\bm x_{-i,k}}}{\alpha+N-1}
    \endgroup
    \begingroup \color{Green}
        p(x_i \vert \bm x_{-i, k}, \bm\mu)
    \endgroup
    & \text{for existing } k
    \\
    \begingroup \color{OrangeRed}
        \frac{\alpha}{\alpha+N-1}
    \endgroup
    \begingroup \color{Green}
        p(x_i\vert \bm\mu)
    \endgroup
    & \text{otw.}
\end{cases}$

$\bm x_{-i,c} \coloneqq \brace{ x_j \mid z_j {=} c, j {\neq} i }$ data assigned to clust. $c$

% ===


\chapter{Conditional Probability}

Let's say there is a virus going around, and there is a vaccine for it
that requires 2 shots. You are working at a school, and you are
wondering how effective the vaccines are. Some students are
unvaccinated, some have had one shot, and some have had two shots. One
day you test all 644 students to see who has the virus. You end up
with the following table:

\begin{tabular}{c | c c c}
  & $V_0$ & $V_1$ & $V_2$ \\
  \hline
  $T_{+}$ & 88 students & 36 students & 96 students \\
  $T_{-}$ & 92 students & 76 students & 256 students \\
\end{tabular}

Here are what the symbols mean:

\begin{itemize}
\item $V_0$: student has had zero vaccination shots
\item $V_1$: student has had one vaccination shot
\item $V_2$: student has had both vaccination shots
\item $T_{+}$: student tested positive for the virus
\item $T_{-}$: student tested negative for the virus
\end {itemize}

So, for example, your data indicates that there were 76 students who
had only one of the two shots and tested negative for the virus.

Your principal has a few questions. The first is ``If I put five
randomly choosen students in a study group together, what is the
probabilty that one of them has the virus?''

The first thing you might do is make a new table that shows what is
the probability of a randomly choosen student being in any particular
group. You just divide each entry by 644 (the total number of
students).

\begin{tabular}{c | c c c}
  & $V_0$ & $V_1$ & $V_2$ \\
  \hline
  $T_{+}$ & $p(V_0 \text{ AND } T_{+}) = 13.7\%$ & $p(V_1 \text{ AND } T_{+}) = 5.6\%$ & $p(V_2 \text{ AND } T_{+}) = 14.9\%$\\
  $T_{-}$ & $p(V_0 \text{ AND } T_{-}) = 14.3\%$ & $p(V_1 \text{ AND } T_{-}) = 11.8\%$ & $p(V_2 \text{ AND } T_{-}) = 39.8\%$
\end{tabular}

(In this table, I expressed the number as a percentage with a decimal
point -- you had to round off the numbers. If you wanted exact answers, you
would have keep each as a fraction: 36 students represents
$\frac{9}{161}$ of the student body.)

\section{Marginalization}

Now we can sum across the columns and rows.

\begin{tabular}{c | c c c | c}
  & $V_0$ & $V_1$ & $V_2$ & sum \\
  \hline
  $T_{+}$ & 0.137 & 0.056 & 0.149 & $p(T_{+}) = 0.342$\\
  $T_{-}$ & 0.143 & 0.118 & 0.398 & $p(T_{+}) = 0.547$\\
  \hline
  sum & $p(V_0) = 0.280$ & $p(V_1) = 0.174$ & $p(V_2) = 0.547$ &
\end{tabular}

If a child is choosen randomly from the entire student body, there is
a 34.2\% that the student has tested positive for the virus. And there is
17.4\% chance that the student has one shot of the vaccine.

This summing of the probabilities across one dimension is known as
\textit{marginalizing}. Marginalization is just summing across all the
variable that you don't care about. You don't care who has the virus,
just the probability that a student has not received even one shot of
the vaccine? You marginalize all the vaccine statuses.

To answer the principal's question, the easy thing to do is find the
answer of the opposite ``if I put five randomly choosen students in a
study group together, what is the probabilty that \textit{none} of
them has tested positive for the virus?''

The chance that a randomly choose student doesn't have the virus
($p(T_{-}$) is 54.7\%.  Thus the chance that 5 randomly chosen
students don't have the virus is $0.547 \times 0.547 \times 0.547
\times 0.547 \times 0.547 = 0.0489$ Thus the probability of the
opposite is $1.0 - 0.0489 = 0.951$

The answer, then, is ``If you put 5 kids in a study group together,
there is a 95.1 \% probability that at least one of them has the
virus.''
% KA: https://www.khanacademy.org/math/ap-statistics/probability-ap/stats-conditional-probability/v/testing-independence-from-experimental-data

\section{Conditional Probability}

Now the principal asks you, ``What if I make a group of 5 kids who
have had both shots of the vaccine? What are the odds that one of them
has tested positive for the virus?''

This involves the idea of \textit{Conditional probability}.  You want
to know the odds that a student doesn't have the virus given that
student has had both shots of the vaccine.

There is a mathematical notation for this:

$$p(T_{-} | V_{2})$$

That is the probability that a student who has had both vaccination
shots will test negative for the virus.

How would you calculate this? You would count all the students who had
a positive test \textit{and} both vaccination shots, which you would
divide by the total number of students who had both vaccination shots.

$$p(T_{-} | V_{2}) = \frac{256}{96 + 256} = \frac{8}{11} \approx 72.7\%$$

If we are working from the probabilities, you can get the same result
this way: Divide the probability that a randomly choosen student had a
positive test \textit{and} both vaccination shots by the probabilty
that a student had both vaccination shots:

$$p(T_{-} | V_{2}) = \frac{p(T_{-} \text{ AND } V_{2})}{p(T_{-})} =  \frac{0.398}{0.547} \approx 72.7\%$$

Notice that this is different from $p( V_{2} | T_{-})$, which is the
probability that a student has had both vaccinations, given they
tested negative for the virus.

Back to the principal's question: "If you have 5 students who have had
both vaccinations, what is the probability that all of them tested
negative for the virus?" The probability that one student is virus-free
is $\frac{8}{11}$, so the probability that 5 students are virus-free
is $\frac{8}{11}^5 \approx 0.203$.  So, there is a $79.6\%$ chance
that at least one of the five has the virus.
% KA: https://www.khanacademy.org/math/statistics-probability/probability-library/conditional-probability-independence/v/calculating-conditional-probability

\section{Chain Rule for Probability}

You just used this equality: For any events $A$ and $B$

$$p(A | B) = \frac{p(A \text{ AND } B)}{p(B)}$$

This is more commonly written like this:

$$p(A \text{ AND } B) = \frac{p(A | B)}{p(B)}$$

This is an abstract way of writing the idea, but the idea itself
is pretty intuitive: The probability that I'm going to buy and ticket
and win the lottery is equal to the probability that I buy a ticket
times the probability that I win, given that I have bought a ticket.
(Here $A$ is ``win the lottery'' and $B$ is ``buy a ticket''.)

This is known as \textit{The Chain Rule of Probability}.  And we can
chain together as many events as we want: The probability that you are
going to die in the car that you bought with your winnings from the
lottery ticket you bought is:

$$p(W \text{ AND } X \text{ AND } Y \text{ AND } Z) = p( W | X \text{ AND } Y \text{ AND } Z) p( X |  Y \text{ AND } Z) p (Y | Z) p(Z)$$

where

\begin{itemize}
\item $W =$ Dying in car accident
\item $X =$ Buying a car with lottery winnings
\item $Y =$ Winning the lottery
\item $Z =$ Buying a lottery ticket
\end{itemize}

In English, then, the equation says:

``The probability that you will die in a car accident, buy a car with
lottery winnings, win the lottery, and buy a lottery ticket is equal
to the probability that you buy a lottery ticket times the probability
that you win the lottery (given that you have bought a ticket) times
the probability that buy a car with those lottery winnings (given that
bought a ticket and won) times the probability that you crash that car
(given that you have bought the car, won the lottery, and bought a
ticket).''

% KA: https://www.khanacademy.org/math/ap-calculus-ab/ab-differentiation-2-new/ab-3-1a/v/chain-rule-introduction