% !TeX root = main.tex

\chapter{Prefix Sum and Histogram}
\glsresetall
\label{chapter:prefixsum}

\section{Prefix Sum}
\label{sec:prefixSum}

Prefix sum is a common kernel used in many applications, e.g., recurrence relations,
compaction problems, string comparison, polynomial evaluation, histogram, radix sort, and quick sort \cite{blelloch1990prefix}. Prefix sum requires restructuring in order to create an efficient FPGA design.

The prefix sum is the cumulative sum of a sequence of numbers. Given a sequence of inputs $in_n$, the prefix sum $out_n$ is the summation of the first $n$ inputs, namely $out_n = in_0 + in_1 + in_2 + \cdots + in_{n-1} + in_n$. The following shows the computation for the first four elements of the output sequence $out$.
\begin{align*}
out_0 & = in_0 &\\
out_1 & = in_0 + in_1 &\\
out_2 & = in_0 + in_1 + in_2 \\
out_3 & = in_0 + in_1 + in_2 + in_3 \\
\cdots
\end{align*}

Of course, in practice we don't want to store and recompute the sum of all of the previous inputs, so the prefix sum is often computed by the recurrence equation:
\begin{equation}
out_n = out_{n-1} + in_n
\end{equation}

The disadvantage of the recurrence equation is that we must compute $out_{n-1}$ before computing $out_n$, which fundamentally limits the parallelism and throughput that this computation can be performed.  In contrast, the original equations have obvious parallelism where each output can be computed independently at the expense of a significant amount of redundant computation.  C code implementing the recurrence equation is shown in Figure \ref{fig:prefixsumSW}. Ideally, we'd like to achieve $II=1$ for the loop in the code, but this can be challenging even for such simple code.  Implementing this code with \VHLS results in behavior like that shown in Figure \ref{fig:prefixsumSW}.
\begin{figure}
\begin{minipage}{.5\textwidth}
\lstinputlisting{examples/prefixsumBO.cpp}
\end{minipage}
\begin{minipage}{.5\textwidth}
\centering
\includesvg{prefixsumBO_behavior}
\end{minipage}
\caption{ Code for implementing prefix sum, and its accompanying behavior. }
\label{fig:prefixsumSW}
\end{figure}

The way this code is written, each output is written into the output memory \lstinline|out[]| and then in the next iteration is read back out of the memory again.  Since the memory read is has a latency of one, data read from memory cannot be processed until the following clock cycle.  As a result, such a design can only achieve a loop II of 2.  In this case there is a relatively easy way to rewrite the code: we can simply perform the accumulation on a separate local variable, rather than reading the previous value back from the array.  Avoiding extra memory accesses in favor of register storage is often advantageous in processor code, but in HLS designs it is often more significant since other operations are rarely a performance bottleneck.  Code that does this is shown in Figure \ref{fig:prefixsum_optimized}.

\begin{figure}
\begin{minipage}{.5\textwidth}
\lstinputlisting{examples/prefixsum_optimized.cpp}
\end{minipage}
\begin{minipage}{.5\textwidth}
\centering
\includesvg{prefixsum_optimized_behavior}
\end{minipage}
\caption{ Code for implementing an optimized prefix sum, and its accompanying behavior. }
\label{fig:prefixsum_optimized}
\end{figure}

You might ask why the compiler is not able to optimize the memory loads and stores automatically in order to improve the II of the design.  It turns out that \VHLS is capable of optimizing loads and stores to array, but only for reads and writes within the scope of a single basic block.  You can see this if we unroll the loop, as shown in Figure \ref{fig:prefixsum_unrolled}.  Note that we also have to add appropriate \lstinline{array_partition} s in order to be able to read and write multiple values at the interfaces.   In this case, \VHLS is able to eliminate most of the read operations of the \lstinline{out[]} array within the body of the loop, but we still only achieve a loop II of 2.  In this case the first load in the body of the loop is still not able to be removed.  We can, however, rewrite the code manually to use a local variable rather than read from the \lstinline{out[]} array.

\begin{figure}
\begin{minipage}{.5\textwidth}
\lstinputlisting{examples/prefixsum_unrolled.cpp}
\end{minipage}
\begin{minipage}{.5\textwidth}
\raggedleft
\includesvg{prefixsum_unrolled_behavior}
\end{minipage}
\caption{ Optimizing the prefixsum code using \lstinline{unroll}, \lstinline{pipeline}, and \lstinline{array_partition} directives. }
\label{fig:prefixsum_unrolled}
\end{figure}

%results in the following message:
%\lstinputlisting[basicstyle=\ttfamily\footnotesize]{examples/prefixsumBO.message}

%\begin{figure}
%\lstinputlisting{examples/prefixsumBO.cpp}
%\caption{ The initial code for implementing prefix sum. }
%\label{fig:prefixsumSW}
%\end{figure}

Ideally, when we unroll the inner loop, we the perform more operations per clock and reduce the interval to compute the function. If we unroll by a factor of two, then the performance doubles. A factor of four would increase the performance by factor four, i.e., the performance scales in a linear manner as it is unrolled. While this is mostly the case, as we unroll the inner loop there are often some aspects of the design that don't change.  Under most circumstances, such as when the iteration space of loops execute for a long time, these aspects represent a small fixed overhead which doesn't contribute significantly to the performance of the overall function.  However, as the number of loop iterations decreases, these fixed portions of the design have more impact.  The largest fixed component of a pipelined loop is the depth of the pipeline itself.  The control logic generate by \VHLS for a pipelined loop requires the pipeline to completely flush before code after the loop can execute.

\begin{exercise}
Unroll the \lstinline{for} loop corresponding to the prefix sum code in Figure \ref{fig:prefixsum_optimized} by different factors in combination with array partitioning to achieve a loop II of 1.  How does the \lstinline{prefixsum} function latency change? What are the trends in the resource usages? Why do you think you are seeing these trends? What happens when the loop becomes fully unrolled?
\end{exercise}

\begin{figure}
\centering
\includegraphics[width=  .7\textwidth]{images/architectures_prefixsum}
\caption{ Part a) displays an architecture corresponding to the code in Figure~\ref{fig:prefixsumSW}. The dependence on the \lstinline{out[]} array can prevent achieving a loop II of 1. Computing the recurrence with a local variable, as shown in the code in Figure~\ref{fig:prefixsum_optimized}, is able to reduce the latency in the recurrence and achieve a loop II of 1.}
\label{fig:architecture_prefixsum}
\end{figure}

Figure~\ref{fig:architecture_prefixsum} shows the hardware architecture resulting from synthesizing the code from Figure \ref{fig:prefixsumSW} and Figure~\ref{fig:prefixsum_optimized}.   In part a), we can see that the `loop' in the circuit includes the output memory that stores the \lstinline{out[]} array, whereas in part b), the loop in the circuit includes only a register that stores the accumulated value and the output memory is only written.  Simplifying recurrences and eliminating unnecessary memory accesses is a common practice in optimizing HLS code.

%This \lstinline{A} variable maps to a hardware register. A register is helpful because a it can be read to and written from on every cycle. Unlike a memory, which has limited number of read and write ports, the register value can be read from and subsequently sent to a large number of places that need that data on every cycle with very limited penalty. The only major issues is wire routing, which in the grand scheme of things is not much additional overhead in terms of resources. Nor does it typically incur a large penalty in terms of performance.

\note{Could add a whole part about doing reduction. Probably not necessary.}
\note{Should have something here about what to do with floating point accumulation.  This is fundamentally more problematic than what's above (which is relatively easily handled by improving store->load optimization.}

The goal of this section is to show that even a small changes in the code can sometimes have a significant effect on the hardware design. Some changes may not necessarily be intuitive, but can be identified with feedback from the tool.

\section{Histogram}
\label{sec:histogram}

A Histogram models the probability distribution of a discrete signal. Given a sequence of discrete input values, the histogram counts the number of times each value appears in the sequence.  When normalized by the total number of input values, the histogram becomes the probability distribution function of the sequence.  Creating a histogram is a common function used in image processing, signal processing, database processing, and many other domains.  In many cases, it is common to quantize high-precision input data into a smaller number of intervals or \term{bins} as part of the histogram computation.  For the purpose of this section, we will skip the actual process by which this is done and focus on what happens with the binned data.

\begin{figure}
\centering
\includegraphics[width=  .7\textwidth]{images/histogram_introd}
\caption{ An example of a histogram.  }
\label{fig:histogram_introd}
\end{figure}

Figure~\ref{fig:histogram_introd} provides a simple example of a histogram.  The data set consists of a sequence of binned values, in this case represented by an integer in $[0,4]$. The corresponding histogram, consisting of a count for each bin, is shown below along with a graphical representation of the histogram, where the height of each bar corresponding to the count of each separate value.  Figure~\ref{fig:histogramSW} shows baseline code for the \lstinline{histogram} function.

\begin{figure}
\lstinputlisting{examples/histogramSW.cpp}
\caption{ Original code for calculating the histogram. The \lstinline{for} loop iterates across the input array and increments the corresponding element of the \lstinline{hist} array. }
\label{fig:histogramSW}
\end{figure}

The code ends up looking very similar to the prefix sum in the previous section.  The difference is that the prefix sum is essentially only performing one accumulation, while in the \lstinline|histogram| function we compute one accumulation for each bin.  The other difference is that in the prefix sum we added the input value each time, in this case we only add 1.  When pipelining the inner loops using the \lstinline|pipeline| directive, we return to the same problem as with the code in Figure \ref{fig:prefixsumSW}, where we can only achieve a loop II of 2 due to the recurrence through the memory.   This is due to the fact that we are reading from the \lstinline{hist} array and writing to the same array in every iteration of the loop.
Figure~\ref{fig:architecture_histogram} shows the hardware architecture for the code in Figure~\ref{fig:histogramSW}. You can see that the \lstinline{hist} array has a read and write operation. The \lstinline{val} variable is used as the index into the \lstinline{hist} array, and the variable at that index is read out, incremented, and written back into the same location.

\begin{figure}
\centering
\includegraphics[width=  .5\textwidth]{images/architectures_histogram}
\caption{ An architecture resulting from the code in Figure \ref{fig:histogramSW}. The \lstinline{val} data from the \lstinline{in} array is used to index into the \lstinline{hist} array. This data is incremented and stored back into the same location.}
\label{fig:architecture_histogram}
\end{figure}

\section{Histogram Optimization and False Dependencies}
Let's look deeper at the recurrence here.  In the first iteration of the loop, we read the \lstinline{hist} array at some location $x_0$ and write back to the same location $x_0$.  The read operation has a latency of one clock cycle, so the write has to happen in the following clock.  Then in the next iteration of the loop, we read at another location $x_1$.  Both $x_0$ and $x_1$ are dependent on the input and could take any value, so we consider the worst case when generating the circuit.  In this case, if $x_0 == x_1$, then the read at location $x_1$ cannot begin until the previous write has completed.  As a result, we must alternate between reads and writes.

It turns out that we must alternate between reads and writes as long as $x_0$ and $x_1$ are independent.  What if they are {\em not} actually independent?  For instance, we might know that the source of data never produces two consecutive pieces of data that actually have the same bin.  What do we do now?  If we could give this extra information to the HLS tool, then it would be able to read at location $x_1$ while writing at location $x_0$ because it could guarantee that they are different addresses.   In \VHLS, this is done using the \lstinline|dependence| directive.

The modified code is shown in Figure \ref{fig:histogram_dependence}.  Here we've explicitly documented (informally) that the function has some preconditions.  In the code, we've added an \lstinline|assert()| call which checks the second precondition.
in \VHLS, this assertion is enabled during simulation to ensure that the simulation testvectors meet the required precondition.  The \lstinline|dependence| directive captures the effect of this precondition on the circuit, generated by the tool.  Namely, it indicates to \VHLS that reads and writes to the \lstinline|hist| array are dependent only in a particular way.  In this case, \lstinline|inter|-iteration dependencies consisting of a read operation after a write operation (RAW) have a distance of 2.  In this case a distance of $n$ would indicate that read operations in iteration $i+n$ only depend on write operations in iteration $i$.  In this case, we assert that \lstinline|in[i+1] != in[i]|, but it could be the case that \lstinline|in[i+2] == in[i]| so the correct distance is 2.

\begin{figure}
\lstinputlisting{examples/histogram_dependence.cpp}
\caption{ An alternative function for computing a histogram.  By restricting the inputs and indicating this restriction to \VHLS via the \lstinline|dependence| directive, II=1 can be achieved without significantly altering the code. }
\label{fig:histogram_dependence}
\end{figure}

\begin{exercise}
In Figure \ref{fig:histogram_dependence}, we added a precondition to the code, checked it using an assertion, and indicated the effect of the precondition to the tool using the \lstinline|dependence| directive.  What happens if your testbench violates this precondition?  What happens if you remove the \lstinline|assert()| call?  Does \VHLS still check the precondition?   What happens if the precondition is not consistent with the \lstinline|dependence| directive?
\end{exercise}

Unfortunately, the \lstinline{dependence} directive doesn't really help us if we are unwilling to accept the additional precondition.  It's also clear that we can't directly apply same optimization as with the \lstinline|prefixsum| function, since we might need to use all of the values stored in the \lstinline|hist| array.  Another alternative is implement the \lstinline|hist| array with a different technology, for instance we could partition the \lstinline|hist| array completely resulting in the array being implemented with \gls{ff} resources.  Since the data written into a \gls{ff} on one clock cycle is available immediately on the next clock cycle, this solves the recurrence problem and can be a good solution when a small number of bins are involved.  The architecture resulting from such a design is shown in Figure \ref{fig:histogram_partitioned}.  However, it tends to be a poor solution when a large number of bins are required.  Commonly histograms are constructed with hundreds to thousands of bins and for large data sets can require many bits of precision to count all of the inputs.   This results in a large number of FF resources and a large mux, which also requires logic resources.  Storing large histograms in \gls{bram} is usually a much better solution.

\begin{figure}
\centering
\note{fixme!}
\includegraphics[width=  .5\textwidth]{images/architectures_histogram}
\caption{ An architecture resulting from the code in Figure \ref{fig:histogramSW} when the \lstinline|hist| array is completely partitioned.}
\label{fig:histogram_partitioned}
\end{figure}

Returning to the code in Figure~\ref{fig:histogramSW}, we see that there are really two separate cases that the architecture must be able to handle.  One case is when the input contains consecutive values in the same bin.  In this case, we'd like to use a simple register to perform the accumulation with a minimal amount of delay.  The second case is when the input does not contain consecutive values in the same bin, in which case we need to read, modify, and write back the result to the memory.  In this case, we can guarantee that the read operation of the \lstinline|hist| array can not be affected by the previous write operation.   We've seen that both of these cases can be implemented separately, perhaps we can combine them into a single design.  The code to accomplish this is shown in Figure~\ref{fig:histogramOpt1}. This code uses a local variable \lstinline|old| to store the bin from the previous iteration and another local variable \lstinline{accu} to store the count for that bin.  Each time through the loop we check to see if we are looking at the same bin as the previous iteration.  If so, then we can simply increment \lstinline|accu|.  If not, then we need to store the value in \lstinline|accu| in the \lstinline|hist| array and increment the correct value in the \lstinline|hist| array instead.  In either case, we update \lstinline|old| and \lstinline|accu| to contain the current correct values.  The architecture corresponding to this code is shown in Figure \ref{fig:architecture_histogram_restructured}.

In this code, we still need a \lstinline|dependence| directive, just as in Figure \ref{fig:histogram_dependence}, however the form is slightly different.  In this case the read and write accesses are to two different addresses in the same loop iteration.  Both of these addresses are dependent on the input data and so could point to any individual element of the \lstinline|hist| array.  Because of this, \VHLS assumes that both of these accesses could access the same location and as a result schedules the read and write operations to the array in alternating cycles, resulting in a loop II of 2.  However, looking at the code we can readily see that \lstinline|hist[old]| and \lstinline|hist[val]| can never access the same location because they are in the \lstinline|else| branch of the conditional \lstinline|if(old == val)|.  Within one iteration (an \lstinline|intra|-dependence) a read operation after a write operation (\lstinline|RAW|) can never occur and hence is a \lstinline|false| dependence.  In this case we are not using the \lstinline|dependence| directive to inform the tool about a precondition of the function, but instead about a property of the code itself.

\begin{figure}
\lstinputlisting{examples/histogram_opt1.cpp}
\caption{ Removing the read after write dependency from the \lstinline{for} loop. This requires an \lstinline{if/else} structure that may seem like it is adding unnecessary complexity to the design. However, it allows for more effective pipelining despite the fact that the datapath is more complicated. }
\label{fig:histogramOpt1}
\end{figure}

\begin{exercise}
Synthesize the code from Figure \ref{fig:histogramSW} and Figure \ref{fig:histogramOpt1}. What is the initiation interval (II) in each case? What happens when you remove the \lstinline{dependence} directive from the code in Figure \ref{fig:histogramOpt1}? How does the loop interval change in both cases? What about the resource usage?
\end{exercise}

\begin{aside}
For the code in Figure \ref{fig:histogramOpt1}, you might question why a tool like \VHLS cannot determine this property.  In fact, while in some simple cases like this one better code analysis could propagate the \lstinline|if| condition property into each branch, we must accept that there are some pieces of code where properties of memory accesses are actually undecidable.  The highest performance in such cases will only be achieved in a static schedule with the addition of user information.  Several recent research works have looked to improve this by introducing some dynamic control logic into the design\cite{winterstein13dynamic, liu17elasticflow, dai17dynamic}.
\end{aside}

A pictorial description of the restructured code from Figure \ref{fig:histogramOpt1} is shown in Figure \ref{fig:architecture_histogram_restructured}. Not all of the operations are shown here, but the major idea of the function is there. You can see the two separate \lstinline{if} and \lstinline{else} regions (denoted by dotted lines). The \lstinline{acc} variable is replicated twice in order to make the drawing more readable; the actual design will only have one register for that variable.  The figure shows the two separate datapaths for the \lstinline{if} and the \lstinline{else} clause with the computation corresponding to the \lstinline{if} clause on the top and the \lstinline{else} clause datapath on the bottom.

\begin{figure}
\centering
\includegraphics[width=  .6\textwidth]{images/architectures_histogram_restructured}
\caption{ A depiction of the datapath corresponding to the code in Figure \ref{fig:histogramOpt1}. There are two separate portions corresponding to the \lstinline{if} and \lstinline{else} clauses. The figure shows the important portions of the computation, and leaves out some minor details. }
\label{fig:architecture_histogram_restructured}
\end{figure}

\section{Increasing Histogram Performance}

With some effort, we've achieved a design with a loop II of 1.  Previously we have seen how further reducing the execution time of a design can be achieved by partial unrolling of the inner loop.  However, with the \lstinline|histogram| function this is somewhat difficult for several reasons.  One reason is the challenging recurrence, unless we can break up the input data in some fashion, the computation of one iteration of the loop must be completed with the computation of the next iteration of the loop. A second reason is that with a loop II of 1, the circuit performs a read and a write of the \lstinline|hist| array each clock cycle, occupying both ports of a \gls{bram} resource in the FPGA.   Previously we have considered \gls{arraypartitioning} to increase the number of memory ports for accessing an array, but there's not a particularly obvious way to partition the \lstinline|hist| array since the access order depends on the input data.

All is not lost, however, as there is a way we can expose more parallelism by decomposing the histogram computation into two stages.  In the first stage, we divide the input data into a number of separate partitions.  The histogram for each partition can be computed independently using a separate instance, often called a \gls{pe}, of the histogram solution we've developed previously.  In the second stage, the individual histograms are combined to generate the histogram of the complete data sets.  This partitioning (or mapping) and merging (or reducing) process is very similar to that adopted by the MapReduce framework \cite{dean08mapreduce} and is a common pattern for parallel computation.  The map-reduce pattern is applicable whenever there is recurrence which includes a commutative and associative operation, such as addition in this case. This idea is shown in Figure~\ref{fig:architecture_histogram_parallel}.

\begin{figure}
\centering
\includegraphics[width=  .9\textwidth]{images/architectures_histogram_parallel}
\caption{The histogram computation implemented using a map-reduce pattern.  The processing element (PE) in Part a) is the same architecture as shown in Figure \ref{fig:architecture_histogram_restructured}. The \lstinline|in| array is partitioned and each partition is processed by a separate \gls{pe}. The merge block combines the individual histograms to create the final histogram.  }
\label{fig:architecture_histogram_parallel}
\end{figure}

The code for implementing this architecture is shown in Figure \ref{fig:histogram_parallel}. The \lstinline{histogram_map} function implements the `map' portion of the map-reduce pattern and will be instantiated multiple times. The code is very similar to the code in Figure \ref{fig:histogramOpt1}. The main difference is that we have added the additional code to initialize the \lstinline|hist| array.  The \lstinline{histogram_map} function takes an input array \lstinline{in} which will contain a partition of the data being processed and computes the histogram of that partition in the \lstinline|hist| array.   The \lstinline{histogram_reduce} function implements the `reduce' portion of the pattern.  It takes as input a number of partial histograms and combines them into complete histogram by adding together the count for each histogram bin.  In our code example in Figure \ref{fig:histogram_parallel}, we have only two processing elements, thus the merge has two input arrays \lstinline{hist1} and \lstinline{hist2}. This can easily be extended to handle more processing elements.

The new \lstinline{histogram} function takes as an input two partitions of the input data, stored in the \lstinline{inputA} and \lstinline{inputB} arrays.  It computes the histogram of each partition using the \lstinline {histogram_map} function, which are then stored in the \lstinline{hist1} and \lstinline{hist2} arrays. These are feed into the \lstinline{histogram_reduce} function which combines them and stores the result in the \lstinline{hist} array, which is the final output of the top level function \lstinline{histogram}.

\begin{figure}
\lstinputlisting{examples/histogram_parallel.cpp}
\caption{ Another implementation of histogram that uses task level parallelism and pipelining. The histogram operation is split into two sub tasks, which are executed in the two \lstinline{histogram_map} functions. These results are combined in the final histogram result using the \lstinline{histogram_reduce} function. The \lstinline{histogram} function is the top level function that connects these three functions together. }
\label{fig:histogram_parallel}
\end{figure}

\begin{exercise}
Modify the code in Figure \ref{fig:histogram_parallel} to support a parameterizable number \lstinline|NUM_PE| of \glspl{pe}? Hint: You'll need to combine some of the arrays into a single array that is correctly partitioned and add some loops that depend on \lstinline|NUM_PE|.  What happens to the throughput and task interval as you vary the number of \glspl{pe}?
\end{exercise}

We use the \lstinline{dataflow} directive in the \lstinline{histogram} function in order to enable a design with \gls{taskpipelining}.   In this case there are three processes: two instances of the \lstinline{histogram_map} function and one instance of the \lstinline{histogram_reduce} function. Within a single task, the two \lstinline{histogram_map} processes can execute concurrently since they work on independent data, while the \lstinline{histogram_reduce} function must execute after since it uses the results from the  \lstinline{histogram_map} processes. Thus, the \lstinline{dataflow} directive essentially creates a two stage task pipeline with the \lstinline{histogram_map} functions in the first stage and the \lstinline{histogram_reduce} function in the second stage.   As with any dataflow design, the interval of the entire \lstinline{histogram} function depends upon the maximum initiation interval of the two stages. The two \lstinline{histogram_map} functions in the first stage are the same and will have the same interval ($II_\mathrm{histogram\_map}$). The \lstinline{histogram_reduce} function will have another interval ($II_\mathrm{histogram\_reduce}$). The interval of the toplevel \lstinline{histogram} function $II_\mathrm{histogram}$ is then $\max (II_\mathrm{histogram\_map}, II_\mathrm{histogram\_reduce})$.

\begin{exercise}
What happens when you add or change the locations of the \lstinline{pipeline} directives? For example, is it beneficial to add a \lstinline{pipeline} directive to the \lstinline{for} loop in the \lstinline{histogram_reduce} function? What is the result of moving the \lstinline{pipeline} directive into the \lstinline{histogram_map} function, i.e., hoisting it outside of the \lstinline{for} loop where it currently resides?
\end{exercise}

The goal of this section was to walk through the optimization the histogram computation, another small but important kernel of many applications. The key takeaway is that there are often limits to what tools can understand about our programs.  In some cases we must take care in how we write the code and in other cases we must actually give the tool more information about the code or the environment that the code is executing in.  In particular, properties about memory access patterns often critically affect the ability of HLS to generate correct and efficient hardware.  In \VHLS, these properties can be expressed using the \lstinline|dependence| directive.  Sometimes these optimizations might even be counter-intuitive, such as the addition of the \lstinline{if/else} control structure in \ref{fig:histogramOpt1}.  In other cases optimizations might require some creativity, as in applying the map-reduce pattern in Figures \ref{fig:architecture_histogram_parallel} and \ref{fig:histogram_parallel}.

\section{Conclusion}
In this section, we've looked at the prefix sum and histogram kernels.  Although these functions seem different, they both contain recurrences through a memory access.  These recurrences can limit throughput if the memory access is not pipelined.  In both cases, by rewriting the code we can remove the recurrence.  In the case of the prefix sum, this is much easier since the access patterns are deterministic.  In the case of the histogram we must rewrite the code to address the recurrence or ensure that recurrence never happens in practice.   In either case we needed a way to describe to \VHLS information about the environment or about the code itself that the tool was unable to determine for itself.  This information is captured in the \lstinline{dependence} directive.  Lastly, we looked at ways of parallelizing both algorithms yet further, so that they could process a number of data samples each clock cycle.


\documentclass[11pt,a4paper]{report}

\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{gensymb}
\usepackage{tikz}
\usepackage{pgfplots}
\usetikzlibrary{positioning}
\usepackage{geometry}
\geometry{
    left=2cm,
    right=0.64cm,
    top=0.64cm,
    bottom=2cm
}
\usepackage{multicol}
\setlength{\columnsep}{1cm}
\graphicspath{ {images/} }

\begin{document}

\chapter{Semester 2 Examination 2015-2016\\CZ4041 Machine Learning}

\begin{multicols*}{2}

\section{Question 1}
\noindent \textbf{Question 1a}
\noindent Information given in the question:
\begin{itemize}
  \item $P(D) = 0.001$
  \item $P(\sim D) = 0.999$
  \item $P(T_1|D) = 0.9$
  \item $P(T_2|D) = 0.95$
  \item $P(T_1|\sim D) = 0.01$
  \item $P(T_2|\sim D) = 0.1$
\end{itemize}

\begin{equation*}
\begin{split}
P(D|T_1 T_2) &= \frac{P(T_1 | D) P(T_2 | D) P(D)}{P(T_1 T_2)}\\
&= \frac{0.9 \times 0.95 \times 0.001}{P(T_1 T_2)}\\
&= \frac{0.855 \times 10^{-3}}{P(T_1 T_2)}
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
P(\sim D|T_1 T_2) &= \frac{P(T_1 | \sim D) P(T_2 | \sim D) P(\sim D)}{P(T_1 T_2)}\\
&= \frac{0.01 \times 0.1 \times 0.999}{P(T_1 T_2)}\\
&= \frac{0.999 \times 10^{-3}}{P(T_1 T_2)}
\end{split}
\end{equation*}

\noindent Since $P(\sim D|T_1 T_2) > P(D|T_1 T_2)$, we predict that the patient does not have the disease.\\

\noindent \textbf{Question 1b}

\noindent Let action $a_1$ be the action predicting the patient to have the disease, and action $a_2$ be the action predicting the patient to not have the disease.

$$R(a_1|T_1 T_2) = 1 - P( D | T_1 T_2)$$
$$R(a_2|T_1 T_2) = 1 - P( \sim D | T_1 T_2)$$

\noindent We choose the action with minimum risk. Since $P(\sim D|T_1 T_2) > P(D|T_1 T_2)$, thus $R(a_2|T_1 T_2) < R(a_1|T_1 T_2)$, so the action $a_2$ has lower risk. As a result, we predict that the patient does not have the disease.\\

\noindent \textbf{Question 1c}

\noindent Information given in the question:
\begin{itemize}
  \item $\lambda_{12} = 0.05$
  \item $\lambda_{21} = 1$
\end{itemize}

\noindent Where $\lambda_{12}$ is the lost occur when the action is predict the patient to have disease $a_1$, but the ground truth is the patient do not have the disease $\sim D$. The lost $\lambda_{21}$ is defined in similar way. The risks are:

\begin{equation*}
\begin{split}
R(a_1|T_1 T_2) &= \lambda_{12} P(\sim D | T_1 T_2)\\
&= 0.05 \times \frac{0.999 \times 10^{-3}}{P(T_1 T_2)}\\
&= \frac{0.4995 \times 10^{-3}}{P(T_1 T_2)}
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
R(a_2|T_1 T_2) &= \lambda_{21} P( D | T_1 T_2)\\
&= 1 \times \frac{0.855 \times 10^{-3}}{P(T_1 T_2)}\\
&= \frac{0.855 \times 10^{-3}}{P(T_1 T_2)}\\
\end{split}
\end{equation*}

\noindent Since $R(a_1|T_1 T_2) < R(a_2|T_1 T_2)$, we predict that the patient has the disease.

\section{Question 2}
\noindent \textbf{Question 2a}

\noindent We define the inputs to Hidden Layer as:

$$X = \begin{bmatrix} X_1 \\ X_2 \end{bmatrix}$$

\noindent The weight from Input Layer to Hidden Layer is:

$$W =
\begin{bmatrix}
w_{13} & w_{23} \\
w_{14} & w_{24}
\end{bmatrix}$$

\noindent The activation function is:

$$
f(u) =
\begin{cases}
1 & u \ge 0\\
-1 & u < 0
\end{cases}
$$

\noindent The output of the Hidden Layer (which is also the input to Output Layer) is:

$$Z = f(W X)$$

\noindent The weight from Hidden Layer to Output Layer is:

$$V =
\begin{bmatrix}
w_{35} & w_{45}
\end{bmatrix}$$

\noindent The output of Output Layer is:

$$y = f(V Z)$$

\noindent For each $X$, we calculate $y$:

\begin{center}
\begin{tabular}{|c | c | c | c |}
\hline
$X_1$ & $X_2$  & $y$ & Predict \\ \hline
2     & -0.5   & 1   & +       \\
1     & 1      & 1   & -       \\
3     & 1      & 1   & -       \\
2     & -2     & -1  & +       \\
1.5   & 2      & 1   & -       \\ \hline
\end{tabular}
\end{center}

\noindent The error rate is $80\%$.\\

\noindent \textbf{Question 2b}

\noindent We define $X$ as:

$$X =
\begin{bmatrix}
2 & -0.5 \\
1 & 1 \\
3 & 1 \\
2 & -2 \\
1.5 & 2
\end{bmatrix}$$

\noindent The pairwise inner products between the five data points is:

$$X X^{T} =
\begin{bmatrix}
4.25 & 1.50 &  5.50 &  5.00  &  2.00\\
1.50 & 2.00 &  4.00 &  0.00  &  3.50\\
5.50 & 4.00 & 10.00 &  4.00  &  6.50\\
5.00 & 0.00 &  4.00 &  8.00  & -1.00\\
2.00 & 3.50 &  6.50 & -1.00  &  6.25
\end{bmatrix}$$

\noindent \textbf{Question 2c}

\begin{tikzpicture}
\begin{axis}[
    enlargelimits=false,
    axis equal
]
\addplot+[
    nodes near coords,
    only marks,
    point meta=explicit symbolic
]
table[meta=label] {
    x   y   label
    2  -0.5 ID1
    1   1   ID2
    3   1   ID3
    2  -2   ID4
    1.5 2   ID5
    7   4   ID6
    5   0.2 Q2d
};
\addplot [
    domain=0:7,
    samples=2,
    color=red,
]
{0.25};
\end{axis}
\end{tikzpicture}

\noindent The decision boundary is $y=0.25$.\\

\noindent \textbf{Question 2d}

\noindent Predict the test data point to be + class.

\section{Question 3}

\noindent \textbf{Question 3a}

\noindent If we discard the last eigenvector to reduce the dimension to 2:

$$\text{POV} = \frac{9.78 + 2.11}{9.78 + 2.11 + 0.11} = 0.991$$

\noindent If we discard the last eigenvector to reduce the dimension to 1:

$$\text{POV} = \frac{9.78}{9.78 + 2.11 + 0.11} = 0.815$$

\noindent \textbf{Question 3b}

$$z = U_{\text{reduce}} x^{T}$$
$$U_{\text{reduce}} = \begin{bmatrix}
0.78 & -0.54 & 0.32 \\
0.04 & 0.56 & 0.83
\end{bmatrix}$$

\begin{center}
\begin{tabular}{| c | c | c |}
\hline
Data  & Original    & Reduced\\ \hline
$x_1$ & $(2,0,2)$   & $(2.20,1.74)$   \\
$x_2$ & $(-1,0,-1)$ & $(-1.10,0.87)$  \\
$x_3$ & $(3,-3,0)$  & $(3.96,-1.56)$  \\
$x_4$ & $(-3,2,-2)$ & $(-4.06,-0.66)$ \\
$x_5$ & $(-1,1,1)$  & $(-1.00,1.35)$  \\ \hline
\end{tabular}
\end{center}

\noindent \textbf{Question 3c}

\begin{center}
\begin{tabular}{| c | c  c  c  c  c |} \hline
      & $x_1$  & $x_2$   & $x_3$  & $x_4$  & $x_5$   \\ \hline
$x_1$ & 0      & 3.4128  & 3.7400 & 6.7043 & 3.2237  \\
$x_2$ & 3.4128 & 0       & 5.6132 & 3.3320 & 0.49031 \\
$x_3$ & 3.7400 & 5.6132  & 0      & 8.0703 & 5.7506  \\
$x_4$ & 6.7043 & 3.3320  & 8.0703 & 0      & 3.6611  \\
$x_5$ & 3.2237 & 0.49031 & 5.7506 & 3.6611 & 0       \\ \hline
\end{tabular}
\end{center}

\noindent We merge $x_2$ and $x_5$:

\begin{center}
\begin{tabular}{| c | c  c  c  c  |} \hline
               & $x_2 \cap x_5$  & $x_1$   & $x_3$  & $x_4$  \\ \hline
$x_2 \cap x_5$ & 0               & 3.2237  & 5.6132 & 3.3320 \\
$x_1$          & 3.2237          & 0       & 3.7400 & 6.7043 \\
$x_3$          & 5.6132          & 3.7400  & 0      & 8.0703 \\
$x_4$          & 3.3320          & 6.7043  & 8.0703 & 0      \\ \hline
\end{tabular}
\end{center}

\noindent We merge $x_2 \cap x_5$ with $x_1$:

\begin{center}
\begin{tabular}{| c | c  c  c |} \hline
                        & $x_2 \cap x_5 \cap x_1$ & $x_3$  & $x_4$  \\ \hline
$x_2 \cap x_5 \cap x_1$ & 0                       & 3.7400 & 3.3320 \\
$x_3$                   & 3.7400                  & 0      & 8.0703 \\
$x_4$                   & 3.3320                  & 8.0703 & 0      \\ \hline
\end{tabular}
\end{center}

\noindent We merge $x_2 \cap x_5 \cap x_1$ with $x_4$:

\begin{center}
\begin{tabular}{| c | c  c |} \hline
                                 & $x_2 \cap x_5 \cap x_1 \cap x_4$ & $x_3$  \\ \hline
$x_2 \cap x_5 \cap x_1 \cap x_4$ & 0                                & 3.7400 \\
$x_3$                            & 3.7400                           & 0      \\ \hline
\end{tabular}
\end{center}

\noindent Finally we merge $x_3$ to $x_2 \cap x_5 \cap x_1 \cap x_4$.

\begin{tikzpicture}[sloped]
\node (a) at (0,0) {a};
\node (b) at (1,0) {b};
\node (c) at (2,0) {c};
\node (d) at (3,0) {d};
\node (e) at (4,0) {e};
\node (dummy) at (3.5,5) {};

\node (ab)    at (0.5,0.4903) {};
\node (abc)   at (1.5,3.00) {}; % change the value abit from 3.22
\node (abcd)  at (2.5,3.30) {};
\node (abcde) at (3.5,3.74) {};

\draw  (a)    |- (ab.center);
\draw  (b)    |- (ab.center);
\draw  (ab)   |- (abc.center);
\draw  (c)    |- (abc.center);
\draw  (abc)  |- (abcd.center);
\draw  (d)    |- (abcd.center);
\draw  (abcd) |- (abcde.center);
\draw  (e)    |- (abcde.center);
\draw (abcde) |- (dummy.center);

\end{tikzpicture}

\noindent \textbf{Question 3d}
\noindent [Help wanted!]\\

\section{Question 4}

\noindent \textbf{Question 4a}
\noindent [Copied and pasted from lecture notes] Steps:
\begin{itemize}
\item Train C1 with dataset D1
\item Test C1 with dataset D2
\item Find error samples and an equal amount of correct samples and form D3
\item Train C2 with dataset D3
\item Find error samples and equal amount of correct samples and form D4
\item Train C1and C2 with dataset D4
\item Find errors and equal number of correct samples and form D5
\item Train C3 with D5...
\end{itemize}

\noindent \textbf{Question 4b}
\noindent [Help wanted!]\\

\noindent \textbf{Question 4c i}

\begin{equation*}
\begin{split}
P(A) &= \sum_{B,E} P(A,B,E)\\
     &= \sum_{B,E} P(A | B,E) P(B,E)\\
     &= \sum_{B,E} P(A | B,E) P(B) P(E)\\
     &=       P(A |      B,     E) P(     B) P(     E) +\\
     &\ \ \ \ P(A |      B,\sim E) P(     B) P(\sim E) +\\
     &\ \ \ \ P(A | \sim B,     E) P(\sim B) P(     E) +\\
     &\ \ \ \ P(A | \sim B,\sim E) P(\sim B) P(\sim E)\\
     &=       0.95 \times 0.01 \times 0.02 +\\
     &\ \ \ \ 0.90 \times 0.01 \times 0.98 +\\
     &\ \ \ \ 0.25 \times 0.99 \times 0.02 +\\
     &\ \ \ \ 0.01 \times 0.99 \times 0.98  \\
     &= 0.02366\\
P(\sim A) &= 0.9763
\end{split}
\end{equation*}

\noindent \textbf{Question 4c ii}

\begin{equation*}
\begin{split}
P(S) &= \sum_{A} P(S,A)\\
     &= \sum_{A} P(S|A)P(A)\\
     &=      P(S|     A)P(     A) +\\
     &\ \ \ \ P(S|\sim A)P(\sim A)\\
     &= 0.70 \times 0.02366 + 0.05 \times 0.9763\\
     &= 0.06538
\end{split}
\end{equation*}

\noindent \textbf{Question 4c iii}

\begin{equation*}
\begin{split}
P(S|B) &= \sum_{A} P(S,A|B)\\
       &= \sum_{A} \frac{P(S,A,B)}{P(B)}\\
       &= \sum_{A} \frac{P(S|A,B)P(A,B)}{P(B)}\\
       &= \sum_{A} \frac{P(S|A,B)P(A|B)P(B)}{P(B)}\\
       &= \sum_{A} P(S|A,B)P(A|B)\\
       &= \sum_{A} P(S|A)P(A|B)\\
       &=      P(S|     A)P(     A|B)+\\
       &\ \ \ \ P(S|\sim A)P(\sim A|B)\\
       &= 0.70 \times P(A|B) + 0.05 \times P(\sim A|B)\\
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
P(A|B)      &= \sum_{E} P(A,E|B)\\
            &= \sum_{E} \frac{P(A,E,B)}{P(B)}\\
            &= \sum_{E} \frac{P(A|E,B)P(E,B)}{P(B)}\\
            &= \sum_{E} \frac{P(A|E,B)P(E)P(B)}{P(B)}\\
            &= \sum_{E} P(A|E,B)P(E)\\
            &= P(A|E,B)P(E) + P(A|\sim E,B)P(\sim E)\\
            &= 0.95 \times 0.02 + 0.90 \times 0.98\\
            &= 0.901\\
P(\sim A|B) &= 0.099\\
P(S|B)      &= 0.70 \times 0.901 + 0.05 \times 0.099\\
            &= 0.6357
\end{split}
\end{equation*}

\begin{equation*}
\begin{split}
P(B|S) &= \frac{P(S|B)P(B)}{P(S)}\\
       &= \frac{0.6357 \times 0.01}{0.06538}\\
       &= 0.09723
\end{split}
\end{equation*}

\end{multicols*}
\end{document}


\documentclass[a4paper]{scrartcl}
\usepackage{amssymb, amsmath} % needed for math
\usepackage[utf8]{inputenc} % this is needed for umlauts
\usepackage[ngerman]{babel} % this is needed for umlauts
\usepackage[T1]{fontenc}    % this is needed for correct output of umlauts in pdf
\usepackage[margin=2.5cm]{geometry} %layout
\usepackage{hyperref}   % links im text
\usepackage{parskip} % no indentation on new paragraphs
\usepackage{color}
\usepackage{framed}
\usepackage{enumerate}  % for advanced numbering of lists
\usepackage{algorithm,algpseudocode}
\usepackage{braket} % needed for \Set
\clubpenalty  = 10000   % Schusterjungen verhindern
\widowpenalty = 10000   % Hurenkinder verhindern

\hypersetup{
  pdfauthor   = {Martin Thoma},
  pdfkeywords = {Google Code Jam, Round 1C 2013, Pogo},
  pdftitle    = {Proof of correctness for an algorithm for pogo}
}

% From http://www.matthewflickinger.com/blog/archives/2005/02/20/latex_mod_spacing.asp
% Thanks!
\makeatletter
\def\imod#1{\allowbreak\mkern10mu({\operator@font mod}\,\,#1)}
\makeatother

\renewcommand{\algorithmicrequire}{\textbf{Input: }}
\renewcommand{\algorithmicensure}{\textbf{Output: }}

\newenvironment{myindentpar}[1]%
 {\begin{list}{}%
         {\setlength{\leftmargin}{#1}}%
         \item[]%
 }
 {\end{list}}

\begin{document}
\section{The Problem}
You're on a two-dimensional grid $\mathbb{Z} \times \mathbb{Z}$ and
have to find a way to get to one coordinate $(x,y) \in \mathbb{Z} \times \mathbb{Z}$. You start at
$(0, 0)$.

In your
$i$-th step you move either $\underbrace{(+i,0)}_{=: E}$,
$\underbrace{(-i,0)}_{=: W}$, $\underbrace{(0,+i)}_{=: N}$ or
$\underbrace{(0,-i)}_{=: S}$.

\section{The algorithm}
\begin{algorithm}
    \begin{algorithmic}
        \Function{calculateSteps}{$x \in \mathbb{Z}$, $y \in \mathbb{Z}$}
            \State $s \gets 0$
            \State $dist \gets |x| + |y|$
            \\
            \While{$\overbrace{\frac{s^2 + s}{2} < dist}^\text{condition 1}$ or $\overbrace{\frac{s^2 + s}{2} \not\equiv dist \imod{2}}^\text{condition 2}$}
                \State $s \gets s + 1$
            \EndWhile
            \\
            \State \Return $s$
        \EndFunction
    \end{algorithmic}
    \caption{Algorithm to calculate the minimum amount of steps}
    \label{alg:calculateSteps}
\end{algorithm}

\clearpage

\begin{algorithm}[ht!]
    \begin{algorithmic}[ht!]
        \Function{solvePogo}{$x \in \mathbb{Z}$, $y \in \mathbb{Z}$}
            \State $s_{\min} \gets$ \Call{calculateSteps}{$x, y$}
            \\
            \State $solution \gets \varepsilon$
            \For{$i$ in $s_{\min}, \dots, 1$}
                \If{$|x| > |y|$}
                    \If{$x > 0$}
                        \State $solution \gets solution + E$
                        \State $x \gets x - i$
                    \Else
                        \State $solution \gets solution + W$
                        \State $x \gets x + i$
                    \EndIf
                \Else
                    \If{$y > 0$}
                        \State $solution \gets solution + N$
                        \State $y \gets y - i$
                    \Else
                        \State $solution \gets solution + S$
                        \State $y \gets y + i$
                    \EndIf
                \EndIf
            \EndFor
            \\
            \State \Return \Call{reverse}{$solution$}
        \EndFunction
    \end{algorithmic}
    \caption{Algorithm to solve the pogo problem}
    \label{alg:solvePogo}
\end{algorithm}

\section{Correctness}
\subsection{calculateSteps}
Let $x,y \in \mathbb{Z}$ and $s := \Call{calculateSteps}{x, y}$.

Let $s_{\min}$ be the minimum amount of necessary steps to get from $(0,0)$
to $(x,y)$ when you move $i$ units in your $i$'th step.

\textbf{Theorem: } $s = s_{\min}$

It's enough to proof $s \geq s_{\min}$ and $s \leq s_{\min}$.

\begin{myindentpar}{1cm}
\textbf{Theorem: } $s \leq s_{\min}$ (we don't make too many steps)

\textbf{Proof: }
\begin{myindentpar}{1cm}
We have to get from $(0,0)$ to $(x, y)$. As we may only move in
taxicab geometry we have to use the taxicab distance measure $d_1$:
\[d_1 \left (p, q \right ) := \sum_{i=1}^2 |p_i -q_i|\]

So in our scenario:
\[d_1 \left ((0,0), (x,y) \right ) = |x| + |y|\]

This means we have to move at least $|x| + |y|$ units to get
from $(0,0)$ to $(x, y)$. As we move $i$ units in the $i$'th step,
we have to solve the following equations for $s_{\min1}$:
\begin{align}
    \sum_{i=1}^{s_{\min1}} i          &\geq |x| + |y| &&\text{ and } &|x| + |y|      &> \sum_{i=1}^{s_{\min1} - 1} i\\
    \frac{s_{\min1}^2 + s_{\min1}}{2} &\geq |x| + |y| &&             &               &> \sum_{i=1}^{s_{\min1} - 1} i &
\end{align}

This is what algorithm \ref{alg:calculateSteps} check with \texttt{condition 1}.
As the algorithm increases $s$ only by one in each loop, it makes
sure that $\sum_{i=1}^{s_{\min1} - 1} i$ is bigger than $|x| + |y|$.

You can undo moves by going back. But this will always make an even
number undone. When you go $(+i, 0)$ and later $(-j, 0)$ it is the
same as if you've been going $(i-j, 0)$. So $2\cdot i$ steps got undone.
But $2\cdot i$ is an even number. You will never be able to undo
an odd number of moved units. This means, the parity of the minimum
number of units you would have to move if you would move one unit per
step has to be the same as the parity of the moves you actually do.
This is exactly what \texttt{condition 2} makes sure.

So we need at least $s$ steps $\Rightarrow s \leq s_{\min} \square$
\end{myindentpar}

\textbf{Theorem: } $s \geq s_{\min}$ (we make enough steps)

\textbf{Proof: }
\begin{myindentpar}{1cm}
We chose $s$ in a way that \texttt{condition 1} is true.
As we have to go $i \in 1,\dots,s$, we can get every possible sum $\Sigma \in \Set{-\frac{s^2+s}{2}, \dots +\frac{s^2+s}{2}}$
with a subset of $\Set{1, \dots, s}$\footnote{This can easily be proved by induction over $\Sigma$.}.
This means we can make a partition $(A, \underbrace{\Set{1, \dots, s} \setminus A}_{=: B})$
such that $|\sum_{i \in A} i| = |x|$ and $|\sum_{i \in B} i|-2\cdot j = |y|$.
This means, we can reach $(x,y)$ from $(0,0)$.
\end{myindentpar}
\end{myindentpar}

\subsection{solvePogo}
\textbf{Theorem: } \Call{solvePogo}{$x,y$} returns a valid, minimal sequence of steps to get from $(0, 0)$ to $(x,y)$

\textbf{Proof: }
\begin{myindentpar}{1cm}
As $s_{\min}$ is the minimum amount of steps you need to get from
$(0,0)$ to $(x,y)$, \Call{solvePogo}{$x,y$} will return a minimal
sequence of steps to get from $(0, 0)$ to $(x,y)$ (see proof above).

We only have to prove that the sequence of steps that \Call{solvePogo}{$x,y$}
is valid, i.e. that you will get from $(0,0)$ to $(x,y)$ with the given
sequence.

TODO.
\end{myindentpar}

\end{document}


\documentclass[a4paper]{article}
\usepackage[colorlinks]{hyperref}
\usepackage{amsmath,esint,bm}
\title{Divergence Theorem}
\author{Naitree Zhu}
\date{Last modified: \today}
\begin{document}
\maketitle
In vector calculus, the Divergence theorem\footnote{For more information:\url{http://en.wikipedia.org/wiki/Divergence_theorem}}, also known as Gauss's theorem or Ostrogradsky's theorem,is a result that relates the flow (that is, flux) of a vector field through a surface to the behavior of the vector field inside the surface.

More precisely, the divergence theorem states that the outward flux of a vector field through a closed surface is equal to the volume integral of the divergence over the region inside the surface. Intuitively, it states that \emph{the sum of all sources minus the sum of all sinks gives the net flow out of a region.}

In physics and engineering, the divergence theorem is usually applied in three dimensions. However, it generalizes to any number of dimensions. In one dimension, it is equivalent to the fundamental theorem of calculus. In two dimensions, it is equivalent to Green's theorem.

The theorem is a special case of the more general Stokes' theorem.

\part{Mathematical statement}
Suppose \textit{V} is a subset of $R^{n}$ (in the case of n = 3, \textit{V} represents a volume in 3D space) which is compact and has a piecewise smooth boundary \textit{S}. If $\boldsymbol{F}$ is a continuously differentiable vector field defined on a neighborhood of \textit{V}, then we have:
\begin{equation}
\iiint_V \nabla\cdot\boldsymbol{F}\mathrm{d}V = \oiint_S \boldsymbol{F}\cdot\mathrm{d}\boldsymbol{S}
\end{equation}
The closed manifold $\partial V$ is quite generally the boundary of \textit{V} oriented by outward-pointing normals.
\section*{Corollaries}
By applying the divergence theorem in various contexts, other useful identities can be derived:
\begin{itemize}
\item Applying the divergence theorem to the product of a scalar function \textit{g} and a vector field $\boldsymbol{F}$, the result is
\begin{equation}
\iiint_V \left[\boldsymbol{F}\cdot\left(\nabla g\right) + g\left(\nabla\cdot\boldsymbol{F}\right)\right]\mathrm{d}V=\oiint_S g\boldsymbol{F}\cdot\mathrm{d}\boldsymbol{S}
\end{equation}
A special case of this is $\mathbf{F}=\nabla f$, in which case the theorem is the basis for Green's identities.
\item Applying the divergence theorem to the product of a scalar function, \textit{f}, and a non-zero constant vector, the following theorem can be proven(Proof is \hyperlink{proof1}{here}):
\begin{equation}
\iiint_V \nabla f \mathrm{d}V=\oiint_S f\mathrm{d}\boldsymbol{S}
\end{equation}
\item Similarly, applying the divergence theorem to the cross-product of a vector field $\mathbf{F}$ and a non-zero constant vector, the following theorem can be proven(Proof is \hyperlink{proof2}{here}):
\begin{equation}
\iiint_V \nabla\times\boldsymbol{F}\mathrm{d}V=\oiint_S \mathrm{d}\boldsymbol{S}\times\boldsymbol{F}
\end{equation}
\end{itemize}
\part{Generalizations}
\section{Multiple dimensions}
One can use the \emph{general} Stokes' Theorem to equate the n-dimensional volume integral of the divergence of a vector field \textbf{F} over a region \textit{U} to the (n-1)-dimensional surface integral of \textbf{F} over the boundary of \textit{U}:
\begin{equation}
\int_U \nabla\cdot\boldsymbol{F}\mathrm{d}V_{n}=\oint_{\partial U} \boldsymbol{F}\cdot\hat{n}\mathrm{d}S_{n-1}
\end{equation}
This equation is also known as the Divergence theorem.

When n = 2, this is equivalent to Green's theorem.

When n = 1, it reduces to the Fundamental theorem of calculus.
\part{History}
The theorem was first discovered by Lagrange in 1762, then later independently rediscovered by Gauss in 1813, by Ostrogradsky, who also gave the first proof of the general theorem, in 1826, by Green in 1828, etc. Subsequently, variations on the divergence theorem are correctly called Ostrogradsky's theorem, but also commonly Gauss's theorem, or Green's theorem.
\part{2-dimensional version}
\begin{equation}
\iint_S \nabla\cdot\bm{F}\mathrm{d}S=\oint_L \bm{F}\cdot\hat{n}\mathrm{d}l
\end{equation}
where $\hat{n}$ is the outward-pointing unit normal vector, and the direction of line integral observes right-hand rule.

By expansion in Cartesian coordinates, it's easily seen that Green theorem is essentially an identical formula with a different form.
\part{Proofs}
\section{Proof to eq.~(3)}
\[
\iiint_V \nabla f \mathrm{d}V=\oiint_S f\mathrm{d}\boldsymbol{S}
\]
\hypertarget{proof1}{Proof}:

If the vector field $\mathbf{F}=f\mathbf{C}$, where $\mathbf{C}$ is an arbitrary nonzero constant vector, then
\[
\nabla\cdot\boldsymbol{F}=\boldsymbol{C}\cdot\nabla f
\]
Apply divergence theorem
\begin{align*}
\int_V \nabla\cdot\boldsymbol{F}\mathrm{d}V
&=\boldsymbol{C}\cdot\oint_S f\mathrm{d}\boldsymbol{S}\\
&=\boldsymbol{C}\cdot\int_V \nabla f\mathrm{d}V
\end{align*}
Since \textbf{C} is nonzero and arbitrary, it cannot always be perpendicular to $\oint_S f\mathrm{d}\boldsymbol{S}$ and $\int_V \nabla f\mathrm{d}V$. So it's only logical that
\[
\iiint_V \nabla f \mathrm{d}V=\oiint_S f\mathrm{d}\boldsymbol{S}
\]
\section{Proof to eq.~(4)}
\hypertarget{proof2}{Proof}:
By setting $\mathbf{G}=\mathbf{C}\times\mathbf{F}$, where \textbf{C} is arbitrary nonzero vector, eq.~(4) can be proved, similar to the proof above.

Using:
\[
\nabla\cdot\left(\boldsymbol{a}\times\boldsymbol{b}\right)=\boldsymbol{b}\cdot\left(\nabla\times\boldsymbol{a}\right)-\boldsymbol{a}\cdot\left(\nabla\times\boldsymbol{b}\right)
\]
then,
\begin{align*}
\int_V \boldsymbol{G}\cdot\mathrm{d}V&=-\boldsymbol{C}\cdot\int_V \nabla\times\boldsymbol{F}\mathrm{d}V\\&=\int_S \left(\boldsymbol{C}\times\boldsymbol{F}\right)\cdot\mathrm{d}\boldsymbol{S}\\&=\boldsymbol{C}\cdot\int_S \boldsymbol{F}\times\mathrm{d}\boldsymbol{S}
\end{align*}
With the similar logic applied with \textbf{C} in the proof above, we get
\[
\iiint_V \nabla\times\boldsymbol{F}\mathrm{d}V=\oiint_S \mathrm{d}\boldsymbol{S}\times\boldsymbol{F}
\]
\end{document}


\documentclass[a4paper,10pt]{article}
%\documentclass[a4paper,10pt]{scrartcl}

\usepackage[utf8]{inputenc}
\usepackage{lineno}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{times}
\usepackage[T1]{fontenc}  % to get the appropriate fonts for icelandic thorn
\usepackage{textcomp}
\usepackage{natbib}
\usepackage{placeins}
\usepackage{listings}
\usepackage{xcolor}

\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codered}{rgb}{1.0,0.0,0.0}
\definecolor{codeblue}{rgb}{0.0,0,1.0}
\definecolor{codeyellow}{rgb}{1.0,0.7,0.0}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}

\lstdefinestyle{mystyle}{
	backgroundcolor=\color{backcolour},
	commentstyle=\color{codered},
	keywordstyle=\color{codeblue},
	numberstyle=\tiny\color{codegreen},
	stringstyle=\color{codeyellow},
	basicstyle=\ttfamily\footnotesize,
	breakatwhitespace=false,
	breaklines=true,
	captionpos=b,
	keepspaces=true,
	numbers=left,
	numbersep=5pt,
	showspaces=false,
	showstringspaces=false,
	showtabs=false,
	tabsize=2
}

\lstset{style=mystyle}
\DeclareMathOperator{\atantwo}{arctan2}

\title{Lomb-Scargle Algorithm Documentation}
\author{Matt James}
\date{}

\pdfinfo{%
  /Title    ()
  /Author   ()
  /Creator  ()
  /Producer ()
  /Subject  ()
  /Keywords ()
}

\begin{document}
\maketitle

\section{Description of Lomb-Scargle Periodogram}

Everything in this document is pieced together from \citet{Lomb1976,Scargle1983,Hocke1998}. This algorithm will calculate the power, $P$, amplitude, $A$, and phase, $\phi$, of a wave with a given frequency, $f$, within an irregularly sampled time series of length $n$ samples.

Let us consider sinusoidal waves of the form
\begin{equation}
	y_f(t_i) = a \cos{\omega(t_i - \tau)} + b \sin{\omega(t_i - \tau)}, \label{EqWave}
\end{equation}
where $\omega = 2 \pi f$ and $1 \le i \le n$.

$\tau$ is defined in \citet{Scargle1983} as
\begin{equation}
	\tan{2 \omega \tau} = \frac{\sum_{i=1}^{n} \sin{2 \omega t_i}}{\sum_{i=1}^{n} \cos{2 \omega t_i}},
\end{equation}
which can easily be rewritten as
\begin{equation}
	\tau = \frac{\atantwo{\left( \sum_{i=1}^{n} \sin{2 \omega t_i}, \sum_{i=1}^{n} \cos{2 \omega t_i} \right)}}{2 \omega}
\end{equation}

The constants $a$ and $b$ are defined as
\begin{equation}
	a = \frac{\sqrt{\frac{2}{n}}\sum_{i=1}^{n}y_i \cos{\omega (t+i - \tau)}}{\left( \sum_{i=1}^{n} \cos^2{\omega (t+i - \tau)} \right)^{\frac{1}{2}}}
\end{equation}
and
\begin{equation}
b = \frac{\sqrt{\frac{2}{n}}\sum_{i=1}^{n}y_i \sin{\omega (t+i - \tau)}}{\left( \sum_{i=1}^{n} \sin^2{\omega (t+i - \tau)} \right)^{\frac{1}{2}}}.
\end{equation}

The periodogram is calculated using
\begin{equation}
	P(\omega) = \frac{1}{2\sigma^2}\frac{n}{2} (a^2 + b^2),
\end{equation}
where $\sigma = \frac{1}{n-1}\sum_{i=1}^{n} y_i^2$ is the variance (assuming that the mean has been subtracted from the data already).

The amplitude is calculated using
\begin{equation}
	A(\omega) = \sqrt{\frac{4\sigma^2}{n}P(\omega)}
\end{equation}
or, equivalently,
\begin{equation}
	A(\omega) = \sqrt{a^2 + b^2}.
\end{equation}

Equation \ref{EqWave} can be expressed differently:
\begin{equation}
	y_f(t_i) = A(\omega) \cos{\left[\omega (t_i - \tau) + \phi \right]},
\end{equation}
where the wave phase is given by,
\begin{equation}
	\phi = -\atantwo(b,a).
\end{equation}

\section{Example Python Code}
The code used in this module is based on the following Python code.

\begin{lstlisting}[language=Python]
def LombScargle(t,x,f):
	'''
	Calculates the Lomb-Scargle periodogram using the method defined in
	Hocke 1998. This method assumes that any mean is removed from the
	data.
	'''

	#preformat the input variables
	t = np.array([t],dtype='float64').flatten()
	x = np.array([x],dtype='float64').flatten()
	f = np.array([f],dtype='float64').flatten()

	#get array sizes
	nf = np.size(f)
	n = np.size(t)

	#create output arrays (Power, amplitude, phase, a, b)
	P = np.zeros((nf,),dtype='float64')
	A = np.zeros((nf,),dtype='float64')
	phi = np.zeros((nf,),dtype='float64')
	a = np.zeros((nf,),dtype='float64')
	b = np.zeros((nf,),dtype='float64')

	#convert f to omega
	w = 2*np.pi*f

	#calculate variance (sigma**2)
	o2 = np.sum(x**2)/(n-1)

	#loop through each frequency
	for i in range(0,nf):
		#calculate sums to get Tau
		ss2w = np.sum(np.sin(2*w[i]*t))
		sc2w = np.sum(np.cos(2*w[i]*t))

		#calculate Tau
		Tau = np.arctan2(ss2w,sc2w)/(2*w[i])

		#calculate w(t - Tau)
		wtT = w[i]*(t - Tau)

		#calculate some more sums
		syc = np.sum(x*np.cos(wtT))
		sys = np.sum(x*np.sin(wtT))
		sc2 = np.sum(np.cos(wtT)**2)
		ss2 = np.sum(np.sin(wtT)**2)

		#get a  and b
		rt2n = np.sqrt(2/n)
		a[i] = rt2n*syc/np.sqrt(sc2)
		b[i] = rt2n*sys/np.sqrt(ss2)

	#calculate the periodogram
	a2b2 = a**2 + b**2
	P[:] = (n/(4*o2))*a2b2

	#calculate amplitude
	A[:] = np.sqrt(a2b2)

	#calculate phase
	phi[:] = -np.arctan2(b,a)

	return P,A,phi,a,b
\end{lstlisting}
\bibliographystyle{agu08}
\bibliography{references}
\end{document}


\section{Introduction}

A skeleton is a lower dimensional entity which represents shape of it's parent object. It being simpler than the parent object, operations like pattern recognition, approximation, similarity estimation, collision detection, animation, matching and deformation can be performed efficiently on it than on the parent object.

Skeletons can be computed via various mathematical formulations such as Medial Axis Transform (MAT),  Chordal Axis Transform (CAT), Thinning etc. Table \ref{Medials} briefly summarizes these methods of Midcurves creation and their strengths-weaknesses.

\begin{table}
\caption{Current Medial Computation Methods}
\begin{tabular}[htbp]{@{} p{0.14\linewidth}  p{0.22\linewidth}  p{0.22\linewidth}  p{0.23\linewidth} @{}} \toprule
{\bf Method } & {\bf Medial }  & {\bf Description} & {\bf Comments}\\
\midrule
%------------------------------------------------------------------------------------------------------------------------------------
\raisebox{-.9\height}{MAT \cite{Ramanathan2004}} &
\raisebox{-.9\height}{\includegraphics[scale=0.3]{..//Common/images/MAT.png} }&
Locii of centers of maximal disk traversing within boundary &
Computable for any shape. But has unwanted branches. Sensitive to boundary perturbations. \\

%------------------------------------------------------------------------------------------------------------------------------------
\raisebox{-.9\height}{CAT \cite{Quadros2008}}&
\raisebox{-.9\height}{\includegraphics[scale=0.45]{..//Common/images/CAT.png}}&
Creates triangulation first then joins midpoints of sides &
Gaps at end. Expensive triangulation. \\

%------------------------------------------------------------------------------------------------------------------------------------
{Straight Skeleton \cite{Henrik2004}} &
\raisebox{-.9\height}{\includegraphics[scale=0.3]{..//Common/images/Straight.png}} &
Goes on thinning from boundary. &
Bisectors mot equidistant. Has unnecessary branches.\\

\bottomrule
\end{tabular}
\label{Medials}
\end{table}

In this paper we focus on 2D planar sketch profiles.  Even in 2D profiles, shapes vary enormously. At the first level of simplification, we would deal with 2D polygons only(with an assumption that curved shapes can be converted to polygonal shape by faceting). Divide-and-Conquer is one of the widely used strategy for dealing with complex models. Shape decomposition partitions given shape into sub-shapes and then skeletonization can be performed on simpler sub-polygons.

A polygon can be decomposed into convex regions by dividing at all reflex(concave) vertices. Generally criterion for decomposition is to produce a minimum number of convex components or to minimize the total length of the boundary of these components. Within the minimum component criterion methods further classification could be based on whether or not Steiner points (brand new, non polygonal vertices) are allowed.


\chapter{Abelian categories}
In this chapter I'll translate some more familiar concepts into categorical language;
this will require some additional assumptions about our category,
culminating in the definition of a so-called ``abelian category''.
Once that's done, I'll be able to tell you what this ``diagram chasing'' thing is all about.

Throughout this chapter, ``$\injto$'' will be used for monic maps and ``$\surjto$'' for epic maps.

\section{Zero objects, kernels, cokernels, and images}
\prototype{In $\catname{Grp}$, the trivial group and homomorphism
are the zero objects and morphisms.
If $G$, $H$ are abelian then the cokernel
of $\phi : G \to H$ is $H/\img \phi$.}

A \vocab{zero object} of a category is an object $0$ which is both initial and terminal;
of course, it's unique up to unique isomorphism.
For example, in $\catname{Grp}$ the zero object is the trivial group,
in $\catname{Vect}_k$ it's the zero-dimensional vector space consisting of one point, and so on.
\begin{ques}
	Show that $\catname{Set}$ and $\catname{Top}$ don't have zero objects.
\end{ques}
For the rest of this chapter, all categories will have zero objects.

In a category $\AA$ with zero objects, any two objects $A$ and $B$ thus have a distinguished morphism
\[ A \to 0 \to B \]
which is called the \vocab{zero morphism} and also denoted $0$.
For example, in $\catname{Grp}$ this is the trivial homomorphism.

We can now define:
\begin{definition}
	Consider a map $A \taking f B$.
	The \vocab{kernel} is defined as the equalizer of this map and the map $A \taking 0 B$.
	Thus, it's a map $\ker f : \Ker f \injto A$ such that
	\begin{diagram}
		\Ker f && \\
		\dInj^{\ker f} & \rdDashed^0 & \\
		A & \rTo_f & B
	\end{diagram}
	commutes, and moreover any other map with the same property factors uniquely through $\Ker A$
	(so it is universal with this property).
	By \Cref{prob:equalizer_monic}, $\ker f$ is a monic morphism,
	which justifies the use of ``$\injto$''.
\end{definition}
Notice that we're using $\ker f$ to represent the map and $\Ker f$ to represent the object
Similarly, we define the cokernel, the dual notion:
\begin{definition}
	Consider a map $A \taking f B$.
	The \vocab{cokernel} of $f$ is a map $\coker f: B \surjto \Coker f$ such that
	\begin{diagram}
		A & \rTo^f & B \\
		& \rdDashed_0 & \dSurj_{\coker f} \\
		&& \Coker f
	\end{diagram}
	commutes, and moreover any other map with the same property factors
	uniquely through $\Coker f$ (so it is universal with this property).
	Thus it is the ``coequalizer'' of this map and the map $A \taking 0 B$.
	By the dual of \Cref{prob:equalizer_monic}, $\coker f$ is an epic morphism,
	which justifies the use of ``$\surjto$''.
\end{definition}
Think of the cokernel of a map $A \taking f B$ as ``$B$ modulo the image of $f$'', e.g.
\begin{example}
	[Cokernels]
	Consider the map $\Zc6 \to D_{12} = \left\langle r,s \mid r^6=s^2=1, rs=sr\inv\right\rangle$.
	Then the cokernel of this map in $\catname{Grp}$ is $D_{12} / \left\langle r \right\rangle \cong \Zc 2$.
\end{example}
This doesn't always work out quite the way we want since in general the image of
a homomorphism need not be normal in the codomain.
Nonetheless, we can use this to define:
\begin{definition}
	The \vocab{image} of $A \taking f B$ is the kernel of $\coker f$.
	We denote $\Img f = \Ker(\coker f)$.
	This gives a unique map $\img f : A \to \Img f$.
\end{definition}
When it exists, this coincides with our concrete notion of ``image''.
Picture:
\begin{diagram}
	A & \rTo^f & B \\
	& \rdTo(1,2)~{\exists!} \rdDashed(3,2)^{0\qquad\qquad} \ruInj(1,2) & & \rdSurj(1,2)^{\coker f} \\
	& \Img f & \rDashed^0 & \Coker f
\end{diagram}
Note that by universality of $\Img f$,
we find that there is a unique map $\img f : A \to \Img f$ that makes the entire diagram commute.

\section{Additive and abelian categories}
\prototype{$\catname{Ab}$, $\catname{Vect}_k$, or more generally $\catname{Mod}_R$.}
We can now define the notion of an additive and abelian category,
which are the types of categories where this notion is most useful.

\begin{definition}
	An \vocab{additive category} $\AA$ is one such that:
	\begin{itemize}
		\ii $\AA$ has a zero object, and any two objects have a product.
		\ii More importantly: every $\Hom_\AA(A, B)$ forms an \emph{abelian group} (written additively)
		such that composition distributes over addition:
		\[ (g+h)\circ f = g\circ f + h\circ f
			\quad\text{and}\quad
			f\circ(g+h) = f\circ g + f \circ h. \]
		The zero map serves as the identity element for each group.
\end{itemize}
\end{definition}
\begin{definition}
	An \vocab{abelian category} $\AA$ is one with the additional properties that
	for any morphism $A \taking f B$,
	\begin{itemize}
		\ii The kernel and cokernel exist, and
		\ii The morphism factors through the image so that $\img(f)$ is epic.
	\end{itemize}
	So, this yields a diagram
	\begin{diagram}
		\Ker(f) & \rInj^{\quad\ker(f)} & A & \rSurj^{\img(f)\quad} & \Img(f) & \rInj & B & \rSurj^{\coker(f)} \Coker(f).
	\end{diagram}
\end{definition}

\begin{example}[Examples of abelian categories]
	\listhack
	\begin{enumerate}[(a)]
		\ii $\catname{Vect}_k$, $\catname{Ab}$ are abelian categories,
		where $f+g$ takes its usual meaning.
		\ii Generalizing this, the category $\catname{Mod}_R$ of $R$-modules is abelian.
		\ii $\catname{Grp}$ is not even additive, because there is no way to assign
		a commutative addition to pairs of morphisms.
	\end{enumerate}
\end{example}

In general, once you assume a category is abelian, all the properties you would want
of these kernels, cokernels, \dots\ that you would guess hold true.
For example,
\begin{proposition}[Monic $\iff$ trival kernel]
	A map $A \taking f B$ is monic if and only if its kernel is $0 \to A$.
	Dually, $A \taking f B$ is epic if and only if its cokernel is $B \to 0$.
\end{proposition}
\begin{proof}
	The easy direction is:
	\begin{exercise}
		Show that if $A \taking f B$ is monic, then $0 \to A$ is a kernel.
		(This holds even in non-abelian categories.)
	\end{exercise}
	Of course, since kernels are unique up to isomorphism, monic $\implies$ $0$ kernel.
	On the other hand, assume that $0 \to A$ is a kernel of $A \taking f B$.
	For this we can exploit the group structure of the underlying homomorphisms now.
	Assume the diagram
	\begin{diagram}
		Z & \pile{\rTo^g \\ \rTo_h} & A & \rTo^f & B
	\end{diagram}
	commutes.
	Then $(g - h) \circ f = g \circ f - h \circ f = 0$, and we've arrived at a commutative diagram.
	\begin{diagram}
		Z && \\
		\dTo^{g-h} & \rdDotted^0 & \\
		A & \rTo_f & B
	\end{diagram}
	But since $0 \to A$ is a kernel it follows that $g-h$ factors through $0$,
	so $g-h = 0 \implies g = h$, which is to say that $f$ is monic.
\end{proof}
\begin{proposition}[Isomorphism $\iff$ monic and epic]
	In an abelian category,
	a map is an isomorphism if and only if it is monic and epic.
\end{proposition}
\begin{proof}
	Omitted. (The Mitchell embedding theorem
	presented later implies this anyways for
	most situations we care about,
	by looking at a small sub-category.)
\end{proof}

\section{Exact sequences}
\prototype{$0 \to G \to G \times H \to H \to 0$ is exact.}
Exact sequences will seem exceedingly unmotivated until you learn about homology groups,
which is one of the most natural places that exact sequences appear.
In light of this, it might be worth trying to read the chapter on homology groups
simultaneously with this one.

First, let me state the definition for groups, to motivate the general categorical definition.
A sequence of groups
\[ G_0 \taking{f_1} G_1 \taking{f_2} G_2 \taking{f_3} \dots \taking{f_n} G_n \]
is \emph{exact} at $G_k$ if the image of $f_k$ is the kernel of $f_{k+1}$.
We say the entire sequence is exact if it's exact at $k=1,\dots,n-1$.
\begin{example}
	[Exact sequences]
	\listhack
	\begin{enumerate}[(a)]
		\ii The sequence
		\[ 0 \to \Zc 3
			\overset{\times 5}{\injto} \Zc{15}
			\surjto \Zc{5}
			\to 0 \]
		is exact.
		Actually, $0 \to G \injto G \times H \surjto H \to 0$ is exact in general.
		(Here $0$ denotes the trivial group.)
		\ii For groups, the map $0 \to A \to B$ is exact if and only if $A \to B$ is injective.
		\ii For groups, the map $A \to B \to 0$ is exact if and only if $A \to B$ is surjective.
	\end{enumerate}
\end{example}

Now, we want to mimic this definition in a general \emph{abelian} category $\AA$.
So, let's write down a criterion for when $A \taking f B \taking g C$ is exact.
First, we had better have that $g \circ f = 0$,
which encodes the fact that $\img(f) \subseteq \ker(g)$.
Adding in all the relevant objects, we get the commutative diagram below.
\begin{diagram}
	A && \rDashed^0 && C \\
	\dSurj^{\img f} & \rdTo(2,1)^f & B & \ruTo(2,1)^g & \uDashed_0 \\
	\Img f & \ruInj(2,1)_{\iota} & \rDotted_{\exists!} & \luInj(2,1) & \Ker g \\
\end{diagram}
Here the map $A \surjto \Img f$ is epic since we are assuming $\AA$ is an abelian category.
So, we have that
\[ 0 = (g \circ \iota) \circ \img f = g \circ (\iota \circ \img f) = g \circ f = 0 \]
but since $\img f$ is epic, this means that $g \circ \iota = 0$.
So there is a \emph{unique} map $\Img f \to \Ker g$, and we require that this diagram commutes.
In short,
\begin{definition}
	Let $\AA$ be an abelian category. The sequence
	\[ \dots \to A_{n-1} \taking{f_n} A_n \taking{f_{n+1}} A_{n+1} \to \dots \]
	is \vocab{exact} at $A_n$ if $f_n \circ f_{n+1} = 0$ and
	the canonical map $\Img f_n \to \Ker f_{n+1}$ is an isomorphism.
	The entire sequence is exact if it is exact at each $A_i$.
	(For finite sequences we don't impose condition on the very first and very last object.)
\end{definition}

\begin{exercise}
	Show that, as before, $0 \to A \to B$ is exact $\iff$ $A \to B$ is monic.
\end{exercise}

\section{The Freyd-Mitchell embedding theorem}
We now introduce the Freyd-Mitchell embedding theorem,
which essentially says that any abelian category can be realized as a concrete one.

\begin{definition}
	A category is \vocab{small} if $\obj(\AA)$ is a set (as opposed to a class),
	i.e.\ there is a ``set of all objects in $\AA$''.
	For example, $\catname{Set}$ is not small because there is no set of all sets.
\end{definition}

\begin{theorem}
	[Freyd-Mitchell embedding theorem]
	Let $\AA$ be a small abelian category.
	Then there exists a ring $R$ (with $1$ but possibly non-commutative)
	and a full, faithful, exact functor onto the category of left $R$-modules.
\end{theorem}
Here a functor is \vocab{exact} if it preserves exact sequences.
This theorem is good because it means
\begin{moral}
	You can basically forget about all the weird definitions
	that work in any abelian category.
\end{moral}
Any time you're faced with a statement about an abelian category,
it suffices to just prove it for a ``concrete'' category
where injective/surjective/kernel/image/exact/etc.\
agree with your previous notions.
A proof by this means is sometimes called \emph{diagram chasing}.

\begin{remark}
	The ``small'' condition is a technical obstruction
	that requires the objects $\AA$ to actually form a set.
	I'll ignore this distinction,
	because one can almost always work around it
	by doing enough set-theoretic technicalities.
\end{remark}

For example, let's prove:
\begin{lemma}
	[Short five lemma]
	In an abelian category, consider the commutative diagram
	\begin{diagram}
		0 & \rTo & A & \rInj^p & B & \rSurj^q & C & \rTo & 0 \\
		&& \dTo^\cong_\alpha && \dTo_\beta && \dTo^\cong_\gamma && \\
		0 & \rTo & A' & \rInj^{p'} & B' & \rSurj^{q'} & C' & \rTo & 0
	\end{diagram}
	and assume the top and bottom rows are exact.
	If $\alpha$ and $\gamma$ are isomorphisms, then so is $\beta$.
\end{lemma}

\begin{proof}
	We prove that $\beta$ is epic (with a similar proof to get monic).
%	One can show that it's possible to take a small subcategory of $\AA$
%	containing the $10$ elements and $13$ arrows above, as well as all necessary
%	kernels, cokernels, et cetera.
%	(Essentially, let $\BB_0$ be the diagram above and let $\BB_{i+1}$ add in any needed objects;
%	then $\bigcup \BB_i$ is a set-sized category).
	By the embedding theorem we can treat the category as $R$-modules over some $R$.
	This lets us do a so-called ``diagram chase'' where we move elements around the picture,
	using the concrete interpretation of our category as $R$-modules.

	Let $b'$ be an element of $B'$.
	Then $q'(b') \in C'$, and since $\gamma$ is surjective, we have a $c$ such that $\gamma(c) = b'$,
	and finally a $b \in B$ such that $q(b) = c$.
	Picture:
	\begin{diagram}
		b \in B & \rMapsto^q & c \in C \\
		\dDashed_\beta && \dMapsto^\cong_\gamma \\
		b' \in B' & \rMapsto^{q'} & c' \in C'
	\end{diagram}
	Now, it is not necessarily the case that $\beta(b) = b'$.
	However, since the diagram commutes we at least have that
	\[ q'(b') = q'(\beta(b)) \]
	so $b' - \beta(b) \in \Ker q' = \Img p'$, and there is an $a' \in A'$ such that
	$p'(a') = b' - \beta(b)$;
	use $\alpha$ now to lift it to $a \in A$.
	Picture:
	\begin{diagram}
		a \in A && b \in B && \\
		\dMapsto && && \\
		a' \in A' & \rMapsto & b'-\beta(b) \in B' & \rMapsto & 0 \in C'
	\end{diagram}
	Then, we have
	\[
		\beta(b + q(a)) = \beta b + \beta p a
		= \beta b + p' \alpha a
		= \beta b + (b' - \beta b)
		= b'
	\]
	so $b' \in \Img \beta$ which completes the proof that $\beta'$ is surjective.
\end{proof}

\section{Breaking long exact sequences}
\prototype{First isomorphism theorem.}

In fact, it turns out that any exact sequence breaks into short exact sequences.
This relies on:
\begin{proposition}[``First isomorphism theorem'' in abelian categories]
	\label{prop:break_exact}
	Let $A \taking f B$ be an arrow of an abelian category.
	Then there is an exact sequence
	\[ 0 \to \Ker f \taking{\ker f} A \taking{\img f} \Img f \to 0. \]
\end{proposition}

\begin{example}
Let's analyze this theorem in our two examples of abelian categories:
\begin{enumerate}[(a)]
	\ii In the category of abelian groups,
	this is basically the first isomorphism theorem.
	\ii In the category $\catname{Vect}_k$,
	this amounts to the rank-nullity theorem, \Cref{thm:rank_nullity}.
\end{enumerate}
\end{example}
Thus, any exact sequence can be broken into short exact sequences, as
\begin{diagram}
	&& 0 & & 0 && 0 && 0 \\
	&&& \rdDotted(1,1) C_n \ruDotted(1,1) &&&& \rdDotted(1,1) C_{n+2} \ruDotted(1,1) & \\
	\dots & \rTo & A_{n-1} \ruDotted(1,1) & \rTo^{f_{n-1}} & \rdDotted(1,1) A_n & \rTo^{f_n} &
	A_{n+1} \ruDotted(1,1)  & \rTo^{f_{n+1}} & \rdDotted(1,1) \dots \\
	& \rdDotted(1,1) C_{n-1} \ruDotted(1,1) &&&& \rdDotted(1,1) C_{n+1} \ruDotted(1,1) &&& \\
	0 \ruDotted(1,1) && \rdDotted(1,1) 0 && 0 \ruDotted(1,1) && \rdDotted(1,1) 0 &&
\end{diagram}
where $C_k = \img f_{k-1} = \ker f_k$ for every $k$.

\section\problemhead

\begin{problem}
	[Four lemma]
	In an abelian category, consider the commutative diagram
	\begin{diagram}
		A & \rTo^p & B & \rTo^q & C & \rTo^r & D \\
		\dSurj_\alpha && \dInj_\beta && \dTo_\gamma && \dInj_\delta \\
		A' & \rTo_{p'} & B' & \rTo_{q'} & C' & \rTo_{r'} & D'
	\end{diagram}
	where the first and second rows are exact.
	Prove that if $\alpha$ is epic, and $\beta$ and $\delta$ are monic,
	then $\gamma$ is epic.
	\begin{soln}
		Let $c \in C$ with $\gamma(c) = 0$.
		We show $c = 0$.
		This proceeds in a diagram chase:
		\begin{itemize}
			\ii Note that $0 = r'(\gamma(c)) = \delta(r(c))$, and since $\delta$
			is injective, it follows that $r(c) = 0$.
			\ii Since the top row is exact,
			it follows $c = q(b)$ for some $b \in B$.
			\ii Then $q'(\beta(b)) = 0$,
			so if we let $b' = \beta(b)$,
			then $b' \in \ker(q')$.
			As the bottom row is exact, there exists $a'$ with $p'(a') = b'$.
			\ii Since $\alpha$ is injective,
			there is $a \in A$ with $\alpha(a') = a$.
			\ii Since $\beta$ is injective,
			it follows that $p(a) = b$.
			\ii Since the top row is exact, and $b$ is in the image of $p$,
			it follows that $0 = q(b) = c$ as needed.
		\end{itemize}

	\end{soln}<++>
\end{problem}

\begin{problem}
	[Five lemma]
	\gim
	In an abelian category, consider the commutative diagram
	\begin{diagram}
		A & \rTo^p & B & \rTo^q & C & \rTo^r & D & \rTo^s & E \\
		\dSurj_\alpha && \dTo^\cong_\beta && \dTo_\gamma && \dTo^\cong_\delta && \dInj_\eps \\
		A' & \rTo_{p'} & B' & \rTo_{q'} & C' & \rTo_{r'} & D' & \rTo_{s'} & E'
	\end{diagram}
	where the first and second rows are exact.
	Prove that if $\alpha$ is epic, $\eps$ is monic, and $\beta$, $\delta$ are isomorphisms,
	then $\gamma$ is an isomorphism as well.
	Thus this is a stronger version of the short five lemma.
\end{problem}

\begin{sproblem}
	[Snake lemma]
	\yod
	In an abelian category, consider the diagram
	\begin{diagram}
		&& A & \rTo^f & B & \rSurj^g & C & \rTo & 0 \\
		&& \dTo_a && \dTo_b && \dTo_c && \\
		0 & \rTo & A' & \rInj_{f'} & B' & \rTo_{g'} & C' &&
	\end{diagram}
	where the first and second rows are exact sequences.
	Prove that there is an exact sequence
	\[ \Ker a \to \Ker b \to \Ker c \to \Coker a \to \Coker b \to \Coker c. \]
\end{sproblem}