\chapter{Functions and Graphs}

\section{Slope}
Slope defines the average rate of change on a function in a given interval,
\(\Delta x\) (this expression is defined as $x_2-x_1$).

Consider a function $f(x)$.  $f$ takes in x-coordinates, and spits out
corresponding y-coordinates, creating a set of ordered pairs that produces a
line, curve, or otherwise.

The formal definition of slope is given as follows:
\begin{equation}
m=\frac{f(x_2)-f(x_1)}{x_2-x_1}
\end{equation}

\subsection{Point-slope form}
Given the coordinate pair of some point in the x-y plane, and the slope of any
arbitrary line $m$, we can develop an equation for the line that has the slope
$m$ and goes through the coordinate $(x, y)$.

The general equation of that line is given as follows:
\begin{equation}
y-y_1=m(x-x_1)
\end{equation}

\subsection{Sketching lines}
The general equation of a linear line is as follows:
\begin{equation}
y=mx+b
\end{equation}

Therefore, the y-intercept is given when $x=0$.  To solve for the y-intercept of
the line given above, we evaluate it, plugging $0$ in for $x$, producing a
coordinate pair of $(0, f(0))$.

To sketch a line, preform the following steps:
\begin{enumerate}
\item{Find the coordinate of the y-intercept of the graph, according to the rule
given above}
\item{Sketch that location into the coordinate plane}
\item{Construct the two adjacent points on the plane according to the slope $m$
and the definition of slope}
\item{Connect the dots}
\end{enumerate}

\subsection{Properties of Slope}
Two lines are parallel iff the following condition is met: $m_1=m_2$.

\section{Functions}
A function $f$ maps a set of x-coordinates ($A$) to a set of y-coordinates
($B$).  Each element in $A$ is mapped to \textit{exactly one} point from the set
$B$.

\begin{itemize}
\item{Each element of $A$ must be matched to an element of $B$}
\item{Some elements of $B$ may not be matched with any elements of $A$}
\item{Two or more elements of $A$ may be matched with the same element $B$}
\item{An element of $A$ can not be matched with more than one distinct element
of $B$}
\end{itemize}

The difference quotient is defined as follows, and becomes important later:
\begin{equation}
\frac{f(x+h)-f(x)}{h}, h \ne 0
\end{equation}

Some formal definitions follow:
\begin{description}
\item[Function]{a relationship between two variables such that to each value of
the independent variable there is exactly one value of the dependent variable}
\item[Function notation]{$y=f(x)$}
\item[Domain]{the set of values $x$ for which $f$ can produce real values}
\item[Range]{the set of values that $f$ can produce for all given $x$ in the
domain}
\item[Implied domain]{the set of real numbers that satisfy the expression for
which $f$ is defined upon}
\end{description}

\section{Graphs of functions}
A function can be defined on any given interval as one of three things:
increasing, decreasing, or constant. The classifications of each follow:

For any interval $x_1$, $x_2$...
\begin{itemize}
\item The function is increasing for all $x$ such that $f(x_1)<f(x_2)$
\item The function is decreasing for all $x$ such that $f(x_1)>f(x_2)$
\item The function is constant for all $x$ such that $f(x_1)=f(x_2)$
\end{itemize}

A function has a relative minimum when there exists an interval $(x_1,x_2)$ such
that there is an $a$ that $x_1<x<x_2$ and $f(a) \leq f(x)$.

Conversly, a function has a relative maximum when there exists an interval
$(x_1,x_2)$ such that there is an $a$ that $x_1<x<x_2$ and $f(a) \geq f(x)$.

\subsection{Even and Odd functions}
\begin{itemize}
\item A function is even such that for all $x$ in the domain of $f$,
$f(-x)=f(x)$
\item A function is odd such that for all $x$ in the domain of $f$,
$f(-x)=-f(x)$
\end{itemize}

\section{Translations}
There are several types of translations that can be applied to parent functions
and what will become their "children".  For a parent function $f$, you can
produce a child function $g$ such that $g$ is defined in terms of $f(x)$.  The
simple translations follow:

\begin{enumerate}
\item Vertical shift $c$ units upwards: $g(x)=f(x)+c$
\item Vertical shift $c$ units downwards: $g(x)=f(x)-c$
\item Horizontal shift $c$ units to the left: $g(x)=f(x+c)$
\item Horizontal shift $c$ units to the right: $g(x)=f(x-c)$
\end{enumerate}

To reflect a function across an axis, take the following advice:

\begin{enumerate}
\item A reflection in the $x$-axis for a function $f$: $g(x)=-f(x)$
\item A reflection in the $y$-axis for a function $f$: $g(x)=f(-x)$
\end{enumerate}

To stretch or shirnk a function $g(x)$ in terms of $f(x)$, simply multiply $f$
by a scalar $c$.

\section{Combinations of Functions}
There are several ways that you can combine two functions $f$ and $g$ (for
example) to produce a new, composite function of the two:

\begin{enumerate}
\item Sum: $(f+g)(x)=f(x)+g(x)$
\item Difference: $(f-g)(x)=f(x)-g(x)$
\item Product: $(fg)(x)=f(x)*g(x)$
\item Quotient: $\frac{f}{g}(x)=\frac{f(x)}{g(x)}$
\end{enumerate}

\subsection{Composite Functions}
A composite function of $f$ and $g$ is defined as follows: $f(g(x))$.
Therefore, the range of $g$ must be limited to the domain of $f$ so that no $x$
values can be passed to the composite function that are invalid.

\section{Inverse Functions}
Two functions $f$, and $g$ are considered to be inverses of each other, iff both
of the following two conditions are met:

\begin{enumerate}
\item $f(g(x))=x$ for all $x$ in the domain of $g(x)$.
\item $g(f(x))=x$ for all $x$ in the domain of $f(x)$.
\end{enumerate}

To find an inverse function algebraically, preform the following steps:

\begin{enumerate}
\item Use the Horizontal Line Test to determine if $f$ has an inverse ($f^-1$)
  \begin{itemize}
  \item (If it doesn't, limit the domain of $f$ so that it does.)
  \end{itemize}
\item{Plug in $f^-1(x)$ into the original function and isolate}
\end{enumerate}


\newpage
\section{Taylor series}

\begin{Introduction}{TAYLOR}
This short note describes a package of REDUCE procedures that allow
Taylor expansion in one or more variables and efficient manipulation
of the resulting Taylor series. Capabilities include basic operations
(addition, subtraction, multiplication and division) and also
application of certain algebraic and transcendental functions. To a
certain extent, Laurent expansion can be performed as well.
\end{Introduction}

\begin{Operator}{taylor}
  The \name{taylor} operator is used for expanding an expression into a
  Taylor series.

  \begin{Syntax}
    \name{taylor}\(\meta{expression}
                   \name{,}\meta{var}\name{,}
                   \meta{expression}\name{,}\meta{number}\\
                   \{\name{,}\meta{var}\name{,}
                   \meta{expression}\name{,}\meta{number}\}\optional\)
  \end{Syntax}
  \meta{expression} can be any valid REDUCE algebraic expression.
  \meta{var} must be a \nameref{kernel}, and is the expansion
  variable.  The \meta{expression} following it denotes the point
  about which the expansion is to take place. \meta{number} must be a
  non-negative integer and denotes the maximum expansion order.  If
  more than one triple is specified \name{taylor} will expand its
  first argument independently with respect to all the variables.
  Note that once the expansion has been done it is not possible to
  calculate higher orders.

  Instead of a \nameref{kernel}, \meta{var} may also be a list of
  kernels.  In this case expansion will take place in a way so that
  the {\em sum\/} of the degrees of the kernels does not exceed the
  maximum expansion order.  If the expansion point evaluates to the
  special identifier \name{infinity}, \name{taylor} tries to expand in
  a series in 1/\meta{var}.

  The expansion is performed variable per variable, i.e.\ in the
  example above by first expanding
  \IFTEX{$\exp(x^{2}+y^{2})$}{exp(x^2+y^2)}
  with respect to
  \name{x} and then expanding every coefficient with respect to \name{y}.

  \begin{Examples}
    taylor(e^(x^2+y^2),x,0,2,y,0,2);     &
                1 + Y^{2} + X^{2} + Y^{2}*X^{2} + O(X^{2},Y^{2})   \\
    taylor(e^(x^2+y^2),{x,y},0,2); & 1 + Y^{2} + X^{2} + O(\{X^{2},Y^{2}\})\\
    \explanation{The following example shows the case of a non-analytical function.}\\
    taylor(x*y/(x+y),x,0,2,y,0,2); & ***** Not a unit in argument to QUOTTAYLOR \\
  \end{Examples}

  \begin{Comments}
    Note that it is not generally possible to apply the standard
    reduce operators to a Taylor kernel. For example, \nameref{part},
    \nameref{coeff}, or \nameref{coeffn} cannot be used. Instead, the
    expression at hand has to be converted to standard form first
    using the \nameref{taylortostandard} operator.

    Differentiation of a Taylor expression is possible.  If you
    differentiate with respect to one of the Taylor variables the
    order will decrease by one.

    Substitution is a bit restricted: Taylor variables can only be
    replaced by other kernels.  There is one exception to this rule:
    you can always substitute a Taylor variable by an expression that
    evaluates to a constant.  Note that REDUCE will not always be able
    to determine that an expression is constant: an example is
    sin(acos(4)).

    Only simple taylor kernels can be integrated. More complicated
    expressions that contain Taylor kernels as parts of themselves are
    automatically converted into a standard representation by means of
    the \nameref{taylortostandard} operator. In this case a suitable
    warning is printed.

  \end{Comments}

\end{Operator}

\begin{Switch}{taylorautocombine}
  If you set \name{taylorautocombine} to \name{on}, REDUCE
  automatically combines Taylor expressions during the simplification
  process.  This is equivalent to applying \nameref{taylorcombine} to
  every expression that contains Taylor kernels.  Default is
  \name{on}.
\end{Switch}

\begin{Switch}{taylorautoexpand}
  \name{taylorautoexpand} makes Taylor expressions ``contagious'' in
  the sense that \nameref{taylorcombine} tries to Taylor expand all
  non-Taylor subexpressions and to combine the result with the rest.
  Default is \name{off}.
\end{Switch}

\begin{Operator}{taylorcombine}
  This operator tries to combine all Taylor kernels found in its
  argument into one.  Operations currently possible are:
  \begin{itemize}
    \item Addition, subtraction, multiplication, and division.
    \item Roots, exponentials, and logarithms.
    \item Trigonometric and hyperbolic functions and their inverses.
  \end{itemize}

  \begin{Examples}
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    taylorcombine log hugo; & X + O(X^{3})\\
    taylorcombine(hugo + x); & (1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})) + X\\
    on taylorautoexpand; \\
    taylorcombine(hugo + x); & 1 + 2*X + \rfrac{1}{2}*X^{2} + O(X^{3})
  \end{Examples}

  \begin{Comments}
    Application of unary operators like \name{log} and \name{atan}
    will nearly always succeed. For binary operations their arguments
    have to be Taylor kernels with the same template. This means that
    the expansion variable and the expansion point must match.
    Expansion order is not so important, different order usually means
    that one of them is truncated before doing the operation.

    If \nameref{taylorkeeporiginal} is set to \name{on} and if all
    Taylor kernels in its argument have their original expressions
    kept \name{taylorcombine} will also combine these and store the
    result as the original expression of the resulting Taylor kernel.
    There is also the switch \nameref{taylorautoexpand}.

    There are a few restrictions to avoid mathematically undefined
    expressions: it is not possible to take the logarithm of a Taylor
    kernel which has no terms (i.e. is zero), or to divide by such a
    beast.  There are some provisions made to detect singularities
    during expansion: poles that arise because the denominator has
    zeros at the expansion point are detected and properly treated,
    i.e.\ the Taylor kernel will start with a negative power.  (This
    is accomplished by expanding numerator and denominator separately
    and combining the results.)  Essential singularities of the known
    functions (see above) are handled correctly.
  \end{Comments}
\end{Operator}

\begin{Switch}{taylorkeeporiginal}
  \name{taylorkeeporiginal}, if set to \name{on}, forces the
  \nameref{taylor} and all Taylor kernel manipulation operators to
  keep the original expression, i.e.\ the expression that was Taylor
  expanded.  All operations performed on the Taylor kernels are also
  applied to this expression which can be recovered using the operator
  \nameref{taylororiginal}.  Default is \name{off}.
\end{Switch}

\begin{Operator}{taylororiginal}
  Recovers the original expression (the one that was expanded) from
  the Taylor kernel that is given as its argument.

  \begin{Syntax}
    \name{taylororiginal}\(\meta{expression}\) or
    \name{taylororiginal} \meta{simple_expression}
  \end{Syntax}

  \begin{Examples}
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    taylororiginal hugo; &
       ***** Taylor kernel doesn't have an original part in TAYLORORIGINAL\\
    on taylorkeeporiginal; \\
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    taylororiginal hugo; & E^{X}
  \end{Examples}

  \begin{Comments}
    An error is signalled if the argument is not a Taylor kernel or if
    the original expression was not kept, i.e.\ if
    \nameref{taylorkeeporiginal} was set \name{off} during expansion.
  \end{Comments}

\end{Operator}

\begin{Switch}{taylorprintorder}
  \name{taylorprintorder}, if set to \name{on}, causes the remainder
  to be printed in big-O notation.  Otherwise, three dots are printed.
  Default is \name{on}.
\end{Switch}

\begin{Variable}{taylorprintterms}
  Only a certain number of (non-zero) coefficients are printed. If
  there are more, an expression of the form \name{n terms} is printed
  to indicate how many non-zero terms have been suppressed.  The
  number of terms printed is given by the value of the shared
  algebraic variable \name{taylorprintterms}.  Allowed values are
  integers and the special identifier \name{all}. The latter setting
  specifies that all terms are to be printed. The default setting is
  5.

  \begin{Examples}
    taylor(e^(x^2+y^2),x,0,4,y,0,4); &
           1 + Y^{2} + \rfrac{1}{2}*Y^{4} + X^{2} + Y^{2}*X^{2} +
             (4 terms) + O(X^{5},Y^{5})\\
    taylorprintterms := all; & TAYLORPRINTTERMS := ALL \\
    taylor(e^(x^2+y^2),x,0,4,y,0,4); &
    \begin{multilineoutput}{}
1 + Y^{2} + \rfrac{1}{2}*Y^{4} + X^{2} + Y^{2}*X^{2} +%
 \rfrac{1}{2}*Y^{4}*X^{2} + \rfrac{1}{2}*X^{4} +%
 \rfrac{1}{2}*Y^{2}*X^{4}\\
 + \rfrac{1}{4}*Y^{4}*X^{4} + O(X^{5},Y^{5})
    \end{multilineoutput}
  \end{Examples}

\end{Variable}

\begin{Operator}{taylorrevert}
  \name{taylorrevert} allows reversion of a Taylor series of a
  function f, i.e., to compute the first terms of the expansion of the
  inverse of $f$ from the expansion of $f$.

  \begin{Syntax}
    \name{taylorrevert}\(\meta{expression}\name{,}
                         \meta{var}\name{,}\meta{var}\)
  \end{Syntax}

  The first argument must evaluate to a Taylor kernel with the second
  argument being one of its expansion variables.

  \begin{Examples}
    taylor(u - u**2,u,0,5); &    U - U^{2} + O(U^{6}) \\
    taylorrevert (ws,u,x);  &    X + X^{2} + 2*X^{3} + 5*X^{4} + 14*X^{5} + O(X^{6})
  \end{Examples}

\end{Operator}

\begin{Operator}{taylorseriesp}
  This operator may be used to determine if its argument is a Taylor
  kernel.

  \begin{Syntax}
    \name{taylorseriesp}\(\meta{expression}\) or \name{taylorseriesp}
    \meta{simple_expression}
  \end{Syntax}

  \begin{Examples}
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    if taylorseriesp hugo then OK;& OK \\
    if taylorseriesp(hugo + y) then OK else NO; & NO
  \end{Examples}

  \begin{Comments}
    Note that this operator is subject to the same restrictions as,
    e.g., \name{ordp} or \name{numberp}, i.e.\ it may only be used in
    boolean expressions in \name{if} or \name{let} statements.
  \end{Comments}

\end{Operator}

\begin{Operator}{taylortemplate}
  The template of a Taylor kernel, i.e.\ the list of all variables
  with respect to which expansion took place together with expansion
  point and order can be extracted using

  \begin{Syntax}
    \name{taylortemplate}\(\meta{expression}\) or
    \name{taylortemplate} \meta{simple_expression}
  \end{Syntax}

  This returns a list of lists with the three elements
  (VAR,VAR0,ORDER).  An error is signalled if the argument is not a
  Taylor kernel.

  \begin{Examples}
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    taylortemplate hugo; & \{\{X,0,2\}\}
  \end{Examples}

\end{Operator}

\begin{Operator}{taylortostandard}
  This operator converts all Taylor kernels in its argument into
  standard form and resimplifies the result.

\begin{Syntax}
  \name{taylortostandard}\(\meta{expression}\) or
  \name{taylortostandard} \meta{simple_expression}
\end{Syntax}

  \begin{Examples}
    hugo := taylor(exp(x),x,0,2); & HUGO := 1 + X + \rfrac{1}{2}*X^{2} + O(X^{3})\\
    taylortostandard hugo; & \rfrac{X^{2} + 2*X + 2}{2}
  \end{Examples}

\end{Operator}

\endinput

\section{Warnings and error messages}
\index{errors ! TAYLOR package}
\begin{itemize}

\item \name{Branch point detected in ...}\\
    This occurs if you take a rational power of a Taylor kernel
    and raising the lowest order term of the kernel to this
    power yields a non analytical term (i.e.\ a fractional power).

\item \name{Cannot expand further... truncation done}\\
    You will get this warning if you try to expand a Taylor kernel to
    a higher order.

\item \name{Converting Taylor kernels to standard representation}\\
    This warning appears if you try to integrate an expression that
    contains Taylor kernels.

\item \name{Error during expansion (possible singularity)}\\
    The expression you are trying to expand caused an error.
    As far as I know this can only happen if it contains a function
    with a pole or an essential singularity at the expansion point.
    (But one can never be sure.)

\item \name{Essential singularity in ...}\\
    An essential singularity was detected while applying a
    special function to a Taylor kernel.
    This error occurs, for example, if you try to take
    the logarithm of a Taylor kernel that starts with a negative
    power in one of its variables, i.e.\ that has a pole
    at the expansion point.

\item \name{Expansion point lies on branch cut in ...}\\
    The only functions with branch cuts this package knows of are
    (natural) logarithm, inverse circular and hyperbolic tangent and
    cotangent.  The branch cut of the logarithm is assumed to lie on
    the negative real axis.  Those of the arc tangent and arc
    cotangent functions are chosen to be compatible with this: both
    have essential singularities at the points $\pm i$.  The branch
    cut of arc tangent is the straight line along the imaginary axis
    connecting $+1$ to $-1$ going through $\infty$ whereas that of arc
    cotangent goes through the origin.  Consequently, the branch cut
    of the inverse hyperbolic tangent resp.\ cotangent lies on the
    real axis and goes from $-1$ to $+1$, that of the latter across
    $0$, the other across $\infty$.

    The error message can currently only appear when you try to
    calculate the inverse tangent or cotangent of a Taylor
    kernel that starts with a negative degree.
    The case of a logarithm of a Taylor kernel whose constant term
    is a negative real number is not caught since it is
    difficult to detect this in general.

\item \name{Not a unity in ...}\\
    This will happen if you try to divide by or take the logarithm of
    a Taylor series whose constant term vanishes.

\item \name{Not implemented yet (...)}\\
    Sorry, but I haven't had the time to implement this feature.
    Tell me if you really need it, maybe I have already an improved
    version of the package.

\item \name{Reversion of Taylor series not possible: ...}\\
\ttindex{TAYLORREVERT}
    You tried to call the \name{TAYLORREVERT} operator with
    inappropriate arguments. The second half of this error message
    tells you why this operation is not possible.

\item \name{Substitution of dependent variables ...}\\
    You tried to substitute a variable that is already present in the
    Taylor kernel or on which one of the Taylor variables depend.

\item \name{Taylor kernel doesn't have an original part}\\
\ttindex{TAYLORORIGINAL} \ttindex{TAYLORKEEPORIGINAL}
    The Taylor kernel upon which you try to use \name{TAYLORORIGINAL}
    was created with the switch \name{TAYLORKEEPORIGINAL}
    set to \name{OFF}
    and does therefore not keep the original expression.

\item \name{Wrong number of arguments to TAYLOR}\\
    You try to use the operator \name{TAYLOR} with a wrong number of
    arguments.

\item \name{Zero divisor in TAYLOREXPAND}\\
    A zero divisor was found while an expression was being expanded.
    This should not normally occur.

\item \name{Zero divisor in Taylor substitution}\\
    That's exactly what the message says.  As an example consider the
    case of a Taylor kernel containing the term \name{1/x} and you try
    to substitute \name{x| by \verb|0}.

\item \name{... invalid as kernel}\\
    You tried to expand with respect to an expression that is not a
    kernel.

\item \name{... invalid as order of Taylor expansion}\\
    The order parameter you gave to \name{TAYLOR} is not an integer.

\item \name{... invalid as Taylor kernel}\\
\ttindex{TAYLORORIGINAL} \ttindex{TAYLORTEMPLATE}
    You tried to apply \name{TAYLORORIGINAL| or \verb|TAYLORTEMPLATE}
    to an expression that is not a Taylor kernel.

\item \name{... invalid as Taylor variable}\\
    You tried to substitute a Taylor variable by an expression that is
    not a kernel.

\item \name{... invalid as value of TaylorPrintTerms}\\
\ttindex{TAYLORPRINTTERMS}
    You have assigned an invalid value to \name{TAYLORPRINTTERMS}.
    Allowed values are: an integer or the special identifier
    \name{ALL}.

\item \name{TAYLOR PACKAGE (...): this can't happen ...}\\
    This message shows that an internal inconsistency was detected.
    This is not your fault, at least as long as you did not try to
    work with the internal data structures of \REDUCE. Send input
    and output to me, together with the version information that is
    printed out.

\end{itemize}

\section{Comparison to other packages}

At the moment there is only one \REDUCE{} package that I know of:
the truncated power series package by Alan Barnes and Julian Padget.
In my opinion there are two major differences:
\begin{itemize}
  \item The interface. They use the domain mechanism for their power
        series, I decided to invent a special kind of kernel. Both
        approaches have advantages and disadvantages: with domain
        modes, it is easier
        to do certain things automatically, e.g., conversions.
  \item The concept of a truncated series. Their idea is to remember
        the original expression and to compute more coefficients when
        more of them are needed. My approach is to truncate at a
        certain order and forget how the unexpanded expression
        looked like.  I think that their method is more widely
        usable, whereas mine is more efficient when you know in
        advance exactly how many terms you need.
\end{itemize}

\end{document}


% This is part of the TFTB Reference Manual.
% Copyright (C) 1996 CNRS (France) and Rice University (US).
% See the file refguide.tex for copying conditions.

\markright{tfrcw}
\section*{\hspace*{-1.6cm} tfrcw}

\vspace*{-.4cm}
\hspace*{-1.6cm}\rule[0in]{16.5cm}{.02cm}
\vspace*{.2cm}

{\bf \large \sf Purpose}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
Choi-Williams time-frequency distribution.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Synopsis}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
\begin{verbatim}
[tfr,t,f] = tfrcw(x)
[tfr,t,f] = tfrcw(x,t)
[tfr,t,f] = tfrcw(x,t,N)
[tfr,t,f] = tfrcw(x,t,N,g)
[tfr,t,f] = tfrcw(x,t,N,g,h)
[tfr,t,f] = tfrcw(x,t,N,g,h,sigma)
[tfr,t,f] = tfrcw(x,t,N,g,h,sigma,trace)
\end{verbatim}
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Description}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
        {\ty tfrcw} computes the Choi-Williams distribution of a
        discrete-time signal {\ty x}, or the cross Choi-Williams
        representation between two signals. This distribution has the
        following expression :

\[CW_x(t,\nu)=2
\iint_{-\infty}^{+\infty} \dfrac{\sqrt{\sigma}}{4\sqrt{\pi}|\tau|}\
e^{-v^2\sigma/(16\tau^2)}\ x(t+v+\frac{\tau}{2})\
x^*(t+v-\frac{\tau}{2})\ e^{-j2\pi \nu \tau}\ dv\ d\tau.\]

\hspace*{-.5cm}\begin{tabular*}{14cm}{p{1.5cm} p{8cm} c}
Name & Description & Default value\\
\hline
        {\ty x}     & signal if auto-CW, or {\ty [x1,x2]} if cross-CW {\ty
			(Nx=length(x))}\\
        {\ty t}     & time instant(s)          & {\ty (1:Nx)}\\
        {\ty N}     & number of frequency bins & {\ty Nx}\\
        {\ty g}     & time smoothing window, {\ty G(0)} being forced to {\ty 1}, where {\ty G(f)} is the Fourier transform of {\ty g(t)}
              &                           {\ty window(odd(N/10))}\\
        {\ty h}     & frequency smoothing window, {\ty h(0)} being forced to {\ty 1}
              &                           {\ty window(odd(N/4))}\\
        {\ty sigma} & kernel width             & {\ty 1}\\
        {\ty trace} & if nonzero, the progression of the algorithm is shown
              &                           {\ty 0}\\
     \hline {\ty tfr}   & time-frequency representation\\
        {\ty f}     & vector of normalized frequencies\\

\hline
\end{tabular*}
\vspace*{.2cm}

When called without output arguments, {\ty tfrcw} runs {\ty tfrqview}.
\end{minipage}

\newpage

{\bf \large \sf Example}
\begin{verbatim}
         sig=fmlin(128,0.05,0.3)+fmlin(128,0.15,0.4);
         g=window(9,'Kaiser'); h=window(27,'Kaiser');
         t=1:128; tfrcw(sig,t,128,g,h,3.6,1);
\end{verbatim}
\vspace*{.5cm}

{\bf \large \sf See Also}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
all the {\ty tfr*} functions.
\end{minipage}
\vspace*{.5cm}

{\bf \large \sf Reference}\\
\hspace*{1.5cm}
\begin{minipage}[t]{13.5cm}
[1] H. Choi, W. Williams ``Improved Time-Frequency Representation of
Multicomponent Signals Using Exponential Kernels'', IEEE Trans. on
Acoustics, Speech and Signal Processing, Vol. 37, No. 6, June 1989.
\end{minipage}


\section{Stokes' Theorem}\label{sec:StokesTheorem}

Recall that one version of Green's Theorem (see
Equation~\ref{eq:greens theorem second form}) is
$$\int_{\partial D} \vect{f}\cdot d\vect{r}
=\iint_{D}(\nabla\times \vect{f})\cdot\vect{k}\,dA.
$$
Here $D$ is a region in the $x$-$y$ plane and $\vect{k}$ is a unit normal
to $D$ at every point. If $D$ is instead an orientable surface in
space, there is an obvious way to alter this equation, and it turns
out still to be true:

\begin{theorem}{Stokes' Theorem}{}
Provided that the quantities involved are
sufficiently nice, and in particular if $D$ is orientable,
$$\int_{\partial D} \vect{f}\cdot d\vect{r}
=\iint_{D}(\nabla\times \vect{f})\cdot\vect{N}\,dS,$$
if $\partial D$ is oriented counter-clockwise relative to $\vect{N}$.\index{Stokes' theorem}
\end{theorem}

The proof of Stokes' Theorem will follow a discussion and several examples of the Theorem in use.

Note how little has changed: $\vect{k}$ becomes $\vect{N}$, a unit normal to
the surface, and $dA$ becomes $dS$, since this is now a general
surface integral. The phrase ``counter-clockwise relative to $\vect{N}$''
means that if we take the direction of $\vect{N}$ to be ``up'', then we
go around the boundary counter-clockwise when viewed from ``above''.

\begin{example}{}{}
Let $\vect{f}=\langle  y^2z,x^2z,xy^2\rangle$
and the surface $D$ be $x=\sqrt{1-y^2-z^2}$, oriented in
the positive $x$ direction.
It quickly becomes apparent that the surface integral in Stokes'
Theorem is intractable, so compute the line integral.
\end{example}

\begin{solution}
The boundary of
$D$ is the unit circle in the $y$-$z$ plane, $\vect{r}=\langle 0,\cos
u,\sin u\rangle$, $0\le u\le 2\pi$. The integral is
$$\int_0^{2\pi} \langle  y^2z,x^2z,xy^2 \rangle\cdot
\langle 0,-\sin u,\cos u\rangle\,du=
\int_0^{2\pi} 0\,du = 0,$$
because $x=0$.
\end{solution}

An interesting consequence of Stokes' Theorem is that if $D$ and $E$
are two orientable surfaces with the same boundary, then
$$
\iint_{D}(\nabla\times \vect{f})\cdot\vect{N}\,dS
=\int_{\partial D} \vect{f}\cdot d\vect{r}
=\int_{\partial E} \vect{f}\cdot d\vect{r}
=\iint_{E}(\nabla\times \vect{f})\cdot\vect{N}\,dS.
$$
Sometimes both of the integrals
$$\iint_{D}(\nabla\times \vect{f})\cdot\vect{N}\,dS
\qquad\hbox{and}\qquad\int_{\partial D} \vect{f}\cdot d\vect{r}$$
are difficult, but you may be able to find a second surface $E$ so
that
$$\iint_{E}(\nabla\times \vect{f})\cdot\vect{N}\,dS$$
has the same value but is easier to compute.

In the previous example, the line integral was easy to
compute. But we might also notice that another surface $E$ with the
same boundary is the flat disk $y^2+z^2\le 1$.

\begin{example}{}{}
Let $\vect{f}=\langle  y^2z,x^2z,xy^2 \rangle$ and the surface $E$ be $y^2+z^2\le 1$. Compute the surface integral.
\end{example}

\begin{solution}
The unit normal $\vect{N}$ for this surface is simply $\vect{i}=\langle 1,0,0\rangle$. We compute
the curl:
$$\nabla\times\vect{f}=\langle 2xy-x^2, 0, 2xz-2yz \rangle.$$
Since $x=0$ everywhere on the surface,
$$(\nabla\times\vect{f})\cdot \vect{N}=
\langle 0, 0, 2xz-2yz \rangle\cdot\langle 1,0,0\rangle=0,$$
so the surface integral is
$$\iint_{E}0\,dS=0,$$
as before. In this case, of course, it is still somewhat easier to
compute the line integral, avoiding $\nabla\times\vect{f}$ entirely.
\end{solution}

%\begin{example}{}{}
%Let $\vect{f}=\langle -y^2,x,z^2\rangle$, and let the curve $C$
%be the intersection of the cylinder $x^2+y^2=1$ with the plane
%$y+z=2$, oriented counter-clockwise when viewed from above.
%Compute $\ds\int_C \vect{f}\cdot d\vect{r}$.
%\end{example}

%\begin{solution}
%We compute $\ds\int_C \vect{f}\cdot d\vect{r}$ in two ways.

%First we do it directly: a vector function for $C$ is
%$\vect{r}=\langle \cos u,\sin u, 2-\sin u\rangle$, so
%$\vect{r}'=\langle -\sin u,\cos u,-\cos u\rangle$, and the integral is then
%$$\int_0^{2\pi} y^2\sin u+x\cos u-z^2\cos u\,du
%=\int_0^{2\pi} \sin^3 u+\cos^2 u-(2-\sin u)^2\cos u\,du
%=\pi.$$

%To use Stokes' Theorem, we pick a surface with $C$ as the boundary;
%the simplest such surface is that portion of the plane $y+z=2$ inside
%the cylinder. This has vector equation $\vect{r}=\langle
%v\cos u,v\sin u,2-v\sin u\rangle$. We compute
%$\vect{r}_u= \langle -v\sin u,v\cos u,-v\cos u\rangle$,
%$\vect{r}_v= \langle \cos u,\sin u, -\sin u\rangle$, and
%$\vect{r}_u\times\vect{r}_v=\langle 0,-v,-v\rangle$. To match the
%orientation of $C$ we need to use the normal $\langle
%0,v,v\rangle$. The curl of $\vect{f}$ is $\langle 0,0,1+2y\rangle=
%\langle 0,0,1+2v\sin u\rangle$, and
%the surface integral from Stokes' Theorem is
%$$\int_0^{2\pi}\int_0^1 (1+2v\sin u)v\,dv\,du=\pi.$$
%In this case the surface integral was more work to set up, but the
%resulting integral is somewhat easier.
%\end{solution}

Now let's look at the proof of Stokes' Theorem.

\begin{proof}
We can prove here a special case of Stokes' Theorem, which perhaps
not too surprisingly uses Green's Theorem.

Suppose the surface $D$ of interest can be expressed in the form
$z=g(x,y)$, and let $\vect{f}=\langle f_1,f_2,f_3\rangle$. Using the vector
function $\vect{r}=\langle x,y,g(x,y)\rangle$ for the surface we get the
surface integral
\begin{align*}
\iint_{D} \nabla\times\vect{f}\cdot d\vect{S}&=
\iint_{E} \langle {\partial f_3 \over \partial y}-{\partial f_2 \over \partial z},{\partial f_1 \over \partial z}-{\partial f_3 \over \partial x},{\partial f_2 \over \partial x}-{\partial f_1 \over \partial y}\rangle\cdot
\langle -g_x,-g_y,1\rangle\,dA	\\
&=\iint_{E} -{\partial f_3 \over \partial x}g_x+{\partial f_2 \over \partial z}g_x-{\partial f_1 \over \partial z}g_y+{\partial f_3 \over \partial x}g_y+{\partial f_2 \over \partial x}-{\partial f_1 \over \partial y}\,dA.
\end{align*}
Here $E$ is the region in the $x$-$y$ plane directly below the surface
$D$.

For the line integral, we need a vector function for $\partial D$. If
$\langle x(t),y(t)\rangle$ is a vector function for
$\partial E$ then we may use $\vect{r}(t)=\langle x(t),y(t),g(x(t),y(t))\rangle$
to represent $\partial D$. Then
$$\int_{\partial D}\vect{f}\cdot d\vect{r}
=\int_a^b f_1{dx\over dt}+f_2{dy\over dt}+f_3{dz\over dt}\,dt
=\int_a^b f_1{dx\over dt}+f_2{dy\over dt}+f_2\left({\partial z\over\partial
    x}{dx\over dt}+{\partial z\over\partial y}{dy\over dt}\right)\,dt.$$
using the chain rule for $dz/dt$. Now we continue to manipulate this:
\begin{align*}
\int_a^b f_1{dx\over dt}+f_2{dy\over dt}+&f_3\left({\partial z\over\partial
    x}{dx\over dt}+{\partial z\over\partial y}{dy\over dt}\right)\,dt	\\
&=\int_a^b \left[\left(f_1+f_3{\partial z\over\partial x}\right){dx\over dt}+
\left(f_2+f_3{\partial z\over\partial y}\right){dy\over dt}\right]\,dt	\\
&=\int_{\partial E} \left(f_1+f_3{\partial z\over\partial x}\right)\,dx+
\left(f_2+f_3{\partial z\over\partial y}\right)\,dy,
\end{align*}
which now looks just like the line integral of Green's Theorem, except
that the functions $f_1$ and $f_2$ of Green's Theorem have been replaced
by the more complicated $f_1+f_3(\partial z/\partial x)$
and $f_2+f_3(\partial z/\partial y)$. We can apply Green's Theorem to get
$$\int_{\partial E} \left(f_1+f_3{\partial z\over\partial x}\right)\,dx+
\left(f_2+f_3{\partial z\over\partial y}\right)\,dy=
\iint_{E} {\partial\over \partial x}\left(f_2+f_3{\partial z\over\partial y}\right)
-{\partial\over \partial y}\left(f_1+f_3{\partial z\over\partial x}\right)\,dA.$$
Now we can use the chain rule again to evaluate the derivatives
inside this integral, and it becomes
\begin{align*}
\iint_{E} &{\partial f_2 \over \partial x}+{\partial f_2 \over \partial z}g_x+{\partial f_3 \over \partial x}g_y+{\partial f_3 \over \partial z}g_xg_y+f_3g_{yx}-
\left({\partial f_1 \over \partial y}+{\partial f_1 \over \partial z}g_y+{\partial f_3 \over \partial y}g_x+{\partial f_3 \over \partial z}g_yg_x+f_3g_{xy}\right)\,dA	\\
&=\iint_{E} {\partial f_2 \over \partial x}+{\partial f_2 \over \partial z}g_x+{\partial f_3 \over \partial x}g_y-{\partial f_1 \over \partial y}-{\partial f_1 \over \partial z}g_y-{\partial f_3 \over \partial y}g_x\,dA,
\end{align*}
which is the same as the expression we obtained for the surface
integral.
\end{proof}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Opensolutionfile{solutions}[ex]
\section*{Exercises for \ref{sec:StokesTheorem}}

\begin{enumialphparenastyle}

\begin{ex}
Let $\vect{f}=\langle z,x,y\rangle$.
The plane $z=2x+2y-1$ and the paraboloid $z=x^2+y^2$ intersect in a
closed curve. Stokes' Theorem implies that
$$\iint_{D_1} (\nabla\times\vect{f})\cdot \vect{N}\,dS=
\oint_C \vect{f}\cdot d\vect{r}=
\iint_{D_2} (\nabla\times\vect{f})\cdot \vect{N}\,dS,
$$
where the line integral is computed over the intersection $C$ of the plane
and the paraboloid, and the two surface integrals are computed over
the portions of the two surfaces that have boundary $C$ (provided, of
course, that the orientations all match). Compute all three integrals.
\begin{sol}
	$-3\pi$
\end{sol}
\end{ex}

\begin{ex}
Let $D$ be the portion of $z=1-x^2-y^2$ above the $x$-$y$
plane, oriented up, and let $\vect{f}=\langle
xy^2,-x^2y,xyz\rangle$. Compute $\ds\iint_{D} (\nabla\times\vect{f})\cdot \vect{N}\,dS$.
\begin{sol}
	$0$
\end{sol}
\end{ex}

\begin{ex}
Let $D$ be the portion of $z=2x+5y$ inside $x^2+y^2=1$,
oriented up, and
let $\vect{f}=\langle y,z,-x\rangle$. Compute
$\ds\int_{\partial D} \vect{f}\cdot d\vect{r}$.
\begin{sol}
	$-4\pi$
\end{sol}
\end{ex}

\begin{ex}
Compute $\ds\oint_C x^2z\,dx + 3x\,dy - y^3\,dz$, where $C$
is the unit circle $\ds x^2+y^2=1$ oriented counter-clockwise.
\begin{sol}
	$3\pi$
\end{sol}
%/Albert
\end{ex}

\begin{ex}
Let $D$ be the portion of $z=px+qy+r$ over a region in the
$x$-$y$ plane that has area $A$, oriented up, and
let $\vect{f}=\langle ax+by+cz,ax+by+cz,ax+by+cz\rangle$. Compute
$\ds\int_{\partial D} \vect{f}\cdot d\vect{r}$.
\begin{sol}
	$A(p(c-b)+q(a-c)+a-b)$
\end{sol}
\end{ex}

\begin{ex}
Let $D$ be any surface and
let $\vect{f}=\langle f_1(x),f_2(y),f_3(z)\rangle$ ($f_1$ depends only on $x$,
$f_2$ only on $y$, and $f_3$ only on $z$). Show that
$\ds\int_{\partial D} \vect{f}\cdot d\vect{r}=0$.
\end{ex}

\begin{ex}
Show that $\ds\int_C f\nabla g+g\nabla f\cdot d\vect{r}=0$, where
$\vect{r}$ describes a closed curve $C$ to which Stokes' Theorem
applies.
\end{ex}

\end{enumialphparenastyle}


\documentclass[12pt, a4paper]{IEEEtran}
\title{Standard for The Set Programming Language}
\author{Jedi Yang}

\usepackage{indentfirst}

\begin{document}
\maketitle

\section{Data Structure}
	The Set Programming Language aims at using sets and assist with vectors to build a Functional Programming Language. The data structure of Set is more important than language and grammar.

\subsection{Atom}
	Atom is the fundamental unit of Set. It means a set with only one number, either an integer or a decimal, but only numbers are stored in atoms.

	An atom can be expressed as $\{A\}$, but it can also be simplified as $A$. However, these two expression are different when doing calculation:

	$\{A\} + \{B\} == \{A, B\}$ while $A + B$ is the sum of number $A$ and $B$.

	An Atom with brackets around accept Set Operations while an atom without brackets accept Basic Operations (see Operations)

	\subsubsection{Number}
	Due to IEEE 754, if an integer is stored in a float or double, it is almost impossible to determine whether the number is an integer. The numbers in Set Programming Language use Irreducible Fractions to store both integers and decimals. An irreducible fraction contains two integers, a denominator and a numerator. Atoms can only store Irreducible Fractions.
	\subsubsection{Boolean}
	Boolean is special number with true and false only. Typically an atom of 0/0 is false. A set containing value 0/0 is also false. Otherwise, valid numbers and sets are seen as true including zero. During comparison, true is returned as {1/1} and false is {0/0}.

\subsection{Set}
	Set is a new data structure that is rare in other programming languages. It is similar to an Array, but there are two restrictions:
	\begin{enumerate}
	\item A set has no two identical numbers
	\item A set is ordered. The smallest is the head of the set.
	\end{enumerate}
	If a set does not meet the restrictions, identical numbers should be reduced and values are sorted automatically. Set can be catagorized into Enumerative Sets and Conditional Sets.

	\subsubsection{Enumerative Sets}
	This branch contains sets such that all numbers within the sets are listed. Enumerative Sets are stored in linked lists, with the smallest number is the head. All enumerative sets are finite.

	For example: $\{1, 3, 5, 17/2, 10\}$ is an enumerative set.

	\subsubsection{Conditional Sets}
	This branch contains sets such that numbers are determined by an operation and conditions. Not all conditional sets are infinite, some of them can be listed out.

	For example: $\{x:2x+7|2<x<7\}$ is a conditional set.

	%If the conditional set has less than one thousand numbers, the runtime should be able to convert the conditional set into an enumerative set
\subsection{Vector}
	Vector is another name for Array. However, Vectors should accept vector operations such as dot product and it only accepts Numbers. Vectors and Sets are similar. Vectors are also stored in linked lists. The difference between Vectors and Sets are that vectors do not have restrictions for sets:
	\begin{enumerate}
	\item A vector can have two identical numbers
	\item A vector can be randomly sorted.
	\end{enumerate}
	There is a special form of Vector. Transformation Vector.

	\subsubsection{Transformation Vector}
	This is vector that describes a series of transformation to one or more than one variables. In transformation vectors, Functions are stored. The return values of the prior function acts as the arguments for the next one.

	\subsubsection{Function}
	It is similar to Lambda Expressions in other programming languages. A function can only support one line of code. Except the return values and arguments of a function, the function cannot use other variables. Functions are better used in transformation vectors to support complex operations.

\subsection{String}
	String is a special vector, similar to most programming languages. However, it should have a different implementation using consequtive memory spaces.

\section{Operation}
\subsection{Numbers}
	Unary Operation
	\begin{enumerate}
		\item -x
	\end{enumerate}
	Binary Operation:
	\begin{enumerate}
		\item x+y
		\item x-y
		\item x*y
		\item x/y
		\item x\%y
		\item x\&y
		\item x$|$y
		\item x==y
		\item x!=y
		\item x>y
		\item x<y
		\item x>=y
		\item x<=y
		\item x\&\&y
		\item x$||$y
	\end{enumerate}
\subsection{Set \& Number}
	Binary Operation:
	\begin{enumerate}
		\item s+x == s.append(x)
		\item s-x == s.remove(x)
		\item s*x == s.forEach({this$\sim$this*x})
		\item s/x == s.forEach({this$\sim$this/x})
		\item s\%x ==s.forEach({this$\sim$this\%x})
	\end{enumerate}
\subsection{Sets}
	Binary Operation:
	\begin{enumerate}
		\item u+v == v.forEach(u.append)
		\item u-v == v.forEach(u.remove)
		\item u*v == v.forEach(u.exist)
		\item u/v == (u+v).forEach(xnor(u.exist,v.exist))
	\end{enumerate}
\end{document}


\input{../common/common.tex}

\title{Math notes - Prison Cells}
\author{Uwe Hoffmann}
\hypersetup{colorlinks, pdftitle={Math notes - Prison Cells}}

\begin{document}

\setcounter{chapter}{1}
\section*{Prison Cells}

\vspace{10 mm}
\begin{problem}
A prison has $n$ cells with all cell doors shut initially. The warden is a little weird so he walks the whole row of cells and opens every cell door. Then he walks the whole row again and shuts every other cell door. Then he walks the whole row again and opens every third door then walks the row again and shuts every 4th door etc. You can assume that the doors are numbered $0$ to $(n - 1)$ and the warden always starts at zero and walks them in order. Which doors will stay open when the warden is done ?
\end{problem}

Each time the warden walks the row of cells he toggles the state (open or close) of some of the cells. It is clear then that the number of toggles to one cell determines if it is open or closed in the end. In the beginning each cell door is closed so if the number of toggles is even then it stays closed, if it is odd then it is open at the end.

The goal then is to calculate the number of toggles for a cell. The cells are numbered $0$ to $(n - 1)$ so lets try to calculate the number of toggles for cell $k$. The first time the warden walks the row of cells he toggles each cell including our cell $k$. The second time he toggles cells $0, 2, 4, \dots$. That means he toggles cell $k$ if k is even. The third time around he toggles cells $0, 3, 6, \dots$ so he toggles cell $k$ if k is a multiple of 3. If we continue we see that the cell $k$ gets toggled on the warden's $d$ walk if $k$ is a multiple of $d$ or said differently if $d$ divides $k$.

It follows that the number of toggles $T(k)$ for cell $k$ is

\[
	T(k) = \sum_{d \mid k} 1.
\]

This is already pretty good but for the expression above it's not so obvious for which $k$ $T(k)$ will be even and for which it will be odd. So we will make a short excursion into basic number theory in the hopes that we can transform the expression into something more revealing.

 \subsection{A little number theory}

We say that two integers $m$ and $n$ are \textit{relatively prime} if the only common divisors are $\pm 1$ and we write $(m, n) = 1$ in that case.

\begin{defn}
A function $f:\mathbb{N} \rightarrow \Omega$ with $\Omega$ a field is said to be \textbf{weakly multiplicative} if

\[
	\forall \ m, n \in \mathbb{N}: (m, n) = 1 \ \Rightarrow \ f(mn) = f(m) f(n).
\]
\end{defn}

\begin{thm}
If $f$ is a weakly multiplicative function then so is the function

\[
	g(n) = \sum_{d \mid n} f(d).
\]
\end{thm}

\begin{proof}

Let $m_1,m_2 \in \mathbb{N}$ with $(m_1, m_2) = 1$. Let's define two sets

\[
	S_1 = \{ d\ :\ d \mid m_1 m_2 \}, \ S_2 = \{ d_1 d_2\ :\ d_1 \mid m_1 \wedge d_2 \mid m_2 \}.
\]

\noindent It is obvious that $S_2 \subseteq S_1$. On the other hand

\[
	\begin{array}{l}
	              \forall x \in S_1 \leadsto x \mid m_1 m_2 \ \mbox{(by definition)} \\
	              \mbox{Let } \ k = (x, m_1), \mbox{ so } x = y k, m_1 = z k,  \mbox{ for some } y, z \in \mathbb{N} \mbox{ and } (y, z) = 1 \\
	              x \mid m_1 m_2 \leadsto y k \mid z k m_2 \leadsto y \mid m_2 \mbox{ because } (y, z) = 1\\
	              \mbox{This means } x = y k \in S_2 \mbox{ because } y \mid m_2 \wedge k \mid m_1.
	 \end{array}
\]

\noindent So we have $S_1 = S_2$. We can now write

\[
   \begin{array}{lcl}
		&&g(m_1 m_2) \\
	      &=& { < \mbox{definition of g} >} \\
                  &&(\sum d: d \mid m_1 m_2 : f(d)) \\
       	      &=& { < \mbox{index sets }S_1 = S_2\mbox{ so we change bounded variables}  >} \\
                  && (\sum d_1, d_2: d_1 \mid m_1 \wedge d_2 \mid m_2 : f(d_1 d_2)) \\
 	       &=& { < \mbox{f is weakly multiplicative and } (d_1, d_2) = 1 > }  \\
	         && (\sum d_1, d_2: d_1 \mid m_1 \wedge d_2 \mid m_2 : f(d_1) f(d_2))\\
	        &=& { < \mbox{nesting}  > }  \\
	        && (\sum d_1: d_1 \mid m_1 : (\sum d_2: d_2 \mid m_2: f(d_1) f(d_2))) \\
	        &=& { < \mbox{multiplication distributes over addition}  > }  \\
	         && (\sum d_1: d_1 \mid m_1 : f(d_1) (\sum d_2: d_2 \mid m_2: f(d_2))) \\
	        &=& { < \mbox{definition of g}  > }  \\
	        && (\sum d_1: d_1 \mid m_1 : f(d_1) g(m_2)) \\
	        &=& { < \mbox{multiplication distributes over addition}  > }  \\
	        	&& (\sum d_1: d_1 \mid m_1 : f(d_1)) g(m_2) \\
		&=& { < \mbox{definition of g}  > }  \\
		&& g(m_1) g(m_2).
   \end{array}
\]
which proves the theorem.
\end{proof}

The theorem tells us that the function $T(k)$ which is the number of toggles for cell $k$
\[
	T(k) = \sum_{d \mid k} 1.
\]
is in fact a weakly multiplicative function because the function inside the sum (the constant function 1) is trivially a weakly multiplicative function.

\subsection{A more detailed solution}

If we use the unique prime factorization of $k$
\[
	k = p_1^{a_1} p_1^{a_1} \dots p_h^{a_h}
\]
and use the fact that $(p_i^{a_i}, p_j^{a_j}) = 1$ we get
\[
	T(k) = \prod_{i  = 1}^h T(p_i^{a_i}).
\]

But it's easy to see that $T(p_i^{a_i}) = a_i + 1$ so we have
\[
	T(k) = \prod_{i  = 1}^h (a_i + 1).
\]

When is $T(k)$ even ? When any of the $a_i$ are odd. To find out if a cell is open or closed do the prime factorization and look at the exponents of the primes. If any of them is odd then the cell stays closed.

\end{document}


% !TeX root = main.tex

\chapter{Matrix Multiplication}
\glsresetall
\label{chapter:matrix_multiplication}

This chapter looks at a bit more complex design -- matrix multiplication. We consider two different versions. We start with a ``straightforward'' implementation, i.e., one that takes two matrices as inputs and outputs the result of their multiplication. We call this complete matrix multiplication. Then, we look at a block matrix multiplication. Here the input matrices are feed into the function in portions, and the function computes partial results.

\section{Background}

Matrix multiplication is a binary operation that combines two matrices into a third. The operation itself can be described as a linear operation on the vectors that compose the two matrices. The most common form of matrix multiplication is call the \term{matrix product}. The matrix product $\mathbf{AB}$ creates an $n \times p$ matrix when matrix $\mathbf{A}$ has dimensions $n \times m$ and matrix $\mathbf{B}$ has dimensions $m \times p$.

More precisely, we define the following: \begin{equation}
\mathbf{A} =
 \begin{bmatrix}
  A_{11} & A_{12}  & \cdots & A_{1m} \\
  A_{21} & A_{22}  & \cdots & A_{2m} \\
  \vdots  & \vdots  &\ddots & \vdots  \\
    A_{n1} & A_{n2}  & \cdots & A_{nm} \\
 \end{bmatrix},  \quad
\mathbf{B} =
 \begin{bmatrix}
  B_{11} & B_{12}  & \cdots & B_{1p} \\
  B_{21} & B_{22}  & \cdots & B_{2p} \\
  \vdots  & \vdots   &\ddots & \vdots  \\
  B_{m1} & B_{m2}  & \cdots & B_{mp} \\
 \end{bmatrix}
\end{equation}

\begin{equation}
\mathbf{AB} = \begin{bmatrix}
 \left(\mathbf{AB}\right)_{11} & \left(\mathbf{AB}\right)_{12} & \cdots & \left(\mathbf{AB}\right)_{1p} \\
 \left(\mathbf{AB}\right)_{21} & \left(\mathbf{AB}\right)_{22} & \cdots & \left(\mathbf{AB}\right)_{2p} \\
\vdots & \vdots & \ddots & \vdots \\
 \left(\mathbf{AB}\right)_{n1} & \left(\mathbf{AB}\right)_{n2} & \cdots & \left(\mathbf{AB}\right)_{np} \\
\end{bmatrix}
\end{equation} where the operation $\left(\mathbf{AB}\right)_{ij}$ is defined as $\left(\mathbf{A}\mathbf{B}\right)_{ij} = \sum_{k=1}^m A_{ik}B_{kj}$.

Now we provide a simple example. Let
\begin{equation}
\mathbf{A} =
 \begin{bmatrix}
 \label{eq:ABmatrix}
  A_{11} & A_{12}  &  A_{13} \\
  A_{21} & A_{22} & A_{23} \\
 \end{bmatrix},  \quad
\mathbf{B} =
 \begin{bmatrix}
  B_{11} & B_{12}  \\
  B_{21} & B_{22}   \\
   B_{31} & B_{32}  \\
 \end{bmatrix}
\end{equation}
The result of the matrix product is
\begin{equation}\label{eq:ABmatrix_product}\mathbf{AB} = \begin{bmatrix}
 A_{11}B_{11} + A_{12}B_{21} + A_{13}B_{31} & A_{11}B_{12} + A_{12}B_{22} + A_{13}B_{32} \\
 A_{21}B_{11} + A_{22}B_{21} + A_{23}B_{31} & A_{21}B_{12} + A_{22}B_{22} + A_{23}B_{32}\\
\end{bmatrix}
\end{equation}

Matrix multiplication is a fundamental operation in numerical algorithms. Computing the product between large matrices can take a significant amount of time. Therefore, it is critically important part of many of problems in numerical computing. Fundamentally, matrices represent linear transforms between vector spaces; matrix multiplication provides way to compose the linear transforms.  Applications include linearly changing coordinates (e.g., translation, rotation in graphics), high dimensional problems in statistical physics (e.g., transfer-matrix method), and graph operations (e.g., determining if a path exists from one vertex to another). Thus it is a well studied problem, and there are many algorithms that aim to increase its performance, and reduce the memory usage.

\begin{figure}
\lstinputlisting[firstline=5]{examples/matrixmultiplication.cpp}
\caption{A common three \lstinline{for} loop structure for matrix multiplication. The outer \lstinline{for} loops, labeled \lstinline{rows} and \lstinline{cols}, iterate across the rows and columns of the output matrix $\mathbf{AB}$.  The innermost loop, labeled \lstinline{product} multiplies the appropriate elements of one row of $\mathbf{A}$ and one column of $\mathbf{B}$ and accumulates them until it has the result for the element in $\mathbf{AB}$ .  }
\label{fig:matrixmultiplication_sw}
\end{figure}

\section{Complete Matrix Multiplication}

We start our optimization process with perhaps the most common method to compute a matrix multiplication -- using three nested \lstinline{for} loops. Figure \ref{fig:matrixmultiplication_sw} provides the code for such an implementation.  The outer \lstinline{for} loops, labeled \lstinline{rows} and \lstinline{cols}, iterate across the rows and columns of the output matrix $\mathbf{AB}$. The innermost \lstinline{for} loop computes a dot product of one row of $\mathbf{A}$ and one column of $\mathbf{B}$. Each dot product is a completely independent set of computations that results in one element of $\mathbf{AB}$.  Conceptually, we are performing \lstinline{P} matrix-vector multiplications, one for each column of $\mathbf{B}$.

In this case, we've applied a \lstinline{pipeline} directive to the \lstinline{col} loop with a target initiation interval of 1. The result is that the innermost \lstinline{for} loop is fully unrolled, and we expect the resulting circuit include roughly $M$ multiply-add operators and to have an interval of roughly $N*P$ cycles.  As discussed in Chapter \ref{chapter:dft}, this is only one reasonable choice.  We could choose to place the \lstinline{pipeline} directive in different locations in the function with the goal of achieving different resource-throughput tradeoffs. For instance, placing the same directive at the top of the function (outside all of the \lstinline{for} loops) will result in all of the loops being completely unrolled, which would take roughly $N*M*P$ multiply-add operators and would have an interval of 1 cycle. Placing it inside the \lstinline{row} loop would result in roughly $M*P$ multiply-add operators and an interval of roughly $N$ cycles. These design points are relatively easy to achieve, given the corresponding array partitioning.  It's also possible to pipeline the innermost loop with the goal of achieving a design with only one multiply-add operator, although achieving an II=1 implementation at high clock frequencies can be difficult because of the recurrence involved in the accumulation of variable \lstinline{ABij}.  We can also partially unroll different loops to achieve yet more design points.  The fundamental tradeoff here is between the complexity of the resulting architecture, i.e., the number of multiply-add operators, and performance, i.e., the number of cycles that the hardware is busy.  In an ideal world, each doubling of the resource usage should result in exactly half the number of clock cycles being required, although in practice such `perfect scaling' is difficult to achieve.

\begin{exercise}
Change the location of the \lstinline{pipeline} directive. How does the location effect the resource usage? How does it change the performance? Which alternative provides the best performance in terms of function interval? Which provides the smallest resource usage? Where do you think is the best place for the directive? Would increasing the size of the matrices change your decision?
\end{exercise}

Executing large numbers of operations every cycle requires being able to supply all of the required operands and to store the results of each operation.   Previously we have used the \lstinline{array_partition} directive to increase the number of accesses that can be performed on each memory.  As long as the partition of the array that each memory access can be determined at compile time, then array partitioning is a simple and efficient way to increase the number of memory accesses that can be performed each clock cycle.  In this case, we use the slightly different \lstinline{array_reshape} directive to perform array partitioning.  This directive not only partitions the address space of the memory into separate memory blocks, but then recombines the memory blocks into a single memory.  This transformation increases the data width of the memory used to store the array, but doesn't change the overall number of bits being stored.  The difference is shown in Figure \ref{fig:matmul_array_reshape}.

\begin{figure}
\centering
\includesvg{matmul_array_reshape}
\caption{Three different implementations of a two-dimensional array.  On the left is the original array consisting of $N*M$ elements.  In the middle, the array has been transformed using the \lstinline{array_partition} directive, resulting in $M$ memories, each with $N$ elements.  On the right, the array has been transformed using the \lstinline{array_reshape} directive, resulting in one memory with $N$ locations and each location contains $M$ elements of the original array.}
\label{fig:matmul_array_reshape}
\end{figure}

Both \lstinline{array_reshape} and \lstinline{array_partition} increase the number of array elements that can be read each clock cycle.  They also support the same options, enabling \lstinline{cyclic} and \lstinline{block} partitions or partitioning along different dimensions of a multi-dimensional array.  In the case of \lstinline{array_reshape}, the elements must each have the same address in the transformed array, whereas with \lstinline{array_partition}, the addresses in the transformed array can be unrelated.  Although it may seem like one would always want to use \lstinline{array_partition} because it is more flexible, it makes each individual memory smaller, which can sometimes result in inefficient memory usage.  The \lstinline{array_reshape} directive results in larger memory blocks, which can sometimes be mapped more efficiently into primitive FPGA resources.   In particular, the smallest granularity of \gls{bram} blocks in Xilinx Virtex Ultrascale+ devices is 18 Kbits with several different supported combination of depths and widths.  When the partitions of an array become smaller than around 18Kbits, then BRAMs are no longer used efficiently.  If we start with an original array which is a 4-bit array with dimensions [1024][4], this array can fit in a single \gls{bram} resource configured as a 4Kbit x 4 memory.  Partitioning this array completely in the second dimension would result in 4 1Kbit x 4 memories, each of which are much smaller than one one \gls{bram} resource.  Reshaping the array instead using the \lstinline{array_reshape} directive results in a memory which is 1Kbit x 16, a supported \gls{bram} configuration.

%  The other \lstinline{array_reshape} directives modify the data layout for the input arrays $\mathbf{A}$ and $\mathbf{B}$. You can think of this directive as performing two operations. The first partitions the array into separate memories. The second operation combines these memories back into a single memory. This is similar to performing the \lstinline{array_partition} directive followed by the \lstinline{array_map} directive. The end result is still mapped into one array, but there are a smaller number of elements that are caused by grouping some of the elements from the original array into one element in the new array.

%There are multiple arguments to the \lstinline{array_reshape} directive. The \lstinline{variable} argument specifies the name of the array to perform the reshaping.The \lstinline{dim} argument indicates the dimension of the array to perform reshaping upon. This is only appropriate for multidimensional arrays. Another argument is \lstinline{type} which is one of \lstinline{block}, \lstinline{cyclic}, or \lstinline{complete}. \lstinline{Block} combines adjacent elements into one larger element; \lstinline{cyclic} combines interleaved elements into larger elements; and \lstinline{complete} makes all of the elements into one very large element. The final argument is \lstinline{factor}. This is only appropriate when using \lstinline{block} and \lstinline{cyclic} reshaping. It determines the size of the new elements.

%The first \lstinline{array_reshape} directive works on the $\mathbf{A}$ array. It states that the second dimension should be completely partitioned. That is, all of the elements in the second dimension should be combined into a single element. This has the same effect as making $\mathbf{A}$ into a one dimensional array where each element consists of the elements corresponding to the entire row of $\mathbf{A}$. The second \lstinline{array_reshape} directive performs a similar operation on the $\mathbf{B}$ array. But here, it does a complete partitioning on the first dimension. This has the effect of combining all of the elements corresponding to one column into a single element. Thus, it is a one dimensional array, and when we index into the array we get a full columns with of data with each index.

%Performing these two \lstinline{array_reshape} directives allows us to fetch an entire row of the $\mathbf{A}$ matrix and an entire column of the $\mathbf{B}$ matrix with only two memory accesses. One access on $\mathbf{A}$; and one access on $\mathbf{B}$.

\begin{aside}
Note that this partitioning is effective because the partitioned dimension (dimension 2 of $\mathbf{A}$ or dimension 1 of $\mathbf{B}$) is indexed by a constant.   In addition, the non-partitioned dimension is indexed by the same (variable) value.  When partitioning multi-dimensional arrays, this is a good rule of thumb to identify which dimension should be partitioned.
\end{aside}

\begin{exercise}
Remove the \lstinline{array_reshape} directives. How does this effect the performance? How does it change the resource usage? Does it make sense to use any other \lstinline{array_reshape} directives (with different arguments) on these arrays?  In this case, how does the result differ if you use the \lstinline{array_reshape} directive instead?
\end{exercise}

The size of the arrays can have a substantial effect on the optimizations that you wish to perform. Some applications might use very small matrices, say $2 \times 2$ or $4 \times 4$. In this cases, it may be desirable to implement a design with the absolute highest performance, which is generally achieved by applying the \lstinline{pipeline} directive on the entire function.   As the size of the arrays increase, in the range of $32 \times 32$, this approach quickly become infeasible because of the resource limits available in a single device.  There will simply not be enough DSP resources to implement that many multiplications every clock cycle or enough external bandwidth to get get data on and off the chip.  Many FPGA designs are often coupled to the data rates of other components in a system, such as an Analog to Digital (A/D) converter, or the symbol rate in a communication system.  In these designs it is common to instead apply the \lstinline|pipeline| directive on inner loops with the goal of matching the interval of the computation with the data rate in a system.  In such cases, we often need to explore different resource-throughput tradeoffs by moving the \lstinline{pipeline} directive into an inner loop or partially unrolling loops. When
 dealing with very large matrices containing thousands or millions of elements, we often need to take into account more complex architectural considerations.  The next section discusses a common approach to scaling matrix multiply to larger designs, called \term{blocking} or \term{tiling}.

\begin{exercise}
Optimize your design for the $128 \times 128$ matrix multiplication. Then start increasing the size of the matrices by a factor of two (to $512 \times 512$, $1024 \times 1024$, $2048 \times 2048$, etc. How does this effect the resource usage and performance? How about the runtime of the tool? What is the best way to optimize for large large matrix sizes?
\end{exercise}

\section{Block Matrix Multiplication}

A \term{block matrix} is interpreted as being partitioned into different submatrices. This can be visualized by drawing different horizontal and vertical lines across the elements of the matrix. The resulting ``blocks'' can be viewed as submatrices of the original matrix.  Alternatively, we can view the original matrix as a matrix of blocks.  This naturally leads to many hierarchical algorithms in Linear Algebra where we compute matrix operations, such as matrix multiply, on large block matrices by decomposing them into smaller matrix operations on the blocks themselves.

For instance, when we talk about the matrix multiplication operation between matrix $\mathbf{A}$ and $\mathbf{B}$ in Equations \ref{eq:ABmatrix} and \ref{eq:ABmatrix_product}, we might normally think of each element of the matrices $\mathbf{A}_{11}$ or $\mathbf{B}_{23}$ as a single number or perhaps a complex number.  Alternatively, we can consider each element in these matrix operations as a block of the original matrix.  In this case, as long as the sizes of the individual blocks are compatible, we simply have to perform the correct matrix operations instead of the original scalar operations.  For instance, to compute $\mathbf{AB}_{11}$, we would need to compute two matrix products and two matrix sums to compute $\mathbf{A}_{11}\mathbf{B}_{11} + \mathbf{A}_{12}\mathbf{B}_{21} + \mathbf{A}_{13}\mathbf{B}_{31}$.

Matrix blocking turns out to be a very useful technique for a number of reasons.  One reason is that blocking is an easy way to find more structure in the algorithm that we can explore.  In fact, some of the optimizations that we have already seen as loop transformations, such as loop unrolling, can be viewed as specific simple forms of blocking.  Another reason is that we can choose to block a matrix according to the natural structure of the matrix.  If a matrix has a large block of zeros, then many individual products may be zero.  If we want to skip these individual products then this can be difficult in a statically schedule pipeline, whereas it may be easier to skip a large block of zeros.  Many matrices are \term{block-diagonal} where the blocks on the diagonal are non-zero and blocks off the diagonal are zero.  Yet another reason is that the blocked decomposition results in lots of smaller problems operating on smaller sets of data.  This increases the data locality of a computation.  In processor systems, it is common to choose block sizes that conveniently match the memory hierarchy of a processor or the natural size of the vector data types supported by the processor.  Similarly, in FPGAs we can choose the blocking sizes to match the available on-chip memory size or to the number of multiply-add operators that we can budget to support.

Until now, we have assumed that accelerators always have all of their data available before the start of a task.  However in designs dealing with large datasets, such as large matrices, this can sometimes be an unneeded constraint.  Since it is unlikely that our accelerator will be able to process all input data immediately, we can build an accelerator that receives input data only right before it is needed.  This allows an accelerator to more efficiently use the available on-chip memory.  We call this a \term{streaming architecture} since we transfer the input data (and potentially the output data) one portion at a time rather than all at once.

Streaming architectures are common in many applications.  In some cases this is because of a conscious design choice that we make to decompose a large computation into multiple smaller computations.  For instance, we may design a matrix multiplication system that reads and processes one block of data at a time from external memory.  In other cases, we might process a stream of data because the data is being sampled in real time from the physical world, for instance, from an A/D converter.  In other cases, the data we are processing may simply be created in sequence from a previous computation or accelerator.  In fact, we've already seen an instance of this in Section \ref{sec:fft_task_pipelining}.

One potential advantage of streaming is a reduction in the memory that we can use to store the input and output data. The assumption here is that we can operate on the data in portions, create partial results, and then we are done with that data, thus we do not need to store it. When the next data arrives, we can overwrite the old data resulting in smaller memories.

In the following, we develop a streaming architecture for matrix multiplication. We divide the input arrays $\mathbf{A}$ and $\mathbf{B}$ into blocks, which are a contiguous set of rows and columns, respectively. Using these blocks, we compute a portion of the product $\mathbf{AB}$. Then we stream the next set of blocks, compute another portion of $\mathbf{AB}$ until the entire matrix multiplication is complete.

\begin{figure}
\centering
\includegraphics[width=  .85\textwidth]{images/blockmm}
\caption{ One possible blocked decomposition of the matrix multiplication of two $4 \times 4$ matrices. The entire $\mathbf{AB}$ product is decomposed into four matrix multiply operations operating on a $2 \times 4$ block of $\mathbf{A}$ and a $4 \times 2$ block of $\mathbf{B}$.}
\label{fig:blockmm}
\end{figure}

Figure \ref{fig:blockmm} provides the a description of the streaming architecture that we create. Our architecture has a variable \lstinline{BLOCK_SIZE} that indicates the number of rows that we take from the $\mathbf{A}$ matrix on each execution, the number of columns taken from the $\mathbf{B}$ matrix, and the \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} result matrix corresponding to the data that we compute each time for the $\mathbf{AB}$ matrix.

The example in Figure \ref{fig:blockmm} uses a \lstinline{BLOCK_SIZE = 2}. Thus we take two rows from $\mathbf{A}$, and two columns from $\mathbf{B}$  on each execution of the streaming architecture that we define. The result of each call to the \lstinline{blockmatmul} function is a $2 \times 2$ matrix for the $\mathbf{AB}$ architecture.

Since we are dealing with $4 \times 4$ matrices in the example, we need to do this process four times. Each time we get a $2 \times 2$ set of results for the $\mathbf{AB}$ matrix. The figure shows a progression of the rows and columns that we send. In Figure \ref{fig:blockmm} a) we send the first two rows of $\mathbf{A}$ and the first two columns of $\mathbf{B}$. The function will compute a $2 \times 2$ matrix corresponding to the first two elements in the rows and columns of the resulting matrix $\mathbf{AB}$.

In Figure \ref{fig:blockmm} b), we use again the first two rows of $\mathbf{A}$, but this time we send the last two columns of $\mathbf{B}$. We do not need to resend the data from the rows of $\mathbf{A}$ since they are the same as the previous data from the previous execution. And we get the results for the $2 \times 2$ matrix corresponding to the data in ``upper left'' corner of $\mathbf{AB}$.

Figure \ref{fig:blockmm} c) sends different data for both the $\mathbf{A}$ and $\mathbf{B}$ matrices. This time we send the last two rows of $\mathbf{A}$ and the first two columns of $\mathbf{B}$. The results from this computation provide the ``lower left'' corner of the $\mathbf{AB}$ matrix.

The final execution of the streaming block matrix multiply, shown in Figure \ref{fig:blockmm} d), uses the same last two rows of the $\mathbf{A}$ matrix from the previous iteration. And it sends the last two columns of the $\mathbf{B}$ matrix. The result provides the elements in the ``lower right'' corner of the $\mathbf{AB}$ matrix.

\begin{figure}
\lstinputlisting{examples/block_mm.h}
\caption{  The header file for the block matrix multiplication architecture. The file defines the data types used within the function, the key constants, and the \lstinline{blockmatmul} function interface.   }
\label{fig:block_mm_h}
\end{figure}

Before we show the code for the block matrix multiplication, we define some data types that we will use. Figure \ref{fig:block_mm_h} shows the header file for the project. We create a custom data type \lstinline{DTYPE} that specifies the type of data that we will multiply in the $\mathbf{A}$ and $\mathbf{B}$ matrices, and the corresponding $\mathbf{AB}$ matrix. This is currently set to an \lstinline{int} data type.

\begin{aside}
It is good coding practice to use a custom data type in your designs. This allows you to easily change the data type, and to have one source of information so that you do not have errors when changing the data type in the future design iterations. And it is quite common to change the data type over the course of the design. For example, first you may start out with a \lstinline{float} or \lstinline{double} type while you get a functionally correct design. This also provides a baseline for error since later you will likely change your design to use fixed point data type.  Fixed point data can reduce the number of resources, and increase the performance potentially at the cost of a reduction in the precision of the resulting data. You will likely try many different fixed point types until you find the right tradeoff between accuracy/error, performance, and resource usage.
\end{aside}

\lstinline{SIZE} determines the number of rows and columns in the matrices to be multiplied. We limit this to square matrices although handling arbitrary matrix sizes could be done by changing the code in a number of different places. We leave that as an exercise for the reader.

\begin{exercise}
Change the code to allow it to handle matrices of arbitrary size.
\end{exercise}

The \lstinline{BLOCK_SIZE} variable defines the number of rows from $\mathbf{A}$ and the number of columns from $\mathbf{B}$ that we operate upon in each execution. This also defines how much data that we stream at one time into the function. The output data that we receive from the function at each execution is an \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} portion of the $\mathbf{AB}$ matrix.

The \lstinline{blockvec} data type is used to transfer the \lstinline{BLOCK_SIZE} rows of $\mathbf{A}$ and columns of $\mathbf{B}$ to the function on each execution. The \lstinline{blockmat} data type is where we store the partial results for the $\mathbf{AB}$ matrix.

Finally, the \lstinline{blockmat} data type is a structure consisting of an \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} array. This holds the resulting values from one execution of the \lstinline{matrmatmul} function.

The function prototype itself takes the two inputs which are both of the type \lstinline{hls::stream<blockvec> &}. These are a sequence of \lstinline{blockvec} data. Remember that a \lstinline{blockvec} is a data type that consists of an array with \lstinline{BLOCK_SIZE} elements.

The \lstinline{hls::stream<>} template class is one way in \VHLS of creating a FIFO data structure that works well in simulation and synthesis. The samples are sent in sequential order using the \lstinline{write()} function, and retrieved using the \lstinline{read()} function. This library was developed since streaming is a common methodology for passing data in hardware design, yet this same operation can be modeled in many different ways using the C programming language, for instance, by using arrays. In particular, it can be difficult for the \VHLS tool to infer streaming behaviors when dealing complex access patterns or multi-dimensional arrays. The built-in stream library enables the programmer to explicitly specify the order of stream accesses, avoiding any limitations of this inference.

\begin{aside}
The \lstinline{hls::stream} class must always be passed by reference between functions, e.g., as we have done in the \lstinline{blockmatmul} function in Figure \ref{fig:block_mm}.
\end{aside}

\begin{figure}
\lstinputlisting[format=none]{examples/block_mm.cpp}
\caption{  The \lstinline{blockmatmul} function takes a \lstinline{BLOCK_SIZE} set of rows from $\mathbf{A}$ matrix, a \lstinline{BLOCK_SIZE} set of columns from the $\mathbf{B}$ matrix, and creates a \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} partial result for the $\mathbf{AB}$ matrix. The first part of the code (denoted by the label \lstinline{loadA}) stores the rows from $\mathbf{A}$ into a local memory, the second part in the nested \lstinline{partialsum for} performs the computation for the partial results, and the final part (with the \lstinline{writeoutput} label) takes these results and puts them the proper form to return from the function.}
\label{fig:block_mm}
\end{figure}

The code for executing one part of the streaming block matrix multiplication is shown in Figure \ref{fig:block_mm}. The code has three portions denoted by the labels \lstinline{loadA}, \lstinline{partialsum}, and \lstinline{writeoutput}.

The first part of the code, denoted by the \lstinline{loadA} label, is only executed on certain conditions, more precisely when \lstinline{it % (SIZE/BLOCK_SIZE) == 0}. This is done to save some time in the cases when we can reuse the data from the $\mathbf{A}$ matrix from the previous execution of the function.

Remember that in each execution of this \lstinline{blockmatmul} function we send \lstinline{BLOCK_SIZE} rows from the $\mathbf{A}$ matrix and \lstinline{BLOCK_SIZE} columns from the $\mathbf{B}$ matrix. We send multiple \lstinline{BLOCK_SIZE} of columns for each \lstinline{BLOCK_SIZE} of rows from $\mathbf{A}$. The variable \lstinline{it} keeps track of the number of times that we have called the \lstinline{blockmatmul} function. Thus, we do a check on each execution of the function to determine if we need to load the rows from $\mathbf{A}$. When we do not, this saves us some time. When it is executed, it simply pulls data from the \lstinline{Arows} stream and puts it into a static local two-dimensional matrix \lstinline{A[BLOCK_SIZE][SIZE]}.

Fully understanding this code requires some explanation about the \lstinline{stream} class, and how we are using it. The \lstinline{stream} variable \lstinline{Arows} has elements of the type \lstinline{blockvec}. A \lstinline{blockvec} is a matrix of size \lstinline{BLOCK_SIZE}. We use this in the following manner; each element in the \lstinline{Arows} stream has an array that holds one element from each of the \lstinline{BLOCK_SIZE} rows of the $\mathbf{A}$ matrix. Thus, in each call the the \lstinline{blockmatmul} function, the \lstinline{Arows} stream will have \lstinline{SIZE} elements in it, each of those holding one of each of the \lstinline{BLOCK_SIZE} rows. The statement \lstinline{tempA = Arows.read()} takes one element from the \lstinline{Arows} stream. Then we load each of these elements into the appropriate index in the local \lstinline{A} matrix.

\begin{aside}
The \lstinline{stream} class overloads the \lstinline{<<} operator to be equivalent to the \lstinline{read()} function. Thus, the statements \lstinline{tempA = Arows.read()} and \lstinline{tempA << Arows} perform the same operation.
\end{aside}

The next part of the computation calculates the partial sums. This is the bulk of the computation in the \lstinline{blockmatmul} function.

The \lstinline{Bcols} stream variable is utilized in a very similar manner to the \lstinline{Arows} variable. However, instead of storing rows of $\mathbf{A}$, it stores the data corresponding the columns of $\mathbf{B}$ that the current execution of the function is computing upon. Every call of the \lstinline{blockmatmul} function will provide new data for the columns of the $\mathbf{B}$ matrix. Thus, we do not need to conditionally load this data as we do with the $\mathbf{A}$ matrix. The function itself works in a very similar manner to that from the \lstinline{matmul} in Figure \ref{fig:matrixmultiplication_sw} except that we are only calculating \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} results from the $\mathbf{AB}$ matrix. Thus we only have to iterate across \lstinline{BLOCK_SIZE} rows of $\mathbf{A}$ and \lstinline{BLOCK_SIZE} columns of $\mathbf{B}$. But each row and column has \lstinline{SIZE} elements, hence the bounds on the outer \lstinline{for} loop.

The final portion of the function moves the data from the local \lstinline{AB} array, which has dimensions of \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE}; this holds the partial results of the $\mathbf{AB}$ output matrix. %The results are written into the \lstinline{out_temp} variable, which has the data type \lstinline{blockmat}. This data type is a \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} two dimension matrix. The \lstinline{out_temp} data is pushed into the \lstinline{ABpartial} stream in the statement \lstinline{ABpartial.write(out_temp);}

Of the three parts of the function, the middle part, which calculates the partial sum, requires the most computation. By inspecting the code, we can see that this part has three nested \lstinline{for} loops with a total of \lstinline{SIZE} $\times$ \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} iterations. The first part has \lstinline{SIZE} $\times$ \lstinline{BLOCK_SIZE} iterations; and the last part has \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} iterations. Thus, we should focus our optimizations on the middle part, i.e., the \lstinline{partialsum} nested \lstinline{for} loops.

The common starting point for optimizations of nested \lstinline{for} loops is to pipeline the innermost \lstinline{for} loop. Then, if that does not require too many resources, the designer can move the \lstinline{pipeline} directive into higher level \lstinline{for} loops. Whether the resulting design consume too many resource depends upon the specified \lstinline{BLOCK_SIZE}; if this is small, then it is likely worth moving the \lstinline{pipeline} directive. It may even be worthwhile to move it inside the outermost \lstinline{for} loop. This will unroll the two inner \lstinline{for} loops and thus very likely increase the resource usage by a substantial amount. However, it will increase the performance.

\begin{exercise}
How does changing the \lstinline{BLOCK_SIZE} effect the performance and resource usage? How about changing the \lstinline{SIZE} constant? How does moving the \lstinline{pipeline} directive across the three different nested \lstinline{for} loops in the \lstinline{partialsum} portion of the function change the performance and resource usage?
\end{exercise}

The \lstinline{dataflow} directive at the start of the function creates a pipeline across the portions of the function, i.e., the \lstinline{loadA for} loop, the \lstinline{partialsum} nested \lstinline{for} loop, and the \lstinline{writeoutput for} loop. Using this directive will decrease the interval of the \lstinline{blockmatmul} function. However, this is limited by the largest interval of all three of the portions of the code. That is, the maximum interval for the \lstinline{blockmatmul} function -- let us call it interval(\lstinline{blockmatmul}) -- is greater than or equal to the the interval of the three parts which are defined as interval(\lstinline{loadA}), interval(\lstinline{partialsum}), and interval(\lstinline{writeoutput}). More formally,

\begin{align}
\label{eq:interval}
interval(\texttt{blockmatmul}) \ge \max(&interval(\texttt{loadA}), interval(\texttt{partialsum}), \nonumber \\
 & interval(\texttt{writeoutput}))
 \end{align}

We need to keep Equation \ref{eq:interval} in mind as we optimize the \lstinline{blockmatmul} function. For example, assume that interval(\lstinline{partialsum}) is much larger than the other two portions of the function. Any performance optimizations that minimize interval(\lstinline{loadA}) and interval(\lstinline{writeoutput}) are useless since the function interval, i.e., interval(\lstinline{blockmatmul}) would not decrease. Thus, the designer should focus any performance optimization effort to decrease interval(\lstinline{partialsum}), i.e., target performance optimizations on those three nested \lstinline{for} loops.

It is important to note that this only applies to performance optimizations. The designer can (and should) optimize the resource usage of these other two parts. In fact, they are ripe for such optimizations since reducing the resource usage often increases the interval and/or latency. In this case, it is ok to increase the interval as it will not effect the overall performance of the \lstinline{blockmatmul} function. In fact, the ideal case is to optimize all three parts of the function such that they all have the same interval, assuming that we can easily tradeoff between the interval and resource usage (which is not always the case).

The testbench for the \lstinline{blockmatmul} function is shown in Figures \ref{fig:block_mm_init} and \ref{fig:block_mm_final}. We split it across two figures to make it more readable since it is a longer piece of code. Up until this point, we have not shown the testbenches. We show this testbench for several reasons. First, it provides insight into how the \lstinline{blockmatmul} function works. In particular, it partitions the input matrices into blocks and feeds them into the \lstinline{blockmatmul} function in a block by block manner. Second, it gives a complex usage scenario for using \lstinline{stream} template for simulation. Finally, it gives the reader an idea about how to properly design testbenches.

The \lstinline{matmatmul_sw} function is a simple three \lstinline{for} loop implementation of matrix multiplication. It takes two two-dimensional matrices as inputs, and outputs a single two-dimensional matrix. It is very similar to what we have seen in the \lstinline{matrixmul} function in Figure \ref{fig:matrixmultiplication_sw}. We use this to compare our results from the blocked matrix multiplication hardware version.

Let us focus on the first half of the testbench shown in Figure \ref{fig:block_mm_init}. The beginning block of code initializes variables of the rest of the function. The variable \lstinline{fail} keeps track of whether the matrix multiplication was done correctly. We will check this later in the function. The variables \lstinline{strm_matrix1} and \lstinline{strm_matrix2} are \lstinline{hls:stream<>} variables that hold the rows and columns of the $\mathbf{A}$ and $\mathbf{B}$ matrices, respectively. Each element of these \lstinline{stream} variables is a \lstinline{<blockvec>}. Referring back at the \lstinline{block_mm.h} file in Figure \ref{fig:block_mm_h}, we recall that a \lstinline{blockvec} is defined as an array of data; we will use each \lstinline{blockvec} to store one row or column of data.

\begin{aside}
The \lstinline{stream} variable resides in the \lstinline{hls} namespace. Thus, we can use that namespace and forgo the \lstinline{hls::stream} and instead simply use \lstinline{stream}. However, the preferred usage is to keep the \lstinline{hls::} in front of the \lstinline{stream} to insure code readers that the stream is relevant to \VHLS and not C construct from another library. Also, it avoids having to deal with any potential conflicts that may occur by introducing a new namespace.
\end{aside}

The next definitions in this beginning block of code are the variables \lstinline{strm_matrix1_element} and \lstinline{strm_matrix2_element}. These two variables are used as placeholders to populate each \lstinline{blockvec} variable that we write into the \lstinline{strm_matrix1} and \lstinline{strm_matrix2} stream variables. The \lstinline{block_out} variable is used to store the output results from the \lstinline{blockmatmul} function. Note that this variable uses the data type \lstinline{blockmat} which is a two-dimensional array of \lstinline{BLOCK_SIZE} $\times$ \lstinline{BLOCK_SIZE} as defined in the \lstinline{block_mm.h} header file (see Figure \ref{fig:block_mm_h}). The final definitions are \lstinline{A},  \lstinline{B},  \lstinline{matrix_swout},  and\lstinline{matrix_hwout}. These are all \lstinline{SIZE} $\times$ \lstinline{SIZE} two-dimensional arrays with the \lstinline{DTYPE} data type.

\begin{aside}
You can name the streams using an initializer. This is good practice as it gives better error messages. Without the name, the error message provides a generic reference to the stream with the data type. If you have multiple stream declarations with the same data type, then you will have to figure out which stream the error is referring to. Naming the stream variable is done by giving the variable an argument which is the name, e.g., \lstinline{hls::stream<blockvec> strm_matrix1("strm_matrix1");}.
\end{aside}

The next set of nested \lstinline{initmatrices for} loops sets the values of the four two-dimensional arrays \lstinline{A}, \lstinline{B}, \lstinline{matrix_swout}, and \lstinline{matrix_hwout}. The variables \lstinline{A} and \lstinline{B} are input matrices. These are initialized to a random value between [0, 512). We picked the number 512 for no particular reason other than it can fit any 9 bit value. Keep in mind that while the \lstinline{DTYPE} is set as an \lstinline{int}, and thus has significantly more range than [0,512), we often move to fixed point values with much smaller ranges later in the design optimization process. The \lstinline{matrix_swout} and \lstinline{matrix_hwout} are both initialized to $0$. These are filled in later by calls to the functions \lstinline{matmatmul_sw} and \lstinline{blockmatmul}.

\begin{figure}
\lstinputlisting{examples/blockmatmul_test_init.cpp}
\caption{  The first part of the testbench for block matrix multiplication. The function is split across two figures since it is too long to display on one page. The rest of the testbench is in Figure \ref{fig:block_mm_final}. This has a ``software'' version of matrix multiplication, and variable declarations and initializations. }
\label{fig:block_mm_init}
\end{figure}

The second part of the testbench is continued in Figure \ref{fig:block_mm_final}. This has the last portion of the code from the \lstinline{main} function.

\begin{figure}
\lstinputlisting[format=none]{examples/blockmatmul_test.cpp}
\caption{  The second portion of the block matrix multiply testbench. The first part is shown in Figure \ref{fig:block_mm_init}. This shows the computation required to stream the data to the \lstinline{blockmatmul} function, and the code that tests that this function matches a simpler three \lstinline{for} loop implementation. }
\label{fig:block_mm_final}
\end{figure}

The first part of this figure has a complex set of nested \lstinline{for} loops. The overall goal of the computation in these \lstinline{for} loops is to set up the data from the input matrices $\mathbf{A}$ and $\mathbf{B}$ so that it can be streamed to the \lstinline{blockmatmul} function. Then the results of the \lstinline{blockmatmul} function are stored in the \lstinline{matrix_hwout} array.

The outer two \lstinline{for} loops are used to step across the input arrays in a blocked manner. You can see that these both iterate by a step of \lstinline{BLOCK_SIZE}. The next two \lstinline{for} loops write rows from $\mathbf{A}$ into \lstinline{strm_matrix1_element} and the columns from $\mathbf{B}$ into \lstinline{strm_matrix2_element}. It does this in an element by element fashion by using the variable \lstinline{k} to access the individual values from the rows (columns) and write these into the one dimensional array for each of these ``elements''. Remember that both \lstinline{strm_matrix1_element} and \lstinline{strm_matrix2_element} have the datatype \lstinline{blockvec}, which is a one dimensional array of size \lstinline{BLOCK_SIZE}. It is meant to hold \lstinline{BLOCK_SIZE} elements from each row or column. The inner \lstinline{for} loop iterates \lstinline{BLOCK_SIZE} times. The \lstinline{strm_matrix1} and \lstinline{strm_matrix2} stream variables are written to \lstinline{SIZE} times. That means that has a buffer of the entire row (or column) and each element in the buffer holds \lstinline{BLOCK_SIZE} values.

\begin{aside}
The \lstinline{stream} class overloads the \lstinline{>>} operator to be equivalent to the \lstinline{write(data)} function. This is similar to overloading the \lstinline{read()} function to the \lstinline{<<} operator. Thus, the statements \lstinline{strm_matrix1.write(strm_matrix1_element)} and \lstinline{strm_matrix1_element >> strm_matrix1} perform the same operation.
\end{aside}

The final part of this portion of the code to highlight is the \lstinline{if} statements. These are correspond to the values $\mathbf{A}$ matrix. Essentially, these are there so that we do not constantly write the same values to \lstinline{strm_matrix1}. Recall that the values from the $\mathbf{A}$ matrix are used across several calls to the \lstinline{blockmatmul} function. See Figure \ref{fig:blockmm} for a discussion on this. These \lstinline{if} statements are placed there to highlight the fact that you should not continually write the same data over and over. This is important because the internal code of the \lstinline{blockmatmul} only does a read of this data when it is necessary. So if we continued to write this consistently, the code would not function correctly do to the fact that this stream is written to more than it is read from.

Now that the input data, the testbench calls the \lstinline{blockmatmul} function. After the function call, it receives the partial computed results in the \lstinline{block_out} variable. The next two \lstinline{for} loops but these results into the appropriate locations in the \lstinline{matrix_hwout} array.

After this complex set of \lstinline{for} loops, the block matrix multiplication is complete. And the testbench continues to insure that the code is written correctly. It does this by comparing the results from the multiple calls to the \lstinline{blockmatmul} function to results that were computed in the \lstinline{matmatmul_sw}, which is a much simpler version of matrix matrix multiplication. After this function call, the testbench iterates through both two-dimensional matrices \lstinline{matrix_hwout} and \lstinline{matrix_swout} and makes sure that all of the elements are equivalent. If there is one or more element that is not equal, it sets the \lstinline{fail} flag equal to 1. The testbench completes by printing out \lstinline{failed} or \lstinline{passed}.

\begin{aside}
It is important that note that you cannot directly compare the performance of the function \lstinline{blockmatmul} with that of code for matrix multiplication, such as the code in Figure \ref{fig:matrixmultiplication_sw}. This is because it takes multiple calls to the \lstinline{blockmatmul} function in order to perform the entire matrix multiplication. It is important to always compare apples to apples.

\begin{exercise}
Derive a function to determine the number of times that \lstinline{blockmatmul} must be called in order to complete the entire matrix multiplication. This function should be generic, e.g., it should not be assume a specific value of \lstinline{BLOCK_SIZE} or size of the matrix (i.e., \lstinline{SIZE}).
\end{exercise}

\begin{exercise}
Compare the resource usage of block matrix multiplication versus matrix multiplication. How do the resources change as the size of the matrices increases? Does the block size play a role in the resource usage? What are the general trends, if any?
\end{exercise}

\begin{exercise}
Compare the performance of block matrix multiplication versus matrix multiplication. How does the performance change as the size of the matrices increases? Does the block size play a role in the performance? Pick two architectures with similar resource usage. How does the performance for those architectures compare?
\end{exercise}

\end{aside}

\section{Conclusion}

Block matrix multiplication provides a different way to compute matrix multiplication. It computes partial results of the result matrix by streaming a subset of the input matrices to a function. This function is then computed multiple times in order to complete the entire matrix multiplication computation.


%!TEX root = ../main.tex
%-------------------------------------------------------------------------------
\subsection{Mathematical formulation}\label{Mathematics}
%-------------------------------------------------------------------------------
EKW models are set up as a standard Markov decision process (MDP) \citep{Puterman.1994, White.1993}. When making sequential decisions under uncertainty, the task is to determine the optimal policy $\pi^*$ with the largest expected total discounted utilities $v^{\pi^*}_1(s_1)$ as formalized in equation (\ref{Objective Risk}). In principle, this requires evaluating the performance of all policies based on all possible sequences of utilities, each weighted by the probability with which they occur. Fortunately, however, the multistage problem can be solved by a sequence of simpler inductively defined single-stage problems.\footnote{Optimal decisions in an MDP are a deterministic function of the current state $s$ only, i.e., an optimal decision rule is always deterministic and Markovian. We restrict our notation to this special case right from the beginning.}\\

\noindent The value function $v^\pi_t(s_t)$ captures the expected total discounted utilities under policy $\pi$ from period $t$ onwards for an individual experiencing state $s_t$:
%
\begin{align*}
  v^\pi_t(s_t) \equiv \E_{s_t}^\pi\left[\left.\sum^{T - t}_{j = 0}  \delta^j\, u_{t + j}(s_{t + j}, a^\pi_{t + j}(s_{t + j})) \,\right\vert\,\mathcal{I}_t\,\right].
\end{align*}
%
Then we can determine $v_1^\pi(s_1)$ for any policy by recursively evaluating equation (\ref{MDP Policy Equations}):
%
\begin{align}\label{MDP Policy Equations}
v^\pi_t(s_t) = u_t(s_t,  a^\pi_t(s_t)) + \delta\,\E^\pi_{s_t} \left[\left.v^\pi_{t + 1}(s_{t + 1})  \,\right\vert\,\mathcal{I}_t\,\right].
\end{align}
%
Equation (\ref{MDP Policy Equations}) expresses the total value $v^\pi_t(s_t)$ of adopting policy $\pi$ going forward as the sum of its immediate utility and all expected discounted future utilities.\\

\noindent The principle of optimality \citep{Bellman.1954} allows to construct $\pi^*$ by solving the optimality equations (\ref{MPD Optimality})  for all $s$ and $t$ recursively:
%
\begin{align}\label{MPD Optimality}
v^{\pi^*}_t(s_t)  & = \max_{a_t \in A}\bigg\{ u_t(s_t, a_t) + \delta\, \E^{\pi^*}_{s_t} \left[\left.v^{\pi^*}_{t + 1}(s_{t + 1})\,\right\vert\,\mathcal{I}_t\,\right] \bigg\}.
\end{align}

\noindent The optimal value function $v^{\pi^*}_t$ is the sum of the expected discounted utilities in $t$ over the remaining time horizon assuming the optimal policy is implemented going forward. The optimal action is choosing the alternative with the highest total value:
%
\begin{align*}
a^{\pi^*}_t(s_t) = \underset{a_t\in A}{\argmax} \bigg\{ u_t(s_t, a_t) + \delta\,\E^{\pi^*}_{s_t} \left[\left.v^{\pi^*}_{t + 1}(s_{t + 1})\right\vert\,\mathcal{I}_t\,\right] \bigg\}.\\
\end{align*}

\noindent Algorithm \ref{Backward induction procedure} allows to solve the MDP by a simple backward induction procedure. In the final period $T$, there is no future to take into account, and the optimal action is choosing the alternative with the highest immediate utilities in each state. With the decision rule for the final period at hand, the other optimal decisions can be determined recursively following equation (\ref{MPD Optimality}) as the calculation of their expected future utilities is straightforward given the relevant transition probabilities.

\vspace{0.5cm}\begin{algorithm}
\caption{\strut Backward induction procedure}\label{Backward induction procedure}
\begin{algorithmic}\vspace{0.3cm}
\For{$t = T, \hdots, 1$}
    \If{t == T}
        \State $v^{\pi^*}_T(s_T) =  \underset{a_T\in A}{\max} \bigg\{ u_T(s_T, a_T) \bigg\}\qquad \forall\, s_T\in S$
    \Else
        \State Compute $v^{\pi^*}_t(s_t)$ for each $s_t\in S$ by
        \State $\qquad v^{\pi^*}_t(s_t) = \underset{a_t\in A}{\max} \bigg\{ u_t(s_t, a_t) + \delta\,\E^\pi_{s_t} \left[\left.v^{\pi^*}_{t + 1}(s_{t + 1})\right\vert\,\mathcal{I}_t\,\right] \bigg\}$
        \State and set
        \State $\qquad a^{\pi^*}_t(s_t) = \underset{a_t\in A}{\argmax} \bigg\{ u_t(s_t, a_t) + \delta\,\E^\pi_{s_t} \left[\left.v^{\pi^*}_{t + 1}(s_{t + 1})\right\vert\,\mathcal{I}_t\,\right] \bigg\}$.
    \EndIf
\EndFor
\vspace{0.3cm}\end{algorithmic}
\end{algorithm}\FloatBarrier