\documentclass[11pt,a4paper]{article}

\usepackage[T1]{fontenc}
\usepackage[latin1]{inputenc}
\usepackage{makeidx}
\usepackage{alltt}
\usepackage{times}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{chapterbib}
\usepackage{graphicx}
\usepackage{xspace}
\usepackage{caption}

\hyphenation{unique}
\hyphenation{effect}
\hyphenation{Stand-ard}
\hyphenation{libr-ary}
\hyphenation{direct-ory}
\hyphenation{state-ment}
\hyphenation{argu-ment}
\hyphenation{oper-ators}
\hyphenation{symb-olic}
\hyphenation{needs}
\hyphenation{GVARSLAST}
\hyphenation{ODE-SOLVE}
\hyphenation{hyper-geometric}
\hyphenation{equat-ion}
\hyphenation{equat-ions}
\hyphenation{OFF}
\hyphenation{Opt-ions}
\hyphenation{execu-tion}
\hyphenation{poly-nom-ials}
\hyphenation{func-t-ions}
\hyphenation{Inte-grals}
\hyphenation{Stutt-gart}
\hyphenation{rat-ional}
\hyphenation{irrat-ional}
\hyphenation{par-am-eter}

\setlength{\parindent}{0pt}

\setlength{\parskip}{6pt}

\setlength{\hfuzz}{10pt}  % don't complain about tiny overfull boxes
\setlength{\vfuzz}{1pt}

\renewcommand{\sloppy}{\tolerance=9999\relax%}
                       \setlength{\emergencystretch}{0.2\hsize}}

\tolerance=1000

\raggedbottom

\newlength{\reduceboxwidth}
\setlength{\reduceboxwidth}{4in}

\newlength{\redboxwidth}
\setlength{\redboxwidth}{3.5in}

\newlength{\rboxwidth}
\setlength{\rboxwidth}{2.6in}

\newcommand{\REDUCE}{REDUCE\xspace}
\newcommand{\RLISP}{RLISP\xspace}
\newcommand{\underscore}{\char`\_}
\newcommand{\ttindex}[1]{{\renewcommand{\_}{\protect\underscore}%
                          \renewcommand{\|}{\protect\textbar}%
                          \index{#1@\texttt{#1}}}}
\def\ttindextypehelper#1!#2\relax{\def\blubbera{#1}\def\blubberb{#2}}
\def\capitalizehelper#1#2\relax{\edef\blubba{\uppercase{#1}\lowercase{#2}}}
\def\capitalize#1{\capitalizehelper#1\relax}
\newcommand{\ttindextype}[2]{{\renewcommand{\_}{\protect\underscore}%
                              \renewcommand{\|}{\protect\textbar}%
                              \ttindextypehelper#2!\relax
                              \ifx\\\blubberb\\% true if \blubberb is empty
                                \index{#1 #2@\texttt{#1} #2}%
                              \else
                                \index{#1 \blubbera@\texttt{#1} \blubbera!\blubberb}%
                              \fi}}

\newenvironment{syntax}{%
    \list{}{\setlength{\leftmargin}{1em}\setlength{\listparindent}{0em}}%
    \item\relax\math\displaystyle
    \catcode`\|=\active
}{\endmath\endlist}

\newenvironment{syntaxtable}{
    \syntax
    \begin{array}{lcl}%
}{\end{array}\endsyntax}

\newcommand{\meta}[1]{\mbox{$\langle$\textit{#1}$\rangle$}}

\title{{\bf Rational Approximations Package for REDUCE}}
\author{Lisa Temme\\Wolfram Koepf\\ e-mail: {\tt koepf@zib.de}\\\& Alan Barnes\\ email: {\tt Alan.Barnes45678@gmail.com}}
\date{August 1995 \& February 2019}
\begin{document}
\maketitle

\section{Periodic Decimal Representation}

\index{Periodic decimal representation}
The division of one integer by another often results in
a period in the decimal part. The \texttt{rational2periodic}
function in this package can recognise and represent
such an answer in a periodic representation. The inverse
function, \texttt{periodic2rational}, converts a
periodic representation back to a rational number.

\ttindextype{RATIONAL2PERIODIC}{operator}
\ttindextype{PERIODIC2RATIONAL}{operator}
\ttindextype{PERIODIC}{operator}
\begin{tabbing}
\textbf{\underline{Periodic Representation of a Rational Number}}\\[\baselineskip]

\textbf{SYNTAX:} \hspace{3mm}
       \= \texttt{rational2periodic(n);}\\
\> {\tt rational2periodic(n, b);}\\[\baselineskip]

\textbf{INPUT:}
\> \texttt{n} \hspace{3mm} is a rational number\\
\> {\tt b} \hspace{3mm} is the number base, if absent the default is 10.\\[\baselineskip]

\textbf{RESULT:}
     \> {\tt periodic(\{a1,...,an\},\{b1,...,bm\},\{c1,...,ck\},$\pm{\rm b}$)}\\
 \> where  {\tt\{a1,...,an\}} is a list of the digits in the integer part,\\
 \> {\tt\{b1,...,bm\}} is a list of the digits in the non-periodic part,\\
 \> {\tt\{c1,...,ck\}} is a list of the digits in the periodic part\\
 \> and $\pm${\tt b} where {\tt b} is the number base $2 \leq {\rm b} \leq 16$, \\
 \> a minus indicating the rational number {\tt n} was negative. \\[\baselineskip]

\textbf{EXAMPLES:}\\
    \> $-59/70$ written as $-0.8\overline{428571}$\\
    \>  \texttt{1: rational2periodic(-59/70);}\\
    \> \texttt{ periodic(\{0\}, \{8\}, \{4,2,8,5,7,1\}, -10)}\\[\baselineskip]
    \> $1/80$ written as a hexadecimal is $0.0\overline{3}$\\
    \>  \texttt{2: rational2periodic(1/80,16);}\\
    \> \texttt{ periodic(\{0\}, \{0\}, \{3\}, 16)}\\[\baselineskip]

    Normally the operator {\tt periodic} will not be seen as the output will
    be prettyprinted\\
    as $-0.8\overline{428571}$ and $0.0\overline{3}\ {\rm (base\ 16)}$
    respectively. Currently pretty-printed output \\
    looks better when the switch \texttt{FANCY} is \texttt{OFF}.\\
\end{tabbing}
\begin{tabbing}
  \textbf{\underline{Rational Number of a Periodic Representation}}\\[\baselineskip]
\textbf{SYNTAX:}\\
\hspace{3mm} \= \texttt{periodic2rational(periodic(\{a1,...,an\},\{b1...bm\},\{c1,...,ck\},$\pm{\rm b}$)}\\
   \> \texttt{periodic2rational(\{a1,...,an\},\{b1...bm\},\{c1,...,ck\},$\pm{\rm b}$)}\\[\baselineskip]

\textbf{INPUT:}\\
     \> \texttt{\{a1,...,an\}} is a list of the digits in the integer part,\\
     \> \texttt{\{b1,...,bm\}} is a list of the digits in the non-periodic part,\\
     \>\texttt{\{c1,...,ck\}} is a list of the digits in the periodic part\\
     \> and {\tt b} is the number base $2 \leq {\rm b} \leq 16$, a minus\\
     \> indicating the rational number result should be negative. \\
     \> If the base is omitted, 10 is assumed. \\[\baselineskip]

\textbf{RESULT:}\\
     \> A rational number.\\[\baselineskip]

\textbf{EXAMPLES:}\\[\baselineskip]
\hspace{10mm} \= $0.8\overline{428571}$ written as $59/70$ \\
    \> \texttt{3: periodic2rational(periodic(\{0\},\{8\},\{4,2,8,5,7,1\}));}
\\[\baselineskip]
    \> \hspace{1mm} {\tt 59}\\
    \> {\tt ----}\\
    \> \hspace{1mm} {\tt 70}\\[\baselineskip]
    \> \texttt{4: periodic2rational(\{0\},\{8\},\{4,2,8,5,7,1\}, -10);}
\\[\baselineskip]
    \> \hspace{4mm} \texttt{59}\\
    \> \texttt{- ----}\\
    \> \hspace{4mm} \texttt{70}
\end{tabbing}

Note that \texttt{periodic2rational} will produce the correct rational result
when passed a parameter for the periodic part which is not minimal.
Similarly, a parameter for the periodic part which consists of all 9's
(or in base $b$, all $(b-1)$'s) is treated correctly although such periodic
parts are not canonical and are never generated by calls to
\texttt{rational2periodic}.

\begin{tabbing}
For example,\\
\hspace{10mm} \= \texttt{periodic2rational(\{0\}, \{\}, \{1, 2, 1, 2\});} \\
    \> \texttt{periodic2rational(\{0\}, \{1\}, \{2, 1\});} \\
    \> \texttt{periodic2rational(\{0\}, \{1, 2\}, \{1, 2, 1, 2\});} \\[\baselineskip]
all produce the same rational result, namely $\frac{4}{33}$, as the canonical input\\
    \> \texttt{periodic2rational(\{0\}, \{\}, \{1, 2\});}\\[\baselineskip]
Similarly,\\
    \> \texttt{periodic2rational(\{0\}, \{\}, \{9\});} \\
    \> \texttt{periodic2rational(\{0\}, \{9\}, \{9\});} \\
    \> \texttt{periodic2rational(\{0\}, \{\}, \{9, 9, 9, 9\});} \\
all produce the same rational result, namely $1$, as the  canonical input\\
    \> \texttt{periodic2rational(\{1\}, \{\}, \{\});}
\end{tabbing}

Although the operators \texttt{periodic2rational} and \texttt{rational2periodic} work even when \texttt{ROUNDED} is \texttt{ON}, they
are best used when \texttt{ROUNDED} is \texttt{OFF}. The input
to \texttt{rational2periodic} should not be a rounded number, otherwise an error results.

For example, the input \texttt{rational2periodic(1/7);} will produce the
intended periodic representation even with \texttt{ROUNDED ON}. However,
 the input

\texttt{a := 1/7; rational2periodic(a);}

will result in an error as the simplifier is applied in the assignment and
rounds the rational number.

Similarly, although the result of \texttt{periodic2rational} will always be
a rational number (represented by a \texttt{QUOTIENT} prefix form), if the
simplifier is applied to the result a rounded value will be produced.

%
%
%
%\newpage
\section{Continued Fractions}

\index{Continued fractions}
A continued fraction (see ~\cite{CF}) has the general form
{\Large
\[a_0 + \frac{a_1}{b_1 +
         \frac{a_2}{b_2+
          \frac{a_3}{b_3 + \ldots
        }}}
\;.\]
}

A more compact way of writing this is as
\[a_0 + \frac{a_1|}{|b_1} + \frac{a_2|}{|b_2} + \frac{a_3|}{|b_3} + \ldots\,.\]
\\
Even more succinctly:
\[\{a_0,\ \{a_1, b_1\},\ \{a_2, b_2\},\ \ldots\}\]

%
\ttindextype{CONTFRAC}{operator}
This is represented in {\small REDUCE} as
\[{\tt
  contfrac(\textsl{Expression},
    \textsl{Rational approximant},
                \{a0, \{a1,b1\}, \{a2,b2\},.....\})
}\]

\ttindextype{CFRAC}{operator}
The operator \texttt{CFRAC} is used to generate a generalised continued
fraction expansion of an algebraic expression.\\
\begin{syntaxtable}
  \texttt{cfrac(}\meta{num}\texttt{)} \\
  \texttt{cfrac(}\meta{num}\texttt{,}\meta{length}\texttt{)}\\
  \texttt{cfrac(}\meta{func}\texttt{,}\meta{var}\texttt{)}\\
  \texttt{cfrac(}\meta{func}\texttt{,}\meta{var}\texttt{,}
  \meta{length}\texttt{)}
\end{syntaxtable}

\begin{tabbing}

\textbf{INPUT:}\hspace{5mm}
\= \meta{num} \hspace{3mm} \= \ is any real number\\
\> \meta{func}                   \> \ is a function\\
\> \meta{var}                 \> \ is the function main variable\\
\> \meta{length}     \> \ is the maximum number of terms (continuents)\\
\>    \> \ to be generated and is \textbf{optional}.
\end{tabbing}

For non-rational function or irrational number input the \meta{length}
argument specifies the number of continuents (ordered pairs, $\{a_i,b_i\}$),
to be returned. Its default value is five.
For rational function or rational number input the
\texttt{length} argument can only truncate the answer, it cannot
return additional pairs even if the precision is increased.
The default for rational function or rational number input is the
complete continued fraction.

\ttindextype{CF\_TAYLOR}{switch}

For a non-rational function, power series expansion is necessary. The new
switch \texttt{cf\_taylor} controls whether the TAYLOR or the TPS package is
used to produce the power series required. By default this switch is OFF and
so the TPS package is normally employed.
In most cases the choice is not important, but the TPS option is somewhat
better at handling cases where the series expansion is rather sparse.
In a few cases TPS may fail to produce a series expansion when TAYLOR succeeds
and vice-versa.

For numerical input the default value is exact for
rational number arguments whilst for irrational or rounded input it is
dependent on the precision of the session. The
\texttt{length} argument will only take effect if is smaller
than the number of ordered pairs which the default value would return.

If the number of continuent pairs returned does not exceed
twelve, the result will usually be pretty-printed as a two element list
consisting of the convergent followed by a rendering of the traditional
continued fraction expansion. For a larger number of pairs the output is
of the second element is printed as a list of pairs. Thus, usually the
operator \texttt{contfrac} is not seen in the output.

%\newpage
\large{\textbf{EXAMPLES}}
\begin{verbatim}
cfrac(pi, 4);

     355             1
{pi,-----,3 + ----------------}
     113               1
               7 + ----------
                          1
                    15 + ---
                          1

cfrac(sqrt 2, 5);

          41                1
{sqrt(2),----,1 + ---------------------}
          29                  1
                   2 + ---------------
                                1
                        2 + ---------
                                  1
                             2 + ---
                                  2

cfrac(23.696, 4);

  2962   237              1
{------,-----,23 + ---------------}
  125    10                 1
                    1 + ---------
                              1
                         2 + ---
                              3

cfrac((x+2/3)^2/(6*x-5), x, 10);

     2
  9*x  + 12*x + 4
{-----------------,   exact,
     54*x - 45

   6*x + 13          1
 ---------- + -------------}
     36         24*x - 20
               -----------
                    9

cfrac(e^x, x);

       3      2
  x   x  + 9*x  + 36*x + 60
{e , -----------------------,
           2
        3*x  - 24*x + 60

                  x
 1 + ---------------------------}
                    x
      1 - ---------------------
                      x
           2 + ---------------
                        x
                3 - ---------
                          x
                     2 + ---
                          5
\end{verbatim}

The operator \texttt{CF} \ttindextype{CF}{operator} is a
synonym for the operator \texttt{CONTINUED\_FRACTION}.
\begin{syntaxtable}
  \texttt{cf(}\meta{num}\texttt{)} \hspace{5mm}\\
  \texttt{cf(}\meta{num}\texttt{,}\meta{size}\texttt{)} \\
  \texttt{cf(}\meta{num}\texttt{,}\meta{size}
  \texttt{,}\meta{numterms}\texttt{)}
\end{syntaxtable}

The meaning of the arguments is the same as for the operator
\texttt{CONTINUED\_FRACTION}: the original number to be expanded
\meta{num}, an optional maximum size \meta{size} permitted for the
denominator of the convergent and an optional maximum number of continuents
\meta{numterms} to be generated.

The output is in the same format as that of \texttt{CFRAC} described above.
As with the operator \texttt{CFRAC} output of \texttt{CF} is normally
pretty-printed so the operator \texttt{confract} will not be seen.

The accessor operators \texttt{CF\_EXPRESSION},
\ttindextype{CF\_EXPRESSION}{operator}
\texttt{CF\_CONVERGENT} \ttindextype{CF\_CONVERGENT}{operator} and \\
\texttt{CF\_CONTINUENTS} \ttindextype{CF\_CONTINUENTS}{operator} allow
the various parts of a continued fraction object \meta{cf\_object}
(as returned by any of the operators \texttt{cf}, \texttt{cfrac},
\texttt{continued\_fraction} and \texttt{cf\_euler}) to be extracted.
\ttindextype{CF\_EULER}{operator}

These three operators return, respectively, the originating
expression of the continued fraction object, the last convergent of the
continued fraction, a list of its continuents
(that is a list of pairs of partial numerators and denominators).

The operator \texttt{CF\_CONVERGENTS} \ttindextype{CF\_CONVERGENTS}{operator}
returns a list of all the convergents of the expansion.
\begin{syntaxtable}
  \texttt{cf\_expression(}\meta{cf\_object}\texttt{)} \\
  \texttt{cf\_convergent(}\meta{cf\_object}\texttt{)} \\
  \texttt{cf\_continuents(}\meta{cf\_object}\texttt{)} \\
  \texttt{cf\_convergents(}\meta{cf\_object}\texttt{)}
\end{syntaxtable}

\large{\textbf{EXAMPLES}}
\begin{verbatim}
2: cf(6/11);

  6    6          1
{----,----,---------------}
  11   11           1
            1 + ---------
                      1
                 1 + ---
                      5

3: a := cf(pi,1000);

          355             1
a := {pi,-----,3 + ----------------}
          113               1
                    7 + ----------
                               1
                         15 + ---
                               1

4: cf_convergents a;

    22   333   355
{3,----,-----,-----}
    7    106   113

5: cf_continuents a;

{3,7,15,1}

6: precision 20;

12

7: cf pi;

     21053343141
{pi,-------------,{3,7,15,1,292,1,1,1,2,1,3,1,14,2,1,1,2,2,2,2,1}}
     6701487259
\end{verbatim}

\ttindextype{CF\_EULER}{operator}
The operator \texttt{CF\_EULER} is used to generate a generalised
continued fraction expansion of an algebraic expression using a formula due to
Leonhard Euler (see ~\cite{LE})).
\begin{syntaxtable}
 \texttt{cf\_euler(}\meta{func}\texttt{,}\meta{var}\texttt{)}\\
 \texttt{cf\_euler(}\meta{func}\texttt{,}\meta{var}\texttt{,}
  \meta{length}\texttt{)}
\end{syntaxtable}

\begin{tabbing}
\textbf{INPUT:}\hspace{5mm}
\= \meta{func} \hspace{15mm} \= \ is a function\\
\> \meta{var}                 \> \ is the function main variable\\
\> \meta{length}     \> \ is the maximum number of continuents\\
\>    \> \ to be generated and is \textbf{optional}.
\end{tabbing}

The meaning of the parameters is similar to those of \texttt{CFRAC}, but the
continued fraction expansion generated will usually be different. Note that
unlike \texttt{CFRAC}, \texttt{CF\_EULER} cannot currently generate
continued fraction expansion of numbers and for a rational function argument
(with a non-constant denominator) the expansion will not be exact.

\ttindextype{CF\_UNIT\_NUMERATORS}{operator}
\ttindextype{SCF_UNIT_DENOMINATORS}{operator}
\ttindextype{CF\_REMOVE\_FRACTIONS}{operator}
\ttindextype{CF\_REMOVE\_CONSTANT}{operator}
A number of operators are provided for transforming their continued
fraction argument \meta{cf\_object} into an equivalent expansion,
that is one with exactly the same convergents.
They all accept as their single argument any continued fraction
object \meta{cf\_object}.
These are:\\
\begin{tabbing}
  \= \texttt{cf\_unit\_denominators}
  \hspace{10mm} \= converts all partial denominators to 1. \\
  \> \texttt{cf\_unit\_numerators}
  \> converts all partial numerators to 1. \\
  \> \texttt{cf\_remove\_fractions}
  \> converts the denominators of the partial\\
  \> \ \    \> numerators and partial denominators in\\
  \>  \> the continuents to 1.\\
  \> \texttt{cf\_remove\_constant}
  \> removes the zeroth continuent (if non-zero)\\
  \> \ \  \>  absorbing it into the first continuent pair.\\
\end{tabbing}

\ttindextype{CF\_TRANSFORM}{operator}
The operator \texttt{CF\_TRANSFORM} is a general purpose function
for transforming its continued fraction argument \meta{cf\_object} into
an equivalent expansion. Unlike the four preceding operators it requires
a second argument: a list of multipliers used to modify the partial
numerators and denominators of the original expansion.
\begin{syntaxtable}
  \texttt{cf\_transform(}\meta{cf\_object}\texttt{,}
  \meta{multiplier-list}\texttt{)}
\end{syntaxtable}
To understand the operation of \texttt{cf\_transform} consider first the
special case where \meta{multiplier-list} is a list of the
form $\{1, 1, \ldots, 1, l_n, 1, \ldots, 1\}$ whose nth element is $l_n$.
Only the nth continuent pair $\{a_n,\ b_n\}$ and (n+1)th partial numerator
$a_{n+1}$ are altered and become $\{l_na_n,\ l_nb_n\}$ and $l_na_{n+1}$
respectively. For a \meta{multiplier-list} that has more than one non-unit
element, the above transformations are applied sequentially from left to
right.

If the number of continuent pairs in the \meta{cf\_object}
is greater than the length of the \meta{multiplier-list}, the latter is
(in effect) padded with 1's. Conversely if it is shorter, the surplus
elements of \meta{multiplier-list} are ignored.

\ttindextype{CF\_EVEN\_ODD}{operator}
The operator \texttt{CF\_EVEN\_ODD} splits its continued fraction argument
\meta{cf\_object} into two continued fraction objects: namely its even and
odd parts (in that order) which are returned as a two-element list.
\begin{syntaxtable}
  \texttt{cf\_even\_odd(}\meta{cf\_object}\texttt{)}
\end{syntaxtable}
The convergents of the even part are the even-numbered convergents of the
original expansion and those of the odd part are the odd-numbered ones
(except the zeroth convergent which is necessarily zero).  For the continued
fraction expansions generated by the operators \texttt{cf} and
\texttt{cfrac} with a numerical first argument \meta{num}. The convergents of
the even part form a monotonically increasing sequence whilst those of the
odd part (after the zeroth) form a monotonically decreasing sequence.

\large{\textbf{EXAMPLES}}
\begin{verbatim}
cf_remove_fractions(cf_euler(e^x, x, 4));

        3      2
  x    x  + 3*x  + 6*x + 6
{e , ---------------------,
              6

                   1
 -------------------------------------}
                     x
  1 - -------------------------------
                          x
       (x + 1) - -------------------
                              2*x
                  (x + 2) - -------
                             x + 3

a := cf_remove_fractions(cf_euler(4*atan x, x, 4));

a := {4*atan(x),

              7       5        3
        - 60*x  + 84*x  - 140*x  + 420*x
      -----------------------------------,
                      105

                               4*x
      -----------------------------------------------------}
                                  2
                                 x
       1 + -----------------------------------------------
                                          2
                 2                     9*x
            ( - x  + 3) + -------------------------------
                                                   2
                                  2            25*x
                           ( - 3*x  + 5) + -------------
                                                  2
                                             - 5*x  + 7

b := (a where x => 1);

          304            4
b := {pi,-----,----------------------}
          105              1
                1 + ----------------
                             9
                     2 + ----------
                               25
                          2 + ----
                               2

c := cf(pi, 0, 6);

          104348                    1
c := {pi,--------,3 + ------------------------------}
          33215                       1
                       7 + ------------------------
                                         1
                            15 + -----------------
                                           1
                                  1 + -----------
                                              1
                                       292 + ---
                                              1

cf_remove_constant c;

     104348                22
{pi,--------,-------------------------------}
     33215                    1
              7 + -------------------------
                                22
                   333 + -----------------
                                   1
                          1 + -----------
                                      1
                               292 + ---
                                      1

c:= cf(pi, 0, 8)$
d := cf_even_odd c;

           208341                15
d := {{pi,--------,3 + ----------------------},
           66317                   292
                        106 - --------------
                                       15
                               4687 - -----
                                       585

           312689             22
      {pi,--------,-------------------------}}
           99532                 1
                    7 + -------------------
                                   22
                         355 - -----------
                                       1
                                294 - ---
                                       3
cf_convergents c;

    22   333   355   103993   104348   208341   312689
{3,----,-----,-----,--------,--------,--------,--------}
    7    106   113   33102    33215    66317    99532

cf_convergents first d;

    333   103993   208341
{3,-----,--------,--------}
    106   33102    66317

cf_convergents second d;

    22   355   104348   312689
{0,----,-----,--------,--------}
    7    113   33215    99532
\end{verbatim}

%\newpage
\section{Pad\'{e} Approximation}
\index{Pad\'{e} Approximation}

The Pad\'{e} approximant represents a function by the ratio of two
polynomials (see ~\cite{PA} \S 4.2). The coefficients of the powers occuring
in the polynomials
are determined by the coefficients in the Taylor series
expansion of the function (see ~\cite{PA}). Given a power series
\[ f(x) = c_0 + c_1 (x-h) + c_2 (x-h)^2 \ldots \]
and the degree of numerator, $n$, and of the denominator, $d$,
the \texttt{pade} function finds the unique coefficients
$a_i,\, b_i$ in the Pad\'{e} approximant
\[ \frac{a_0+a_1 x+ \cdots + a_n x^n}{b_0+b_1 x+ \cdots + b_d x^d} \; .\]

\ttindextype{PADE}{operator}
\begin{tabbing}
\textbf{SYNTAX:} \hspace{5mm}\= \texttt{pade(f, x, h, n, d);}\\[\baselineskip]

\textbf{INPUT:}
\> \texttt{f} \hspace{3mm} \= the funtion to be approximated\\
\> \texttt{x}             \> the function variable\\
\> \texttt{h}             \> the point at which the approximation is evaluated\\
\> \texttt{n}             \> the (specified) degree of the numerator\\
\> \texttt{d}             \> the (specified) degree of the denominator\\[\baselineskip]
\textbf{RESULT:}
\> Pad\a'{e} Approximant, ie. a rational function.\\[\baselineskip]
\end{tabbing}
\textbf{ERROR MESSAGES:}\\
%
\texttt{***** not yet implemented}\\
%
The Taylor series expansion for the function, f, has not yet
been implemented in the {\small REDUCE} Taylor Package.\\[\baselineskip]
%
%
\texttt{***** no Pade Approximation exists}\\
%
A Pad\'{e} Approximant of this function does not exist.\\[\baselineskip]
%
%\newpage
\texttt{***** Pade Approximation of this order does not exist}\\
%
A Pad\'{e} Approximant of this order (ie. the specified
numerator and denominator orders) does not exist but one
of a different order may exist.\\[\baselineskip]
%
%
\large{\textbf{EXAMPLES}}

\begin{verbatim}

23: pade(sin(x),x,0,3,3);

          2
 x*( - 7*x  + 60)
------------------
       2
   3*(x  + 20)

24: pade(tanh(x),x,0,5,5);

     4        2
 x*(x  + 105*x  + 945)
-----------------------
      4       2
 15*(x  + 28*x  + 63)

25: pade(atan(x),x,0,5,5);

        4        2
 x*(64*x  + 735*x  + 945)
--------------------------
         4       2
 15*(15*x  + 70*x  + 63)

26: pade(exp(1/x),x,0,5,5);

***** no Pade Approximation exists

27: pade(factorial(x),x,1,3,3);

***** not yet implemented

28: pade(asech(x),x,0,3,3);

            2                        2                 2
- 3*log(x)*x  + 8*log(x) + 3*log(2)*x  - 8*log(2) + 2*x
--------------------------------------------------------
                          2
                       3*x  - 8

29: taylor(ws-asech(x),x,0,10);

               11
log(x)*(0 + O(x  ))

     13    6     43    8    1611    10      11
 + (-----*x  + ------*x  + -------*x   + O(x  ))
     768        2048        81920

30: pade(sin(x)/x^2,x,0,10,0);

***** Pade Approximation of this order does not exist

31:  pade(sin(x)/x^2,x,0,10,2);

     10        8         6           4            2
( - x   + 110*x  - 7920*x  + 332640*x  - 6652800*x

  + 39916800)/(39916800*x)

32: pade(exp(x),x,0,10,10);

  10        9         8           7            6
(x   + 110*x  + 5940*x  + 205920*x  + 5045040*x

              5               4                3
  + 90810720*x  + 1210809600*x  + 11762150400*x

                 2
  + 79394515200*x  + 335221286400*x + 670442572800)/

      10        9         8           7            6
    (x   - 110*x  + 5940*x  - 205920*x  + 5045040*x

                     5               4
         - 90810720*x  + 1210809600*x

                        3               2
         - 11762150400*x + 79394515200*x

         - 335221286400*x + 670442572800)

33: pade(sin(sqrt(x)),x,0,3,3);

(sqrt(x)*
            3            2
  (56447*x  - 4851504*x  + 132113520*x - 885487680))\

              3         2
    (7*(179*x  - 7200*x  - 2209680*x - 126498240))
\end{verbatim}

\begin{thebibliography}{9}
\bibitem{CF} Jones, W B.; Thron, W.J., \\
{\it Continued fractions. Analytic Theory and Applications},
 (Encyclopedia of Mathematics and its Applications, Vol 11),
Addison-Wesley Publishing Company, Reading, Massachusetts, 1980.
\bibitem{LE} L Euler L., {\it Introductio in analysin infinitorum, Vol 1,
 Chapter 18}, 1748.
\bibitem{PA} Baker(Jr.), G.A. and Graves-Morris, P.,\\
{\it Pad\'{e} Approximants, Part I: Basic Theory},
(Encyclopedia of Mathematics and its Applications, Vol 13),
Addison-Wesley Publishing Company, Reading, Massachusetts, 1981.
\end{thebibliography}

\end{document}


\documentclass[12pt]{article}
\usepackage[utf8]{inputenc}
\usepackage{float}
\usepackage{amsmath}

\usepackage[hmargin=3cm,vmargin=6.0cm]{geometry}
%\topmargin=0cm
\topmargin=-2cm
\addtolength{\textheight}{6.5cm}
\addtolength{\textwidth}{2.0cm}
%\setlength{\leftmargin}{-5cm}
\setlength{\oddsidemargin}{0.0cm}
\setlength{\evensidemargin}{0.0cm}

%misc libraries goes here
\usepackage{tikz}
\usetikzlibrary{automata,positioning}

\begin{document}

\section*{Student Information }
%Write your full name and id number between the colon and newline
%Put one empty space character after colon and before newline
Full Name : Yavuz Selim Yesilyurt \\
Id Number : 2259166 \\

% Write your answers below the section tags
\section*{Answer 1}

\subsection*{a.}
We can construct the bottom-up parser for $G$ as the following: \\

\begin{align*}
1-&((q_0,a,e),(q_0,a)) \\
2-&((q_0,b,e),(q_0,b)) \\
3-&((q_0,c,e),(q_0,S)) \\
4-&((q_0,e,e),(q_0,X)) \\
5-&((q_0,e,XbXSb),(q_0,S)) \\
6-&((q_0,e,XaXSa),(q_0,S)) \\
7-&((q_0,a,Xa),(q_0,X)) \\
8-&((q_0,a,Xb),(q_0,X)) \\
9-&((q_0,e,S),(q_1,e))
\end{align*}
\subsection*{b.}
\begin{table}[H]
	\small
	\centering
	\begin{tabular}{|c|c|c|c|c|}
	\hline
	$Step$ & $State$ & $Unread \ Input$ & $Stack$ & $Transition \ Used$\\
	\hline
	1 & $q_0$ & $abbcbabbaa$ & $e$ & -\\
	2 & $q_0$ & $bbcbabbaa$ & $a$ & 1\\
	3 & $q_0$ & $bcbabbaa$  & $ba$ & 2\\
	4 & $q_0$ & $cbabbaa$   & $bba$ & 2\\
	5 & $q_0$ & $babbaa$   & $Sbba$ & 3\\
	6 & $q_0$ & $babbaa$ & $XSbba$ & 4\\
	7 & $q_0$ & $abbaa$ & $bXSbba$ & 2\\
	8 & $q_0$ & $abbaa$  & $XbXSbba$ & 4\\
	9 & $q_0$ & $abbaa$   & $Sba$ & 5\\
	10 & $q_0$ & $bbaa$ & $aSba$ & 1\\
	11 & $q_0$ & $bbaa$ & $XaSba$ & 4\\
	12 & $q_0$ & $bbaa$  & $XSba$ & 7\\
	13 & $q_0$ & $baa$   & $bXSba$ & 2\\
	14 & $q_0$ & $baa$   & $XbXSba$ & 4\\
	15 & $q_0$ & $baa$ & $Sa$ & 5\\
	16 & $q_0$ & $aa$ & $bSa$ & 2\\
	17 & $q_0$ & $aa$  & $abSa$ & 1\\
	18 & $q_0$ & $a$   & $XabSa$ & 4\\
	19 & $q_0$ & $a$ & $XbSa$ & 7\\
	20 & $q_0$ & $a$ & $XSa$ & 8\\
	21 & $q_0$ & $a$  & $aXSa$ & 1\\
	22 & $q_0$ & $e$   & $XaXSa$ & 4\\
	23 & $q_0$ & $e$   & $S$ & 6\\
	24 & $q_1$ & $e$ & $e$ & 9\\
	\hline
	\end{tabular}
	\end{table}

\newpage
\section*{Answer 2}

\subsection*{a.}
We can define the Turing Machine which computes $f(x)$ as $M = (K,\Sigma,\delta_1, s, H)$, where
$K = \{s_0,odd,even,q_0,q_1,q_2,q_3,q_4,h\}$, $\Sigma = \{\sqcup,1,0,\triangleright\}$, $s = s_0$, $H = \{h\}
$ and the transition function is defined as follows;
\begin{align*}
&\delta_1(s_0,\sqcup) = (odd,\rightarrow) &&\delta_1(q_1,0) = (q_2,\rightarrow)\\
&\delta_1(odd,1) = (even,\rightarrow) &&\delta_1(q_2,1) = (q_2,0)\\
&\delta_1(odd,\sqcup) = (q_0,\leftarrow) &&\delta_1(q_2,0) = (q_3,\rightarrow) \\
&\delta_1(even,1) = (odd,\rightarrow) &&\delta_1(q_3,1) = (q_3,\rightarrow)\\
&\delta_1(even,\sqcup) = (h,1) &&\delta_1(q_3,\sqcup) = (q_0,\leftarrow) \\
&\delta_1(q_0,1) = (q_0,\sqcup) &&\delta_1(q_4,0) = (q_4,1) \\
&\delta_1(q_0,\sqcup) = (q_1,\leftarrow) &&\delta_1(q_4,1) = (q_4,\leftarrow)\\
&\delta_1(q_0,0) = (q_4,1) &&\delta_1(q_4,\sqcup) = (h,\sqcup)\\
&\delta_1(q_1,1) = (q_1,\leftarrow) \\
&\delta_1(q_1,\sqcup) = (q_2,\rightarrow)
\end{align*}
Note that the transition function also includes the functions of the form $\delta_1(y,\triangleright)
= (s_1, \rightarrow)$ for every $y\in K_1-H_1$ trivially.
The machine defined above performs like this;\\\\
First, it goes right to find a 1. It keeps going right with reading 1 until it read a blank symbol. After reading blank symbol if machine is in the even state then it makes that blank symbol 1 and halts, but if the machine is in the odd state it goes one left to get the last 1, then it makes that 1 a blank symbol and keeps going left until it reads a blank symbol (actually it reaches to the blank symbol at the start), then it goes one right to access the first 1. After reaching the first 1 it replaces it with a 0 and again it goes to the last 1 to its right and again it replaces the last 1 with a blank symbol and comes to the first 1 again and replaces it with a 0. The machine keeps doing that until it sees blank symbol next to 0. After seeing this it just replaces all the 0's to its right to 1's and when it reaches to the blank symbol at first it halts.

\section*{Answer 3}
The set of languages decided by a semi-infinite tape, Move-restricted Turing Machines is equivalent to set of languages recognized by Finite Automatas, since the Move-restricted Turing Machines have neither the ability of moving its head left nor the ability of storing the information given by input string (stack behaviour). It can only read the input word from left to right and any modifications it would make onto input string would also be intraceable since it can not go back and inspect the value there and this is similar to behaviour of Finite Automatas, they also cant keep track of the memory of the word and cant read other parts of word. Therefore, it can be inferred that the class of languages decided by them is actually equivalent to class of languages recognized by Finite Automatas.

\section*{Answer 4}

\subsection*{a.}
We can define the Queue-based deterministic Turing Machine as $M = (K,\Sigma,\delta_1, s, H)$, where
$K = \{s_0,q_0,enqueue,dequeue,h\}$, $\Sigma = \{\sqcup,a,b,\triangleright\}$, $s = s_0$, $H = \{h\}
$ and the transition function is defined as follows (Note that this is just a general definition, for distinct languages, these definitions-K,$\Sigma$,s,H,$\delta$- may differ):
\begin{align*}
&\delta_1(s_0,input \ symbol) = (q_0,front \ symbol,rear \ symbol)\\
&\delta(q_0,X) = (dequeue,\sqcup,rear \ symbol) \\
&\delta(dequeue,input \ symbol) = (q_0,\rightarrow,rear \ symbol) \\
&\delta(q_0,a) = (enqueue,front \ symbol,\rightarrow) \\
&\delta(enqueue,a) = (q_0,front symbol,a) \\
&\delta(q_0,b) = (enqueue,front \ symbol,\rightarrow) \\
&\delta(enqueue,b) = (q_0,front symbol,b) \\
&\delta(q_0,\sqcup) = (h,front \ symbol,rear \ symbol)
\end{align*}

where $front \ symbol$ and $rear \ symbol$ $\in \Sigma$ and indicates what is under the front and rear heads at that moment respectively. \\

Note that we assume initially tape of the machine is filled with some string already stored inside and it has 2 heads front and rear, initially pointing to the start and the end of the word inside respectively. As the machine reads input instructions, it updates its content and heads accordingly. Machine has a main state $q_0$ and if the input instruction gives an X symbol, then machine will go to dequeue state and it will fulfill dequeue operation and return to the main state $q_0$. If the input instruction gives a symbol from $\Sigma - \{\sqcup,\triangleright\}$, the machine will go to enqueue state and it will fulfill enqueue operation with given symbol word and return to the main state $q_0$. And, if input instruction gives a $\sqcup$ symbol, the machine will halt. Also note that $Front$ and $Rear$ operations can be fulfilled through directly reading the symbol under $Front$ and $Rear$ heads at that moment.

\subsection*{b.}
Notion of configuration of Queue-based deterministic Turing Machine can be written as: \\
\begin{center}
$K \times \{\Sigma - \{\triangleright ,\sqcup \} \} \times \{\Sigma - \{\triangleright ,\sqcup \} \}^* \times \{\Sigma - \{\triangleright ,\sqcup \} \}$
\end{center}
Note that it refers to: (Current State, Symbol Under Front Head, Everything To The Right of the Front Head, Symbol Under Rear Head)
\subsection*{c.}
Yields-in-one-step relation of the machine can be written as: \\
\begin{center}
$(q_1,a_1,w_1,b_1) \vdash_M (q_2,a_2,w_2,b_2)$ \\
where $a_1,a_2,b_1,b_2\in \Sigma$ , and $\delta(q_1,input \ symbol) = (q_2,a_2,b_2)$
\end{center}

\subsection*{d.}
Church-Turing Thesis states that all computational devices-machines have the same power regarding computability. Any language, that is decided, semi-decided or any function that is computed by Queue-based deterministic Turing Machine can be decided, semi-decided or computed by a standard Turing Machine. To prove, we can mimick the transitions and operations that can be done in Queue-based TM, in standard TM as: \\

For front operation, we can give the first element of the string on the tape by going left until the reading head reads the start symbol or a blank symbol and then going one right, for rear operation, we can give the last element of the string on the tape by going right until the reading head reads a blank symbol and then going one left, For enqueue operation, we can operate rear operation first and go one more right and replace the blank symbol by the symbol that is needed to be enqueued, For dequeue operation, we can operate front operation first and replace the symbol under the reading head with a blank symbol and going one right.

\subsection*{e.}
We can define the Queue-based deterministic Turing Machine for $L = \{wcw : w \in \{a,b\}^*\}$ as $M = (K,\Sigma,\delta, s, H)$, where
$K = \{s_0,enqueue,dequeue,h\}$, $\Sigma = \{\sqcup,a,b,c,\triangleright\}$, $s = s_0$, $H = \{h\}
$ and the transition function is defined as follows;
\begin{align*}
&\delta(s_0,a,\triangleright,\triangleright) = (enqueue,a,\rightarrow,\rightarrow)\\
&\delta(s_0,b,\triangleright,\triangleright) = (enqueue,b,\rightarrow,\rightarrow)\\
&\delta(enqueue,a,\sqcup,\sqcup) = (enqueue,\rightarrow,a,\rightarrow)\\
&\delta(enqueue,b,\sqcup,\sqcup) = (enqueue,\rightarrow,b,\rightarrow)\\
&\delta(enqueue,a,front \ symbol,rear \ symbol) = (enqueue,\rightarrow,front \ symbol,\rightarrow)\\
&\delta(enqueue,b,front \ symbol,rear \ symbol) = (enqueue,\rightarrow,front \ symbol,\rightarrow)\\
&\delta(enqueue,a,front \ symbol,\sqcup) = (enqueue,\rightarrow,front \ symbol,a)\\
&\delta(enqueue,b,front \ symbol,\sqcup) = (enqueue,\rightarrow,front \ symbol,b)\\
&\delta(enqueue,c,front \ symbol,rear \ symbol) = (dequeue,\rightarrow,front \ symbol,rear \ symbol)\\
&\delta(dequeue,a,a,rear \ symbol) = (dequeue,\rightarrow,\sqcup,rear \ symbol)\\
&\delta(dequeue,b,b,rear \ symbol) = (dequeue,\rightarrow,\sqcup,rear \ symbol)\\
&\delta(dequeue,a,\sqcup,rear \ symbol) = (dequeue,\rightarrow,\rightarrow,rear \ symbol)\\
&\delta(dequeue,b,\sqcup,rear \ symbol) = (dequeue,\rightarrow,\rightarrow,rear \ symbol)\\
&\delta(dequeue,\sqcup,\sqcup,\sqcup) = (h,\sqcup,\sqcup,\sqcup) \\
\end{align*}

where $front \ symbol$ and $rear \ symbol$ $\in \Sigma$ and indicates what is under the front and rear heads at that moment respectively. \\

Note that initially front and rear heads are pointing to the start symbol. Firstly, as the Machine reads input words, it enqueues each of them until it reads a c symbol and updates its heads and content accordingly. After reading a c, it will go to dequeue state and as it reads the other half of the input, it will check if the symbol under the front head and input matches and if they match it will dequeue the and update its front head accordingly. When machine reads a blank symbol it will check if the symbol under its front and rear head is a blank symbol (i.e. the heads point the same symbol and tape is empty) and if it is, then the machine will halt.

\section*{Answer 5}

\subsection*{a.}

\begin{center}
\begin{tikzpicture}[scale=0.2]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (6.3,-20) circle (3);
\draw (6.3,-20) node {$R$};
\draw [black] (14.8,-13.5) circle (3);
\draw (14.8,-13.5) node {$d$};
\draw [black] (23.9,-13.5) circle (3);
\draw (23.9,-13.5) node {$R$};
\draw [black] (32.5,-13.5) circle (3);
\draw (32.5,-13.5) node {$x$};
\draw [black] (41.8,-13.5) circle (3);
\draw (41.8,-13.5) node {$R$};
\draw [black] (51.7,-13.5) circle (3);
\draw (51.7,-13.5) node {$L$};
\draw [black] (60.5,-13.5) circle (3);
\draw (60.5,-13.5) node {$d$};
\draw [black] (68.5,-13.5) circle (3);
\draw (68.5,-13.5) node {$L_x$};
\draw [black] (76.3,-13.5) circle (3);
\draw (76.3,-13.5) node {$R$};
\draw [black] (33.7,-36.1) circle (3);
\draw (33.7,-36.1) node {$n$};
\draw [black] (74.7,-26.6) circle (3);
\draw (74.7,-26.6) node {$L_{\bar{x}}$};
\draw [black] (63.4,-26.6) circle (3);
\draw (63.4,-26.6) node {$R$};
\draw [black] (53,-26.6) circle (3);
\draw (53,-26.6) node {$b$};
\draw [black] (74.7,-36.1) circle (3);
\draw (74.7,-36.1) node {$d$};
\draw [black] (64.2,-36.1) circle (3);
\draw (64.2,-36.1) node {$R$};
\draw [black] (74.7,-46.3) circle (3);
\draw (74.7,-46.3) node {$d$};
\draw [black] (64.2,-46.3) circle (3);
\draw (64.2,-46.3) node {$R$};
\draw [black] (51.7,-54.4) circle (3);
\draw (51.7,-54.4) node {$d$};
\draw [black] (6.3,-54.4) circle (3);
\draw (6.3,-54.4) node {$L_{\sqcup}$};
\draw [black] (15.5,-39.2) circle (3);
\draw (15.5,-39.2) node {$R$};
\draw [black] (23.3,-48.6) circle (3);
\draw (23.3,-48.6) node {$y$};
\draw [black] (4.3,-16.2) -- (4.9,-17.35);
\draw (2.09,-15.69) node [above] {$start$};
\fill [black] (4.9,-17.35) -- (4.97,-16.4) -- (4.09,-16.87);
\draw [black] (8.68,-18.18) -- (12.42,-15.32);
\fill [black] (12.42,-15.32) -- (11.48,-15.41) -- (12.09,-16.21);
\draw (9.6,-16.25) node [above] {$a$};
\draw [black] (17.8,-13.5) -- (20.9,-13.5);
\fill [black] (20.9,-13.5) -- (20.1,-13) -- (20.1,-14);
\draw [black] (22.577,-10.82) arc (234:-54:2.25);
\draw (23.9,-6.25) node [above] {$a,d$};
\fill [black] (25.22,-10.82) -- (26.1,-10.47) -- (25.29,-9.88);
\draw [black] (26.9,-13.5) -- (29.5,-13.5);
\fill [black] (29.5,-13.5) -- (28.7,-13) -- (28.7,-14);
\draw (28.2,-14) node [below] {$b$};
\draw [black] (35.5,-13.5) -- (38.8,-13.5);
\fill [black] (38.8,-13.5) -- (38,-13) -- (38,-14);
\draw [black] (44.214,-15.261) arc (81.6215:-206.3785:2.25);
\draw (45.92,-19.98) node [below] {$b$};
\fill [black] (41.87,-16.49) -- (41.26,-17.21) -- (42.25,-17.35);
\draw [black] (40.79,-16.32) -- (34.71,-33.28);
\fill [black] (34.71,-33.28) -- (35.45,-32.69) -- (34.51,-32.35);
\draw (38.51,-25.57) node [right] {$a,\sqcup$};
\draw [black] (25.09,-16.25) -- (32.51,-33.35);
\fill [black] (32.51,-33.35) -- (32.65,-32.41) -- (31.73,-32.81);
\draw (28.07,-25.77) node [left] {$c,\sqcup$};
\draw [black] (44.8,-13.5) -- (48.7,-13.5);
\fill [black] (48.7,-13.5) -- (47.9,-13) -- (47.9,-14);
\draw (46.75,-13) node [above] {$c,d$};
\draw [black] (54.7,-13.5) -- (57.5,-13.5);
\fill [black] (57.5,-13.5) -- (56.7,-13) -- (56.7,-14);
\draw [black] (63.5,-13.5) -- (65.5,-13.5);
\fill [black] (65.5,-13.5) -- (64.7,-13) -- (64.7,-14);
\draw [black] (71.5,-13.5) -- (73.3,-13.5);
\fill [black] (73.3,-13.5) -- (72.5,-13) -- (72.5,-14);
\draw [black] (34.816,-11.595) arc (126.80351:53.19649:32.69);
\fill [black] (34.82,-11.6) -- (35.76,-11.52) -- (35.16,-10.72);
\draw (54.4,-4.58) node [above] {$b$};
\draw [black] (75.94,-16.48) -- (75.06,-23.62);
\fill [black] (75.06,-23.62) -- (75.66,-22.89) -- (74.66,-22.77);
\draw (74.83,-19.92) node [left] {$d$};
\draw [black] (71.7,-26.6) -- (66.4,-26.6);
\fill [black] (66.4,-26.6) -- (67.2,-27.1) -- (67.2,-26.1);
\draw [black] (55.693,-25.31) arc (106.09686:73.90314:9.042);
\fill [black] (55.69,-25.31) -- (56.6,-25.57) -- (56.32,-24.61);
\draw (58.2,-24.46) node [above] {$x$};
\draw [black] (60.543,-27.492) arc (-79.39967:-100.60033:12.737);
\fill [black] (60.54,-27.49) -- (59.66,-27.15) -- (59.85,-28.13);
\draw [black] (61.033,-28.442) arc (-54.19775:-90.3269:41.199);
\fill [black] (36.7,-36.23) -- (37.49,-36.73) -- (37.5,-35.73);
\draw (53.45,-34.96) node [below] {$a,b,\sqcup$};
\draw [black] (62.077,-23.92) arc (234:-54:2.25);
\draw (63.4,-19.35) node [above] {$d$};
\fill [black] (64.72,-23.92) -- (65.6,-23.57) -- (64.79,-22.98);
\draw [black] (65.7,-28.53) -- (72.4,-34.17);
\fill [black] (72.4,-34.17) -- (72.11,-33.27) -- (71.47,-34.04);
\draw (70,-30.86) node [above] {$c$};
\draw [black] (71.7,-36.1) -- (67.2,-36.1);
\fill [black] (67.2,-36.1) -- (68,-36.6) -- (68,-35.6);
\draw [black] (61.475,-37.352) arc (-67.90323:-112.09677:33.296);
\fill [black] (36.43,-37.35) -- (36.98,-38.12) -- (37.35,-37.19);
\draw (48.95,-40.3) node [below] {$a,b,\sqcup$};
\draw [black] (66.35,-38.19) -- (72.55,-44.21);
\fill [black] (72.55,-44.21) -- (72.32,-43.29) -- (71.63,-44.01);
\draw (70.41,-40.72) node [above] {$c$};
\draw [black] (71.7,-46.3) -- (67.2,-46.3);
\fill [black] (67.2,-46.3) -- (68,-46.8) -- (68,-45.8);
\draw [black] (61.263,-46.905) arc (-81.27787:-135.7047:29.491);
\fill [black] (35.68,-38.35) -- (35.88,-39.27) -- (36.6,-38.57);
\draw (43.37,-46.39) node [below] {$a,b,\sqcup$};
\draw [black] (62.341,-48.65) arc (-42.90766:-71.20587:18.855);
\fill [black] (54.6,-53.66) -- (55.52,-53.88) -- (55.2,-52.93);
\draw (59.73,-52.14) node [below] {$c$};
\draw [black] (48.7,-54.4) -- (9.3,-54.4);
\fill [black] (9.3,-54.4) -- (10.1,-54.9) -- (10.1,-53.9);
\draw [black] (6.3,-51.4) -- (6.3,-23);
\fill [black] (6.3,-23) -- (5.8,-23.8) -- (6.8,-23.8);
\draw [black] (5.487,-22.876) arc (11.95432:-276.04568:2.25);
\draw (0.77,-25.5) node [below] {$d$};
\fill [black] (3.52,-21.1) -- (2.64,-20.78) -- (2.84,-21.76);
\draw [black] (8.89,-21.52) -- (31.11,-34.58);
\fill [black] (31.11,-34.58) -- (30.68,-33.74) -- (30.17,-34.61);
\draw (16.2,-28.55) node [below] {$c,\sqcup$};
\draw [black] (7.6,-22.71) -- (14.2,-36.49);
\fill [black] (14.2,-36.49) -- (14.31,-35.56) -- (13.41,-35.99);
\draw (10.19,-30.67) node [left] {$b$};
\draw [black] (13.276,-41.196) arc (-20.35775:-308.35775:2.25);
\draw (8.23,-41.57) node [left] {$d$};
\fill [black] (12.56,-38.65) -- (11.99,-37.9) -- (11.64,-38.84);
\draw [black] (18.46,-38.7) -- (30.74,-36.6);
\fill [black] (30.74,-36.6) -- (29.87,-36.25) -- (30.04,-37.23);
\draw (27.14,-38.68) node [below] {$\bar{\sqcup}$};
\draw [black] (17.42,-41.51) -- (21.38,-46.29);
\fill [black] (21.38,-46.29) -- (21.26,-45.36) -- (20.49,-45.99);
\draw (18.85,-45.34) node [left] {$\sqcup$};
\end{tikzpicture}
\end{center}

\subsection*{b.}
Let $G = (V,\Sigma,R,S)$ be the grammar and $V = (a,b,c,S,X,R)$, $\Sigma = (a,b,c)$, $R \subseteq (V - \Sigma) \times V^*$, The rules are:

\begin{align*}
S &\rightarrow b | abbcccX \\
X &\rightarrow RX | e \\
cR &\rightarrow Rcccc \\
cR &\rightarrow Rc \\
bR &\rightarrow Rbb \\
aR &\rightarrow Raa \\
aR &\rightarrow Ra \\
R &\rightarrow e \\
\end{align*}

\section*{Answer 6}

\subsection*{a.}
With the information given to us, let's find out more about these 5 languages: \\

Since a \textbf{regular grammar} can generate all strings in $L_1$ we can deduce that $L_1$ is a regular language. Since a Finite Automata can recognize $L_1$, a proper $M_1$ can be written for $L_1$. \\

If a \textbf{deterministic top-down parser} can be built for CFG whose language is $L_2$ then we can say that $L_2$ is a deterministic context-free language. Since a Deterministic Push Down Automata can recognize $L_2$, a proper $M_2$ can be written for $L_2$.\\

$L_3$ is a context-free language since a (inherently ambiguous) context-free grammar can be provided for this language. Since a Push Down Automata can recognize $L_3$, a proper $M_3$ can be written for $L_3$.\\

Since a TM M decides the resulting language ($\bar{L_4}\cap \bar{L(a^*b^*)}$) we can say it is recursive. Since $\bar{L(a^*b^*)}$ is a regular language, for result to be recursive, left-handside ($\bar{L_4}$) must be recursive. Since recursive languages are closed under complementation, we can deduce that $L_4$ is a recursive language. Since it is recursive, it can be decided by some TM $M_4$.\\

We know that the class of languages generated by unrestricted grammars is equivalent to class of languages that is semi-decided by Turing Machines. Given an unrestricted grammar, a Turing machine is simple enough to construct, as a two-tape nondeterministic Turing machine. Since $L_5$ is also generated by an unrestricted grammar, we can deduce that it is a recursively enumerable language. Since it is recursively enumerable, it can be semi-decided by some TM $M_5$.\\

Therefore, yes, all 5 TM's for languages exists.

\subsection*{b.}
Church-Turing Thesis states that an algorithm corresponds to a TM that halts on every input. Also we know that, everything that can be computed has an algorithm. In part a we showed that, for these 5 languages corresponding TM's exists and they are all in the class of languages that is eventually decidable/semi-decidable. Therefore all of them can be generated by a (unrestricted) grammar, we can make the computation, so we  deduce that there always exists algorithms for these languages, they are computable.
\subsection*{c.}
Let's first decide whether $L$ is undecidable or not: \\

We know that $L_1$ is regular and $L_2$ is DCFL. Since DCFLs are closed under complementation $\bar{L_2}$ is also a DCFL. Since DCFLs are also closed under concatenation, $\bar{L_2}L_1$ is also a DCFL. We know that $L_5$ is recursively enumerable, since recursively enumerable languages are closed under intersection and Kleene Star $(\bar{L_2}L_1 \cap L_5)^*$ is also a recursively enumerable language. We know that $L_3$ is CFL and $L_4$ is recursive. Since CFLs are closed under Kleene Star $L_3^*$ is also a CFL and since recursive languages are closed under complementation $\bar{L_4}$ is also a recursive language. Since recursive languages are closed under concatenation $L_3^*\bar{L_4}$ is also recursive. Finally, since recursively enumerable languages are closed under union, $L = (\bar{L_2}L_1 \cap L_5)^* \cup L_3^*\bar{L_4}$ is also a recursively enumerable language. So $L$ is semi-decidable by some TM $M$. $M$ will accept a string, for given inputs, it also may not halt.
\subsection*{d.}
We can't. Because complement of recursively enumerable languages are not necessarily recursively enumerable, they may be undecidable. Let us justify that by reaching a contradiction for assuming that an undecidable language is a recursively enumerable language: \\

Let $H = \{"M""w"|\textit{TM M halts on input w}\}$, $H$ is semi-decided by Universal Turing Machine, therefore it is recursively enumerable. Suppose it is recursive, then there exists a TM $M_0$ that decides $H$, consider $H_1 = \{"M"|\textit{TM M halts on "M"}\}$, if $H$ is recursive, then $H_1$ is also recursive, if $H_1$ is recursive, then so does its complement, $\bar{H_1}=\{w|\textit{w is not encoding of a TM or}$ $\textit{does not halt on itself}\}$. There must be some TM $M^*$ that semi-decides $\bar{H_1}$ (if it is recursive, it also must be recursively enumerable). But at this point we reach a contradiction since, $"M^*"\in \bar{H_1}$ iff $M^*$ does halt on $"M^*"$ and also by definition of $\bar{H_1}$, $"M^*"\in \bar{H_1}$ iff $M^*$ does not halt on $"M^*"$. So we prove that $\bar{H_1}$ is not recursively enumerable, $H_1$ is not recursive and $H$ is not recursive, yet the only conclusion that we are interested is that $\bar{H_1}$ is not recursively enumerable and we can't come up with a TM for $\bar{H_1}$, which also proves the statement that "We always come up with a TM that semi-decides $\bar{L}$" is not true.

%Do not submit solutions for Question 7, yet do solve it.

\end{document}


\documentclass[a4paper,10pt]{article}
%\documentclass[a4paper,10pt]{scrartcl}
\usepackage{hyperref}
% \usepackage{caption}
\hypersetup{pdfborder={0 0 0}}
\usepackage[all]{hypcap}
\usepackage{default}
\usepackage{graphicx}
\usepackage{array}
\usepackage{amsmath}
\usepackage{listings}
\usepackage {apacite}
\lstset{language=Python,% general command to set parameter(s)
basicstyle =\small\ttfamily,          % print whole listing small
showstringspaces=false,
commentstyle=\ttfamily}

\title{Backpropagation in Feedforward Networks}
\author{Dr. Olivia Guest}
\date{\today}

\begin{document}
\maketitle
\section{Overview}
Last week we ran and trained a 2-layer perceptron. For reasons that became obvious (i.e., if no linear separation of the target states can be accomplished) 2-layer perceptrons are not as useful as one might initially assume. So we must turn to adding a 3rd layer of units between the input and the output units. This as you might expect is called the hidden layer because they are neither input nor output units --- their activations are not directly in contact with the environment the network is in. Adding hidden units produces a network called a 3-layer perceptron. As we shall see it can now be taught classification problems like exclusive-or which are not linearly separable.

Backpropagation in a feedforward network works by running the network \textbf{forwards} (as we did with the 2-layer perceptron) and then running the network \textbf{backwards}. This second phase is required because we need to calculate targets not only for the output units but also for the hidden units. Their targets are calculated based on the output units' activations and the output targets. Using these hidden unit targets we can then calculate the error of the hidden units and subsequently update the weights from the input units to the hidden units, as well as being able to update the weights from the hidden units to the output units. In other words, running the network backwards allows us to calculate errors for every single unit (apart from those in the input layer, which have no targets).  The errors for every unit are then used to calculate the connection weight updates, thus adjusting our network to conform to the required output states given an input.

\section{Forwards Phase: Propagation of Activations}
During the feedforward phase, each unit, apart from the input units, receives activations from units in the previous layer. Thus, the input to each unit, $i$, is:
\begin{equation}
\label{eq:ff_eta}
 \eta_{i}= \sum_{j} s_jw_{ji} + b_i
\end{equation}
where $b_i$ is the bias of unit $i$, and $s_j$ is the state of a unit $j$ that projects onto $i$.
% \cite{rumelhart86}.
When unit $i$ receives input $\eta_{i}$, its state changes to:
\begin{equation}
\label{eq:logistic}
  s_i = \frac{1}{1 + e^{-\eta_{i}}}
\end{equation}
as usual by means of the \textbf{logistic function} (previously we called the activation function $f$, when discussing the 2-layer perceptron, and we did not use the logistic function). The logistic function is the activation function that is most commonly used with backpropagation, although $tanh(\eta_i)$ (the hyperbolic tangent function) is also common.

In \autoref{fig:logistic}, we can see a graphical depiction of what the sigmoid activation function, in this case a logistic curve. The values on the $x$ axis represent the values that $\eta_i$ (the pre-synaptic input to unit $i$) takes on, while the $y$ axis represents the output of the activation function that will be the state of the unit: $s_i$ (the post-synaptic state).

\begin{enumerate}
 \item What does this function do to the pre-synaptic input $\eta_i$ (as calculated in \autoref{eq:ff_eta})?

\item What does a high value of $b_i$ do to unit $i$'s post-synaptic state? What about a low bias?

 \item How is this different to how we defined our activation function $f$ last week?

 \item What is the constant $e$ in \autoref{eq:logistic}?

 \item What are the differences and similarities between running the 3-layer and the 2-layer perceptrons?

 \item What type of flow control statement is appropriate for calculating the $\sum$ in \autoref{eq:ff_eta}?

 \item Fill in the \texttt{Feedforward()} function in the file \texttt{network\_missing.py} to reflect any changes when compared to the 2-layer network.

 \item How can you check that your code is correct? Is it?  Hint: Use a very simple input (target is irrelevant because you are not training yet) and calculate yourself what a few of the states of the network should be using the above equations. Do they match up with the code you have written when you propagate the pattern?

\end{enumerate}

\begin{figure}[hb]
 \centering
 \includegraphics[scale=.5]{fig/Logistic-curve.pdf}
 \caption{The shape of the logistic curve. }
 \label{fig:logistic}
\end{figure}

\section{Backwards Phase: Propagation of Error Signal}
\label{sub:backwards_phase}
After the feedforward phase, the network is run in reserve in order to gather the weight adjustments. Unit $i$ sends the error signal:
\begin{equation}
  \delta_i = s_i - t_i %self.output_units[i] - self.targets[p][i]
\label{eq:bp_error}
\end{equation}
where $t_i$ is the target state. If $i$ is an output unit $t_i$ is the pre-set target pattern. This metric for calculating error is called cross-entropy error, other types (e.g., mean squared error) can also be used.

% Notice how the pre- and post-synaptic states of the unit are used to measure the error --- compare to how the output of the 2-layer perceptron was evaluated.
% \cite<using cross entropy, for rationale see:>{golik13}.
However, if $i$ is a hidden unit, its target is:

\begin{equation}
\label{eq:ff_targets}
  t_i = \sum_{j} \delta_j w_{ij}
\end{equation}
meaning that the target for hidden unit $i$ is a function of the error signal emitted by the output units and the weights that connect them.

Now that we know how to calculate the targets for both output and hidden units, we need to calculate their respective deltas. For the output unit the error is defined in \autoref{eq:bp_error}. For hidden units we use a slightly different way of calculating the error:

\begin{equation}
\label{eq:hidden_error}
\delta_i = s_i (1 - s_i)  t_i
\end{equation}

Now we have both types of $\delta_i$s (the errors for the output and hidden units), we may calculate the \emph{proposed} changes to the weights:

\begin{equation}
\label{eq:delta_weights}
\Delta w_{ij}= \delta_j s_i
\end{equation}
gh

These are merely suggestions to update the connection weights and do not represent the actual changes we will make, as we may decide to scale them by the learning rate, apply momentum, etc.

\begin{enumerate}
 \item If the pre-synaptic activation of an output unit $i$ is $-6$ and its target $t_i$ is $1.0$, what error signal $\delta_i$ will it emit? (Use equations \ref{eq:ff_eta}, \ref{eq:logistic}, and \ref{eq:bp_error}.)

 \item What type of flow control statement is appropriate for calculating the $\sum$ in \autoref{eq:ff_targets}?

 \item What type of data is $i$ and what type of data is $s_i$?

 \item What are the main differences and similarities between the backwards and the feedforward phases of running a network?

 \item Why is the backwards phase required?

 \item How many types of target $t_i$ are there and how do we calculate them?

  \item Fill in the \texttt{Backprop()} function to run the network backwards. Remeber to run it forwards first!  However, do not change the weights just yet. Merely accumulate the proposed changes to the weights by calculating \autoref{eq:delta_weights}.

 \item Check if your code is working correctly. Are the output errors correct, are the hidden targets correct, ...?

 \end{enumerate}

\section{Weight Adjustments}
Once the network has been run forwards and backwards for an epoch, $\epsilon$, weight updates are calculated using:
% \begin{equation}
% \label{eq:ff_delta_weight}
% \Delta w_{ij} = \delta_j s_i
% \end{equation}
% for both the input-to-hidden and the hidden-to-output connection weights. The $\Delta w_{ij}$s are then applied to the current weights (at epoch $\epsilon$), to produce the new weights:
\begin{equation}
\label{eq:ff_adjusting_weights}
%\Delta w_{ij}^{\epsilon} = \nu \Delta w_{ij}^{\epsilon-1} - \mu \delta_j s_i
w_{ij}^{\epsilon +1} = w_{ij}^{\epsilon} +  \nu \Delta w_{ij}^{\epsilon - 1}  - \mu \Delta w_{ij}^{\epsilon}
\end{equation}
taking into account the momentum, $\nu$, and the learning rate, $\mu$.  $\Delta w_{ij}^{\epsilon}$ is the proposed weight update, from \autoref{eq:delta_weights}, while $\Delta w_{ij}^{\epsilon - 1}$  is the actual weight update applied last time! $\Delta w_{ij}^{\epsilon - 1}$ is the change to the weights including learning rate and momentum from the previous epoch.

\begin{enumerate}

\item Apply the weight updates as in \autoref{eq:ff_adjusting_weights}. Fill in \texttt{Apply\_Deltas()}. Think carefully about how you will keep track of the previous actual (not proposed) weight change $\Delta w_{ij}^{\epsilon-1}$. Remember that $\delta_i$s have been being accumulated, so they will need to be set to zero when you have applied them, in order for them to be ready to receive the new epoch's.

\item Now  that your network learns, compare its performance on not with a 2-layer perceptron. Is ti faster or slower? What parameters play a role (learning rate, momentum, initial values of weights, etc.)?

 \end{enumerate}

% \cite{attohokine99, hertz91}.

\end{document}


%%!TEX TS-program = latex

%This template gives a nice gray-sober environment.
\documentclass[red,handout]{beamer}
\usepackage{etex}
\usepackage[utf8]{inputenc}
\input ../AuxFiles/PreambleSlides.tex

\title{{\scshape Econometrics I}}
\author{{\scshape Jos\'e Luis Montiel Olea}}
\date{}

%---------------------------------------------------Begin Document-------------------------------------
\begin{document}
\setbeamerfont{alerted text}{series=\normalfont}
\setbeamercolor{alerted text}{fg=blue}

\frame{\titlepage}

\frame{
\begin{center}
Introduction to Probability and Statistics for Economists \\
(Ph.D. in Economics, 1st year)
\end{center}
}

\frame{
\begin{center}
\textbf{Lectures 1 and 2}
\end{center}
}

\section{Introduction}

\frame{
\frametitle{\normalsize \scshape Two Main Blocks}
\justifying
\begin{enumerate}
\item \alt<2>{\textcolor{blue}{{\scshape Probability Theory}}}{{\scshape Probability Theory}} \\
\vspace{.3cm}
\begin{center}\invisible<1-2>{\textcolor{blue}{Basic tools to model uncertainty}}
\end{center}
\vspace{.2cm}
\item \alt<4>{\textcolor{blue}{{\scshape Mathematical Statistics}}}{{\scshape Mathematical Statistics }} \\
\vspace{.3cm}
\begin{center}
\invisible<1-4>{\textcolor{blue}{Data $\rightarrow$ Decisions}}
\end{center}
\only<5>{}
\end{enumerate}
}

\section{Probability Space/Random Variables}

\frame{
\begin{center}
\textbf{Probability Theory}
\end{center}
\begin{center}
How to model `randomness'?
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Things we will learn in Lecture 1:}}
\begin{itemize} [<alert@+>]
\item [$\star$] What is a probability space? \vspace{.4cm}
\begin{enumerate}
\item What is measurable space? \vspace{.3cm}
\item What is a probability space? \vspace{.4cm}
\end{enumerate}
\item [$\star$] What is a random variable? \vspace{.4cm}
\item [$\star$] What is the `distribution’ or `law’ of a random variable?
\end{itemize}
}

\frame{
\begin{center}
\textbf{What is a probability space?}
\end{center}
$$(\Omega, \mathcal{F}, \prob)  $$
}

\frame{
\frametitle{\normalsize {\scshape Probability Space: Two components}}
$$(\Omega, \mathcal{F}, \prob)  $$
\begin{enumerate} [<+- | alert@+>]
\item $(\Omega, \mathcal{F})$ measurable space.
\item [] $$\Omega: \textrm{ States of the world}, \quad \mathcal{F}: \textrm{ Set of events } (\subseteq \Omega).$$
\vspace{.1cm}
\item $\prob: \mathcal{F} \rightarrow [0,1].$ Probability Measure
\item [] $$\text{`How likely is an event in $\mathcal{F}$'}$$
\end{enumerate}
}

\frame{
\frametitle{\normalsize \caps{Measurable Space}}
\begin{center}
See notes.
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Probability Measures}}
\begin{enumerate} [<+-|alert@+>]
\item [$\star$] $\prob(\phi)=0, \: \prob(\Omega)=1$ (Normalization) \\ \vspace{.5cm}
\item [$\star$] For any finite collection $A_1, A_2, \ldots A_m$ such that $A_i \cap A_j = \emptyset$

$$\prob\Big( \cup_{i=1}^{m} A_i \Big) = \sum_{i=1}^{m} \prob(A_i) $$
\item [] This property is called \textcolor{blue}{additivity}. \vspace{.3cm}
\item [$\star$] If you replace finite by \emph{countably infinite}, Property 2 is called \textcolor{blue}{$\sigma$-additivity}.
\end{enumerate}
}

\frame{
\frametitle{\normalsize {\scshape Important}}
\begin{center}
\textcolor{blue}{Normalization and $\sigma$-additivity \underline{define} a probability measure}
\end{center}
}

\frame{
\begin{center}
\textbf{What is a random variable?}
\end{center}
$$ X: \Omega \rightarrow S$$
}

\frame{
\frametitle{\normalsize {\scshape Random Variable}}
$$\textcolor{blue}{X: \Omega \rightarrow S} $$ \pause
\begin{enumerate}[<+- | alert@+>]
\item[$\star$] $\Omega:$ Set of states of the world.\\ \vspace{.5cm}
\item[$\star$] $S:$ Image Space
\vspace{.5cm}
\item[$\star$] $X:$ Random Variable \vspace{.5cm}
\end{enumerate}
}

\frame{
\begin{center}
\textbf{What is the distribution or law of a random variable?}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape `Induced’ Probability of a Random Variable}}
\begin{enumerate}
\item [$\star$]  \alt<2>{\textcolor{blue}{The probability $\prob$ on $\Omega$ induces a probability on subsets of $S$:}}{The probability $\prob$ on $\Omega$ induces a probability on subsets of $S$:}

$$\prob_{X} [F]  \equiv \prob [ \: \{\omega \: | \: X(\omega) \in F \} \:  ], \quad F \subseteq S $$ \vspace{.2cm}

\item [$\star$] \alt<3>{\textcolor{blue}{How likely are the states of the world in which F occurs?}} {How likely are the states of the world in which F occurs?}
\end{enumerate}
}

\frame{
\begin{center}
\textcolor{blue}{The induced probability of a random variable is usually called its {\scshape Distribution or Law}}
\end{center}
}

\frame{
\begin{center}
\textcolor{blue}{Different random variables can induce the same probability on $S$.}
\end{center}
}

\frame{
\frametitle{\normalsize \alt<2>{$P_{X_2}(1)=.5$}{$P_{X_1}(1)=.5$}}
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(-3,-3)(-3,-3)(3,3)
\uput[25](3.5,-3.4){\footnotesize $\Omega$}
\uput[25](-3.3,-3.4){\footnotesize $0$}
\uput[25](2.5,-3.4){\footnotesize $1$}
\uput[25](-3.7,3.2){\footnotesize $S=\{0,1\}$}
\only<1>{\psline[linecolor=red, linewidth=.1] (-3,2)(-.2,2)}
\only<1>{\psline[linecolor=red, linewidth=.1] (-.2,-3)(3,-3)}
\uput[25](-.4,-3.4){\footnotesize $0.5$}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize \alt<2>{$P_{X_2}(1)=.5$}{$P_{X_1}(1)=.5$}}
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(-3,-3)(-3,-3)(3,3)
\uput[25](3.5,-3.4){\footnotesize $\Omega$}
\uput[25](-3.3,-3.4){\footnotesize $0$}
\uput[25](2.5,-3.4){\footnotesize $1$}
\uput[25](-3.7,3.2){\footnotesize $S=\{0,1\}$}
\only<1>{\psline[linecolor=blue, linewidth=.1] (-3,-3)(-.2,-3)}
\only<1>{\psline[linecolor=blue, linewidth=.1] (-.2,2)(3,2)}
\uput[25](-.4,-3.4){\footnotesize $0.5$}
\end{pspicture*}
\end{center}
}

\section{C.D.F.}

\frame{
\begin{center}
\textbf{The Cumulative Distribution Function of Real-valued Random Variables }
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape c.d.f of an $\R$-valued random variable}}
\begin{enumerate}
\item [$\star$] \alt<2>{\textcolor{blue}{How likely is a realization of the random variable X below x?}}{How likely is a realization of the random variable X below x?} \vspace{.3cm}

\item [$\star$] \alt<3>{\textcolor{blue}{The c.d.f. summarizes this information}}{The c.d.f. summarizes this information}
$$F_{X}: \R \rightarrow [0,1] $$

$$F_{X}(x) \equiv \prob\Big\{ \omega \in \Omega \: | \: X(\omega) \leq x \Big\} $$
\end{enumerate}
}

\frame{
\begin{center}
\textbf{Examples of c.d.f.}\\
\hyperlink{Uniform-cdf}{\beamergotobutton{Uniform distribution}}\\
\hyperlink{Bernoulli-cdf}{\beamergotobutton{Bernoulli distribution}} \\
\hyperlink{Properties}{\beamerskipbutton{Skip to Properties}}
\end{center}
}

\frame{
\label{Uniform-cdf}
$$F_{X}(x) =
\left \{
\begin{array}{ccc}
0  & \text{if}  & x < a   \\
& &\\
(x-a)/b-a &  \text{if} & x \in [a,b)  \\
 & &\\
 1 & \text{if}  & x \geq b
\end{array}
\right.
 $$

}

\frame{
\frametitle{\normalsize \alt<2>{\scshape Uniform Distribution on $[a,b]$}{c.d.f. of  $X(\omega)=a + \omega [b-a]$}}
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(0,-3)(-4,-3)(4,3)
\uput[25](3.5,-3.4){\footnotesize $\R$}
\uput[25](-.3,-3.4){\footnotesize $0$}
\uput[25](-.5,-1){\footnotesize $1$}
\uput[25](-.5,3.2){\footnotesize $F_X(\cdot)$}
\psline[linecolor=blue, linewidth=.1] (-4,-3)(-1,-3)
\psline[linecolor=blue, linewidth=.1] (1,-1)(4,-1)
\psPolynomial[coeff=-2 1, linewidth=.1, linecolor=blue]{-1}{1}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $X(\omega)=a + \omega [b-a]$}
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(-3,-3)(-3,-3)(3,3)
\uput[25](3.5,-3.4){\footnotesize $\Omega$}
\uput[25](-3.3,-3.4){\footnotesize $0$}
\uput[25](2.5,-3.4){\footnotesize $1$}
\uput[25](-3.7,3.2){\footnotesize $S=\R$}
\uput[25](-3.7,-3){\footnotesize $a$}
\uput[25](-3.7,2.8){\footnotesize $b$}
\psPolynomial[coeff=0 1,linecolor=red]{-3}{2.8}
\end{pspicture*}
\end{center}
}

\frame{
\label{Bernoulli-cdf}
$$F_{X}(x) =
\left \{
\begin{array}{ccc}
0  & \text{if}  & x < 0   \\
& &\\
1-p &  \text{if} & x \in [0,1)  \\
 & &\\
 1 & \text{if}  & x \geq 1
\end{array}
\right.
 $$
}

\frame{
\frametitle{\normalsize \alt<2>{{\scshape Bernoulli Distribution with parameter $p$}}{c.d.f. of  $X(\omega)=\mathbf{1}[\omega \geq 1-p ]$}}
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(0,-3)(-4,-3)(4,3)
\uput[25](3.5,-3.4){\footnotesize $\R$}
\uput[25](-.3,-3.4){\footnotesize $0$}
\uput[25](-.5,-1){\footnotesize $1$}
\uput[25](2,-3.4){\footnotesize $1$}
\uput[25](-1,-2.7){\footnotesize $1-p$}
\uput[25](-.5,3.2){\footnotesize $F_X(\cdot)$}
\psline[linecolor=blue, linewidth=.1] (-4,-3)(0,-3)
\psline[linecolor=blue, linewidth=.1] (0,-2.5)(2.25,-2.5)
\psline[linecolor=blue, linewidth=.1] (2.25,-1)(4,-1)
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $X(\omega) = \mathbf{1}[\omega \geq 1-p ]$ }

\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(-3,-3)(-3,-3)(3,3)
\uput[25](3.5,-3.4){\footnotesize $\Omega$}
\uput[25](-3.3,-3.4){\footnotesize $0$}
\uput[25](2.5,-3.4){\footnotesize $1$}
\uput[25](-3.7,3.2){\footnotesize $S=\R$}
\psline[linecolor=red] (-3,-3)(-2,-3)
\psline[linecolor=red] (-2,2)(3,2)
\uput[25](-2.3,-3.4){\footnotesize $1-p$}

\end{pspicture*}
\end{center}
}

\frame{
\label{Properties}
\begin{center}
\textbf{What are the common properties of c.d.f?}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Characterization}}

\begin{enumerate}
\item $F_{X}$ is non-decreasing \vspace{.3cm}
\item $\lim_{x \rightarrow \infty} F_{X}(x) =1$ \vspace{.3cm}
\item $\lim_{x \rightarrow -\infty} F_{X}(x) =0$ \vspace{.3cm}
\item $\lim_{h \rightarrow 0^{+}} F_{X}(x+h) = F_{X}(x)$ \vspace{.3cm} \pause
\end{enumerate}
\begin{center}
\item [] \textcolor{blue}{In fact, these 4 properties characterize the induced c.d.f. of a real-valued random variable!}
\end{center}
}

\section{Discrete/Continuous Type}

\frame{
\begin{center}
\textbf{Discrete and Continuous Type of Real-Valued Random Variables}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Discrete Distributions / Discrete r.v.s }}
{\scshape Distribution:} Disributions for which
$\exists$ a countable set
$$\text{Supp}=\{x_1, x_2, \ldots\}, x_i \in \R,$$
such that \vspace{.3cm}
\begin{enumerate}[a)]
\item $\prob_{X}(X=x_i)>0 \quad \forall \quad x_i \in \text{Supp}$ \vspace{.2cm}
\item  $\sum_{x_i \in \text{Supp}} \prob_{X}(X=x_i) = 1$
\end{enumerate} \vspace{.3cm}
are called \textcolor{blue}{discrete}.
}

\frame{
\frametitle{\normalsize {\scshape p.m.f.}}
\begin{center}
\alt<2>{\textcolor{blue}{We will identify discrete distributions/r.v.s by its support and its p.m.f.}}{The function $\prob_{X}(X=x_i)$ is called the \textcolor{blue}{Probability Mass Function} (p.m.f.)}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape (Absolutely) Continuous Distributions}}
Random Variables for which
$$ \textcolor{blue}{ F_X(x) = \int_{-\infty}^{x} f(z) dz   }  $$
\noindent for some $f(z) \geq 0$ $\forall z \in \R$ are called: \pause \vspace{.3cm}
\begin{center}
\textcolor{blue}{(Absolutely) Continuous} \pause
\end{center}
The function $f(z)$ is called \pause \vspace{.3cm}
\begin{center}
\textcolor{blue}{Probability Density Function (p.d.f.)}
\end{center}
}

\frame{
\begin{center}
\begin{center}
\textcolor{blue}{We will identify continuous distributions/r.v.s by its p.d.f.}
\end{center}
\end{center}
}

\frame{
\begin{center}
The set
$$\{z \in \R \: | \: f(z) > 0 \} \subseteq \R $$
is called the \textcolor{blue}{support} of the continuous r.v. \vspace{.5cm}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape From p.d.f. to probabilities}}
Let $X$ be a real-valued random variable with p.d.f. $f(z)$ \pause \vspace{.5cm}
\begin{itemize}  [<+->]
\item [$\star$] \alert<2>{$\prob_{X}[X \leq a] = \int_{-\infty}^{a} f(z) dz$} \vspace{.5cm}
\item [$\star$] \alert<3>{$\prob_{X}[a \leq X \leq b] = \int_{a}^{b} f(z) dz$} \vspace{.5cm}
\item [$\star$] \alert<4>{$\prob_{X}[X > a ] = \int_{a}^{\infty} f(z) dz$}
\end{itemize}
}

\frame{
\label{Examples-Discrete}
\begin{center}
\textbf{Examples of (Univariate) Discrete Distributions}\\
\vspace{.2cm}
\hyperlink{Bernoulli}{\beamergotobutton{Bernoulli}}
\hyperlink{Binomial}{\beamergotobutton{Binomial}} \\
\hyperlink{Examples-Continuous}{\beamerskipbutton{Skip to Continuous Distributions}}
\end{center}
}

\frame{
\label{Bernoulli}
\frametitle{\normalsize {\scshape Bernoulli Distribution $(p)$,  $p \in (0,1)$}}
\begin{itemize}
\item [$\star$] The Bernoulli distribution with parameters $p$ has support:
$$\text{Supp}=\{0, 1\} $$ \pause
and p.m.f. given by: \pause
$$\prob_{X}(X=x)=p^{x} (1-p)^{1-x} \quad x \in \{0,1\}$$
\end{itemize}
}

\frame{
\frametitle{\normalsize {\scshape How do we know it is a p.m.f.?}}
Two parts: \vspace{.3cm}
\begin{enumerate}[a)]
\item $\prob_{X}(X=x)>0 \quad \forall \: x \in \{0,1\}$. Easy to verify:
$$\textcolor{blue}{p^{x} (1-p)^{1-x}>0}$$ \pause
\item
$$\sum_{x \in \{0,1\}} p^{x} (1-p)^{1-x} = (1-p) + p$$
\end{enumerate}
\hyperlink{Examples-Discrete}{\beamerreturnbutton{Examples}}
}

\frame{
\label{Binomial}
\frametitle{\normalsize {\scshape Binomial Distribution $(n,p)$, $n \in \mathbb{N}, p \in (0,1)$}}
\begin{itemize}
\item [$\star$] The binomial distribution with parameters $(n,p)$ has support:
$$\text{Supp}=\{0,1, 2, \ldots n\} $$ \pause
and p.m.f. given by: \pause
$$\prob_{X}(X=x) \equiv \frac{n!}{(n-x)! x!} p^{x} (1-p)^{n-x} \quad x \in \text{Supp}$$
\end{itemize}
}

\frame{
\frametitle{\normalsize {\scshape How do we know it is a p.m.f.?}}
Two parts: \vspace{.3cm}
\begin{enumerate}[a)]
\item $\prob_{X}(X=x)>0 \quad \forall \: x \in \{0,1,2, \ldots n\}$. Easy to verify:
$$\textcolor{blue}{\frac{n!}{(n-x)! x!} p^{x} (1-p)^{n-x}>0}$$ \pause
\item
$$\sum_{x \in \{0,1, \ldots n\}} \frac{n!}{(n-x)! x!} p^{x} (1-p)^{n-x} =1 ?$$
\end{enumerate}
}

\frame{
\frametitle{\normalsize {\scshape How do we know it is a p.m.f.?}}
Use the Binomial Theorem
$$(a+b)^n = \sum_{x \in \{0,1, \ldots n\}} \frac{n!}{(n-x)! x!} a^{x} b^{n-x} =1$$ \pause
$$a=p, \quad b=1-p $$
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(3,.1)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(0,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(4,0)
\uput[-90](4,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{3}{0.1}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(3,.3)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(4,0)
\uput[-90](4,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{3}{0.3}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(3,.5)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(4,0)
\uput[-90](4,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{3}{0.5}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(3,.7)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(4,0)
\uput[-90](4,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{3}{0.7}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(3,.9)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(4,0)
\uput[-90](4,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{3}{0.9}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(5,.1)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(6,0)
\uput[-90](6,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{5}{0.1}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(5,.3)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(6,0)
\uput[-90](6,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{5}{0.3}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(5,.5)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(6,0)
\uput[-90](6,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{5}{0.5}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(5,.7)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(6,0)
\uput[-90](6,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{5}{0.7}
\end{pspicture}
\end{center}
}

\frame{
\frametitle{\normalsize {\scshape Binomial $(5,.9)$ }}
\begin{center}
\psset{xunit=1cm,yunit=5cm}
\begin{pspicture}(-1,.5)(7,0.55)
\psaxes[Dy=0.2,dy=0.2\psyunit]{->}(0,0)(-.5,0)(6,0)
\uput[-90](6,0){$\text{Supp}$} \uput[90](0,0.5){}
\psBinomial[markZeros,printValue,fillstyle=vlines]{5}{0.9}
\end{pspicture}
\end{center}
}

\frame{
\begin{center}
\hyperlink{Examples-Discrete}{\beamerreturnbutton{Examples}}
\end{center}
}

\frame{
\label{Examples-Continuous}
\begin{center}
\textbf{Examples of (Univariate) Continuous Distributions}\\
\hyperlink{Uniform}{\beamergotobutton{Uniform}}
\hyperlink{Normal}{\beamergotobutton{Normal}} \\
\hyperlink{Moments}{\beamerskipbutton{Skip to Moments}}
\end{center}
}

\frame{
\label{Uniform}
\frametitle{\normalsize {\scshape Uniform [a,b]}}
\begin{itemize}
\item [$\star$] The uniform distribution with parameters $[a,b]$ has p.d.f.
$$f(z)=\frac{1}{b-a} \mathbf{1}\{z \in [a,b]\}$$ \pause
\item [$\star$] and support $[a,b]$
\end{itemize}
}

\frame{
\frametitle{\normalsize \scshape p.d.f. of the Uniform Distribution on $[a,b]$ }
\begin{center}
\begin{pspicture*}(-4,-4) (4,4)
\psaxes[ticks=none, labels=none]{->}(0,-3)(-4,-3)(4,1.3)
\uput[25](-3.5,2){\footnotesize $f(z) =
\left \{
\begin{array}{ccc}
0  & \text{if}  & z < a   \\
& &\\
1/(b-a) &  \text{if} & z \in [a,b]  \\
 & &\\
 0 & \text{if}  & z > b
\end{array}
\right.
 $}
\uput[25](3.5,-3.4){\footnotesize $\R$}
\uput[25](-.3,-3.4){\footnotesize $0$}
\uput[25](-1.2,-3.4){\footnotesize $a$}
\uput[25](.8,-3.4){\footnotesize $b$}
\uput[25](1.2,-1.1){\footnotesize $1/(b-a)$}
\psline[linecolor=blue, linewidth=.1] (-4,-3)(-1,-3)
\psline[linecolor=blue, linewidth=.1] (1,-3)(4,-3)
\psPolynomial[coeff=-1 0, linewidth=.1, linecolor=blue]{-1}{1}
\end{pspicture*}
\end{center}
}

\frame{
\label{Normal}
\frametitle{\normalsize {\scshape Normal Distribution $(\mu, \sigma^2)$}}
\begin{itemize}
\item [$\star$] The normal distribution with parameters $(\mu, \sigma^2)$ has p.d.f.
$$f(z)=\frac{1}{\sigma \sqrt{2 \pi}} \exp\Big( -\frac{1}{2 \sigma^2} (z-\mu)^2 \Big)$$ \pause
\item [$\star$] and support $\R$
\end{itemize}
}

\frame{
\frametitle{\normalsize {\scshape Normal Distribution $(\mu, \sigma^2)$}}
\begin{itemize}
\item [$\star$] $f(z)> 0 \pause \checkmark$ \pause \vspace{.5cm}
\item [$\star$] How do we know that:
$$ \int_{-\infty}^{\infty} \frac{1}{\sigma \sqrt{2 \pi}} \exp\Big( -\frac{1}{2 \sigma^2} (z-\mu)^2 \Big) = 1? $$ \pause
\textcolor{blue}{Euler-Poisson Integral/Gaussian Integral}
\end{itemize}
}

\frame{
\frametitle{\normalsize $N(0,1)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=0, sigma=1, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $N(0,.5)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=0, sigma=.5, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $N(0,.35)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=0, sigma=.35, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $N(.5,.35)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=.5, sigma=.35, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $N(-5,.35)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=-.5, sigma=.35, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $N(0,.35)$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=0, sigma=.35, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\frame{
\frametitle{\normalsize $\textbf{N(0,1)}$}
\begin{center}
\psset{yunit=4cm,xunit=2}
\begin{pspicture*}(-2,-.25) (2,1.25)
\uput[25](-.25,.5){\footnotesize $.5$}
\uput[25](-2,-.2){\footnotesize $-4$}
\uput[25](1.8,-.2){\footnotesize $4$}
\psaxes[ticks=none, labels=none]{->}(0,0)(-2.5,0)(2.5,1.25)
\psGauss[mue=0, sigma=1, linecolor=red, linewidth=2pt]{-2}{2}
\end{pspicture*}
\end{center}
}

\section{Moments}

\frame{
\label{Moments}
\begin{center}
\textbf{Moments}
\end{center}
}

\frame{
\frametitle{\normalsize \scshape{Mean of a discrete r.v.}}
\begin{itemize}[<+-|alert@+>]
\item [$\star$] {Let $X$ be a discrete r.v. with support $S$ and p.m.f. $\prob_{X}$} \vspace{.3cm}
\item [$\star$] {The mean or expected value of $X$ is defined as:}
$$\expec_{\prob_{X}}[X] \equiv \mu \equiv \sum_{x_n \in S} x_n \: \prob_{X}[X = x_n ] $$
\item [$\star$] {The variance of $X$ is defined as:}
$$\expec_{\prob_{X}}[(X-\mu)^2] \equiv \sigma^2 \equiv \sum_{x_n \in S} (x_n-\mu)^2 \: \prob_{X}[X = x_n ] $$

\end{itemize}
}

\frame{
\frametitle{\normalsize {\scshape Mean of a Continuous Type r.v.}}
\begin{itemize} [<+->]
\item [$\star$] For a continuous-type random variable $g(X)$, where $X \sim f_X$:
\alert<2>{$$\expec_{f_X}[g(X)] \equiv \int_{-\infty}^{\infty} g(z) f_X(z) dz $$}
\item [$\star$] Therefore, the mean and the variance of X are given by:
\begin{eqnarray*}
\mu &=& \int_{-\infty}^{\infty} z f_X(z)dz \\
\sigma^2 &=& \int_{-\infty}^{\infty} (z-\mu)^2 f_X(z) dz
\end{eqnarray*}
\end{itemize}
}

\frame{
\begin{center}
\textbf{Examples}\\
\hyperlink{Mean-Uniform}{\beamergotobutton{Mean of a Uniform [a,b]}} \\
\hyperlink{Variance-Uniform}{\beamergotobutton{Second Moment of a Uniform [a,b]}}\\
\hyperlink{MGF}{\beamerskipbutton{Skip to Moment Generating Function}}
\end{center}
}

\frame{
\label{Mean-Uniform}
\frametitle{\normalsize {\scshape U[a,b]}: $\mu$}
\begin{itemize}
\item [$\star$] $\mu = \frac{b+a}{2}$ \vspace{.3cm}
\item [$\star$] $f(z)=\frac{1}{b-a} \mathbf{1}\{z \in [a,b]\}$ \vspace{.3cm}
\item [$\star$] Derivation:
\begin{eqnarray*}
\mu = \int_{-\infty}^{\infty} z f(z) dz  &=& \frac{1}{b-a}\int_{a}^{b} z dz \\ \pause
&=& \frac{1}{2}\frac{1}{b-a} z^2 \Big]_{a}^{b} \\ \pause
&=& \frac{1}{2} \frac{1}{b-a} (b^2-a^2) \\ \pause
&=& \frac{b+a}{2}
\end{eqnarray*}
\end{itemize}
}

\frame{
\label{Variance-Uniform}
\frametitle{\normalsize {\scshape U[a,b]}: $\mathbb{E}[X^2]$}
\begin{itemize}
\item [$\star$] $\mathbb{E}[X^2] = \frac{b^2 + ab + a^2}{3}$ \vspace{.3cm}
\item [$\star$] Derivation:
\begin{eqnarray*}
\expec[X^2] = \int_{-\infty}^{\infty} z^2 f(z) dz  &=& \frac{1}{b-a}\int_{a}^{b} z^2 dz \\ \pause
&=& \frac{1}{3}\frac{1}{b-a} z^3 \Big]_{a}^{b} \\ \pause
&=& \frac{1}{3} \frac{1}{b-a} (b^3-a^3) \\ \pause
&=& \frac{b^2 + ab + a^2}{3}
\end{eqnarray*}
\end{itemize}
}

\frame{
\noindent Hence, $\sigma^2$ is given by:
$$ \mathbb{E}[X^2] - \mathbb{E}[X]^2 = \frac{b^2 + ab + a^2}{3} - \frac{b^2 + 2ab + a^2}{4} = \frac{(b-a)^2}{12} $$
}

\frame{
\label{MGF}
\begin{center}
\textbf{Moment Generating Function}
\end{center}
}

\frame{
\frametitle{\normalsize \caps{MGF}}
\begin{itemize}[<+-|alert@+>]
\justifying
\item [$\star$] The $\mathbb{R}$-valued r.v. $X$ has moment generating function $m_{X}(\cdot)$ if
$$ m_{x}(t) \equiv E_{F}\Big[\exp \Big( t X \Big)\Big] < \infty, \quad \forall \: t \in (-\epsilon,\epsilon).$$
\item [$\star$] The k-th moment of $X$ is the $k$-th derivative of $m_{x}(t)$ at $t=0$.
\end{itemize}
}

\frame{
\frametitle{\normalsize \caps{Example}}
\begin{itemize}[<+-|alert@+>]
\item [$\star$] $X \sim \text{Bernoulli}(p)$
\item []
\begin{eqnarray*}
m_{X}(t) &=& E_{F}\Big[\exp \Big( t X \Big)\Big]  \\
&=& p \exp(t) + (1-p) \exp(0) \\
\end{eqnarray*}
\item [$\star$] Moments:
\item [] $ E_{F}[X]= p$ \\
\item [] $E_{F}[X^2] = p $ \\
\item [] $\vdots$
\item [] $E_{F}[X^k]=p$
\end{itemize}
}

\end{document}


\section{Ocamlyacc}
We mainly cover menhir here.

A grammar is mainly composed of four elements(terminals,
non-terminals, production rulls, start symbol)

\textbf{Syntax}

\begin{bluetext}
    % {header
    % }
    %%
    Grammar rules
    %%
    trailer
\end{bluetext}

A tiny example as follows (It has a subtle bug, readers should find it)
  \begin{ocamlcode}
% {
  open Printf
  let parse_error s =
    print_endline "error\n";
    print_endline s ;
    flush stdout
%}

%token <float> NUM
%token PLUS MINUS MULTIPLY DIVIDE CARET UMINUS
%token NEWLINE

%start input
%type <unit> input
%type <float> exp
%% /* rules and actions */

input: /* empty */ {}
    | input line {}
;

line: NEWLINE {}
    |exp NEWLINE  {printf "\t%.10g\n" $1 ; flush stdout}
;

exp: NUM { $1 }
    |exp exp PLUS {$1 +. $2 }
    |exp exp MINUS {$1 -. $2 }
    |exp exp MULTIPLY {$1 *. $2 }
    |exp exp DIVIDE {$1 /. $2 }
    |exp exp CARET {$1 ** $2 }
    |exp UMINUS {-. $1 }
;

%%
\end{ocamlcode}

Notice that start non-terminal can be given \textit{several}, then you will
have a different .mli file, notice that it's different from ocamllex,
ocamlyacc will generate a .mli file, so here we get the output
interface as follows:

\begin{bluetext}
  %type <type> nonterminal ... nonterminal
  %start symbol ... symbol
\end{bluetext}

\begin{ocamlcode}
type token =
  | NUM of (float)
  | PLUS
  | MINUS
  | MULTIPLY
  | DIVIDE
  | CARET
  | UMINUS
  | NEWLINE
val input :
  (Lexing.lexbuf  -> token) -> Lexing.lexbuf -> unit
val exp :
  (Lexing.lexbuf  -> token) -> Lexing.lexbuf -> float
\end{ocamlcode}

Notice that we may use character strings as implicit terminals as in
\begin{ocamlcode}
expr : expr "+" expr {}
     | expr "*" expr {}
     | ... ;
\end{ocamlcode}
They are directly processed by the parser without passing through the
lexer. But it breaks the uniformity

\textbf{Contextual Grammar}

\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/context.ml}

First gammar
\begin{ocamlcode}
  /* empty corresponds Ctrl-d.*/
  input : /*empty*/ {} | input line {};
\end{ocamlcode}

Notice here we \textbf{preferred left-recursive} in yacc.
The underlying theory for LALR prefers LR. because all the elements
must be shifted onto the stack \textit{before} the rule can be applied even once.

\begin{ocamlcode}
  exp : NUM | exp exp PLUS | exp exp MINUS  ... ;
\end{ocamlcode}

Here is our lexer
\begin{ocamlcode}
{
  open Rpcalc
  open Printf
  let first = ref true
}
let digit = ['0'-'9']
rule token = parse
  |[' ' '\t' ] {token lexbuf}
  |'\n' {NEWLINE}
  | (digit+ | "." digit+ | digit+ "." digit*) as num
      {NUM (float_of_string num)}
  |'+' {PLUS}
  |'-' {MINUS}
  |'*' {MULTIPLY}
  |'/' {DIVIDE}
  |'^' {CARET}
  |'n' {UMINUS}
  |_ as c  {printf "unrecognized char %c" c ; token lexbuf}
  |eof {
    if !first then begin first := false; NEWLINE end
    else raise End_of_file }

{
  let main ()  =
    let file = Sys.argv.(1) in
    let chan = open_in file in
    try
      let lexbuf = Lexing.from_channel chan in
      while true do
        Rpcalc.input token lexbuf
      done
    with End_of_file -> close_in chan

 let _ = Printexc.print main ()

}
\end{ocamlcode}

We write driver function in lexer for convenience, since lexer depends
on yacc. \textit{Printex.print}

\textbf{precedence associatitvity }

Operator precedence is determined by the line ordering of the
declarations;

\textit{\%prec} in the grammar section, the \textit{\%prec} simply
instructs ocamlyacc that the rule \textit{|Minus exp } has the same
precedence as NEG
\textit{\%left,\%right,\%nonassoc}

\begin{enumerate}
  \item The associatitvity of an operator op determines how repeated
    uses of the operator nest: whether \textit{x op y op z} is parsed
    by grouping \textit{x} with \textit{y} or. nonassoc will consider
    it as an error
  \item All the tokens declared in a single precedence declaration
    have equal precedence and nest together according to their
    associatitvity
  \end{enumerate}

\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml} {code/calc/simple.mly}

Notice here the \textit{NEG} is a place a holder, it takes the
place, but it's not a token. since here we need \textit{MINUS} has
different levels.

%% The interface file is as follows
%% \inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/calc/simple.mli}

\textbf{Error Recovery}

By default, the parser function raises exception after calling \textit{parse\_error}
The ocamlyacc reserved word \textit{error}

  \begin{ocamlcode}
    line: NEWLINE |exp NEWLINE | error NEWLINE {}
  \end{ocamlcode}

If an expression that cannot be evaluated is read, the error will be
recognized by the third rule for line, and parsing will continue
(parse\_error is still called). This form of error recovery deals
with syntax errors. There are also other kinds of errors.

\textbf{Location Tracking}

It's very easy. First, remember to use \textit{Lexing.new\_line} to
track your line number, then use  \textit{rhs\_start\_pos, rhs\_end\_pos} to
 track the symbolposition.  1 is for the leftmost component.

\begin{ocamlcode}
            Parsing.(
              let start_pos = rhs_start_pos 3 in
              let end_pos = rhs_end_pos 3 in
              printf "%d.%d --- %d.%d: dbz"
                start_pos.pos_lnum (start_pos.pos_cnum -start_pos.pos_bol)
                end_pos.pos_lnum (end_pos.pos_cnum - end_pos.pos_bol);
              1.0
            )
\end{ocamlcode}

For groupings, use the following function \textit{symbol\_start\_pos,
  symbol\_end\_pos}

\textit{symbol\_start\_pos} is set to the beginning of the leftmost
component, and \textit{symbol\_end\_pos} to the end of the rightmost component.

\textbf{A complex Example}

\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/calc/calc_compl.mly}

In my opinion, the best practice is first modify .mly file, then
change .mll file later

\textbf{SHIFT REDUCE }

A very nice tutorial
\href{http://www.cs.uiuc.edu/class/sp10/cs421/lectures/lecture%2010%20supp.pdf}{shift-reduce}

\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/shift_reduce/s1.mly}
\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/shift_reduce/s2.mly}
\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/shift_reduce/s3.mly}
\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/shift_reduce/s4.mly}
\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/shift_reduce/s5.mly}

The prec trick is covered not correctly in this tutorial.

The symbols are declared to associate to the left, right,
nonassoc. The symbols are \textit{usually} tokens, they can
also be \textit{dummy} nonterminals, for use with the \%prec
directive in the rule.

\begin{enumerate}
\item Tokens and rules have precedences. The precedence of a
\textit{rule} is the precedence of its \textit{rightmost}
terminal. you can override this default by using the \textit{\%prec}
directive in the rule
\item A reduce/reduce conflict is resolved in favor of the first
ruel(in the order given by the source file)
\item A shift/reduce conflict is resolved by comparing the
\textit{predecence of the rule to be reduced} with the
\textit{precedence of the token to be shifted}. If the predecence of
the rule is higher, then the rule will be reducecd; if the predecence
of the token is higher then token will be shifted.
\item A shift/reduce conflict between a rule and a token with
the same precedence will be resolved using the associativity.
\item when a shift/reduce can not be resolved, a warning, and
in favor of \textit{shift}
\end{enumerate}

\section{MENHIR Related}

\begin{enumerate}
\item Syntax
  \begin{bluetext}
specification ::= declaration . . . declaration %% rule . . . rule [ %% Objective Caml code ]
 declaration ::= %{ Objective Caml code %}
                         % parameter < uid : Objective Caml module type >
                         %token [ < Objective Caml type > ] uid . . . uid
                         %nonassoc uid . . . uid
                         %left uid . . . uid
                         %right uid . . . uid
                         %type < Objective Caml type > lid . . . lid
                         %start [ < Objective Caml type > ] lid . . . lid
rule ::= [%public] [%inline] lid [( id, ..., id)] : [|] group | ... | group
group ::= production | . . . | production { Objective Caml code } [ %prec id ]
production ::= producer . . . producer [ %prec id ]
 producer ::= [ lid = ] actual
actual ::= id [( actual, ..., actual)] [? | + | *]

\item parameter

  \begin{bluetext}
%parameter <uid: Objective module types>
  \end{bluetext}

This causes the entire parser to be parameterized over the module.

\item multiple files (private and public,tokens aside)

\item parameterized rules

\item inline

\item standard library

  \begin{tabular}{|c|c|c|c|}
\hline
Name & Recognizes & Produces & Comment \\
\hline
option(X) & $\epsilon$ | X    & $\alpha$ option, if X : $\alpha$  &  \\
ioption(X) &                          &                            & inlined \\
boption(X)  &                        &                            & bool \\
loption(X)  &         $\epsilon$ |X & $\alpha$ list, if X : $\alpha$ list  & \\
pair(X,Y) &  X Y & $\alpha \times \beta$ & \\
separated\_pair(X,sep,Y) & X sep Y  & $\alpha \times \beta$     & \\
preceded(opening,X) & opening X & $\alpha$, if X : $\alpha$ & \\
terminated(X,closing) & X closing & $\alpha$, if X : $\alpha$ & \\
delimited(opening, X closing) & opening X closing & $\alpha$, if X : $\alpha$ & \\
list(X) &  & & \\
nonempty\_list(X) & & &  \\
separated\_list(sep,X) & & & \\
sepearted\_nonempty\_list(sep,X) & & & \\
\hline
  \end{tabular}

\item combined with ulex

A typical tags file is as follows
  \begin{bluetext}
true:use_menhir, pkg_ulex, pkg_pcre, pkg_menhirLib, pkg_batteries
<scanner.ml>: pkg_ulex, syntax_camlp4o
  \end{bluetext}

You have to use \mint{ocaml}|Menhirlib.Convert| API, here
\inputminted[fontsize=\scriptsize, fontsize=\scriptsize, ]{ocaml}{code/menhir/convert.mli}

\item example csss project

\end{enumerate}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "master"
%%% End: