\subsection{Banach space interpolation}\label{subsec:banach_space_interpolation}

\begin{definition}\label{def:interpolated_topological_vector_space}\mcite[24]{BerghLofstrom1976}
  Let \( \BbbK \) be either the \hyperref[def:set_of_real_numbers]{field \( \BbbR \) of real numbers} or the \hyperref[def:set_of_real_numbers]{field \( \BbbC \) of complex numbers}.

  \begin{thmenum}
    \thmitem{def:interpolated_topological_vector_space/compatibility} We say that two \hyperref[def:topological_vector_space]{topological vector spaces} \( \mscrX_0 \) and \( \mscrX_1 \) are \term{compatible} if they can both be \hyperref[def:morphism_invertibility/left_cancellative]{embedded} \hyperref[def:global_continuity]{continuously} into a \hyperref[def:separation_axioms/T2]{Hausdorff} topological vector space \( \mscrU \), in which case we can regard them as subspaces of \( \mscrU \).

    In particular, both \( \mscrX_0 \) and \( \mscrX_1 \) are Hausdorff. We write \( \overline{\mscrX} \coloneqq (\mscrX_0, \mscrX_1) \).

    \thmitem{def:interpolated_topological_vector_space/intersection} Denote by
    \begin{equation*}
      \Delta \overline{\mscrX} \coloneqq \mscrX_0 \cap \mscrX_1
    \end{equation*}
    the \term{intersection} of \( \mscrX_0 \) and \( \mscrX_1 \) (when regarded as subspaces of \( \mscrU \)).

    \thmitem{def:interpolated_topological_vector_space/sum} Denote by
    \begin{equation*}
      \Sigma \overline{\mscrX} \coloneqq ( \mscrX_0 + \mscrX_1 )
    \end{equation*}
    the \term{sum} of \( \mscrX_0 \) and \( \mscrX_1 \). If \( x \in \Sigma \overline{\mscrX} \), then there exist (possibly nonunique) vectors \( x_0 \in X_0 \) and \( x_1 \in X_1 \) such that \( x = x_0 + x_1 \).

    \thmitem{def:interpolated_topological_vector_space/intermediate_space} Let \( \overline{\mscrX} \) be a pair of compatible spaces. We say that the space \( \mscrX \) is an \term{intermediate} space for \( \overline{\mscrX} \) if \( \Delta \overline{\mscrX} \subseteq X \subseteq \Sigma \overline{\mscrX} \) with continuous linear inclusions.

    \thmitem{def:interpolated_topological_vector_space/morphisms} We introduce \hyperref[def:category/morphisms]{morphisms} between two compatible pairs \( \overline{\mscrX} \) and \( \overline{\mscrY} \) that are, strictly speaking, not \hyperref[def:function]{functions} between the pairs themselves. We define an \term{operator} \( T: \overline{\mscrX} \to \overline{\mscrY} \) between compatible pairs to be a function \( T \) from \( \Sigma \overline{\mscrX} \) to \( \Sigma \overline{\mscrY} \) that satisfies the additional conditions
    \begin{align*}
      T(\mscrX_0) \subseteq Y_0
      &&
      T(\mscrX_1) \subseteq Y_1.
    \end{align*}

    \thmitem{def:interpolated_topological_vector_space/category} If \( \cat{C} \) is a \hyperref[def:subcategory]{subcategory} of the category \hyperref[def:category_of_topological_vector_spaces]{\( \cat{TopVect}_{\BbbK} \)} of topological vector spaces. We define the category \( \cat{Interp}_{\cat{C}} \) as the product category \( \cat{TopVect}_{\BbbK} \times \cat{TopVect}_{\BbbK} \). More explicitly:
    \begin{refenum}
      \refitem{def:category/objects} The \hyperref[def:set]{class} of objects is the class of all pairs of \hyperref[def:interpolated_topological_vector_space/compatibility]{compatible spaces}.
      \refitem{def:category/morphisms} The morphisms between two compatible pairs are the \hyperref[def:interpolated_topological_vector_space/morphisms]{continuous linear operators} \( T: \overline{\mscrX} \to \overline{\mscrY} \) between them.
      \refitem{def:category/composition} Composition of morphisms is the usual \hyperref[def:multi_valued_function/composition]{function composition} if we regard a morphism \( T: \overline{\mscrX} \to \overline{\mscrY} \) as a function from \( \Sigma \overline{\mscrX} \) to \( \Sigma \overline{\mscrY} \).
    \end{refenum}

    \thmitem{def:interpolated_topological_vector_space/interpolation_space} We say that the intermediate spaces \( \mscrX \) for \( \overline{\mscrX} \) and \( \mscrY \) for \( \overline{\mscrY} \) are a pair of \term{interpolation spaces} with respect to \( \overline{\mscrX} \) and \( \overline{\mscrY} \) if, for any continuous linear \hyperref[def:interpolated_topological_vector_space/morphisms]{operator} \( T: \overline{\mscrX} \to \overline{\mscrY} \) between the compatible pairs, we have \( T(\mscrX) \subseteq Y \).
  \end{thmenum}
\end{definition}

\begin{proposition}\label{def:banach_space_sum_and_intersection_norms}\mcite[24]{BerghLofstrom1976}
  Let \( \mscrX \coloneqq (\mscrX_0, \mscrX_1) \) be a \hyperref[def:interpolated_topological_vector_space/compatibility]{compatible pair} of \hyperref[def:banach_space]{Banach spaces}.

  \begin{thmenum}
    \thmitem{def:banach_space_sum_and_intersection_norms/intersection} The intersection \( \Delta \overline{\mscrX} = \mscrX_0 \cap \mscrX_1 \) is a Banach space with norm
    \begin{equation}\label{eq:def:banach_space_sum_and_intersection_norms/intersection}
      \norm{x}_{\Delta \overline{\mscrX}} \coloneqq \max \set{ \norm{x}_{\mscrX_0}, \norm{x}_{\mscrX_1} }.
    \end{equation}

    \thmitem{def:banach_space_sum_and_intersection_norms/sum} The sum \( \Sigma \overline{\mscrX} = \mscrX_0 + \mscrX_1 \) is a Banach space with norm
    \begin{equation}\label{eq:def:banach_space_sum_and_intersection_norms/sum}
      \norm{x}_{\Delta \overline{\mscrX}} \coloneqq \inf \set{ \norm{x_0}_{\mscrX_0} + \norm{x_1}_{\mscrX_1} : x_0 + x_1 = x }.
    \end{equation}
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{def:banach_space_sum_and_intersection_norms/intersection} We will first show that \( \norm{x}_{\Delta \overline{\mscrX}} \) is indeed a norm.
  \begin{refenum}
    \refitem{def:norm/N1} We have
    \begin{equation}\label{eq:def:banach_space_sum_and_intersection_norms/intersection/zero}
      \norm{x}_{\Delta \overline{\mscrX}} = \max \set{ \norm{x}_{\mscrX_0}, \norm{x}_{\mscrX_1} } = 0 \T{if and only if} \norm{x}_{\mscrX_0} = \norm{x}_{\mscrX_1} = 0.
    \end{equation}

    Clearly \( 0 \) belongs to both \( \mscrX_0 \) and \( \mscrX_1 \) hence to their intersection. Therefore, \eqref{eq:def:banach_space_sum_and_intersection_norms/intersection/zero} is satisfied if and only if \( x = 0 \).

    \refitem{def:norm/N2} Absolute homogeneity follows from
    \begin{equation*}
      \norm{tx}_{\Delta \overline{\mscrX}}
      =
      \max \set{ \norm{tx}_{\mscrX_0}, \norm{tx}_{\mscrX_1} }
      \reloset {\ref{def:norm/N2}} =
      \abs{t} \max \set{ \norm{x}_{\mscrX_0}, \norm{x}_{\mscrX_1} }
      =
      \abs{t} \norm{x}_{\Delta \overline{\mscrX}}.
    \end{equation*}

    \refitem{def:norm/N3} Subadditivity follows from
    \begin{balign*}
      \norm{x + y}_{\Delta \overline{\mscrX}}
      &=
      \max \set{ \norm{x + y}_{\mscrX_0}, \norm{x + y}_{\mscrX_1} }
      \reloset {\ref{def:norm/N3}} \leq \\ &\leq
      \max \set{ \norm{x}_{\mscrX_0} + \norm{y}_{\mscrX_0}, \norm{x}_{\mscrX_1} + \norm{y}_{\mscrX_1} }
      \reloset {\ref{eq:thm:preordered_magma_max_distributivity}} \leq \\ &\leq
      \max \set{ \norm{x}_{\mscrX_0}, \norm{x}_{\mscrX_1} } + \max \set{ \norm{y}_{\mscrX_0}, \norm{y}_{\mscrX_1} }
      = \\ &=
      \norm{x}_{\Delta \overline{\mscrX}} + \norm{y}_{\Delta \overline{\mscrX}}.
    \end{balign*}
  \end{refenum}

  We will now show the completeness of \( \norm{\cdot}_{\Delta \overline{\mscrX}} \) directly. Let \( \{ x_k \}_{k=1}^\infty \subseteq \Delta \overline{\mscrX} \) be a \hyperref[def:fundamental_net]{fundamental sequence}. Both \( \mscrX_0 \) and \( \mscrX_1 \) are complete, therefore \( \{ x_k \}_{k=1}^\infty \) converges to the same value. Both are subspaces of \( \mscrU \), therefore the limit of the sequence is the same in both. In particular, it belongs to the intersection \( \Delta \overline{\mscrX} \).

  Denote the limit of \( \{ x_k \}_{k=1}^\infty \) by \( x \). Let \( \varepsilon > 0 \) and let \( k_0 \) be an index such that both \( \norm{x_k - \xi_0}_{\mscrX_0} < \varepsilon \) and \( \norm{x_k - \xi_1}_{\mscrX_1} < \varepsilon \) whenever \( k \geq k_0 \). Then, for any \( k \geq k_0 \),
  \begin{equation*}
    \norm{x_k - x}_{\Delta \overline{\mscrX}}
    =
    \max\set{\norm{x_k - x}_{\mscrX_0}, \norm{x_k - x}_{\mscrX_1}}
    <
    \varepsilon.
  \end{equation*}

  Therefore, the sequence \( \{ x_k \}_{k=1}^\infty \) converges to \( x_0 \) in \( \Delta \overline{\mscrX} \).

  \SubProofOf{def:banach_space_sum_and_intersection_norms/sum} Again, we will first show that \( \norm{x}_{\Sigma \overline{\mscrX}} \) is indeed a norm.
  \begin{refenum}
    \refitem{def:norm/N1} Analogously to \ref{def:banach_space_sum_and_intersection_norms/sum},
    \begin{equation*}
      \norm{x}_{\Sigma \overline{\mscrX}} = \inf \set{ \norm{x_0}_{\mscrX_0} + \norm{x_1}_{\mscrX_1} : x_0 + x_1 = x } = 0
    \end{equation*}
    if and only if
    \begin{equation*}
      \norm{x}_{\mscrX_0} = \norm{x}_{\mscrX_1} = 0.
    \end{equation*}

    \refitem{def:norm/N2} Absolute homogeneity follows from
    \begin{equation*}
      \norm{tx}_{\Sigma \overline{\mscrX}}
      =
      \inf \set{ \norm{tx_0}_{\mscrX_0} + \norm{tx_1}_{\mscrX_1} | x_0 + x_1 = x }
      \reloset {\ref{def:norm/N2}} =
      \abs{t} \norm{x}_{\Sigma \overline{\mscrX}}.
    \end{equation*}

    \refitem{def:norm/N3} Subadditivity follows from
    \begin{align*}
      &\phantom{{}={}}
      \norm{x + y}_{\Sigma \overline{\mscrX}}
      = \\ &=
      \inf \set{ \left( \norm{x_0}_{\mscrX_0} + \norm{x_1}_{\mscrX_1} \right) + \left( \norm{y_0}_{\mscrX_0} + \norm{y_1}_{\mscrX_1} \right) | \substack{\textstyle{x_0 + x_1 = x} \\ \textstyle{y_0 + y_1 = y}} }
      \leq \\ &\leq
      \norm{x}_{\Sigma \overline{\mscrX}} + \norm{y}_{\Sigma \overline{\mscrX}}.
    \end{align*}
  \end{refenum}

  It remains to prove the completeness of \( \norm{\cdot}_{\Sigma \overline{\mscrX}} \). Let \( \{ x_0^{(k)} + x_1^{(k)} \}_{k=1}^\infty \subseteq \Sigma \overline{\mscrX} \) be a \hyperref[def:fundamental_net]{fundamental sequence}. Fix \( \varepsilon > 0 \). Then there exists an index \( m_0 \) such that \( k, m \geq k_0 \) implies
  \begin{equation*}
    \norm{x_0^{(k)} + x_1^{(k)} - x_0^{(m)} + x_1^{(m)}}_{\Sigma \overline{\mscrX}} < \varepsilon.
  \end{equation*}

  But
  \begin{equation*}
    \norm{x_0^{(k)} - x_0^{(m)}}_{\mscrX_0}
    \leq
    \norm{\left(x_0^{(k)} + x_1^{(k)} \right) - \left( x_0^{(m)} + x_1^{(m)} \right)}_{\Sigma \overline{\mscrX}}
    <
    \varepsilon,
  \end{equation*}
  hence the sequence \( \{ x_0^{(k)} \}_{k=1}^\infty \) is fundamental. Since \( \mscrX_0 \) is complete, this sequence has a limit, which we will denote by \( \xi_0 \). We define \( \xi_1 \) analogously.

  With the same \( \varepsilon \), denote by \( k_0 \) an index such that both \( \norm{x_0^{(k)} - \xi_0}_{\mscrX_0} < \tfrac \varepsilon 2 \) and \( \norm{x_1^{(k)} - \xi_1}_{\mscrX_1} < \tfrac \varepsilon 2 \) whenever \( k \geq k_0 \).

  Then
  \begin{balign*}
    &\phantom{{}={}}
    \norm{\left( x_0^{(k)} + x_1^{(k)} \right) - \left( \xi_0 + \xi_1 \right)}_{\Sigma \overline{\mscrX}}
    = \\ &=
    \inf \set{ \norm{x_0}_{\mscrX_0} + \norm{x_1}_{\mscrX_1} : x_0 + x_1 = \left( x_0^{(k)} + x_1^{(k)} \right) - \left( \xi_0 + \xi_1 \right) }
    \leq \\ &\leq
    \norm{x_0^{(k)} - \xi_0}_{\mscrX_0} + \norm{x_1^{(k)} - \xi_1}_{\mscrX_1}
    <
    \tfrac \varepsilon 2 + \tfrac \varepsilon 2
    =
    \varepsilon.
  \end{balign*}

  Therefore, \( \xi_0 + \xi_1 \) is the limit of the sequence \( \{ x_0^{(k)} + x_1^{(k)} \}_{k=1}^\infty \subseteq \Sigma \overline{\mscrX} \) in \( \Sigma \overline{\mscrX} \).
\end{proof}

\begin{example}\label{thm:lp_interpolation_spaces/definition}
  The spaces \( L^p(\BbbR) \) are interpolation spaces for the pair \( (L^1(\BbbR), L^\infty(\BbbR)) \). The pair is compatible because both are subspaces of the space \( S(\BbbR) \) of all Lebesgue-measurable real function with metric
  \begin{equation*}
    \rho(f, g) \coloneqq \int_{\BbbR} \frac {\abs{f(x) - g(x)}} {1 + \abs(f(x) - g(x))} d\lambda.
  \end{equation*}
\end{example}

\begin{definition}\label{def:lebesgue_space}\cite[6]{BerghLofstrom1976}
  Let \( \mu: U \to [0, \infty] \) be a positive measure and \( p \) be a positive real number. The \term{Lebesgue space} \( L_p \) is defined as the set of bounded functions \( f: U \mapsto \BbbK \) such that the norm
  \begin{equation*}
    \norm{f}_{L_p} \coloneqq \begin{dcases}
      \parens[\Big]{\int_U \abs{f(t)}^p dt}^{1/p}, &0 < p < \infty \\
      \ess\sup_{t \in U} \abs{f(t)} , &p = \infty
    \end{dcases}
  \end{equation*}
\end{definition}

\begin{theorem}[The Riesz-Thorin interpolation theorem]\label{thm:riesz_thorin}\mcite[24]{BerghLofstrom1976}
  Fix two measure spaces \( (U, \mu) \) and \( (V, \nu) \). Let \( T: S(U, \mu) \to S(V, \nu) \) be a continuous linear map between the corresponding spaces of measurable functions.

  Suppose that for some real numbers \( p_0, p_1, q_0, q_1 \geq 1 \) we have
  \begin{equation*}
    T(L^{p_j}(U, \mu)) \subseteq T(L^{q_j}(V, \nu)), j = 0, 1.
  \end{equation*}

  Additionally, let \( \theta \in (0, 1) \) and
  \begin{align*}
    \frac 1 p = \frac {1 - \theta} {p_0} + \frac {\theta} {p_1}
    &&
    \frac 1 q = \frac {1 - \theta} {q_0} + \frac {\theta} {q_1}.
  \end{align*}

  Then
  \begin{equation*}
    T(L^p(U, \mu)) \subseteq L^q(V, \nu)
  \end{equation*}
  and
  \begin{equation*}
    \norm{T}_{\hom(L^p, L^q)} \leq \norm{T}_{\hom(L^{p_0}, L^{q_0})}^{1 - \theta} \norm{T}_{\hom(L^{p_1}, L^{q_1})}^\theta.
  \end{equation*}
\end{theorem}

\begin{definition}\label{def:distribution_function}\cite[6]{BerghLofstrom1976}
  Let \( \mu: U \to [0, \infty] \) be a positive measure and \( p \) be a positive real number.

  \begin{thmenum}
    \thmitem{def:distribution_function/distribution_function} Given a scalar-valued function \( f: U \mapsto \BbbK \), we define its \term{distribution function} as
    \begin{align*}
      &m_f: [0, \infty] \to \BbbK \\
      &m_f(\sigma) \coloneqq \mu(\set{ x :  > \sigma }).
    \end{align*}

    \thmitem{def:distribution_function/rearrangement} We define the \term{decreasing rearrangement} of \( f \) as
    \begin{equation*}
      f^*(t) \coloneqq \inf\set{ \sigma : m_f(\sigma) \leq t }.
    \end{equation*}

    \thmitem{def:distribution_function/lorenz_space} The \( (p, q)-\)Lorenz space, for potentially infinite positive \( q > 0 \), is the set of functions \( f: U \mapsto \BbbK \) for which the quasinorm
    \begin{equation*}
      \norm{f}_{L_{p,q}} \coloneqq \begin{dcases}
        \parens[\Big]{\int_0^\infty \parens[\Big]{\frac {f^*(\tau)} {\tau^p}}^q \frac {d t} t}^{\frac 1 q}, &1 \leq q < \infty \\
        \ess\sup\parens[\Big]{\frac {f^*(t)} {t^p}}, &q = \infty
      \end{dcases}
    \end{equation*}
    is finite.

    In particular, when \( q = \infty \), we use the notation
    \begin{equation*}
      \norm{f}_{L^{p*}} \coloneqq \parens[\Big]{p \int_0^\infty \sigma^p m_f(\sigma) \frac {d \sigma} \sigma}^{\frac 1 p}
    \end{equation*}
  \end{thmenum}
\end{definition}

\begin{theorem}[The Marcinkiewicz interpolation theorem]
  Fix two measure spaces \( (U, \mu) \) and \( (V, \nu) \). Let \( T: S(U, \mu) \to S(V, \nu) \) be a continuous linear map between the corresponding spaces of measurable functions.

  Suppose that for some real numbers \( p_0, p_1, q_0, q_1 \geq 1 \) we have
  \begin{equation*}
    T(L^{p_j}(U, \mu)) \subseteq T(L^{q_j *}(V, \nu)), j = 0, 1.
  \end{equation*}

  Additionally, let \( \theta \in (0, 1) \) and
  \begin{align*}
    \frac 1 p = \frac {1 - \theta} {p_0} + \frac {\theta} {p_1}
    &&
    \frac 1 q = \frac {1 - \theta} {q_0} + \frac {\theta} {q_1}.
  \end{align*}

  Then, if \( p \leq q \),
  \begin{equation*}
    T(L^p(U, \mu)) \subseteq L^q(V, \nu)
  \end{equation*}
  and
  \begin{equation*}
    \norm{T}_{\hom(L^p, L^q)} \leq C_\theta \norm{T}_{\hom(L^{p_0}, L^{q_0*})}^{1 - \theta} \norm{T}_{\hom(L^{p_1}, L^{q_1*})}^\theta
  \end{equation*}
  for some constant \( C_\theta \).
\end{theorem}

\begin{definition}\label{def:banach_interpolation_space_exponent}\mcite[27]{BerghLofstrom1976}
  Let \( \overline{\mscrX} \coloneqq ( \mscrX_0, \mscrX_1 ) \) and \( \overline{\mscrY} \coloneqq ( \mscrY_0, \mscrY_1 ) \) be compatible pairs of Banach spaces. If \( \mscrX \) and \( \mscrY \) are a pair of interpolation spaces and, additionally, the inequality
  \begin{equation}\label{da:def:banach_interpolation_space_exponent}
    \norm{T}_{\hom(\mscrX, \mscrY)} \leq C \norm{T}_{\hom(\mscrX_0, \mscrY_0)}^{1-\theta} \cdot \norm{T}_{\hom(\mscrX_1, \mscrY_1)}^{\theta}
  \end{equation}
  holds for some constant \( C > 0 \) and \( \theta \in [0, 1] \), we say that the pair \( (\mscrX, \mscrY) \) are \term{interpolation spaces of exponent} \( \theta \).

  If, additionally, \( C = 1 \), we say that \( (\mscrX, \mscrY) \) is an \term{exact pair} of interpolation spaces.
\end{definition}

\begin{definition}\label{def:k_functional}\mcite[38]{BerghLofstrom1976}
  Let \( \overline{\mscrX} \coloneqq ( \mscrX_0, \mscrX_1 ) \) be a compatible pair of Banach spaces. Instead of the norm \( \norm{\cdot}_{\mscrX_1} \) in \( \mscrX_1 \), we can consider \hyperref[def:equivalent_metrics]{equivalent norms} of the type \( t\norm{\cdot}_{\mscrX_1} \) for \( t \geq 0 \). Furthermore, we can also introduce equivalent norms in \( \Sigma \overline{\mscrX} \) via the \term{\( K \)-functional}
  \begin{alignedeq}\label{eq:def:k_functional}
    &K: (0, \infty) \times {\Sigma \overline{\mscrX}} \\
    &K(t, x) \coloneqq \inf \set{ \norm{x_0}_{\mscrX_0} + t\norm{x_1}_{\mscrX_1} : x_0 + x_1 = x }.
  \end{alignedeq}

  See \fullref{def:k_functional_properties/equivalent_norm} for a proof that \( x \mapsto K(t, x) \) for a fixed \( t \geq 0 \) is an equivalent norm in the sum \( \Sigma \overline{\mscrX} \).
\end{definition}

\begin{proposition}\label{def:k_functional_properties}\mcite[38]{BerghLofstrom1976}
  The \hyperref[def:k_functional]{\( K \)-functional} has the following basic properties:

  \begin{thmenum}
    \thmitem{def:k_functional_properties/basic} For any fixed \( x \in \Sigma \overline{\mscrX} \), the function \( t \mapsto K(t, x) \) is positive, \hyperref[def:partially_ordered_set/homomorphism]{monotone} and \hyperref[def:convex_functions]{concave}.

    \thmitem{def:k_functional_properties/inequality} For positive real numbers \( t, s > 0 \), we have the following inequality:
    \begin{equation}\label{eq:def:k_functional_properties/inequality}
      K(t, x) \leq \max\set{1, \frac t s} K(s, x).
    \end{equation}

    \thmitem{def:k_functional_properties/equivalent_norm} For any fixed \( t > 0 \), the function \( x \mapsto K(t, x) \) is an \hyperref[def:equivalent_metrics]{equivalent norm} in the sum \( \Sigma \overline{\mscrX} \).
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{def:k_functional_properties/basic} That \( t \mapsto K(t, x) \) is positive is a slight generalization of \fullref{def:norm/N1}, which can be proved as in \fullref{def:banach_space_sum_and_intersection_norms/sum}.

  Monotonicity follows from the monotonicity of the infimum.

  To see that \( t \mapsto K(t, x) \) is concave, fix \( x \), \( \lambda \in [0, 1] \) and \( t, s > 0 \). We have
  \begin{align*}
    &\phantom{{}={}}
    K(\lambda t + (1 - \lambda) s, x)
    = \\ &=
    \inf \set{ \norm{x_0}_{\mscrX_0} + (\lambda t + (1 - \lambda) s)\norm{x_1}_{\mscrX_1} | x_0 + x_1 = x }
    = \\ &=
    \inf \set{ \lambda \left(\norm{x_0}_{\mscrX_0} + t \norm{x_1}_{\mscrX_1} \right) + (1 - \lambda) \left(\norm{x_0}_{\mscrX_0} + s \norm{x_1}_{\mscrX_1} \right) | x_0 + x_1 = x }
    \geq \\ &\geq
    \lambda K(t, x) + (1 - \lambda) K(s, x).
  \end{align*}

  \SubProofOf{def:k_functional_properties/inequality} Fix positive real numbers \( t, s > 0 \).
  \begin{itemize}
    \item If \( t \leq s \), by monotonicity we have
    \begin{equation}\label{eq:def:k_functional_properties/inequality/monotonicity}
      K(t, x) \leq K(s, x)
    \end{equation}

    \item If \( t > s \), we use concavity with
    \begin{equation*}
      s = \frac s t t + \left(1 - \frac s t \right) 0
    \end{equation*}
    to obtain
    \begin{equation*}
      K(s, x) \geq \frac s t K(t, x) + \left(1 - \frac s t \right) K(0, x).
    \end{equation*}

    By positivity of \( K \), we have \( K(t, x) = 0 \) if and only if \( t = 0 \), hence
    \begin{equation}\label{eq:def:k_functional_properties/inequality/concavity}
      K(t, x) \leq \frac t s K(s, x).
    \end{equation}
  \end{itemize}

  Combining \eqref{eq:def:k_functional_properties/inequality/monotonicity} and \eqref{eq:def:k_functional_properties/inequality/concavity}, we obtain \eqref{eq:def:k_functional_properties/inequality}.

  \SubProofOf{def:k_functional_properties/equivalent_norm} That \( x \mapsto K(t, x) \) for a fixed \( t > 0 \) is a slight generalization of the proof in \fullref{def:banach_space_sum_and_intersection_norms/sum}.

  That the norms \( \norm{\cdot}_{\Sigma \overline{\mscrX}} \) and \( K(t, \cdot) \) are equivalent follows from \eqref{eq:def:k_functional_properties/inequality} with \( s = 1 \) for the upper bound and \( t = 1, s = t \) for the lower bound. That is,
  \begin{equation*}
    \min\set{1, t} \underbrace{K(1, x)}_{\norm{x}_{\Sigma \overline{\mscrX}}} \leq K(t, x) \leq \max\set{1, t} \underbrace{K(1, x)}_{\norm{x}_{\Sigma \overline{\mscrX}}}.
  \end{equation*}
\end{proof}

\begin{example}\label{thm:lp_interpolation_spaces/k_functional}
  The \hyperref[def:k_functional]{\( K \)-functional} for the pair \( (L_1(\BbbR), L_\infty(\BbbR)) \) from \fullref{thm:lp_interpolation_spaces/definition} is
  \begin{equation*}
    K(t, f) \coloneqq \int_0^t f^*(\tau) d\tau.
  \end{equation*}
\end{example}

\begin{definition}\label{def:lorenz_quasinorm}
  For \( \theta \in \BbbR \), \( q \in (0, \infty] \) and nonnegative functions \( g: [0, \infty) \to [0, \infty] \) we define
  \begin{equation}\label{eq:def:lorenz_quasinorm}
    \Phi_{\theta,q}(g) \coloneqq \begin{dcases}
      \left( \int_0^\infty \left( \frac {g(\tau)} {\tau^\theta} \right)^q \frac {d\tau} \tau \right)^{\tfrac 1 q}, &0 < q < \infty \\
      \ess\sup_{t \geq 0} \left( \frac {g(t)} {t^\theta} \right),                                                &q = \infty
    \end{dcases}
  \end{equation}
  and
  \begin{equation}\label{eq:def:lorenz_quasinorm/gamma}
    \gamma_{\theta,q} \coloneqq \Phi_{\theta,q}(\min(t, 1)).
  \end{equation}
\end{definition}

\begin{proposition}\label{thm:def:lorenz_quasinorm/properties}
  The function \hyperref[def:lorenz_quasinorm]{\( \Phi_{\theta,q} \)} has the following basic properties:

  \begin{thmenum}
    \thmitem{thm:def:lorenz_quasinorm/properties/reciprocal} For \( s > 0 \) and \( h(t) \coloneqq g(\tfrac t s) \) we have
    \begin{equation}\label{eq:thm:def:lorenz_quasinorm/properties/reciprocal}
      \Phi_{\theta,q}(h) = \frac 1 {s^{\theta}} \Phi_{\theta,q}(g).
    \end{equation}

    \thmitem{thm:def:lorenz_quasinorm/properties/gamma} For finite \( q \) we have
    \begin{equation}\label{eq:thm:def:lorenz_quasinorm/properties/gamma}
      \gamma_{\theta,q} = \left( \frac 1 {q \theta (1 - \theta)} \right)^{\tfrac 1 q}.
    \end{equation}
  \end{thmenum}
\end{proposition}
\begin{proof}
  \SubProofOf{thm:def:lorenz_quasinorm/properties/reciprocal} The case \( q = \infty \) is obvious. For \( 0 < q < \infty \), we have
  \begin{balign*}
    \Phi_{\theta,q}(h)
    &=
    \left( \int_0^\infty \left( \frac {h(\tau)} {\tau^\theta} \right)^q \frac {d\tau} \tau \right)^{\tfrac 1 q}
    = \\ &=
    \left( \frac 1 {s^{\theta q}} \int_0^\infty \left( \frac {g(\tfrac \tau s)} {\left(\tfrac \tau s \right)^\theta} \right)^q \frac {d{\tfrac \tau s}} {\tfrac \tau s} \right)^{\tfrac 1 q}
    = \\ &=
    \frac 1 {s^{\theta}} \Phi_{\theta,q}(g).
  \end{balign*}

  \SubProofOf{thm:def:lorenz_quasinorm/properties/gamma} We can raise \( \gamma_{\theta,q} \) to the \( q \)-th power for brevity of notation:
  \begin{balign*}
    \gamma_{\theta,q}^q
    &=
    \Phi_{\theta,q}(\min(t, 1))^q
    = \\ &=
    \int_0^1 \left( \frac {\tau} {\tau^\theta} \right)^q \frac {d\tau} \tau + \int_1^\infty \left( \frac {1} {\tau^\theta} \right)^q \frac {d\tau} \tau
    = \\ &=
    \int_0^1 \tau^{(1 - \theta) q - 1} d\tau + \int_1^\infty \tau^{-\theta q - 1} d\tau
    = \\ &=
    \frac {1 - 0} {(1 - \theta) q} + \frac {\lim_{\tau \to \infty} \tau^{-\theta q} - 1} {-\theta q}
    = \\ &=
    \frac 1 {(1 - \theta) q} - \frac 1 {-\theta q}
    = \\ &=
    \frac {-\theta q - (1 - \theta) q} {(1 - \theta) (-\theta) q^2}
    = \\ &=
    \frac 1 {(1 - \theta) \theta q}
  \end{balign*}
\end{proof}

\begin{definition}\label{def:k_functional_interpolation_space}\mcite[40]{BerghLofstrom1976}
  Let \( \overline{\mscrX} \coloneqq ( \mscrX_0, \mscrX_1 ) \) be a compatible pair of Banach spaces.

  For \( \theta \in (0, \infty) \), \( q \in (0, \infty] \), we introduce the following norm:
  \begin{equation}\label{eq:def:k_functional_interpolation_space/norm}
    \norm{x}_{\theta,q,K} \coloneqq \Phi_{\theta,q}(K(t, x)).
  \end{equation}

  The subspace of \( \Sigma\overline{\mscrX} \) for which this norm is finite is denoted by either
  \begin{align*}
    K_{\theta,q}(\overline{\mscrX})
    &&
    \mscrX_{\theta,q,K}.
  \end{align*}
\end{definition}

\begin{theorem}\label{thm:k_functional_interpolation}\mcite[thm. 3.1.2]{BerghLofstrom1976}
  Let \( \theta \in (0, 1) \) and \( q \in (0, \infty] \). The space \( \mscrX_{\theta,q,K} \) defined in \fullref{eq:def:k_functional_interpolation_space/norm} is an \hyperref[def:banach_interpolation_space_exponent]{exact interpolation space} of exponent \( \theta \). Furthermore,
  \begin{equation}\label{eq:thm:k_functional_interpolation/inequality}
    K(s, x) \leq (\gamma_{\theta,q})^{-1} s^\theta \norm{x}_{\theta,q,K}.
  \end{equation}
\end{theorem}
\begin{proof}
  Note that \( K(s, x) \) is a norm on \( \Sigma \overline{X} \) by \fullref{def:k_functional_properties/equivalent_norm}. Therefore, \( \norm{\cdot}_{\theta,q,K} \), the composition of \( K(s, x) \) with the \hyperref[def:lorenz_quasinorm]{Lorenz quasinorm} \( \Phi_{\theta,q} \), is a norm.

  We denote by \( \mscrX_{\theta,q,K} \) the space consisting of all vectors from \( \Sigma \overline{\mscrX} \) where the norm \eqref{eq:def:k_functional_interpolation_space/norm} is finite.

  From \eqref{eq:def:k_functional_properties/inequality} it follows that
  \begin{equation*}
    \min(1, \tfrac t s) K(s, x) \leq K(t, x)
  \end{equation*}
  and hence
  \begin{equation*}
    \underbrace{\Phi_{\theta,q}}_{\text{depends on } t} \parens[\Big]{ \min(1, \tfrac t s) K(s, x) } \leq \underbrace{\Phi_{\theta,q}(K(s, x))}_{\text{norm in } \mscrX_{\theta,q,K}}.
  \end{equation*}

  By \eqref{eq:thm:def:lorenz_quasinorm/properties/reciprocal}, we have
  \begin{equation*}
    \Phi_{\theta,q}(\min(1, \tfrac t s)) = \tfrac 1 {s^\theta} \underbrace{\Phi_{\theta,q}(\min(1, t))}_{\hyperref[eq:def:lorenz_quasinorm/gamma]{\gamma_{\theta,q}}},
  \end{equation*}
  and \eqref{eq:thm:k_functional_interpolation/inequality} follows.

  It remains to show that \( \mscrX_{\theta,q,K} \) is an exact interpolation space of exponent \( \theta \).

  Note that \( K(1, x) = \norm{x}_{\Sigma \overline{\mscrX}} \) and thus \eqref{eq:thm:k_functional_interpolation/inequality} with \( s = 1 \) implies that
  \begin{equation*}
    \gamma_{\theta,q} \norm{x}_{\Sigma \overline{\mscrX}} \leq \norm{x}_{\theta,q,K},
  \end{equation*}
  which shows that \( \mscrX_{\theta,q,K} \) can be embedded continuously into \( \Sigma \overline{\mscrX} \).

  On the other hand, for \( x \in \Delta \overline{\mscrX} \) we have
  \begin{equation*}
    K(t, x) \leq \norm{x} \leq \norm{x}_{\Delta \overline{\mscrX}} \T{since} x = x + 0
  \end{equation*}
  and
  \begin{equation*}
    K(t, x) \leq \norm{x} \leq t \norm{x}_{\Delta \overline{\mscrX}} \T{since} x = 0 + x.
  \end{equation*}

  Therefore,
  \begin{equation*}
    K(t, x) \leq \min(1, t) \norm{x}_{\Delta \overline{\mscrX}},
  \end{equation*}
  which after applying \( \Phi_{\theta,q} \) becomes
  \begin{equation*}
    \norm{x}_{\theta,q,K} \leq \gamma_{\theta,q} \norm{x}_{\Delta \overline{\mscrX}}.
  \end{equation*}

  Hence, we have the chain of continuous linear inclusions of Banach spaces
  \begin{equation*}
    \Delta \overline{\mscrX} \subseteq X \subseteq \Sigma \overline{\mscrX}.
  \end{equation*}

  Finally, to show that \( \mscrX \) is an interpolation space of exponent \( \theta \), fix a linear operator \( T: \overline{\mscrX} \mapsto \overline{CY} \) between compatible pairs and let \( \mscrY \) be an intermediate space for \( \overline{CY} \).

  Then
  \begin{align*}
    K(t, Tx)_{\overline{CY}}
    &\leq
    \inf \set{ \norm{y_0}_{\mscrY_0} + t \norm{y_1}_{\mscrY_1} : y_0 + y_1 = Tx }
    \leq \\ &\leq
    \inf \set{ \norm{T}_{\hom(\mscrX_0, \mscrY_0)} \norm{x_0}_{\mscrX_0} + t \norm{T}_{\hom(\mscrX_1, \mscrY_1)} \norm{x_1}_{\mscrY_1} : x_0 + x_1 = x }
    = \\ &=
    \norm{T}_{\hom(\mscrX_0, \mscrY_0)} K\parens[\Bigg]{\frac {\norm{T}_{\hom(\mscrX_1, \mscrY_1)}} {\norm{T}_{\hom(\mscrX_0, \mscrY_0)}} t, x}.
  \end{align*}

  By applying \( \Phi_{\theta,q} \) to both sides and using \eqref{eq:thm:def:lorenz_quasinorm/properties/reciprocal}, we obtain
  \begin{equation*}
    \norm{Tx}_{\overline{\mscrY}_{\theta,q,K}}
    \leq
    {\norm{T}_{\hom(\mscrX_1, \mscrY_1)}}^{1 - \theta} {\norm{T}_{\hom(\mscrX_0, \mscrY_0)}}^{\theta} \norm{x}_{\overline{\mscrY}_{\theta,q,K}}.
  \end{equation*}

  Thus, \( \mscrX \) satisfies \fullref{def:banach_interpolation_space_exponent} for being an exact interpolating space with exponent \( \theta \).
\end{proof}

\begin{definition}\label{def:discrete_k_interpolation_space}
  For positive numbers \( \theta \) and \( q \), we denote by \( \lambda^{\theta,q} \) the set of all doubly-infinite real sequences \( \{ x_k \}_{k=-\infty}^\infty \) such that the norm
  \begin{equation}\label{eq:def:discrete_k_interpolation_space}
    \norm{\{ x_k \}_{k=-\infty}^\infty}_{\lambda^{\theta,q}} \coloneqq \left( \sum_{k=-\infty}^\infty \left( \frac {\abs{x_k}} {2^{k\theta}} \right)^q \right)^{\tfrac 1 q}
  \end{equation}
  is finite.
\end{definition}

\begin{theorem}\label{thm:discrete_k_interpolation}\mcite[lemma 3.1.3]{BerghLofstrom1976}
  The vector \( x \in \Sigma\overline{\mscrX} \) belongs to \hyperref[def:k_functional_interpolation_space]{\( \mscrX_{\theta,q,K} \)} if and only if the sequence \( \{ x_k \}_{k=-\infty}^\infty \) defined as
  \begin{equation}\label{eq:thm:discrete_k_interpolation/sequence}
    x_k \coloneqq K(2^k, x)
  \end{equation}
  belongs to \hyperref[def:discrete_k_interpolation_space]{\( \lambda^{\theta,q} \)}.

  Furthermore, for any integer \( k \) the following inequalities hold:
  \begin{equation}\label{eq:thm:discrete_k_interpolation/inequalities}
    \frac 1 {2^\theta} \ln 2 \norm{x_k}_{\lambda^{\theta,q}}
    \leq
    \norm{x}_{\theta,q,K}
    \leq
    2 \cdot \ln 2 \norm{x_k}_{\lambda^{\theta,q}}.
  \end{equation}
\end{theorem}
\begin{proof}
  We have
  \begin{equation*}
    \norm{x}_{\theta,q,K}^q
    =
    \int_0^\infty \left( \frac {K(\tau, x)} {\tau^\theta} \right)^q \frac {d\tau} \tau
    =
    \sum_{k=-\infty}^\infty \int_{2^k}^{2^{k+1}} \left( \frac {K(\tau, x)} {\tau^\theta} \right)^q \frac {d\tau} \tau.
  \end{equation*}

  By \fullref{def:k_functional_properties/inequality}, for each integer \( k \),
  \begin{equation*}
    K(2^k, x) \leq 2 K(2^k, x).
  \end{equation*}

  By the \hyperref[def:k_functional_properties/basic]{monotonicity} of \( K \), for \( t \in [2^k, 2^{k+1}] \) we have
  \begin{equation*}
    K(2^k, x) \leq K(t, x) \leq 2 K(2^k, x).
  \end{equation*}

  Denote \( x_k \coloneqq K(2^k, x) \). For \( 2^k \leq t \leq 2^{k+1} \) we have
  \begin{equation*}
    \frac{x_k}{2^{(k+1)\theta}} \leq \frac{K(t, x)}{t^\theta} \leq 2 \frac{x_k}{2^{k\theta}}
  \end{equation*}

  Therefore,
  \begin{align*}
    \norm{x}_{\theta,q,K}^q
    &=
    \sum_{k=-\infty}^\infty \int_{2^k}^{2^{k+1}} \left( \frac {K(\tau, x)} {\tau^\theta} \right)^q \frac {d\tau} \tau
    \leq \\ &\leq
    2^q \sum_{k=-\infty}^\infty \left(\frac{x_k}{2^{k\theta}} \right)^q \cdot \ln \tau \mid_{\tau=2^k}^{2^{k+1}}
    = \\ &=
    \ln 2 \cdot 2^q \sum_{k=-\infty}^\infty \left(\frac{x_k}{2^{k\theta}} \right)^q
    = \\ &=
    2^q \ln 2 \norm{\{ x_k \}_{k=-\infty}^\infty}_{\lambda^{\theta,q}}^q
  \end{align*}
  and similarly for the lower bound.
\end{proof}

\begin{definition}\label{def:e_functional}\mcite[174]{BerghLofstrom1976}
  Let \( \overline{\mscrX} = (\mscrX_0, \mscrX_1) \) be a compatible pair of Banach spaces. Let \( x \in \Sigma \overline{\mscrX} \). Put
  \begin{alignedeq}\label{eq:def:e_functional}
    &E: (0, \infty) \times {\Sigma \overline{\mscrX}} \\
    &E(t, x) \coloneqq \inf \set{ \norm{x - x_0}_{\mscrX_1} : \norm{x_0}_{\mscrX_0} \leq t }.
  \end{alignedeq}
\end{definition}

\begin{proposition}\label{thm:def:e_functional/properties}\mcite[lemma 7.1.3]{BerghLofstrom1976}
  When \( \overline{\mscrX} \) are quasi-Banach spaces, the \hyperref[def:e_functional]{\( E \)-functional} has the following basic properties:

  \begin{thmenum}
    \thmitem{def:k_functional_properties/decreasing} For fixed \( x \in \Sigma\overline{\mscrX} \), the function \( t \mapsto E(t, x) \) is decreasing.

    \thmitem{def:k_functional_properties/subaditive} For \( \varepsilon \in (0, 1) \), we have
    \begin{equation*}
      E(t, x + y) \leq E(\varepsilon t, x) + E((1 + \varepsilon) t, y).
    \end{equation*}

    \thmitem{def:k_functional_properties/positive} \( x = 0 \) if and only if \( E(t, x) = 0 \) for all \( t > 0 \).

    \thmitem{def:k_functional_properties/k_functional_connection}\mcite[thm. 7.1.4]{BerghLofstrom1976}
    \begin{equation*}
      E(t, x) = \sup \set{ \frac {K(s, x) - t} s : s > 0 }.
    \end{equation*}
  \end{thmenum}
\end{proposition}

\begin{definition}\label{def:approximation_space}\mcite[def. 7.1.5]{BerghLofstrom1976}
  Let \( \overline{\mscrX} = (\mscrX_0, \mscrX_1) \) be a compatible pair of Banach spaces. We define an \term{approximation space} \( E_{\alpha,q}(\overline{\mscrX}) \) for \( x \in \Sigma\overline{\mscrX} \) as the space of all members of \( \Sigma\overline{\mscrX} \) for which the following norm
  \begin{equation}\label{eq:def:approximation_space/norm}
    \norm{x}_{\alpha,q,E} \coloneqq \Phi_{-\alpha,q}(E(t,a))
  \end{equation}
  is finite.

  Here \( \alpha \) and \( q \) are both positive real numbers and \( q \) is potentially \( \infty \).
\end{definition}

\begin{theorem}\label{thm:interpolation_space_and_approximation_space}\mcite[thm. 7.1.6]{BerghLofstrom1976}
  Let \( \mscrX \) be a compatible pair of Banach spaces. Let \( \alpha \) and \( q \) be positive real numbers and define
  \begin{align*}
    \theta \coloneqq \frac 1 {\alpha + 1},
    &&
    r \coloneqq \theta q.
  \end{align*}

  Then
  \begin{equation*}
    (E_{\alpha,\theta q}(\overline{\mscrX}))^\theta = K_{\theta,q}(\overline{\mscrX}).
  \end{equation*}
\end{theorem}

\begin{theorem}\label{thm:interpolation_space_and_approximation_space_reiteration}\mcite[thm. 7.1.8]{BerghLofstrom1976}
  Let \( \mscrX \) be a compatible pair of Banach spaces. Let \( \theta, \alpha_0, \alpha_1, r_0, r_1 \) and \( q \) be positive real numbers such that \( \alpha_0 \neq \alpha_1 \) and define \( r \coloneqq \theta q \) and
  \begin{align*}
    \alpha \coloneqq (1 - \theta) \alpha_0 + \theta \alpha_1,
    &&
    \beta \coloneqq - \frac {\alpha_1 - \alpha} {\alpha_0 - \alpha}.
  \end{align*}

  Then
  \begin{equation*}
    K_{\theta,q}(E_{\alpha_0,r_0}(\overline{\mscrX}), E_{\alpha_1,r_1}(\overline{\mscrX})) = E_{\alpha,q}(\overline{\mscrX})
  \end{equation*}
  and
  \begin{equation*}
    E_{\beta,r}(E_{\alpha_0,r_0}(\overline{\mscrX}), E_{\alpha_1,r_1}(\overline{\mscrX}))^\theta = E_{\alpha,q}(\overline{\mscrX}).
  \end{equation*}
\end{theorem}


\documentclass[11pt]{article}
\usepackage{mathtools}
\usepackage{latexsym}
\usepackage{amsfonts}
\usepackage{enumitem}
\usepackage{amsmath}
\usepackage{graphicx}
\usepackage{color, colortbl}
\usepackage{amssymb}
\setlength{\parskip}{\baselineskip}
\renewcommand{\det}[1]{|#1|}
\newcommand{\mf}{-\frac{1}{2}}
\newcommand{\Mu}{M}
\newcommand{\hp}{\circ}
\newcommand{\fc}{\frac{1}{\sqrt{(2\pi)^{d+m}(\sigma^2)^m }}}
\newcommand{\todo}[1]{\textbf{\textcolor{red}{TODO: #1}}}
\newcommand{\note}[1]{\textbf{\textcolor{red}{NOTE: #1}}}
\newcommand{\ps}{\begin{bmatrix} \psi_1 & 0 \\ 0 & \psi_2 \end{bmatrix}}
\begin{document}
\raggedright
\section{Probabilistic PCA}

When I observe data then the best k orthogonal basis of a projection
space are the eigenvectors of the covariance matrix.

\begin{align}
&\arg\min ||X - UU^\top X ||^@_F \\
=&tr(M^\top M)\\
=&tr(X^TX + X^TUU^TUU^TX - zX^TUU^TX)\\
=& \max tr(X^TUU^TX) \\
=& tr(U^TXX^TU)
\end{align}
Now I can also try and solve the following problem.

I observe data x and I assume that it was generated in the following way:
\begin{align}
z \sim & \mathcal{N}(0, I_d)\\
x \sim & \mathcal{N}(Wz+\mu, \sigma^2 I_m)
\end{align}
And I need to find $W, \mu, \sigma^2$ so that $p(x)$ is maximized. $W
\in \mathcal{R}^{m\times d}$.

\begin{align}
  p(x) &= \prod_{i=1}^n \int p(x_i | z_i) p(z_i)\\
  l(x) &= \sum_{i=1}^n \log(\int p(x_i|z_i) p(z_i)) \label{obj}\\
\end{align}
For simplicity lets drop subscripts and derive $p(x) = \int p(x|z) p(z)$.
\begin{align}
  p(z) &= \frac{1}{\sqrt{(2\pi)^d\det{I_d}}} \exp(-\frac{1}{2} z^T  I_d^{-1} z)\\
  p(x|z) &= \frac{1}{\sqrt{(2\pi)^m(\sigma^2)^m \det{I_m}}}
  \exp(-\frac{1}{2}(x-Wz-u)^T(\sigma^2I_m)^{-1}(x-Wz-\mu))
\end{align}

What is $\int p(x|z) p(z)$. I know it is a guassian. I also can use
mathematica. I can also use the fact that its solution is given in
kevin murphy's book. But lets derive it.
\begin{align}
  \int p(x|z)p(z) &= \fc \int
  \exp{\mf (z^Tz + (x-\mu - Wz)^T(\sigma^{-2})(x-\mu-Wz))}
\end{align}
Let $(x-\mu)/\sigma = y,\; W/\sigma=V$
\begin{align}
  \int p(x|z)p(z) &= \fc \int \exp(\mf (z^Tz + (y - Vz)^T(y-Vz)))\\
  &= \fc\exp(\mf y^Ty) \int \exp(\mf (z^T (I+V^TV) z  - 2z^TV^Ty))
\end{align}
Let $U = (I+V^TV)^{-1}$
\begin{align}
  &= \fc \exp(\mf y^Ty) \int \exp(\mf ( \left(z^TU^{-1}z -2z^TU^{-1}a + a^TU^{-1}a\right) - a^TU^{-1}a  ))
\end{align}
This means that $V^Ty = U^{-1}a \implies a = UV^Ty$
\begin{align}
  &= \fc \exp(\mf (y^Ty - a^TU^{-1}a) ) \int \exp(\mf ((z-a)^T U^{-1}(z-a)))\\
  &= \fc \sqrt{(2\pi)^d \det{U}} \exp(\mf (y^Ty - a^TU^{-1}a) )\\
  &= \fc \sqrt{(2\pi)^d \det{U}} \exp(\mf (y^Ty - y^TVU^TU^{-1}UV^Ty))\\
  &= \fc \sqrt{(2\pi)^d \det{U}} \exp(\mf (y^Ty - y^TVU^TV^Ty) )\\
  &= \fc \sqrt{(2\pi)^d \det{U}} \exp(\mf (y^T(I-VU^TV^T)y)
\end{align}

Just substitute back $y$ and $V$ and $U$ to get the answer
\begin{align}
  \int p(x|z)p(z) &= \frac{\sqrt{(2\pi)^d \det{U}}}{\sqrt{(2\pi)^{d+m}(\sigma^2)^m }} \exp(\mf ((x-\mu)^T\frac{(I-V(I+V^TV)^{-1}V^T)}{\sigma^2}(x-\mu))
\end{align}

Interestingly, $(I-V(I+V^TV)^{-1}V^T) = (I+VV^T)^{-1}$
This means that
\begin{align}
  p(z) = \int p(x|z)p(z) &= \frac{1}{\sqrt{(2\pi)^{m}(\sigma^2)^m \det{U}^{-1}}}  \exp(\mf ((x-\mu)^T (\sigma^2 (I+VV^T))^{-1}  (x-\mu)))
\end{align}
Also note that by Sylvester's theorem $\det{U} = \det{I+V^TV}^{-1} = \det{I+VV^T}^{-1} $.

So finally we have proven that $p(x) = \mathcal{N}(\mu,
\sigma^2I+WW^T)$

Now substitute this results back into \ref{obj} to get

\begin{align}
  l(x) &= \sum_{i=1}^n \log(\frac{1}{\sqrt{(2\pi)^{m}(\sigma^2)^m \det{I+VV^T}}}  \exp(\mf ((x_i-\mu)^T (\sigma^2 (I+VV^T))^{-1}  (x_i-\mu))))\\
  &= \mf \sum_{i=1}^n ((x_i-\mu)^T (\sigma^2I+WW^T))^{-1}  (x_i-\mu)) + \log((2\pi)^{m} \det{\sigma^2I+WW^T})
\end{align}
Now get rid of constant terms like $m\log(2\pi)$ and the constant
multipliers which don't affect the optimization (However now I have to
minimize the quantity since I removed $\mf$)
\begin{align}
  &= \sum_{i=1}^n ((x_i-\mu)^T (\sigma^2I+WW^T)^{-1}  (x_i-\mu)) + \log(\det{\sigma^2I+WW^T})
\end{align}
Now minimize this quantity wrt $\mu, W, \sigma$ to get the MLE
parameters

Wrt $\mu$\\
Let $Y=(\sigma^2I+WW^T))^{-1}$, then wrt $\mu$ the derivative is
\begin{align}
  =& \sum_{i=1}^n \frac{\partial}{\partial \mu}  ((x_i-\mu)^T Y  (x_i-\mu)) \\
  =& \sum_{i=1}^n \frac{\partial}{\partial \mu}  (\mu^T Y\mu -
  2x_i^TY\mu + x_i^TYx_i)) \\
  =& \sum_{i=1}^n  (Y+Y^T)\mu -  (2x_i^TY)^T )) \\
\end{align}
However $Y$ is symmetric, therefore $Y+Y^T = 2Y^T$, which means
$$\mu = \frac{1}{n}{\sum_{i=1}^n x_i}$$

Let $y_i = x_i - \mu$ then wrt $\sigma$ and $W$ the objective to
minimize is: where $\sum y_i = 0$
\begin{align}
  =& \sum_{i=1}^n \left(  (y_i^T
  (\sigma^2I+WW^T)^{-1}  y_i) + \log(\det{\sigma^2I+WW^T}) \right)\\
\end{align}
Let's level up and concatenate all the $y_i$ into a matric $Y$, then
the objective becomes
\begin{align}
  tr(Y^T (\sigma^2I+WW^T)^{-1}Y) + n\log(\det{\sigma^2I+WW^T})
\end{align}
Interestingly $\sigma^2I+WW^T$ are inseparable, replace them by $V = (\sigma^2I+WW^T)^{-1}$ to
get the new objective
\begin{align}
  \arg\min_V tr(Y^TVY) - n\log(\det{V})
\end{align}

\begin{align}
  \frac{\partial}{\partial V} tr(Y^TVY) - n\log(\det{V}) &= YY^T - n
  V^{-1} = 0\\
  V^{-1} = (\sigma^2I+WW^T) &= \frac{1}{n}YY^T
\end{align}

Now to solve this equation we would have to let $\sigma$ equal the
last $d-m$ eigenvalues so they would equal each other and then we can
 do an eigen decomposition
of $C = YY^T/n - \sigma^2I = U\sqrt{S}MM^T\sqrt{S^T}U^T \in \mathcal{R}^{d \times
  d}$ then we derive the PCA solution assuming that we had perfect
knowledge of the ppopulation means.

But if instead of taking derivative wrt $V$ we take derivative wrt $W$
and actually recognize the constraint that $W$ would not be able to
perfectly match $C=YY^T/n$  then we would get: \todo{COMPLETE THIS}
\begin{align}
  (YY^T/n(\sigma^2I+WW^T))W &= W
\end{align}

\section{Probabilistic CCA}
I will use the following objective as my starting point for CCA.
Let $A=\Sigma_{xx}, B=\Sigma_{YY}, C=\Sigma_{XY}$
\begin{align}
  \arg\max_{U,V} \;& tr(U^T C V)\\
  \text{subject to } & U^TAU=I,\; V^TBV=I
\end{align}
We can solve this in two ways.
\subsection{Approach 1}
The usual approach is to first find a single direction and then show
that the remaining directions follow as the remaining eigen-vectors of
a matrix all of which would follow the required constraints. Let's
recap that approach first.

Let $u,\; v$ be two directions, then the lagrangian of the objective
becomes
\begin{align}
  \mathcal{L} &=u^TCv + \lambda (u^tAu-1 ) + \mu (v^TBv-1)\\
  \frac{\partial \mathcal{L}}{\partial u} &= v^TC^T + u^T(2\lambda A)  =0 = Cv + 2\lambda A^Tu\\
  \frac{\partial \mathcal{L}}{\partial v} &= u^TC + v^T(2\mu B)  =0=C^Tu+2\mu B^T v\\
  u &= -\frac{1}{2\lambda}A^{-T}Cv\\
  -C^T (-\frac{1}{2\lambda}A^{-T}Cv) &= 2\mu B^Tv\\
  B^{-T}C^TA^{-T}Cv &=4\lambda \mu v
\end{align}
Now use $B=B^T,\; A=A^T,\; 4\lambda \mu = \kappa$, also use the fact that B can be decomposed
into its cholesky factors $B=DD^T \implies B^{-1}=D^{-T}D^{-1}$ to simplify the objective:
\begin{align}
  D^{-T}D^{-1}C^TA^{-1}CD^{-T}D^Tv &=\kappa v  \\
  D^{-1}C^TA^{-1}CD^{-T}D^Tv &=\kappa D^Tv
\end{align}
Let $v'=D^Tv$ to get
\begin{align}
  D^{-1}C^TA^{-1}CD^{-T}v' &=\kappa v'  \\
  \text{Let } M &= D^{-1}C^TA^{-1}CD^{-T}
\end{align}
Now we claim that the successive orthogonal eigen vectors $v'$ of the
symmateric matrix $M$ would give us the
solutions as $v=D^{-T}v', \; u = -\frac{1}{2\lambda}A^{-T}CD^{-T}v'$.

It is easy to see that $v^TBv = v'^T D^{-1}BD^{-T}v' = 1$ and all
subsequent vectors would also obey this.

We can check that $u^TAu =
\frac{1}{4\lambda^2}v'^TD^{-1}C^TA^{-1}AA^{-T}CD^{-T}v' =
\frac{1}{4\lambda^2}v'^TMv'=\frac{\mu}{\lambda}$
We are free to choose $\lambda, \mu$ so to satisfy the constraints we
should choose $\mu=\lambda$.
\subsection{Approach 2}
This approach adds one lagrange multiplier per constraint
and uses hard algebra to grind through. Specifically the lagrangian becomes
\begin{align}
  L &= tr(U^TCV) + \sum \lambda_{ij}(u_i^TAu_j - \delta_{ij}) + \sum
  \mu_{ij}(v_i^TBv_j - \delta_{ij})\\
  &= tr(U^TCV) + 1^T U^T(A\hp \Lambda)U 1 - tr(\Lambda) + 1^T V^T(B\hp \Mu)V 1 - tr(\Mu) \\
  &= tr(U^TCV + 1^T U^T(A\hp \Lambda)U 1 - \Lambda + 1^T V^T(B\hp  \Mu)V 1 - \Mu)\\
  \frac{\partial L}{\partial U}&= V^TC^T + 11^TU^T(A\hp (\Lambda + \Lambda)^T) = 0
\end{align}
\todo{Confirm that this method would work}
\subsection{Probabilistic CCA}
Basically we have to find the MLE estimates of the parameters of a two
observed, one hidden variable model defined with the following
generative story.

I observe data coming in two pairs and I assume that it was generated
in the following way
\begin{align}
  z &\sim \mathcal{N}(0, I_d)\\
  x &\sim \mathcal{N}(W_1z + u_1, \psi_1)\\
  y &\sim \mathcal{N}(W_2z + u_2, \psi_2)
\end{align}
Now I want to estimate the parameters $W_1, W_2, \psi_1, \psi_2,
u_1, u_2$ on the basis of the data. So basically I need to
maximize the likelihood of the parameters.
\begin{align}
  \mathcal{L} &= \prod_i \int_z p(x_i|z) p(y_i|z) p(z)\\
  \log(\mathcal{L}) &= \sum_i \log(\int_z p(x_i|z) p(y_i|z) p(z))
\end{align}
Lets focus on $I = \int_z p(x|z) p(y|z) p(z)$. Let $\dim(x)=m\; \dim(y)=q$
\begin{align}
  p(z)&= \frac{1}{\sqrt{(2\pi)^d}} \exp(\mf z^Tz)\\
  p(x|z)&= \frac{1}{\sqrt{(2\pi)^m\det{\psi_1}}}  \exp( \mf (x-W_1z-u_1)^T\psi_1^{-1}(x-W_1z-u_1))\\
  p(y|z)&= \frac{1}{\sqrt{(2\pi)^q\det{\psi_2}}}  \exp( \mf (y-W_2z-u_2)^T\psi_2^{-1}(y-W_2z-u_2))\\
  I &= \int_z p(x|z) p(y|z) p(z)\\
  &= \frac{1}{\sqrt{(2\pi)^{d+m+q} \det{\psi_1}\det{\psi_2} }}\int_z \exp(\mf
  \big( z^Tz + (x-W_1z-u_1)^T\psi_1^{-1}(x-W_1z-u_1) + \notag\\
  & \qquad (y-W_2z-u_2)^T\psi_2^{-1}(y-W_2z-u_2) \big) )
\end{align}
Now the basic idea is the same as probabilistic PCA, the final
probability distribution is joint guassian over $x, y$, and then we
want to maximize the likelihood of parameters.

Let $a = x-u_1, \; b = y-u_2$ and focus only on the quadratic term $T$.
\begin{align}
  T &= z^Tz + (a-W_1z)^T\psi_1^{-1}(a-W_1z) +
  (b-W_2z)^T\psi_2^{-1}(b-W_2z)\\
  (a-W_1z)^T\psi_1^{-1}(a-W_1z) &= (a^T\psi_1^{-1}a)+  (z^TW_1^T\psi_1^{-1}W_1z) - 2z^TW_1^T\psi_1^{-1}a\\
  (b-W_2z)^T\psi_2^{-1}(b-W_2z) &= (b^T\psi_2^{-1}b)+  (z^TW_2^T\psi_2^{-1}W_2z) - 2z^TW_2^T\psi_2^{-1}b\\
  T &= z^T(I+W_1^T\psi_1^{-1}W_1+W_2^T\psi_2^{-1}W_2)z -2z^T(W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b) + \notag\\
  & \qquad ((a^T\psi_1^{-1}a) + (b^T\psi_2^{-1}b))
\end{align}
Once again we have to complete the squares which means that we'd have to match $T$ above to an expression like the following
\begin{align}
  (z-\mu)^T V^{-1}(z-\mu)=  (z^T V^{-1}z) -2z^TV^{-1}\mu + (\mu^T V^{-1}\mu)
\end{align}
Which means that $V = (I+W_1^T\psi_1^{-1}W_1+W_2^T\psi_2^{-1}W_2)^{-1}$ and $V^{-1}\mu = (W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b) \implies \mu = V(W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b)$.

Let $c = ((a^T\psi_1^{-1}a) + (b^T\psi_2^{-1}b)) - (\mu^T V^{-1}\mu)$, then basically the integral $I$ becomes the following:
\begin{align}
  I &= \frac{1}{\sqrt{(2\pi)^{d+m+q} \det{\psi_1}\det{\psi_2} }}
  \exp(\mf c) \int \exp(\mf (z-\mu)^T V^{-1}(z-\mu))\\
  &= \frac{\sqrt{\det{V}}}{\sqrt{(2\pi)^{m+q} \det{\psi_1}\det{\psi_2} }} \exp(\mf c)\\
  &= ((2\pi)^{m+q}
  \det{\psi_1}\det{\psi_2}\det{V}^{-1})^{-1/2}\exp(\mf c) \label{ccasimple}
\end{align}

Of course we only want to maximize the real likelihood.
\begin{align}
  \log(\mathcal{L})&= \sum \log(I_i) \\
  &= \mf (\sum_i c_i + n \log((2\pi)^{m+q} \det{\psi_1}\det{\psi_2}\det{V}^{-1}))
\end{align}
After removing constants we get the following objective
\begin{align}
  & \arg\min (\sum_i c_i + n \log(\det{\psi_1}\det{\psi_2}\det{I+W_1^T\psi_1^{-1}W_1+W_2^T\psi_2^{-1}W_2}))\\
  c_i &= ((a_i^T\psi_1^{-1}a_i) + (b_i^T\psi_2^{-1}b_i)) - (\mu_i^T V^{-1}\mu_i)\\
  \mu_i &= V(W_1^T\psi_1^{-1}a_i + W_2^T\psi_2^{-1}b_i)\\
  V &= (I+W_1^T\psi_1^{-1}W_1+W_2^T\psi_2^{-1}W_2)^{-1}\\
  \implies c_i &= a_i^T\psi_1^{-1}a_i + b_i^T\psi_2^{-1}b_i - (W_1^T\psi_1^{-1}a_i + W_2^T\psi_2^{-1}b_i)^T V^T(W_1^T\psi_1^{-1}a_i + W_2^T\psi_2^{-1}b_i)
\end{align}

Now strictly speaking one could muddle his/her way through by computer
assisted algebra but it would be better to simplify the forms.

The trick is to go back to Expression~\ref{ccasimple} and to then
view it as a higher dimensional Guassian. Specifically
\begin{align}
  I &= \frac{1}{\sqrt{(2\pi)^{m+q}\det{\psi_1}\det{\psi_2}\det{V}^{-1}}}\exp(\mf c)\\
  &=
  \frac{1}{\sqrt{(2\pi)^{m+q}\det{\psi_1}\det{\psi_2}\det{V}^{-1}}}\exp(\mf
  ((a^T\psi_1^{-1}a) + (b^T\psi_2^{-1}b)) - (\mu^T V^{-1}\mu))\\
  &= \frac{1}{\sqrt{(2\pi)^{m+q}\det{\psi_1}\det{\psi_2}\det{V}^{-1}}}\exp(\mf ((a^T\psi_1^{-1}a) + (b^T\psi_2^{-1}b)) - (W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b)^T V^T(W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b))
\end{align}
Now consider just the quadratic term Q
\begin{align}
  Q &= a^T\psi_1^{-1}a + b^T\psi_2^{-1}b - (W_1^T\psi_1^{-1}a +
  W_2^T\psi_2^{-1}b)^T V^T(W_1^T\psi_1^{-1}a + W_2^T\psi_2^{-1}b)\\
  Q &= a^T(\psi_1^{-1} - \psi_1^{-T}W_1V^TW_1^T \psi_1^{-1}) a +
  b^T(\psi_2^{-1} - \psi_2^{-T}W_2V^TW_2^T \psi_2^{-1}) b  \notag\\
  & \qquad - b^T\psi_2^{-T}W_2V^TW_1^T\psi_1^{-1}a -
  a^T\psi_1^{-T}W_1V^TW_2^T\psi_2^{-1}b\\
  &= \begin{bmatrix} a^T & b^T \end{bmatrix}
    \begin{bmatrix}
       \psi_1^{-1} - \psi_1^{-T}W_1V^TW_1^T \psi_1^{-1} & -\psi_1^{-T}W_1V^TW_2^T\psi_2^{-1} \\
       -\psi_2^{-T}W_2V^TW_1^T\psi_1^{-1}  & \psi_2^{-1} -\psi_2^{-T}W_2V^TW_2^T\psi_2^{-1}  \end{bmatrix}
    \begin{bmatrix}a    \\ b \end{bmatrix}\\
    &= \begin{bmatrix} a^T & b^T \end{bmatrix}
    (\ps^{-1} - \begin{bmatrix}
        \psi_1^{-T}W_1V^TW_1^T \psi_1^{-1} & \psi_1^{-T}W_1V^TW_2^T\psi_2^{-1} \\
       \psi_2^{-T}W_2V^TW_1^T\psi_1^{-1}  & \psi_2^{-T}W_2V^TW_2^T\psi_2^{-1}  \end{bmatrix})
    \begin{bmatrix}a    \\ b \end{bmatrix}\\
    &= \begin{bmatrix} a^T & b^T \end{bmatrix}
    (\ps^{-1} - \ps^{-1} \begin{bmatrix}
        W_1V^TW_1^T  & W_1V^TW_2^T \\
       W_2V^TW_1^T  & W_2V^TW_2^T  \end{bmatrix}\ps^{-1})
    \begin{bmatrix}a    \\ b \end{bmatrix} \\
    &= \begin{bmatrix} a^T & b^T \end{bmatrix}
    (\ps^{-1} - \ps^{-1}
    \begin{bmatrix} W_1 \\ W_2 \end{bmatrix}
    V
    \begin{bmatrix} W_1^T & W_2^T \end{bmatrix}
    \ps^{-1})
    \begin{bmatrix}a    \\ b \end{bmatrix}
\end{align}
Now note that $\psi$ and $V$ are both symmetric. Also note that
\begin{align}
  V &= (I+W_1^T\psi_1^{-1}W_1+W_2^T\psi_2^{-1}W_2)^{-1}\\
  &= (I + \begin{bmatrix} W_1^T W_2^T \end{bmatrix} \begin{bmatrix}
    \psi_1 & 0 \\ 0 & \psi_2 \end{bmatrix}^{-1} \begin{bmatrix} W_1
    \\ W_2 \end{bmatrix})^{-1}
\end{align}
Also consider
\begin{align}
  \Sigma &= \begin{bmatrix} \psi_1 & 0 \\ 0 &
    \psi_2 \end{bmatrix} + \begin{bmatrix} W_1 \\ W_2 \end{bmatrix}  \begin{bmatrix} W_1^T & W_2^T \end{bmatrix}
\end{align}
By the woodbury identity we know its inverse is as follows
\begin{align}
  \Sigma^{-1} &= \ps^{-1} - \ps^{-1}\begin{bmatrix} W_1
    \\ W_2 \end{bmatrix}(I+ \begin{bmatrix} W_1^T &
    W_2^T \end{bmatrix} \ps^{-1} \begin{bmatrix} W_1
    \\ W_2 \end{bmatrix})^{-1} \begin{bmatrix} W_1^T &
    W_2^T \end{bmatrix} \ps^{-1}
\end{align}
The determinant of $\Sigma$ can also be shown to be equal to the form
in the square root by using the fact that $$\det{A+UV^T} = \det{I+V^TA^{-1}U}\det{A}$$

Once we have done this derivation then we can move forward by
calculating the loglikelihood for a general guassian.
Let $u = [u_1; u_2]$, and $v_i = [x_i; y_i]$
\begin{align}
  \mathcal{-L}  &\sim n\log(\det{\Sigma}) + \sum_i tr((v_i-u)^T\Sigma^{-1}(v_i-u))
\end{align}
Now we can cycle the things inside the trace and take out constant
things from the summation and replace $\sum v_i/n = \xi$ and $\sum
(v_iv_i^T - \xi \xi^T)/n = \Xi$ also replace $v_i - u = v_i - \xi +
\xi - u$

\begin{align}
  \mathcal{-L}  &\sim \log(\det{\Sigma}) + tr(\Sigma^{-1}\Xi)+(\xi -
  u)^T\Sigma^{-1}(\xi - u) \label{ccal}
\end{align}

Now again we have to minimize Expression~\ref{ccal} and $\xi, \Xi$ are
given to us through the data. It is easy to set $u$ but $\Sigma$ needs
to be chosen more carefully because it is constrained to be of the
form
\begin{align}
  \Sigma &= \begin{bmatrix} \psi_1 & 0 \\ 0 & \psi_2 \end{bmatrix} + \begin{bmatrix} W_1 \\ W_2 \end{bmatrix}  \begin{bmatrix} W_1^T &    W_2^T \end{bmatrix}\\
  \Sigma &= \begin{bmatrix} \psi_1 & 0 \\ 0 &
    \psi_2 \end{bmatrix} + W W^T
\end{align}

I repeat that the problem only occurs because $\Sigma$ needs to be of
a certain form. If it could have been a general matrix then I wouldn't
have any problems. I'd just take the derivative of
expression~\ref{ccal} and set it to 0. In any case the term
$(\xi - u)^T\Sigma^{-1}(\xi - u)$ is zero so we dont need to worry
about its derivative.

Now let's take derivative of $-\mathcal{L}$ wrt to $\psi_1,
\psi_2, W=[W_1; W_2]$. We must work with $\det{\Sigma}\ne 0$

Wrt $\psi_1$:
\begin{align}
  \partial \mathcal{L} &= tr(\Sigma^{-1} \partial \Sigma - \Xi\Sigma^{-1}\partial\Sigma\Sigma^{-1})\\
  &=  tr((\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1})\partial\Sigma)\\
  &\implies (\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1}) \begin{bmatrix}I & 0\\ 0 & 0\end{bmatrix} = 0
\end{align}

Wrt $\psi_2$:
\begin{align}
  \partial \mathcal{L} &= tr(\Sigma^{-1} \partial \Sigma - \Xi\Sigma^{-1}\partial\Sigma\Sigma^{-1}) = 0\\
  &=  tr((\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1})\partial\Sigma)\\
  &\implies (\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1}) \begin{bmatrix}0 & 0\\ 0 & I\end{bmatrix} = 0
\end{align}

Wrt $W$:
\begin{align}
 \partial \mathcal{L} &= (\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1})W = 0
\end{align}

Now we need to find $\psi, W$ that satisfy these three constraints and
later check which of those is the minima ?
In order to find the solutions Bach first studies the solutions. He
says that assume you that such poins exist. Then those points
first satisfy some conditions.
\begin{enumerate}
\item $WW^T \preccurlyeq \Xi$
\item $\Psi = \Sigma\Xi^{-1}(\Xi - WW^T)$
\item $\psi_1 = \Xi_{11} - W_1 W_1^T$ and $\psi_1 = \Xi_{22} - W_1 W_1^T$
\end{enumerate}

But of course he didnt come up with these relations from thin air. It
is conceivable that he did try to study the solution but the
conditions themselves would have come as a results of search. Anyway
let's derive some of the lemmas to get comfortable with them

\begin{align}
           & (\Sigma^{-1}  - \Sigma^{-1}\Xi\Sigma^{-1})W = 0           \\
  \implies & W = \Xi\Sigma^{-1}W \\
  \implies & W = \Xi(\Psi + WW^T)^{-1}W \\
  \implies & W = \Xi(\Psi^{-1} - \Psi^{-1} W(I+W^T\Psi^{-1}W)^{-1}W^T\Psi^{-1})W \label{eq1}\\
  \implies & W = \Xi(\Psi^{-1}W - \Psi^{-1} W(I+W^T\Psi^{-1}W)^{-1}W^T\Psi^{-1}W) \label{eq2}\\
\end{align}
Now let $W^T\Psi^{-1}W = V$, then Expression~\ref{eq2}
\begin{align}
  W &= \Xi(\Psi^{-1}W - \Psi^{-1} W(I+V)^{-1}V) \\
  W &= \Xi\Psi^{-1}W(I - (I+V)^{-1}V)\\
  W &= \Xi\Psi^{-1}W(I + V)^{-1}\\
  W +WV &= \Xi \Psi^{-1}W\\
  W + WW^T\Psi^{-1}W &= \Xi \Psi^{-1}W\\
  W &= (\Xi - WW^T) \Psi^{-1}W\\
  \implies \Psi &= \Xi - WW^T \;\text{\todo{If we assume that $WW^T$ is invertible}}\\
\end{align}
Now do the cholesky decomposition $\Psi^{-1} = M^TM$ and apply to Expression!\ref{eq1}
\begin{align}
  W &= \Xi(M^TM - M^T M W(I+W^T M^T MW)^{-1}W^T M^T M)W \\
\end{align}
Let $V = MW$ to get
\begin{align}
  W &= \Xi M^TMW - \Xi M^T M W(I+W^T M^T MW)^{-1}W^T M^T MW \\
  &= \Xi M^TV - \Xi M^T V(I+V^TV)^{-1}V^T V \\
  V &= M\Xi M^TV(I - (I+V^TV)^{-1}V^TV)
\end{align}
Since $\Psi$ is invertible so $M$ is invertible.
\begin{align}
  M^{-1}V &= \Xi M^TV(I - (I+V^TV)^{-1}V^TV)
\end{align}

No strike this he derivation goes as follows
\begin{align}
  W &= \Xi(\Psi + WW^T)^{-1}W \\
  W &= \Xi(\Psi^{1/2}\Psi^{1/2} + \Psi^{1/2}\Psi^{-1/2}WW^T\Psi^{-1/2}\Psi^{1/2})^{-1}W \\
  W &= \Xi\Psi^{-1/2}(I + \Psi^{-1/2}WW^T\Psi^{-1/2})^{-1}\Psi^{-1/2}W \\
  \Psi^{-1/2}W &= \Psi^{-1/2}\Xi\Psi^{-1/2}(I + \Psi^{-1/2}WW^T\Psi^{-1/2})^{-1}\Psi^{-1/2}W \\
  \Psi^{-1/2}WW^T\Psi^{-1/2} &= \Psi^{-1/2}\Xi\Psi^{-1/2}(I + \Psi^{-1/2}WW^T\Psi^{-1/2})^{-1}\Psi^{-1/2}WW^T\Psi^{-1/2} \\
\end{align}
Now note that $\Psi^{-1/2}WW^T\Psi^{-1/2}$ is psd therefore it has a eigen-decomposition $USU^T$.
If we ASSUME that it is pd then we can say that
\begin{align}
  I &= \Psi^{-1/2}\Xi\Psi^{-1/2}(I+USU^T)^{-1}\\
  (I+USU^T) &= \Psi^{-1/2}\Xi\Psi^{-1/2} \\
  \implies \Xi  &\succcurlyeq \Psi^{1/2}USU^T\Psi^{1/2} = WW^T
\end{align}

\todo{It is not clear why we can assume that $WW^T$ which is a psd matrix is spd, but go ahead.}

\todo{Actually complete the derivation.}

\section{Incremental Statistical language models}
\subsection{PPCA as a language model}
It is not possible since it only has one view of data. and I do not
want to model the next word as the hidden state rather I want to say
that the next word is a view that is hidden from us and it is
connected to the visible view through a latent state.

\subsection{Using PCCA as a language model}
\todo{Anything that I can do with PGCCA I can do with PCCA. It makes
  sense to derive those things in the easier case and then to forge
  ahead, Think about complexity, performance, advantages etc. at this
  point.}

To do this I need to derive p(view 2| view 1) and ensure that it can
be computed efficiently. p(view 2 | view 1) would become the guassian
language model.
So basically
\begin{align}
  p(y|x) &= \frac{p(y,x)}{p(x)} \\
  &= \frac{\int p(y,x,z)}{p(x)}\\
  &= \frac{\int p(y,x|z)p(z)}{p(x)}\\
  &= \frac{\int p(y|z)p(x|z)p(z)}{p(x)}\\
\end{align}
Both the numerator and the denominator I have calculated before so
plug those values in to get $p(y|x)$ in closed form.

Now the joint pdf of (view-1, view-2) that we calculated was joint normal with
the following multivariate normal distribution.

\pagebreak
Under probabilistic interpretation of CCA two views have the joint pdf as follows
\begin{align}
  \mu &= [\tilde{\mu}_1; \tilde{\mu}_2]\; \text{The sample mean.}\\
  \Sigma &= \begin{bmatrix} W_1 W_1^T + \Psi_1 & W_1 W_2^T \\ W_2 W_1^T & W_2 W_2^T + \Psi_2\end{bmatrix}\\
  \Psi_1 &= \tilde{\Sigma}_{11} - W_1 W_1^T\\
  \Psi_2 &= \tilde{\Sigma}_{22} - W_2 W_2^T\\
  W_1 &= \tilde{\Sigma}_{11}U_{1d}\\
  W_2 &= \tilde{\Sigma}_{22}U_{2d}\\
  U_{1d} &= \tilde{\Sigma}_{11}^{-1/2} \text{lsv}_d(\tilde{\Sigma}_{11}^{-1/2}\tilde{\Sigma}_{12}\tilde{\Sigma}_{22}^{-1/2})\text{lsv=left singular vector}\\
  U_{2d} &= \tilde{\Sigma}_{22}^{-1/2} \text{rsv}_d(\tilde{\Sigma}_{11}^{-1/2}\tilde{\Sigma}_{12}\tilde{\Sigma}_{22}^{-1/2})\\
  \text{Sample Covariance} &= \begin{bmatrix} \tilde{\Sigma}_{11} &
    \tilde{\Sigma}_{12} \\ \tilde{\Sigma}_{21} & \tilde{\Sigma}_{22} \end{bmatrix}
\end{align}
We know that $E[V_2 | V_1]$ is the following:
$$E[V_2 | V_1] = \tilde{\mu}_2 + (W_2W_1^T)(W_1W_1^T+\Psi_1)^{-1}(V_1 - \mu_2)$$
$$p(V_2 | V_1) = \mathcal{N}(E[V_2 | V_1], W_2W_2^T+\Psi_2 - W_2W_1^T(W_1W_1^T+\Psi_1)^{-1}W_1W_2^T)$$
So now if we want to get a probability then we can explicitly calculate it.

So assume that I observe the following two views \note{Both k and n are hyper-param}
\begin{align}
  \text{View-1: }\begin{bmatrix}1 0 1 0 0 0 1 \ldots\end{bmatrix}\\
  \text{View 1 contains words 1 to n-1}\\
  \text{View-2: }\begin{bmatrix}0 0 0 0 1 0 0 \ldots\end{bmatrix}\\
  \text{View 2 contains the nth word}
\end{align}
%% Then we can estimate $\alpha = E[\text{View 2} | \text{View 1}]$ and
%% $\beta = \max(\alpha - \text{View 1})$.
So at test time my observations are encoded in view 1, which are encoded into a
V-long count vector. Now I want to do prediction of view 2 which can then be used for
\section{Probabilistic GCCA}
We want to derive probabilistic GCCA because then we can give a generative semantics to my embeddings and use them for language modelling.
\todo{TODO}
\end{document}


\subsubsection{Resonance ($\omega = \gamma$)}
\noindent
In the case where $\omega = \gamma$, we need to add an extra factor of $t$ to our guess for $y_p$. So,
\begin{equation*}
	y_p = At\cos{(\omega t)} + Bt\sin{(\omega t)}
\end{equation*}
Solving for $A$ and $B$,
\begin{equation*}
	my_p'' + ky_p = 2Bm\omega\cos{(\omega t)} - 2Am\omega\sin{(\omega t)} = F_0\cos{(\omega t)}
\end{equation*}
\begin{equation*}
	\implies A = 0 \text{ and } B = \frac{F_0}{2m\omega}
\end{equation*}
So our solution is,
\begin{equation*}
	y = C_1\cos{(\omega t)} + C_2\sin{(\omega t)} + \frac{F_0}{2m\omega}t\sin{(\omega t)}
\end{equation*}\\

\noindent
Let's look specifically at the IVP where $y(0) = 0$ and $y'(0) = 0$.
\begin{equation*}
	C_1 = C_2 = 0
\end{equation*}
So,
\begin{equation*}
	y = \frac{F_0}{2m\omega}t\sin{(\omega t)}
\end{equation*}\\

\noindent
Here, the amplitude grows with $t$, creating bigger and bigger waves. This is resonance, and you can see mathematically how it is responsible for one string causing another tuned the same to vibrate and the collapsing of bridges.

\begin{center}
	\includegraphics[width=0.75\textwidth]{./higherOrder/forcedVibrs/resonance.png}
\end{center}


\chapter{The two extremal problems}
%\addcontentsline{toc}{chapter}{The two extremal problems}
 \section{The extremal limit problem}
\paragraph{The answer to the extremal limit problem} It turns out that all possible non-degenerate limiting distributions i.e. all extreme values distribution make up a one-parameter family $G_\gamma(x) = \exp(-(1 + \gamma x)^{-\frac{1}{\gamma}})$, where the support of G is the set $\{ x : 1 + \gamma x > 0}$ and $\gamma \in \mathbb{R}$ is the \textbf{E}xtreme \textbf{V}alue \textbf{I}nded or \textbf{EVI}. The three sub-cases are the following :
\begin{itemize}
	\item \underline{$\gamma = 0$ :} \textbf{Gumbel distribution} \newline
	$G_\gamma(u) = \exp(- \exp(- u))$, $u \in \mathbb{R}$
	\item \underline{$\gamma > 0$ :} \textbf{Fréchet distribution} \newline
	$G_\gamma(u) = \exp(- (1 + \gamma u)^{- \frac{1}{\gamma}})$, $u \in ]- \gamma^{-1}, + \infty[$
	\item \underline{$\gamma > 0$ :} \textbf{Weibull distribution} \newline
	$G_\gamma(u) = \exp(- (1 + \gamma u)^{- \frac{1}{\gamma}})$, $u \in ]- \infty, - \gamma^{-1}[$
\end{itemize}
For the derivations and proofs relative to this section, please report to the appendix.
\section{The domain of attraction problem}
\paragraph{Definition} The domain of attraction of an extreme value distribution family (i.e. Gumbel, Fréchet-type or Weibull-type) is the set of distribution functions $F_X$\footnote{$F_X$ being the distribution of the $X_i$ of the sample.} such that the sequence of standardized maxima $(M_n^*)_{n \ge 0}$ will converge in distribution to that extreme value distribution family.
\paragraph{Remark} There are many approaches to characterize the domains of attraction of the extreme value distribution families. We have decided to use Von Mises' theorem to characterize them. This is the historical approach, and a rather straightforward one, still by no means are alternative approaches uninteresting\footnote{In particular, conditions based on the sole behaviour of $F_X$ can be formulated.}.
\paragraph{Hazard function} Let $X$ be a random variable with probability density function/mass function $f_X$ and distribution function $F_X$, then we define the hazard function $r$ as follows : \newline
$r(x) = \frac{f_X(x)}{1 - F_X(x)}$.
\paragraph{A few preliminary notations} $\Phi_\alpha$, $\Psi_\alpha$, $\Delta$ are respectively the symbols used to denote a Fréchet, a Weibull and a Gumbel distributions, with :
\begin{itemize}
	\item $\Phi_\alpha(x) = \exp(- x^\alpha)$
	\item $\Psi_\alpha(x) = \exp(-  \lvert x \rvert^\alpha)$ (let us bear in mind that this is a notation, due to historical reasons).
	\item $\Delta(x) = \exp(- \exp(- x))$
\end{itemize}
\paragraph{Von Mises' theorem}
\begin{enumerate}
	\item If $x^+ = + \infty$ and $x r(x) \xrightarrow[x \rightarrow + \infty]{} \alpha > 0$, then $F_X \in \mathcal{D}(\Phi_\alpha)$.
	\item If $x^+ < + \infty$ and $(x^+ - x) r(x) \xrightarrow[x \rightarrow x^+]{} \alpha > 0$, then $F_X \in \mathcal{D}(\Psi_\alpha)$.
	\item If $r(x)$ is ultimately positive in the neighbourhood of $x^+$ (with $x^+ \le + \infty$), is differentiable on that neighbourhood and is such that $\frac{\mathrm{d}r}{\mathrm{d}x}(x) \xrightarrow[x \rightarrow x^+]{} 0$, then $F_X \in \mathcal{D}(\Delta)$.
\end{enumerate}
\section{Conclusion}
\paragraph{Fisher-Tippett-Gnedenko theorem} The \tetxbf{Fisher-Tippett-Gnedenko} theorem, also known as the \textbf{extremal theorem}, states that if the sequence of standardized maxima converges in distribution to a non-degenerate distribution, then this distribution belongs to one of the three aforementioned extreme value distribution families. The theorem thus provides an answer to the \textit{extremal limit problem}. Von Mises' theorem, encountered in the previous section, provides a complementary answer, that to the \textit{domain of attraction problem}.


\subsection{The Euclidian metric}

For the Euclidian metric:

\(M=I\)

\((dv )^TMdv =(dv)^T dv = d x^2+d y^2 + d z^2\)

\(Action = \int \sqrt {dx^2+dy^2 + dz^2}\)

\(Action = \int \sqrt {\dot x^2+\dot y^2 + \dot z^2}d t\)

\(Action = \int vd t\)

What are symmetries here? Gallilean group and?


\section{Motion Model}

The motion model, $f$, is responsible for predicting the next state given the current state (and the control commands, if applicable).

I use a simpler motion model than Justin's from 2020. Primarily, we do not use GPS coordinates or control parameters for making predictions, and these variables are not included in the state.

\begin{equation}
    f(\boldsymbol{\hat{x}}_{k-1}, \boldsymbol{u}, \Delta t) =
    \begin{pmatrix}
    x_{k-1} + \dot{x}_{k-1} \cdot \Delta t \\
    \dot{x}_{k-1} \\
    y_{k-1} + \dot{y}_{k-1} \cdot \Delta t \\
    \dot{y}_{k-1} \\
    \phi_{k-1} + \dot{\phi}_{k-1} \cdot \Delta t \\
    \dot{\phi}_{k-1} \\
    v_{l,k-1} \\
    v_{r,k-1}
    \end{pmatrix}
\end{equation}

We need to encode this system of equations into a matrix that can be used directly in the EKF. We cannot use simple matrix multiplication to compute $\cos{\phi}$ and $\sin{\phi}$, so these are re-computed and set at the start of the \textit{Predict} phase on every clock cycle, and treated as if they are constants (cos\_phi and sin\_phi). Important robot characteristics are the constants $R$ = WHEEL\_RADIUS and $L$ = WHEELBASE\_LEN.

\begin{equation}
    \boldsymbol{F}_{k} =
    \begin{pmatrix}
    1 & \Delta t & 0 & 0 & 0 & 0 & 0 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & \frac{R}{2} \textrm{cos\_phi} & \frac{R}{2} \textrm{cos\_phi} \\
    0 & 0 & 1 & \Delta t & 0 & 0 & 0 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & \frac{R}{2} \textrm{sin\_phi} & \frac{R}{2} \textrm{sin\_phi} \\
    0 & 0 & 0 & 0 & 1 & \Delta t & 0 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & \frac{R}{L} & -\frac{R}{L} \\
    0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & 0 & 1
    \end{pmatrix}
\end{equation}

This matrix gives us the correct behavior of predicting the next state given the current state, as shown in the following equation.

\begin{equation}
    \boldsymbol{\hat{x}}_{k+1} = \boldsymbol{F}_{k} \cdot \boldsymbol{\hat{x}}_{k}
\end{equation}

We use SymPy to calculate the Jacobean of the motion model. SymPy is a great python package for performing symbolic operations such as this. We obtain the following Jacobean, which is useful for linearizing the EKF.

\begin{equation}
    \begin{pmatrix}
    1 & \Delta t & 0 & 0 & 0 & 0 & 0 & 0 \\
    0 & 0 & 0 & 0 & -\frac{R}{2} (v_l + v_r) \sin{\phi} & 0 & \frac{R}{2} \cos{\phi} & \frac{R}{2} \cos{\phi}  \\
    0 & 0 & 1 & \Delta t & 0 & 0 & 0 & 0 \\
    0 & 0 & 0 & 0 & \frac{R}{2} (v_l + v_r) \cos{\phi} & 0 & \frac{R}{2} \sin{\phi}  & \frac{R}{2} \sin{\phi}  \\
    0 & 0 & 0 & 0 & 1 & \Delta t & 0 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & \frac{R}{L} & -\frac{R}{L} \\
    0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 \\
    0 & 0 & 0 & 0 & 0 & 0 & 0 & 1
    \end{pmatrix}
\end{equation}


% !Mode:: "TeX:UTF-8"
% !TEX program  = xelatex
\newgeometry{margin=1in}

\title{Assignment 6}

\section{Question 1}
\begin{statebox}{Timescale Invariance}{question-1}
    \begin{align*}
        S(t_{i+1}) &= S(t_i) + \mu\delta tS(t_i) + \sigma\delta tY_iS(t_i) \\S(t_{i+1}) &= S(t_i) + \mu\delta t^{1/4}S(t_i) + \sigma\delta tY_iS(t_i)
    \end{align*}
    Please verify or dispute the timescale invariance of the two models above by numerical experiments.
\end{statebox}

Therefore, we have the following code, which verifies the timescale invariance of the models by numerical experiments. And the result of the code below with the arguments $S_0=1$, $\mu=0.05$, $\sigma=0.5$, is Figure~\ref{F:1}.

\lstset{showspaces=false, showtabs=false, tabsize=2, framexleftmargin=5mm, frame=shadowbox, numbers=left, numberstyle=\tiny, breakautoindent=false}
\lstinputlisting[style=Matlab-Pyglike]{code/timescale_invariance_asset_path.m}

\begin{figure}[H]
	\centering
	\includegraphics[width=\textwidth]{figures/2019-10-30-timescale-invariance.png}
	\caption{Timescale Invariance Asset Path}\label{F:1}
\end{figure}


\documentclass{article}
\pagestyle{empty}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{graphicx}
\usepackage{multicol}
\setlength{\oddsidemargin}{0in} \setlength{\evensidemargin}{0in}
\setlength{\topmargin}{0in} \setlength{\textheight}{8.5in}
\setlength{\textwidth}{6.5in}

\makeatletter
\renewcommand*\env@matrix[1][*\c@MaxMatrixCols c]{%
	\hskip -\arraycolsep
	\let\@ifnextchar\new@ifnextchar
	\array{#1}}
\makeatother

\begin{document}
\begin{flushleft}
	\bfseries{MATH 260, Linear Systems and Matrices, Fall `14}\\
	\bfseries{Activity 4:  Matrices, RREF, and Solutions to Systems}\\
	%\bfseries{Honor Code:} \hspace{3.5in}\bfseries{Names:}\\
\end{flushleft}
\begin{flushleft}

\section*{Row Operation Review}

\begin{center}
$\left[\begin{array}{rrr|r}
1 & 1 & 2 & 4\\
2 & -1 & 1 & -4\\
4 & 1 & 5 & 4\\
\end{array}\right]
$\\
\end{center}

There are three basic row operations we can perform (see page 134 of the text):\\
$R_i$ means row \textit{i} before an operation while $R_i^*$ denotes row \textit{i} after an operation.\\
1) Row swap between row \textit{i} and row \textit{j}, denoted: $R_i \leftrightarrow R_j$\\
2) Multiply a row \textit{i} by a constant \textit{c} ( with $c\neq 0$) , denoted: $R_i^*=c R_i$\\
3) Add a (multiple) of a row to another row, denoted: $R_i^*= R_i+c R_j$\\
\hrulefill \\

\vspace{0.2in}
\noindent
Lets try these out with our augmented matrix:\\
a) Perform the following row operations:\\
\vspace{0.2in}
$\begin{array}{c}
R_2^* = R_2 + (-2) R_1\\
\rightarrow \\
R_3^*=R_3+ (-4) R_1\\
\end{array}
$
\hspace{0.25in}
$\begin{bmatrix}[ccc|c]
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\end{bmatrix}$
\hspace{0.2in}
$\begin{array}{c}
\\
\\
R_2^* = - \frac{1}{3} R_2 \\
\rightarrow \\
\\
\end{array}
$
\hspace{0.5in}
$\begin{bmatrix}[ccc|c]
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\end{bmatrix}$\\
\vspace{0.1in}
$\begin{array}{c}
R_1^* = R_1 + (-1) R_2\\
\rightarrow \\
R_3^*=R_3+ (3) R_2\\
\end{array}$
\hspace{0.25in}
$\begin{bmatrix}[ccc|c]
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}& \hspace{0.2in}\\
& & & \\
\end{bmatrix}$\\

\vspace{0.2in}

You should now have a matrix with 5 non-zero entries.  The first non-zero entry in each row should be a positive 1.

\newpage
\section{RREF}
The matrix we got at the end of the warmup was in \textit{reduced row echelon form}. A matrix in RREF has the following traits (see page 136 of the text):
\begin{enumerate}
\item Any zero rows are at the bottom
\item The leftmost non-zero entry of each non-zero row equals 1.\\
 (This entry is called its \textbf{pivot} or \textbf{leading 1}.)
\item Each pivot is further to the right than the pivot in the row above it.
\item Each pivot is the only non-zero entry in its column.\\
\end{enumerate}
Determine if each of the following matrices is in RREF or not. If not, perform row operations to turn it into RREF.\\
\vspace{0.1in}
\begin{center}
$\textbf{M}=\begin{bmatrix}[ccc|c]
1 & 0 & -1 & 1\\
0 & 1 & 1 & 0\\
0 & 0 & 3 & -1
\end{bmatrix}$
\hspace{0.15in}
$\textbf{T}=\begin{bmatrix}[cc|c]
2 & -1 & 0\\
1 & -1 & -3
\end{bmatrix}$
\hspace{0.15in}
$\textbf{A}=\begin{bmatrix}[cccc|c]
1 & 0 & 0 & 2 & 5\\
0 & 1 & 0 & -2 & 2\\
0 & 0 & 1 & 5 & 6\\
0 & 0 & 0 & 0 & 2 \\
\end{bmatrix}$\\
\end{center}

\pagebreak

\section{Number of Solutions}

Two of the previous matrices, have the same traits in RREF, \textbf{M} and \textbf{T}.\\

\vspace{0.2in}

a) Describe the similarities between the two matrices in RREF.

\vspace{1.5in}

b) What's different about \textbf{A} and the matrix from the warmup?  \textit{Hint: Look at where zeros occur.}  Write out what the last line of matrix \textbf{A} represents in equation form.

\vspace{1.5in}

Just like when we dealt with systems of equations, augmented matrices (which are representing systems!) can have 3 types of solutions:  unique, infinitely many, or none.  For the first two (unique and infinitely many) the system is said to be `consistent'.  For the last type, no solutions, the system is `inconsistent'.

\vspace{0.2in}

c) Find which of the above matrices have unique solutions, infinite solutions, or no solutions by converting each back into equations (i.e. with $x_1$ ... $x_4$ as variables).

\newpage

\section{Rank and Pivots}

A \textbf{pivot column} of a matrix is any column that has a leading 1 in it (the rest of the entries in that column are zeros) once the matrix is put into RREF.  Note that if we're looking at augmented matrices, the last column (the one to the right of the vertical bar) is never a pivot column.

\vspace{0.2in}

Ex: For matrix \textbf{M}, all three columns are pivot columns.

\vspace{0.2in}

The \textbf{rank} of a matrix is the number of \textit{pivot columns} it has (again, once it is in RREF).

\vspace{0.2in}

Ex: For matrix \textbf{M} the rank is 3.

\vspace{0.2in}

a) Identify the pivot columns of each of \textbf{T} and \textbf{A} from page 2.

\vspace{2in}

b) Identify the rank of each of \textbf{T} and \textbf{A}.

\end{flushleft}
\end{document}

Fall 2014 - Capaldi A/Activities/Activity04_GaussJordanElim.tex


\section{Model Description}

\subsection{Introduction}

The hinged rigid body class is an instantiation of the state effector abstract class. The state effector abstract class is a base class for modules that have dynamic states or degrees of freedom with respect to the rigid body hub. Examples of these would be reaction wheels, variable speed control moment gyroscopes, fuel slosh particles, etc. Since the state effectors are attached to the hub, the state effectors are directly affecting the hub as well as the hub is back affecting the state effectors.

Specifically, a hinged rigid body state effector is a rigid body that has a diagonal inertia with respect to its $\mathcal{S}_i$ frame as seen in Figure~\ref{fig:FlexFigure}. It is attached to the hub through a hinge with a linear torsional spring and linear damping term. The dynamics of this multi-body problem have been derived and can be seen in Reference~\cite{Allard2016rz}. The derivation is general for $N$ number of panels attached to the hub but does not allow for multiple interconnected panels.

\begin{figure}[htbp]
	\centerline{
		\includegraphics[width=0.8\textwidth]{Figures/Fig4_4_2}}
	\caption{Hinged rigid body frame and variable definitions}
	\label{fig:FlexFigure}
\end{figure}

\subsection{Equations of Motion}

The following equations of motion (EOMs) are pulled from Reference~\cite{Allard2016rz} for convenience. Equation~\eqref{eq:Rbddot3} is the spacecraft translational EOM, Equation~\eqref{eq:Final6} is the spacecraft rotational EOM, and Equation~\eqref{eq:solar_panel_final3} is the hinged rigid body rotational EOM. These are the coupled nonlinear EOMs that need to be integrated in the simulation.

\begin{multline}
m_{\text{sc}} \ddot{\bm r}_{B/N}-m_{\text{sc}} [\tilde{\bm{c}}]\dot{\bm\omega}_{\cal B/N}+\sum_{i}^{N}m_{\text{sp}_i} d_i  \bm{\hat{s}}_{i,3}\ddot{\theta}_i = \bm F_{\textnormal{ext}} - 2 m_{\text{sc}} [\tilde{\bm\omega}_{\cal B/N}]\bm{c}'\\
-m_{\text{sc}} [\tilde{\bm\omega}_{\cal B/N}][\tilde{\bm\omega}_{\cal B/N}]\bm{c}-\sum_{i}^{N}m_{\text{sp}_i}d_i\dot{\theta}_i^2 \bm{\hat{s}}_{i,1}
\label{eq:Rbddot3}
\end{multline}

\begin{multline}
	m_{\text{sc}}[\tilde{\bm{c}}]\ddot{\bm r}_{B/N}+[I_{\text{sc},B}] \dot{\bm\omega}_{\cal B/N} +\sum\limits_{i}^{N}\biggl\lbrace I_{s_i,2}\bm{\hat{h}}_{i,2}+m_{\text{sp}_i}d_i [\tilde{\bm{r}}_{S_i/B}] \bm{\hat{s}}_{i,3}\biggr\rbrace\ddot{\theta}_i = \\
	-[\bm{\tilde{\omega}}_{\cal B/N}] [I_{\text{sc},B}] \bm\omega_{\cal B/N}
	- [I'_{\text{sc},B}] \bm\omega_{\cal B/N} - \sum\limits_{i}^{N}\biggl\lbrace\dot{\theta}_i [\bm{\tilde{\omega}}_{\cal B/N}] \left(I_{s_i,2} \bm{\hat{h}}_{i,2}+m_{\text{sp}_i} d_i [\tilde{\bm{r}}_{S_i/B}] \hat{\bm s}_{i,3}\right)\\ +m_{\text{sp}_i}d_i\dot{\theta}_i^2[\tilde{\bm{r}}_{S_i/B}] \bm{\hat{s}}_{i,1} \biggr\rbrace + \bm{L}_B
	\label{eq:Final6}
\end{multline}

\begin{multline}
m_{\text{sp}_i} d_i \hat{\bm s}_{i,3}^{T} \ddot{\bm r}_{B/N}+ \biggl[\left(I_{s_{i,2}} + m_{\text{sp}_i}d_i^{2}\right) \hat{\bm s}_{i,2}^{T}-m_{\text{sp}_i} d_i \hat{\bm s}_{i,3}^{T} [\tilde{\bm r}_{H_i/B}]\biggr] \dot{\bm\omega}_{\cal B/N} \\
+ \left( I_{s_{i,2}} + m_{\text{sp}_i} d_i^{2} \right) \ddot \theta_i
= - k_i \theta_i - c_i \dot\theta_i + \hat{\bm s}_{i,2}^T \bm \tau_{\text{ext},H_i} + \left( I_{s_{i,3}} - I_{s_{i,1}} + m_{\text{sp}_i}d_i^{2}\right) \omega_{s_{i,3}} \omega_{s_{i,1}}\\
- m_{\text{sp}_i} d_i \hat{\bm s}_{i,3}^{T} [\tilde{\bm\omega}_{\cal B/N}][\tilde{\bm\omega}_{\cal B/N}] \bm r_{H_i/B}
\label{eq:solar_panel_final3}
\end{multline}

\subsection{Back Substitution Method}

In order to integrate the EOMs in a modular fashion, a back substitution method was developed and can be seen in Reference~\cite{Allard2016rz}. The hinged rigid body model must adhere to this analytical form, and the details are briefly summarized in the equations following. First the hinged rigid body EOM is substituted into the translational EOM and rearranged:
\begin{multline}
\Big(m_{\text{sc}} [I_{3\times3}] +\sum_{i=1}^{N}m_{\text{sp}_i}d_i \bm{\hat{s}}_{i,3} \bm a_{\theta_i}^T\Big)\ddot{\bm r}_{B/N}+\Big(-m_{\text{sc}} [\tilde{\bm{c}}] +\sum_{i=1}^{N}m_{\text{sp}_i}d_i \bm{\hat{s}}_{i,3} \bm b_{\theta_i}^T\Big) \dot{\bm\omega}_{\cal B/N} \\
= m_{\text{sc}} \ddot{\bm r}_{C/N} 	- 2 m_{\text{sc}} [\tilde{\bm\omega}_{\cal B/N}] \bm c'
-m_{\text{sc}} [\tilde{\bm\omega}_{\cal B/N}][\tilde{\bm\omega}_{\cal B/N}]\bm{c}
-\sum_{i=1}^{N}\Big(m_{\text{sp}_i}d_i \dot{\theta}_i^2 \bm{\hat{s}}_{i,1}+m_{\text{sp}_i}d_i c_{\theta_i} \bm{\hat{s}}_{i,3} \Big)
\label{eq:Rbddot8}
\end{multline}

Following the same pattern for the hub rotational EOM, Eq.~\eqref{eq:Final6}, yields:
\begin{multline}
\Big[m_{\text{sc}}[\tilde{\bm{c}}] +\sum\limits_{i=1}^{N}\big( I_{s_{i,2}}\bm{\hat{s}}_{i,2}+m_{\text{sp}_i}d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,3}\big)\bm a_{\theta_i}^T \Big]\ddot{\bm r}_{B/N}\\
+\Big[[I_{\text{sc},B}]+\sum\limits_{i=1}^{N}\big( I_{s_{i,2}}\bm{\hat{s}}_{i,2}+m_{\text{sp}_i}d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,3}\big) \bm b_{\theta_i}^T\Big] \dot{\bm\omega}_{\cal B/N}
=
-[\bm{\tilde{\omega}}_{\cal B/N}] [I_{\text{sc},B}] \bm\omega_{\cal B/N}
- [I'_{\text{sc},B}] \bm\omega_{\cal B/N} \\
-\sum\limits_{i=1}^{N}\biggl\lbrace\big(\dot{\theta}_i [\bm{\tilde{\omega}}_{\cal B/N}] +c_{\theta_i} [I_{3\times3}]\big) \left(I_{s_{i,2}} \bm{\hat{s}}_{i,2}+m_{\text{sp}_i} d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \hat{\bm s}_{i,3}\right) +m_{\text{sp}_i}d_i\dot{\theta}_i^2[\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,1} \biggr\rbrace + \bm{L}_B
\label{eq:Final9}
\end{multline}

With the following definitions:

\begin{subequations}
	\begin{align}
	\bm a_{\theta_i} &= - \frac{m_{\text{sp}_i} d_i}{\left( I_{s_{i,2}} + m_{\text{sp}_i} d_i^{2} \right)} \hat{\bm s}_{i,3}
	\label{eq:solar_panel_final11}
	\\
	\bm b_{\theta_i} &= -\frac{1}{\left( I_{s_{i,2}} + m_{\text{sp}_i} d_i^{2} \right)} \bigg[\left(I_{s_{i,2}} + m_{\text{sp}_i}d_i^{2}\right) \hat{\bm s}_{i,2}+m_{\text{sp}_i} d_i [\tilde{\bm r}_{H_i/B}] \hat{\bm s}_{i,3}\bigg]
	\\
	\begin{split}
	c_{\theta_i} &= \frac{1}{\left( I_{s_{i,2}} + m_{\text{sp}_i} d_i^{2} \right)} \Big(- k_i \theta_i - c_i \dot\theta_i + \hat{\bm s}_{i,2} \cdot \bm \tau_{\text{ext},H_i} + \left( I_{s_{i,3}} - I_{s_{i,1}} + m_{\text{sp}_i}d_i^{2}\right) \omega_{s_{i,3}} \omega_{s_{i,1}} \\
	&- m_{\text{sp}_i} d_i \hat{\bm s}_{i,3}^{T} [\tilde{\bm\omega}_{\cal B/N}][\tilde{\bm\omega}_{\cal B/N}] \bm r_{H_i/B}\Big)
	\label{eq:solar_panel_final6}
	\end{split}
	\end{align}
\end{subequations}

The equations can now be organized into the following matrix respresentation:

\begin{equation}
\begin{bmatrix}
[A] & [B]\\
[C] & [D]
\end{bmatrix} \begin{bmatrix}
\ddot{\bm r}_{B/N}\\
\dot{\bm\omega}_{\cal B/N}
\end{bmatrix} = \begin{bmatrix}
\bm v_{\text{trans}}\\
\bm v_{\text{rot}}
\end{bmatrix}
\label{eq:backSub}
\end{equation}

Finally, the hinged rigid body model must make ``contributions" to the matrices defined in Equations~\eqref{eq:backSub}. These contributions are defined in the following equations:

\begin{align}
[A_{\textnormal{contr}}] &= m_{\text{sp}_i}d_i \bm{\hat{s}}_{i,3} \bm a_{\theta_i}^T
\\
[B_{\textnormal{contr}}] &= m_{\text{sp}_i}d_i \bm{\hat{s}}_{i,3} \bm b_{\theta_i}^T
\\
[C_{\textnormal{contr}}] &= \big( I_{s_{i,2}}\bm{\hat{s}}_{i,2}+m_{\text{sp}_i}d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,3}\big)\bm a_{\theta_i}^T
\\
[D_{\textnormal{contr}}] &= \big( I_{s_{i,2}}\bm{\hat{s}}_{i,2}+m_{\text{sp}_i}d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,3}\big) \bm b_{\theta_i}^T
\\
\bm v_{\text{trans,contr}} &= -\Big(m_{\text{sp}_i}d_i \dot{\theta}_i^2 \bm{\hat{s}}_{i,1}+m_{\text{sp}_i}d_i c_{\theta_i} \bm{\hat{s}}_{i,3} \Big)
\\
\bm v_{\text{rot,contr}} &= -\biggl\lbrace\big(\dot{\theta}_i [\bm{\tilde{\omega}}_{\cal B/N}] +c_{\theta_i} [I_{3\times3}]\big) \left(I_{s_{i,2}} \bm{\hat{s}}_{i,2}+m_{\text{sp}_i} d_i [\tilde{\bm{r}}_{S_{c,i}/B}] \hat{\bm s}_{i,3}\right) +m_{\text{sp}_i}d_i\dot{\theta}_i^2[\tilde{\bm{r}}_{S_{c,i}/B}] \bm{\hat{s}}_{i,1} \biggr\rbrace
\end{align}

The final equation that is needed is:

\begin{equation}
\ddot \theta_i = \bm a_{\theta_i}^T \ddot{\bm r}_{B/N} + \bm b_{\theta_i}^T \dot{\bm\omega}_{\cal B/N} + c_{\theta_i}
\label{eq:solar_panel_final5}
\end{equation}


\section*{Maneouver Computations}
\addcontentsline{toc}{section}{Maneouver Computations}
Let us assume symmetric flight (no lateral aerodynamic force $Q=0$ nor lateral thrust force as $\nu=0$), the thrust generated by the engines is parallel to the \textit{x} wind axis (thus $\epsilon=0$),  and that the maneuver takes place flawlessly, this is in the vertical plane too; $\xi=\dot{\xi}=0$.\\
As the maneuver is a rather short, we will also assume that the fuel consumed i negligible and no mass fraction is lost; $\dot{m}=0$.\\
The motion equations follow \cite{tierno}:

\begin{equation*}
	\begin{cases}
	T\cos(\epsilon)\cos(\nu) - D -mg\sin\gamma-m\dot{V}=0\\
	T\cos(\epsilon)\sin(\nu) - Q+mg\cos\gamma\sin\mu+...\\
	...+mV(\dot{\gamma}\sin\mu-\dot{\Xi}\cos\gamma\cos\mu)=0\\
	-T\sin\epsilon-L+mg\cos\gamma\cos\mu+...\\
	...+mV(\dot{\gamma}\cos\mu-\dot{\Xi}\cos\gamma\sin\mu)=0\\
	\dot{x}_e=V\cos\gamma\cos\Xi\\
	\dot{y}_e=V\cos\gamma\sin\Xi\\
	\dot{x}_e=-V\sin\gamma
	\end{cases}
\end{equation*}

And for each of the phases that compose the maneuver, they can be simplified by substituting the flight conditions.

\begin{center}
\begin{tabular}{|l|c|c|c|}\hline
	& $\mu$ & $\gamma=\dot{\gamma}$ & $\xi=\dot{\xi}=\nu=\epsilon=Q$\\ \hline  \hline
	Cruise & 0 & 0 & 0 \\ \hline
	Semicirle & 0 & f(t) & 0 \\ \hline
	Inversion& f(t) & 0 & 0 \\ \hline
\end{tabular}
\end{center}

\subsection*{First phases - cruise}
\addcontentsline{toc}{subsection}{First phases - cruise}
\begin{equation}
	\begin{cases}
		T - D -m\dot{V}=0\\
		0=0\\
		-L+mg=0\\
		\dot{x}_e=V\\
		\dot{y}_e=0\\
		\dot{x}_e=0
	\end{cases}
\end{equation}
There are three equations and four variables; $\alpha$, $\pi$ and V
Total degrees of freedom is one, although in further study both $\alpha$ and $\pi$ will be fixated.

\subsection*{Second phase - semicircle}
\addcontentsline{toc}{subsection}{Second phase - semicircle}
\begin{equation}
	\begin{cases}
		T - D -mg\sin\gamma-m\dot{V}=0\\
		0=0\\
		-L+mg\cos\gamma+mV\dot{\gamma}=0\\
		\dot{x}_e=V\cos\gamma\\
		\dot{y}_e=0\\
		\dot{z}_e=-V\sin\gamma
	\end{cases}
\label{eq:semicircle}
\end{equation}
There are four equations and four variables: $\alpha$, $\pi$, V and $\gamma$. The elevator and gas control lever or throttle are for controlling $\alpha$ and $\pi$ respectively. In further study both of them will be fixed.

\subsection*{Third phase - inversion}
\addcontentsline{toc}{subsection}{Third phase - inversion}
\begin{equation}
	\begin{cases}
		T - D -m\dot{V}=0\\
		mg\sin\mu=0\\
		-L-mg\cos\mu=0\\
		\dot{x}_e=V\\
		\dot{y}_e=0\\
		\dot{z}_e=0
	\end{cases}
\end{equation}
There are three equations and five variables; $\alpha$, $\pi$, V and $\mu$. Except from the velocity, they can be controlled by the pilot trhough the elevator deflection, gas lever and ailerons respectively.
Total degrees of freedom are, therefore, 2. In further study both the elevator deflection and gas elever will be fixated.

\subsection*{Forth phase - cruise}
\addcontentsline{toc}{subsection}{Forth phase - cruise}
\begin{equation}
	\begin{cases}
		T - D -m\dot{V}=0\\
		0=0\\
		-L+mg=0\\
		\dot{x}_e=V\\
		\dot{y}_e=0\\
		\dot{z}_e=0
	\end{cases}
\end{equation}
There are three equations and four variables; $\alpha$, $\pi$ and V. The dynamics are identical to those in the first phase, as both of them are cruise flight.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Semantic segmentation models}
\label{section:semantic_segmentation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
In the last few years, DL techniques were rapidly developed in different life applications such as computer vision.
Image segmentation is a well-known technique employed for computer vision.
It aims to label each pixel in the input image to its matching class and it is applied in many real-life practical applications such as self-driving cars, medical imaging, traffic control systems, video surveillance, and many others.
In this work, we present a comparative study of five DL models based on Fully Convolutional Networks (FCN)~\cite{Long} to detect and localise delamination in composite plates.
Further, these models aim to perform image semantic segmentation by assigning every pixel of the input image as damaged or not damaged.
FCN is created by stacking convolutional layers in an encoder-decoder scheme and skipping dense layers.
The encoder part is responsible for extracting condensed feature maps from the input image at different scale levels by applying cascaded convolutions with strides followed by pooling operations.
The decoder part is responsible for upsampling the condensed feature maps to the same size as the original input image using transposed convolution with strides or upsampling with interpolation.

In this work, the softmax activation function was applied at the output layer for all implemented models in this comparative study.
The softmax calculates the probability for each predicted output of being damaged or undamaged for every single pixel, which implies that the sum of the two probabilities must be one.
Eq.~(\ref{softmax}) depicts the softmax activation function, where \(P(x)_{i}\) is the probability of each target class \(x_{j}\) over all possible target classes \(x_{j}\), C in our case are two classes  (damaged and undamaged).
To predict the label of the output (\(y_{pred}\)) an \(\argmax\) function is applied to select the maximum probability between both of them.
	\begin{equation}
		P(x)_{i} = \frac{e^{x_{i}}}{\sum_{j}^{C} e^{x_{j}}}
		\label{softmax}
	\end{equation}
	\begin{equation}
		y_{pred} = \argmax_{i}\left( P(x)_{i} \right)
		\label{argmax}
	\end{equation}
Selecting a suitable loss function is an important issue because it measures how well the model learns and performs.
Therefore, in all models, we have applied the categorical cross-entropy (CCE) loss function~\cite{Bonaccorso2020}, which is also called \enquote{softmax loss function}.
CCE is used as the objective function to estimate the difference between the actual damage (ground truth) and the predicted damage.
Further, since we have only two classes to be predicted, it is worth mentioning that a Sigmoid activation function at the output layer can be used with a binary cross-entropy (BCE), with no impacts on the predicted outputs.
Eq.~(\ref{CCE}) illustrates the CCE, where \( P(x)_{i}\) is the softmax value of the target class.
	\begin{equation}
	CCE = -\log\left( P(x)_{i} \right)
	\label{CCE}
	\end{equation}

Additionally, it is also important to select a proper accuracy metric of the model, therefore, we have applied intersection over union (\(IoU\)) (Jaccard index)~\cite{Bertels2019} as our accuracy metric.
\(IoU\) is estimated by determining the intersection area between the ground truth and the predicted output.
In this work, we have two classes (damaged and undamaged), the \(IoU\) is computed by taking the \(IoU\) for the damaged class only.
The \(IoU\) metric is defined as in Eq.~(\ref{IoU}):
\begin{equation}
IoU = \frac{Intersection}{Union} = \frac{\hat{Y} \cap Y}{\hat{Y} \cup Y}
\label{IoU}
\end{equation}
where \(\hat{Y}\) represents the predicted vector of damaged and undamaged values, and \(Y\) represents the vector of ground truth values.
The \(IoU\) can be calculated by multiplying the predicted output (matrix of \(zeros\) and \(ones\)) with its ground truth (matrix of \(zeros\) and \(ones\)) to find the intersection, then it is divided over the union which can be calculated by counting all pixels with non-zero values of the predicted output and its ground truth.

Furthermore, Adam optimizer was applied as our optimization method in order to increase the \(IoU\) and to reduce the loss during the training.
In the next subsections, we present five FCN models for pixel-wise semantic segmentation to detect and localise delaminations.