%!TEX root = <JHU-SIMPLEX_proposal.tex>
% \clearpage
\section{Statement of Work}
\label{sec:sow}
% \emph{The SOW must provide a detailed task breakdown, citing specific tasks and their connection to the interim milestones and metrics, as applicable. Each year of the project should be separately defined. The SOW must not include proprietary information. For each defined task/subtask, provide:
% %
% (1) A general description of the objective.
% (2) A detailed description of the approach to be taken to accomplish each defined task/subtask.
% (3) Identification of the primary organization responsible for task execution (prime contractor, subcontractor(s), consultant(s)), by name.
% (4) A measurable milestone, (e.g., a deliverable, demonstration, or other event/activity that marks task completion).
% (5) A definition of all deliverables (e.g., data, reports, software) to be provided to the Government in support of the proposed tasks/subtasks.}
%

\subsection{Phase I}

\subsubsection{Task 1: Mathematical Formalism}
\begin{compactitem}
\item \textbf{Goal:} \emph{RAG Embedding:} Completion of the theoretical development and associated data structures of our RAG representation system. This includes establishing baseline methods for embeddings RAGs and populations thereof.  Specifically, we will explore both JOFC and tensor factorization methodologies, to enable understanding of the computational and statistical advantages and disadvantages of each for embedding high-dimensional non-Euclidean RAGS.

\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstration that RAGs are able to meet TA1 goals, including encoding quantitative and qualitative knowledge, and express functional relationship among entities in complex systems.
\item \textbf{Deliverables:} Description of mathematical framework and preliminary benchmarks evaluating performance on open access data sets and simulations.
\end{compactitem}

\subsubsection{Task 2: Computational Infrastructure}
\begin{compactitem}
\item \textbf{Goal:} \emph{Data Management:} Implementation of baseline algorithms for context-aware reasoning and inference using the representation. Development of an initial computational and data management platform, including establishing common data formats, common methods and format for query and analysis of results, and a common API through which all domain-specific users will access the framework. We will also extend our dense spatial and semantic databases, as well as our graph data format to support time-varying data, along with multi-modal data.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Completion of Phase I prototype software and services.
\item \textbf{Deliverables:} Open source software and documentation for end-to-end prototype.
\end{compactitem}

\subsubsection{Task 3: Datafication}
\begin{compactitem}
\item \textbf{Goal:} \emph{Data Ingest:} Completion of data ingest techniques. Completion of research and design into microscopic and mesoscopic specific analysis tools.  Demonstration of auto-data ingestion and registration of multiple different modalities (functional to structural for both microscopic and mesoscopic data). More specifically, we will have ingested CLARITY, LFM, and M$^3$RI data into the same database schema; all M$^3$RI data will be co-registered.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstration of operational auto-ingestion and registration on two different use-cases.
\item \textbf{Deliverables:} Open source software and documentation for datafication techniques, as well as image datasets ingested and RAGs estimated from all different data modalities.
\end{compactitem}

\subsubsection{Task 4: Discovery}
\begin{compactitem}
\item \textbf{Goal:} \emph{RAG Construction:}
 Completion of research and design into microscopic and mesoscopic specific analysis tools, by designing metrics appropriate for the different data modalities.  This includes both the microscale and mesoscale functional time-series, converting into RAGs via utilizing qualitative information.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstration of RAG construction on both use cases.
\item \textbf{Deliverables:} Open source software and documentation RAG construction techniques, as well as the derived RAGs available via our Web-services.
\end{compactitem}

\subsubsection{Task 5: Program Management}
\begin{compactitem}
\item \textbf{Goal:} \emph{Phase I:} Ensure successful execution of the effort. Manage the proposed effort using a proven methodology for project planning, resource allocation, task specification, and monitoring. Establish a baseline project plan with a list of tasks, specifications, requirements, and timelines; update plan periodically; document updates to the plan and share them with the project team and DARPA PM.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Meet Phase I goals.
\item \textbf{Deliverables:}
(1) Comprehensive quarterly technical reports that include updates systems architecture and progress made on milestones for Phase I;
(2) Brief month reports, including preprints of technical reports;
(3) Final Technical Report;
(4) Monthly Financial Reports.
\end{compactitem}

\subsection{Phase II}

\subsubsection{Task 1: Mathematical Formalism}
\begin{compactitem}
\item \textbf{Goal:} \emph{FlashRAG:} Implementation of all embedding and construction methodologies in FlashGraph to enable scalable implementations and processing.  Moreover, all constructed RAGs will obtain multilevel representations.  We will build R bindings to enable easy use of FlashGraph for data scientists.  We will check that our implementations and bindings yield approximately the same answer as benchmark methods, on data sufficiently small that benchmark methods can run.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Fully operational FlashGraph and R bindings for embedding and constructing methodologies.
\item \textbf{Deliverables:} Open source software and documentation for end-to-end prototype for embedding and constructing RAGs. This includes an R package for FlashGraph.
\end{compactitem}

\subsubsection{Task 2: Computational Infrastructure}
\begin{compactitem}
\item \textbf{Goal:} \emph{Remote Access:} Implementation of prototype platform for remote access. This will include Web-services for uploading the raw data, and downloading the derived data products (RAGs and intermediate data products), as well as both 2D and 3D visualization and annotation tools, which will support multiple kinds of analytic overlays, all of which will support multiple data scales. Moreover, we will have made theoretical and practical refinements to the representation to enable scalable implementation of several foundational algorithms on RAGs, implementing the embedding methodologies developed in Task 1 of Phase I into our semi-external memory formalism.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Fully operational Web-services supporting uploading, visualizing, annotation, querying, downloading, and analyzing the data.
\item \textbf{Deliverables:} Open source software and documentation for end-to-end prototype. This includes an R package for FlashGraph which extends it capabilities to RAGs, rather than simply graphs.
\end{compactitem}

\subsubsection{Task 3: Datafication}
\begin{compactitem}
\item \textbf{Goal:} \emph{Data Register:} Integration of domain-specific computational models across modalities and scales. This includes completion of statistical multi-modal referencing, including alignment of structural and functional imaging data, for both microscopic and mesoscopic data sets.  We will also complete functional inference capabilities. We will align data both via scaling up multidimensional out-of-core image alignment algorithms, and RAG matching, which extends graph matching by incorporating attributes.  This will enable us to determine optimal alignments using data priors and known topological structure, rather than relying on images to align well.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstration of multi-modal registration for both microscopic and mesoscopic use cases.
\item \textbf{Deliverables:} Open source software and documentation for datafication techniques, as well as registered multi-modal image datasets ingested and aligned RAGs from both microscale and mesoscale.
\end{compactitem}

\subsubsection{Task 4: Discovery}
\begin{compactitem}
\item \textbf{Goal:} \emph{RAG Summary Statistics:} Utilize RAG knowledge representation to estimate population moments, motifs, and/or modes from both  micro- and meso-scale RAGs.  More specifically, we will utilize the various joint embedding methodologies developed in Task 1 to estimate these summary statistics.  The different approaches, JOFC versus tensor factorization, will enable incorporating different kinds of prior knowledge and constraints, so they will therefore lead to different bias/variance trade-offs.  We will explore these options empirical on the real data, to complement our experiments in Task 1, to discover both (i) the best methods for estimation these summary statistics, and (ii) the best estimates of the summary statistics for the two different scales.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstration utility of RAG representation of data for estimating summary statistics for multi-modal data.
\item \textbf{Deliverables:} Estimated summary statistics from micro- and meso-scale RAGs available for download in various formats, as well as open source code for the different estimators.
\end{compactitem}

% \subsubsection{Task 4: Discovery}
% \begin{compactitem}
% \item \textbf{Goal:} \emph{RAG Independence:} Instantiation of first-generation analysis tools, including tests for independence between connectivity and graph, vertex, and/or edge attributes, and application of those tools to both use cases. We will leverage the RAG representation system to discover whether mouse or human graph connectivity is independent of graph attributes. For both inference tasks, we will utilize and extending the scalable embedding methodologies developed in Task 2 of Phase II.
% \item \textbf{Primary Site:} JHU
% \item \textbf{Milestone:} Demonstration utility of RAG representation of data for multi-modal neuroscientific discoveries for both microscale and mesoscale.
% \item \textbf{Deliverables:} Discovered multi-modal motifs from microscale data and hypothesis testing results and visualizations from mesoscale data.
% \end{compactitem}

\subsubsection{Task 5: Program Management}
\begin{compactitem}
\item \textbf{Goal:} \emph{Phase II Goals:} Ensure successful execution of the effort. Manage the proposed effort using a proven methodology for project planning, resource allocation, task specification, and monitoring. Establish a baseline project plan with a list of tasks, specifications, requirements, and timelines; update plan periodically; document updates to the plan and share them with the project team and DARPA PM.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Meet Phase II goals.
\item \textbf{Deliverables:}
(1) Comprehensive quarterly technical reports that include updates systems architecture and progress made on milestones for Phase II;
(2) Brief month reports, including preprints of technical reports;
(3) Final Technical Report;
(4) Monthly Financial Reports.
\end{compactitem}

\subsection{Phase III}

\subsubsection{Task 1: Mathematical Formalism}
\begin{compactitem}
\item \textbf{Goal:} \emph{RAG Testing:} Demonstration of capabilities and objectives on both microscale and mesoscale data, as well as one additional SIMPLEX performer. To achieve this, we will extend our embedding methodologies, to derive provably approximately optimal embeddings for conducting one-sample and two-sample tests on RAGs; two fundamental testing procedures in statistics.  Our tests will leverage our ability to efficiently sample RAGs, as they will be resampling based tests, analogs to the classic parametric and non-parametric bootstrap.  We will apply these tests to test, for example, whether our data are sampled from relatively simple RAGs statistical models, and whether population means that we obtained in Phase I are significantly different from one another.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Demonstrate our mechanism for relating qualitative and quantitative knowledge, and relating multiple heterogeneous datasets, on both heterogeneous scales (use cases). Specifically, testing whether multiple heterogeneous datasets are statistically different from one another.
\item \textbf{Deliverables:} Description of capabilities, emphasizing generalizability to multiple use cases, open source code from implementing our tests, visualizations and numerical summaries of test results.
\end{compactitem}

% \subsubsection{Task 1: Mathematical Formalism}
% \begin{compactitem}
% \item \textbf{Goal:} \emph{RAG Learning:} Completion of mathematical tools for hypothesis generation, testing, and model validation using RAGs. This will include context aware algorithms capable of running in interactive time.  More specifically, we will extend the toolbox of inferential techniques to include unsupervised, semi-supervised, and fully supervised methods for clustering RAGs.  All these methods will utilize  the estimation and embedding strategies developed in the previous tasks for this proposal.  Moreover, we will extend such embedding methodologies to optimize them for the particular learning tasks.  Theoretical and numerical results will support the utility of these methodologies.
% \item \textbf{Primary Site:} JHU
% \item \textbf{Milestone:} Demonstration of the full suite of mathematical capabilities on RAGs, including estimating, testing, and un-/semi-/fully-supervised learning methodologies.
% \item \textbf{Deliverables:} Complete description of capabilities of RAGs, as well as open source code to run all analyses in R on commodity hardware.
% \end{compactitem}

\subsubsection{Task 2: Computational Infrastructure}
\begin{compactitem}
\item \textbf{Goal:} \emph{Local Analysis:} Completion of an integrated system on both heterogeneous scales (as well as additional domains).  This system ingests and registers imaging data and semantic and qualitative knowledge, converts them into RAGs, allows query and recall, visualization, hypothesis generation, and analysis. We will also release open source packages containing all of the key resources, such as an R package (which calls igraph or FlashGraph) containing all of the developed methods, and GPU optimized visualization and annotation tools. This will enable anybody to implement analyses locally by running the code on their machine.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Fully operation Web-services to auto-ingest, register, store in compact representation, and query, as well as operate locally.
\item \textbf{Deliverables:} Open source software and documentation for end-to-end prototype. including FlashGraphR and GPU optimized visualization and annotation tool.
\end{compactitem}

\subsubsection{Task 3: Datafication}
\begin{compactitem}
\item \textbf{Goal:} \emph{Quality Control:} Completion of quality control of all datasets.  For each different modality, and each different scale, we will have already converted the raw data into RAGs.  Now, we will build automatic quality controls, so that with each dataset, we automatically generate a quality control report, quantifying the key quality metrics appropriate for that data (see Table \ref{tab:qa}).
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} All datasets have been checked for quality.
\item \textbf{Deliverables:} Open source software and documentation for datafication techniques, including quality control scripts, and resulting outputs.
\end{compactitem}

\subsubsection{Task 4: Discovery}
\begin{compactitem}
\item \textbf{Goal:} \emph{RAG Prediction:} Completion of toolset for analysis, modeling, and data-driven hypothesis generation and testing in both microscale and mesoscale heterogeneous use cases. This will include multiscale prediction; prediction of mouse status from microscale RAGs, and human personality from human RAGs.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone:} Successful integration with TA1 technology and end-of-program demonstrations of our integrated system on both microscale and mesocale.
\item \textbf{Deliverables:} All data-derived products available for visualization utilizing our Web-services and quality control pages, as well as for download and further analysis using our open source code.
\end{compactitem}

\subsubsection{Task 5: Program Management}
\begin{compactitem}
\item \textbf{Goal:} \emph{Phase III:} Ensure successful execution of the effort. Manage the proposed effort using a proven methodology for project planning, resource allocation, task specification, and monitoring. Establish a baseline project plan with a list of tasks, specifications, requirements, and timelines; update plan periodically; document updates to the plan and share them with the project team and DARPA PM.
\item \textbf{Primary Site:} JHU
\item \textbf{Milestone} Meet Phase III goals.
\item \textbf{Deliverables:}
(1) Comprehensive quarterly technical reports that include updates systems architecture and progress made on milestones for Phase III;
(2) Brief month reports, including preprints of technical reports;
(3) Final Technical Report;
(4) Monthly Financial Reports.
\end{compactitem}


% Abstract ====================================================================

\pdfbookmark[1]{Abstract}{Abstract}
\chapter*{Abstract}

Data analysis and machine learning have become an integrative part of the
modern scientific methodology, offering automated procedures for the prediction
of a phenomenon based on past observations, unraveling underlying patterns in
data and providing insights about the problem. Yet, caution should
avoid using machine learning as a black-box tool, but rather consider it as a
methodology, with a rational thought process that is entirely dependent on the
problem under study. In particular, the use of algorithms
should ideally require a reasonable understanding of their
mechanisms, properties and limitations, in order to better apprehend and
interpret their results.

Accordingly, the goal of this thesis is to provide an in-depth
analysis of random forests, consistently calling into
question each and every part of the algorithm, in order to shed new light on
its learning capabilities, inner workings and interpretability. The first
part of this work studies the induction of decision trees and the construction of
ensembles of randomized trees, motivating their design and purpose whenever
possible. Our contributions follow with an original complexity
analysis of random forests, showing their good computational performance
and scalability, along with an in-depth discussion of their
implementation details, as contributed within Scikit-Learn.

In the second part of this work, we analyze and discuss the interpretability of
random forests in the eyes of variable importance measures. The core of our
contributions rests in the theoretical characterization of the Mean Decrease of
Impurity variable importance measure, from which we prove and derive some of
its properties in the case of multiway totally randomized trees and in
asymptotic conditions. In consequence of this work, our analysis  demonstrates
that variable importances as computed from non-totally randomized trees (e.g.,
standard Random Forest) suffer from a combination of defects, due to masking
effects, misestimations of node impurity or due to the binary structure of
decision trees.

Finally, the last part of this dissertation addresses limitations of random
forests in the context of large datasets. Through extensive experiments, we
show that subsampling both samples and features simultaneously provides on par
performance while lowering at the same time the memory requirements. Overall
this paradigm highlights an intriguing practical fact: there is often no need
to build single models over immensely large datasets. Good performance can
often be achieved by building models on (very) small random parts of the data
and then combining them all in an ensemble, thereby avoiding all practical
burdens of making large data fit into memory.


\RequirePackage[l2tabu, orthodox]{nag}
\documentclass{article}

\usepackage[letterpaper]{geometry}
\usepackage{booktabs}
\usepackage{mathtools}
\usepackage[binary-units=true]{siunitx}
\usepackage{tikz}

\title{ECE 487 Assignment 5}
\author{Michael Kwok}
\begin{document}

\maketitle
\subsection*{Stop and Wait ARQ}
Round trip time: \(\frac{600000}{3 \times 10^8} = 2 \times 10^{-3}\)
2 Timeouts: \(0.5 + 0.5\) seconds\\
1 Successful round trip: \(2 \times 10^{-3}\) \\
Total time: \(1.002\) seconds\\
Data sent: \(1000\) bits\\
Throughput: \( \frac{1000}{1.002} = \SI{998}{\bit\per\second}\)

\subsection*{ALOHA station throughput}
\(T_{fr} = \SI{0.4}{\milli\second}\)

\subsubsection*{Pure}
\(G = 0.5\) to get maximum throughput.\\
System frames per second: \( G \cdot \frac{1000}{0.4} = 1250 \text{Frames per Second} \) \\
50 stations, so average frames per second per station: \(\frac{1250}{50} = 25\)

\subsubsection*{Slotted}
\(G = 1\) to get maximum throughput.\\
System frames per second: \( G \cdot \frac{1000}{0.4} = 2500 \text{Frames per Second} \) \\
50 stations, so average frames per second per station: \(\frac{2500}{50} = 50\)

\subsection*{ALOHA system throughput}
\(T_{fr} = \frac{2000}{10^6} = \SI{2}{\milli\second}\)

\(G = \frac{500}{\frac{1000}{2}} = 1\)

\(50 \cdot 10 = 500\) Frames sent per second

\subsubsection*{Pure}
\(S = G \cdot e^{-2} = 0.135 \) \\
System throughput: \(500 \cdot 0.135 = 67.7\) Frames per second

\subsubsection*{Slotted}
\(S = G \cdot e^{-1} = 0.368 \) \\
System throughput: \(500 \cdot 0.368 = 183.9\) Frames per second

\subsection*{CSMA/CD}

\ifx\du\undefined
    \newlength{\du}
\fi
\setlength{\du}{15\unitlength}
\begin{tikzpicture}
    \pgftransformxscale{0.998158}
    \pgftransformyscale{-0.998158}
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \definecolor{dialinecolor}{rgb}{1.000000, 1.000000, 1.000000}
    \pgfsetfillcolor{dialinecolor}
    \definecolor{dialinecolor}{rgb}{1.000000, 1.000000, 1.000000}
    \pgfsetfillcolor{dialinecolor}
    \fill (13.000000\du,8.000000\du)--(13.000000\du,10.000000\du)--(17.000000\du,10.000000\du)--(17.000000\du,8.000000\du)--cycle;
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetmiterjoin
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \draw (13.000000\du,8.000000\du)--(13.000000\du,10.000000\du)--(17.000000\du,10.000000\du)--(17.000000\du,8.000000\du)--cycle;
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node at (15.000000\du,9.194053\du){A};
    \definecolor{dialinecolor}{rgb}{1.000000, 1.000000, 1.000000}
    \pgfsetfillcolor{dialinecolor}
    \fill (23.000000\du,8.000000\du)--(23.000000\du,10.000000\du)--(27.000000\du,10.000000\du)--(27.000000\du,8.000000\du)--cycle;
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetmiterjoin
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \draw (23.000000\du,8.000000\du)--(23.000000\du,10.000000\du)--(27.000000\du,10.000000\du)--(27.000000\du,8.000000\du)--cycle;
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node at (25.000000\du,9.194053\du){B};
    \definecolor{dialinecolor}{rgb}{1.000000, 1.000000, 1.000000}
    \pgfsetfillcolor{dialinecolor}
    \fill (33.000000\du,8.000000\du)--(33.000000\du,10.000000\du)--(37.000000\du,10.000000\du)--(37.000000\du,8.000000\du)--cycle;
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetmiterjoin
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \draw (33.000000\du,8.000000\du)--(33.000000\du,10.000000\du)--(37.000000\du,10.000000\du)--(37.000000\du,8.000000\du)--cycle;
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node at (35.000000\du,9.194053\du){C};
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (15.000000\du,10.000000\du)--(15.000000\du,20.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (25.000000\du,10.000000\du)--(25.000000\du,20.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (35.000000\du,10.000000\du)--(35.000000\du,20.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \pgfsetarrowsend{to}
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (15.000000\du,10.000000\du)--(35.000000\du,18.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \pgfsetarrowsend{to}
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (25.000000\du,12.000000\du)--(15.000000\du,16.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \pgfsetarrowsend{to}
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (25.000000\du,12.000000\du)--(35.000000\du,16.000000\du);
    }
    \pgfsetlinewidth{0.100000\du}
    \pgfsetdash{}{0pt}
    \pgfsetdash{}{0pt}
    \pgfsetbuttcap
    {
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetfillcolor{dialinecolor}
        % was here!!!
        \pgfsetarrowsend{to}
        \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
        \pgfsetstrokecolor{dialinecolor}
        \draw (35.000000\du,14.000000\du)--(15.000000\du,20.000000\du);
    }
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (14.000000\du,10.000000\du){};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (14.000000\du,10.000000\du){};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,10.000000\du){0uS};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,12.000000\du){1uS};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,14.000000\du){2uS};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,16.000000\du){3uS};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,18.000000\du){4uS};
    % setfont left to latex
    \definecolor{dialinecolor}{rgb}{0.000000, 0.000000, 0.000000}
    \pgfsetstrokecolor{dialinecolor}
    \node[anchor=west] at (27.000000\du,20.000000\du){5uS};
\end{tikzpicture}

Transmission rate: \(10\times 10 ^{6} \times 3 \times 10 ^{-6} = \SI{30}{\bit\per\second}\)\\
A will transmit for \SI{3}{\micro\second}, \SI{30}{\bit}\\
B will transmit for \SI{1}{\micro\second}, \SI{10}{\bit}\\
C will transmit for \SI{1}{\micro\second}, \SI{10}{\bit}

\end{document}


\subsection{KENN with multiloss function}
In previous experiments, it has been observed that the pre-KENN network adapts to the presence of KENN regardless of the parameters configuration. From these results comes the intuition to make the pre-KENN network more independent, with the purpose of exploiting the logical knowledge more effectively. To achieve this, the idea is to define a custom multiloss function to simultaneously improve the quality of the pre-KENN and post-KENN predictions.

%\subsubsection{Multiloss function}
The proposed multiloss function is computed by combining the Binary Cross-Entropy (BCE) loss values obtained from the pre-KENN and post-KENN preactivations. To make it possible, the model has been adapted to provide two outputs: one before and one after KENN's layer. The goal is to optimize the post-KENN predictions while preserving a discrete quality of pre-KENN predictions. The final loss is computed by the convex combination of the two losses whose influences are regulated by the parameter $ \alpha $. The resulting formula is the following:

% \begin{gather*}
%     L(y_{pred},y_{true}) = \alpha \cdot BCE(y_{prekenn}, y_{true}) + (1 - \alpha) \cdot BCE(y_{postkenn}, y_{true})
% \end{gather*}

\begin{gather*}
    L(Y, Y', Y_{t}) = \alpha \cdot BCE(Y, Y_{t}) + (1 - \alpha) \cdot BCE(Y', Y_{t})
\end{gather*}
where $Y$ denotes the pre-KENN predictions, $Y'$ the post-KENN predictions, and $Y_{t}$ the values of the ground truth.

\subsubsection{Setup}
The models involved in this experiment are trained following the \textit{Setup B}. The configuration of the other parameters is the following:
\begin{itemize}
    \item \textbf{KB modes:} Bottom Up and Top Down
    \item \textbf{initial clause weights:} 0.5 and variable\footnote{With the term ``variable" we mean that each clause can be initialized with a different value. The original implementation of KENN did not contemplate this possibility when setting clause weights as learnable parameters, so we introduced a small modification to make it possible.}
    \item \textbf{learnable clause weights}
    \item \textbf{encoder:} DistilBERT and BERT, with adapters
    \item \textbf{loss function:} multiloss with $\alpha = 0.5$
\end{itemize}
The total number of configurations obtained by varying these parameters is 8. Since the pre-KENN network is expected to be less influenced by KENN, the clause weights are set as learnable parameters with small initial values to start with a soft influence and let the network establish the relevance of each clause. Indeed, the choice of learnable clause weights allows us to study the weight evolution during the epochs and determine which are the most useful and the least useful clauses. Furthermore, for each clause, the types that are the antecedents of an implication rule (i.e., the types that propagate the information and uniquely characterize the used KBs) have been studied to look for some recurring behavioral patterns. The \textit{Hybrid} mode is excluded from this study because it involves the same clauses of \textit{Bottom Up} and \textit{Top Down}, but with the addition of noise due to the presence of conflicts. For this reason, it would not have been possible to carry out an accurate analysis.

\subsubsection{Results on FIGER}
\paragraph{DistilBERT}
The results obtained using DistilBERT as encoder show some very interesting behaviors of the clause weights. Starting from the Bottom Up, if we look at the distribution of final clause weights in Figure~\ref{fig:weight_distrib_distilbert_figer_bu_multiloss} we can see that almost every weight decreases its initial value. This attitude is not observable in Figure~\ref{fig:weight_distrib_distilbert_figer_bu_learnable} when using KENN with a standard loss and learnable weights. If we look at the figure, we can see widely different results since almost every weight increases its starting value. The same trend is detectable in the Top Down mode, as shown in Figure~\ref{fig:weight_distrib_distilbert_figer_td}.

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.45\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_distrib_distilbert_figer_bu_multiloss.png}
         \caption{Multiloss model - Epoch 55 (563K examples)}
         \label{fig:weight_distrib_distilbert_figer_bu_multiloss}
     \end{subfigure}
     \hspace{10px}
     \begin{subfigure}[b]{0.45\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_distrib_distilbert_figer_bu_learnable.png}
         \caption{Standard loss model - Epoch 42 (430K examples)}
         \label{fig:weight_distrib_distilbert_figer_bu_learnable}
     \end{subfigure}
    \caption{Distributions of the final learned clause weights for the Bottom Up KB mode using DistilBERT-based models - FIGER}
    \label{fig:weight_distrib_distilbert_figer_bu}
\end{figure}

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.45\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_distrib_distilbert_figer_td_multiloss.png}
         \caption{Multiloss model - Epoch 42 (430K examples)}
         \label{fig:weight_distrib_distilbert_figer_td_multiloss}
     \end{subfigure}
     \hspace{10px}
     \begin{subfigure}[b]{0.45\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_distrib_distilbert_figer_td_learnable.png}
         \caption{Standard loss model - Epoch 39 (399K examples)}
         \label{fig:weight_distrib_distilbert_figer_td_learnable}
     \end{subfigure}
    \caption{Distributions of the final learned clause weights for the Top Down KB mode using DistilBERT-based models - FIGER}
    \label{fig:weight_distrib_distilbert_figer_td}
\end{figure}

Moving on to the analysis of the types involved in the clauses we can observe an unexpected behavior. For both Bottom Up and Top Down modes, this study intercepts a remarkable fact: the more a type occurs in the training set, the lower the final weight of its clause. The few clauses that preserve a high weight are those whose antecedents are the rarest in the dataset. Starting from the Bottom Up mode, we can see in Figure~\ref{fig:weight_freq_distilbert_figer_bu_multiloss} a graph showing the relation between the final weight and the frequency of each type S. Looking at the figure, we can clearly observe that there is a strong correlation between weight and frequency. Indeed, the correlation coefficient is -0.77. The Top Down mode (Figure~\ref{fig:weight_freq_distilbert_figer_td_multiloss}) also presents a negative correlation, this time of -0.68 with respect to the frequencies of types F.
%This behavior may be explained by the fact that KENN introduces too much noise to types that the network can already predict discretely without the use of knowledge.

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_distilbert_figer_bu_multiloss.png}
         \caption{Bottom Up - Epoch 55 (563K examples)}
         \label{fig:weight_freq_distilbert_figer_bu_multiloss}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_distilbert_figer_td_multiloss.png}
         \caption{Top Down - Epoch 42 (430K examples)}
         \label{fig:weight_freq_distilbert_figer_td_multiloss}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using DistilBERT - FIGER}
    \label{fig:weight_freq_distilbert_figer}
\end{figure}

By inspecting the weight evolution over the epochs, it emerged that the clause weights keep decreasing until the last epoch, but it is not known for how long they would have continued. To observe in advance the effect of higher epochs and see if some clauses would have reached a weight close to zero, new models were trained starting from variable weights assigned with respect to the frequency of the antecedents of each clause. Clauses involving popular types were assigned a weight of 0.2, while the others kept a weight of 0.5. The weight assignment is based on a frequency threshold of 70K for Bottom Up and 50K for Top Down\footnote{the values are chosen by observing the graphs, without formalizing a mathematical criterion}. The results of these runs are shown in Figure~\ref{fig:weight_freq_distilbert_figer_bu_multiloss_variable} and Figure~\ref{fig:weight_freq_distilbert_figer_td_multiloss_variable} for Bottom Up and Top Down, respectively. As we can see, most clause weights decreased their values even when starting from lower initial values.
%However, by analyzing the weight evolution, it emerged that their decrease over the epochs was slowed down. The reason for this fact is that clauses now have less influence on the final prediction, so they are penalized in a minor way by the loss function.

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_distilbert_figer_bu_multiloss_variable.png}
         \caption{Bottom Up - Epoch 71 (727K examples)}
         \label{fig:weight_freq_distilbert_figer_bu_multiloss_variable}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_distilbert_figer_td_multiloss_variable.png}
         \caption{Top Down - Epoch 42 (430K examples)}
         \label{fig:weight_freq_distilbert_figer_td_multiloss_variable}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using DistilBERT and \textit{variable} clause weights - FIGER}
    \label{fig:weight_freq_distilbert_figer_variable}
\end{figure}

\paragraph{BERT}
The experiments performed using BERT showed an identical behavior. The results obtained with weights set to 0.5 for every clause are shown in Figure~\ref{fig:weight_freq_bert_figer}. The correlation between weight and frequency is similar (-0.76 in Bottom Up, -0.74 in Top Down) as well as the weight evolution during the epochs. The only difference with respect to DistilBERT is that BERT needs fewer epochs to converge, so its weights have decreased less as it performed fewer backward propagation operations. Much lower weights are reached when using the setup with variable weights as reported in Figure~\ref{fig:weight_freq_bert_figer_variable}.

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_figer_bu_multiloss.png}
         \caption{Bottom Up - Epoch 35 (358K training examples)}
         \label{fig:weight_freq_bert_figer_bu_multiloss}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_figer_td_multiloss.png}
         \caption{Top Down - Epoch 35 (358K training examples)}
         \label{fig:weight_freq_bert_figer_td_multiloss}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using BERT - FIGER}
    \label{fig:weight_freq_bert_figer}
\end{figure}

\begin{figure}[bth]
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_figer_bu_multiloss_variable.png}
         \caption{Bottom Up - Epoch 49 (501K training examples)}
         \label{fig:weight_freq_bert_figer_bu_multiloss_variable}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_figer_td_multiloss_variable.png}
         \caption{Top Down - Epoch 33 (337K training examples)}
         \label{fig:weight_freq_bert_figer_td_multiloss_variable}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using BERT and \textit{variable} clause weights - FIGER}
    \label{fig:weight_freq_bert_figer_variable}
\end{figure}

\subsubsection{Results on BBN}
The experiments on BBN were performed directly on BERT. In Figure~\ref{fig:weight_freq_bert_bbn} are represented the graphs of weights and frequencies using the same weight for each clause. The correlation coefficients of the Bottom Up and Top Down modes are -0.73 and -0.55, respectively. The results obtained after repeating the experiments using variable weights are shown in Figure~\ref{fig:weight_freq_bert_bbn_variable}. The weight assignment is based on a frequency threshold of 1700 for Bottom Up and 1500 for Top Down\footnote{the values are chosen by observing the graphs, without formalizing a mathematical criterion}. Looking at the graphs, it is possible to observe that most of the weights continued to decrease.

\begin{figure}
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_bbn_bu_multiloss.png}
         \caption{Bottom Up - Epoch 28 (286K training examples)}
         \label{fig:weight_freq_bert_bbn_bu_multiloss}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_bbn_td_multiloss.png}
         \caption{Top Down - Epoch 19 (194K training examples)}
         \label{fig:weight_freq_bert_bbn_td_multiloss}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using BERT - BBN}
    \label{fig:weight_freq_bert_bbn}
\end{figure}

\begin{figure}
     \centering
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_bbn_bu_multiloss_variable.png}
         \caption{Bottom Up - Epoch 19 (194K training examples)}
         \label{fig:weight_freq_bert_bbn_bu_multiloss_variable}
         \vspace{10px}
     \end{subfigure}
     \begin{subfigure}[b]{0.7\textwidth}
         \centering
         \includegraphics[width=\textwidth]{figures/weight_freq_bert_bbn_td_multiloss_variable.png}
         \caption{Top Down - Epoch 19 (194K training examples)}
         \label{fig:weight_freq_bert_bbn_td_multiloss_variable}
     \end{subfigure}
    \caption{Relation between types frequency (log scale) and final clause weights of multiloss models using BERT and \textit{variable} clause weights - BBN}
    \label{fig:weight_freq_bert_bbn_variable}
\end{figure}

\subsubsection{Conclusion}
The multiloss experiments showed that the pre-KENN network suffers the presence of some logical clauses when it is forced to not adapt its predictions to KENN. In particular, it has been possible to see that the learning process penalized clauses involving the most frequent types much more than the others. This behavior can be motivated by the fact that if a network sees a lot of instances labeled with some types, then it becomes able to correctly classify them without needing logical knowledge. For this reason, it seems that the action of KENN on frequent types is seen as noise added to the final predictions. Conversely, the enhancement of rarer types is not penalized by the learning process, so logical clauses seem to be helpful for the network when few examples labeled with those types are available. For this reason, using a multiloss setup could be a reasonable choice to exploit logical knowledge only where needed, thus avoiding having side effects on the types that are easier to predict.


\documentclass{beamer}

\usetheme{uhh}
\showtotalframenumber
\showuhhlogoeachframe
\showsections

\usepackage{amsmath}
\usepackage{graphicx}
\DeclareMathOperator*{\argmin}{arg\,min}

\usepackage{listings}
\lstset{
  language=python
  }

\title{Part 04: Implementing Word2Vec in Tensorflow}
\author{Fabian Barteld, Benjamin Milde}
\date[20.06.2016]{June 20, 2016}

\AtBeginSection[]
{
   %%%%% section title
   % This is how it would look like in Beamer:
   % \begin{frame}
   %     \frametitle{Overview}
   %     \tableofcontents[sections={2-3},currentsection,sectionstyle=show/hide,subsectionstyle=hide]
   % \end{frame}
  \begin{frame}[plain]
  \begin{tikzpicture}[overlay]
    \relax%
    \fill[blueuhh,opacity=1] (-10,-10)
    rectangle(\the\paperwidth,\the\paperheight);
  \end{tikzpicture}
   \begin{tikzpicture}[overlay]
    \relax%
    \fill[white,opacity=1] (-5,-1.2)
    rectangle(\the\paperwidth,0.5) node[pos=0.5,black]{\LARGE\insertsectionhead};
  \end{tikzpicture}
  \end{frame}

  %%%% add subsection to show navigation dots
  \subsection{}
}

\begin{document}

\maketitle

%\begin{frame}
%  \frametitle{Overview}
%
%  \tableofcontents
%
%\end{frame}

\section{Introduction}

\begin{frame}[fragile]
\frametitle{Training embeddings}
  \begin{itemize}
	\item We will now implement Word2Vec in Tensorflow
	\item (Slides smiliar to https://www.tensorflow.org/tutorials/word2vec)
	\includegraphics[width=0.7\textwidth]{04_skipgram_vs_cbow}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Main concepts I}
  \begin{itemize}

    \item  Neural probabilistic language models are traditionally trained using the maximum likelihood (ML) principle (where $w_t$ is the target word and $h$ is the context):

	 \end{itemize}

\begin{center}
\includegraphics[width=0.5\textwidth]{04_w2v_01}
\end{center}

\end{frame}

\begin{frame}
  \frametitle{Main concepts II}
  \begin{itemize}

    \item  Neural probabilistic language models are traditionally trained using the maximum likelihood (ML) principle (where $w_t$ is the target word and $h$ is the context):

	 \end{itemize}

$$P(w_t | h) = \text{softmax}(\text{score}(w_t, h)) = $$
	$$ \frac{\ exp\{\text{score}(w_t, h) \} } {\sum_\text{Word w' in Vocab} \ exp\{ \text{score}(w', h) \}}$$

\end{frame}

\begin{frame}
  \frametitle{Main concepts III}

    \begin{itemize}

	\item We train this model by maximizing its log-likelihood on the training set, i.e. by maximizing:

	$$  J_\text{ML} = \log P(w_t | h) \ = $$
	$$  \text{score}(w_t, h) - \log \left( \sum_\text{Word w' in Vocab} \exp { \text{score}(w', h) } \right). $$

	\item However this is very expensive, because we need to compute and normalize each probability using the score for all other $V$ words $w'$ in the current context $h$, at every training step.
	\end{itemize}

\end{frame}

\begin{frame}
  \frametitle{Main concepts IV - NCE}

    \begin{itemize}
    \item Noise Contrastive Estimation (NCE)
	\item For feature learning in word2vec we do not need a full probabilistic model. Instead, we train to discriminate the real target words $w_t$ from $k$ imaginary (noise) words w~:

	\end{itemize}

	\begin{center}
	\includegraphics[width=0.6\textwidth]{04_w2v_02}
	\end{center}

\end{frame}

\begin{frame}
  \frametitle{Main concepts V - NCE}

  \begin{itemize}
\item Mathematically, the objective is to maximize:

$$J_\text{NCE} = \log Q_\theta(D=1 |w_t, h) + k \mathop{\mathbb{E}}\limits_{\tilde w \sim P_\text{noise}} \left[ \log Q_\theta(D = 0 |\tilde w, h) \right]$$

\item discriminate the real target words \(w_t\) from \(k\) imaginary (noise) words \(\tilde w\)
\item where $Q_\theta(D=1 | w, h)$ is the binary logistic regression probability
\item under the model of seeing the word $w$ in the context $h$ and assigning the label 1 for datapoint $D$, calculated in terms of the learned embedding vectors $\theta$

\end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Impl. I - Tensorflow W2V}
    \begin{itemize}
    	\item In practice we approximate the expectation by drawing k contrastive words from the noise distribution (i.e. we compute a Monte Carlo average)
   $$J_\text{NCE} \approx \log Q_\theta(D=1 |w_t, h) +  \sum_{i=1, w \sim P_\text{noise}}^{k}  \left[ \log Q_\theta(D = 0 |\tilde w, h) \right]$$
   \item Now we can choose $k \neq |V|$, in practice 5-10 for small datasets, 2-5 for large datasets
   \item Negative sampling, as in the word2vec paper, is a variant of NCE and uses a specific distribution (uniform raised to the power of 3/4)

 \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Impl. II - Tensorflow W2V}
  \begin{lstlisting}

loss = tf.reduce_mean(
  tf.nn.nce_loss(weights=nce_weights,
                 biases=nce_biases,
                 labels=train_labels,
                 inputs=embed,
                 num_sampled=num_sampled,
                 num_classes=vocabulary_size))
    \end{lstlisting}

  \begin{itemize}
  	\item We can use the NCE loss op of Tensorflow to construct a variant of word2vec. Internally, nce\_weights also uses embedding\_lookup and does a form of negative sampling  directly in Tensorflow.
  \end{itemize}
\end{frame}

 \begin{frame}[fragile]
  \frametitle{Impl. III - Tensorflow W2V}
  The embeddings matrix is a variable that we want to optimize:
  \begin{lstlisting}
embeddings = tf.Variable(
tf.random_uniform([vocabulary_size,
embedding_size], -1.0, 1.0))
\end{lstlisting}
\end{frame}

 \begin{frame}[fragile]
 \frametitle{Impl. IIII - Tensorflow W2V}
We also need variables for the nce\_loss:

\begin{footnotesize}
\begin{lstlisting}
nce_weights = tf.Variable(
  tf.truncated_normal([vocabulary_size, embedding_size],
stddev=1.0 / math.sqrt(embedding_size)))
nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
\end{lstlisting}
\end{footnotesize}
\end{frame}

 \begin{frame}[fragile]
  \frametitle{Impl. IV - embedding\_lookup:}

  \begin{footnotesize}
 \begin{lstlisting}
embed = tf.nn.embedding_lookup(embeddings, train_inputs)
\end{lstlisting}
\end{footnotesize}

e.g. If your list of sentences is: $\big[[0, 1], [0, 3]\big]$ (sentence 1 is $[0, 1]$, sentence 2 is $[0, 3]$, the function will compute a tensor of embeddings, which will be of shape $(2, 2, \text{embedding\_size})$ and will look like:
\\
$$[[\text{embedding0, embedding1}], [\text{embedding0, embedding3}]]$$

\end{frame}

 \begin{frame}[fragile]

  \frametitle{Exercise 1 - simple version}
   \begin{itemize}
   		\item Lets put it together: We can use tf.nn.embedding\_lookup for the input projection and tf.nn.nce\_loss for the loss (no other layers needed!).
		\item For simplicity, lets also implement CBOW and Skipgram with a window size of 1.
		\item E.g. for "the quick brown fox jumped over the lazy dog"
		\item (context, target) pairs: ([the, brown], quick), ([quick, fox], brown), ([brown, jumped], fox)
		\item We can simplify to: (the, quick), (brown, quick), (quick, brown), (fox, brown), ... \textbf{CBOW}
		\item or (quick, the), (quick, brown), (brown, quick), (brown, fox), ... \textbf{Skip-gram}
	\end{itemize}
\end{frame}

 \begin{frame}[fragile]

  \frametitle{Exercise 2 - advanced version}
   \begin{itemize}
		\item Lets try to make a version that does not use tf.nn.nce\_loss, as easy as that makes our lives!
		\item We can also do the negative sampling on the host and code up a linear regression as in the previous tutorials
		\item Host will assign labels (1 for true context pairs, 0 for noise pairs)
		\item You have to change the code in the get\_batch function and the inputs to your model and adapt your model accordingly
	\end{itemize}
\end{frame}

 \begin{frame}[fragile]

 \frametitle{Hints}
  \begin{itemize}
		\item Hint1: The negative samples need a second embedding matrix
		\item Hint2: For the loss, to get the logits, use the dot product between embedding pairs.
		\item Hint3: There is no tf.dot(), but you can combine tf.reduce\_sum(x,1) and tf.multiply(a,b).
		\item Hint4: Readable pure Python code with comments: , or if you're feeling masochistic the original uncommented word2vec C impl at:
	\end{itemize}

\end{frame}

 \begin{frame}[fragile]

 \frametitle{Tensorboard}
  \begin{itemize}
		\item Visualize loss, embeddings and much more in your browser
		\item You need to add a few lines of code to tell Tensorboard what to log
		\item Make sure train\_summary\_dir is a new directory for every new experiment!
	\end{itemize}

	\begin{tiny}
\begin{lstlisting}
 loss_summary = tf.summary.scalar('loss', loss)
 train_summary_op = tf.summary.merge_all()
 summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
\end{lstlisting}
\end{tiny}

\end{frame}

\begin{frame}[fragile]

 \frametitle{Tensorboard}
  \begin{itemize}
		\item You need to regularly call the train\_summary\_op in training
		\item Not as often as the training step, because it will otherwise slowdown your training if you have more complex summaries
	\end{itemize}

\begin{tiny}
\begin{lstlisting}
if current_step % 100==0 and current_step != 0:
	summary_str = sess.run(train_summary_op, feed_dict=feed_dict)
	summary_writer.add_summary(summary_str, current_step)
\end{lstlisting}
\end{tiny}

\end{frame}

\begin{frame}[fragile]
 \frametitle{Tensorboard - running it}
 \begin{tiny}
 \begin{lstlisting}
python3 -m tensorflow.tensorboard  --logdir=w2v_summaries_1499773534
--host=127.0.0.1
\end{lstlisting}
\end{tiny}
 \includegraphics[width=0.5\textwidth]{04_loss}
\end{frame}

\begin{frame}[fragile]
 \frametitle{Tensorboard - embeddings}
   \begin{itemize}
		\item Possible to nicely visualize embeddigs, see \url{https://www.tensorflow.org/get_started/embedding_viz}
		\item Also checkout \url{http://projector.tensorflow.org/}, live demo of pretrained embeddings
		\includegraphics[width=0.5\textwidth]{04_embed_viz.png}
	\end{itemize}
\end{frame}

%\begin{frame}[fragile]
%  \frametitle{Layer based APIs vs. Graphs based}
%
%Disadvantage: Difficult to express structures like these:
%
%    \includegraphics[angle=-90,width=0.75\textwidth]{graph_example}
%
%Increasing evidence that these kind of deeply connected networks are very useful.
%
%\end{frame}
%
%\begin{frame}[fragile]
%  \frametitle{Layer based APIs vs. Graphs based}
%  \begin{itemize}
%		\item  Since Tensorflow uses computation graphs, the declaration of the model allows for a higher expressivity
%		\item  Has a steeper learning curve in the beginning
%		\item  In the newer versions of tensorflow, you can also mix layer-like APIs with the computation graph
%		\item  We will focus on not using any short cuts, as this has a higher learning effect and only make use of standard ops in the beginning
%  \end{itemize}
%\end{frame}
%
%\begin{frame}[fragile]
%\frametitle{First steps - Lets open spyder}
%
%\includegraphics[width=1.0\textwidth]{spyder}
%
%\end{frame}
%
%\begin{frame}[fragile]
%\frametitle{First steps - Necessary imports}
%
%\begin{lstlisting}
%import numpy as np
%import tensorflow as tf
%\end{lstlisting}
%
%\begin{itemize}
% 	\item Outside of graph computations, we usually store data in Numpy arrays.
% 	\item Numpy arrays are the main objects to transfer data to inputs of the graph and from outputs of the graph.
% 	\item Numpy arrays are also an abstraction for (homogeneous) multidimensional arrays.
%\end{itemize}
%
%\end{frame}
%
%\begin{frame}[fragile]
%\frametitle{Generating some random data}
%
%\begin{lstlisting}
%#some random test data
%a_data = np.random.rand(256)
%b_data = np.random.rand(256)
%\end{lstlisting}
%
%\begin{itemize}
%	\item Now a and b contain vectors of length 256 with random floats. E.g. print(a\_data) returns:
%\end{itemize}
%
%\begin{lstlisting}
%[ 0.54976368  0.87790201  0.96528541 ...,
% 0.05281365  0.48556404  0.46848266]
%  \end{lstlisting}
%
%\end{frame}
%
%
%\begin{frame}[fragile]
%\frametitle{Declare the computation graph}
%
%\begin{lstlisting}
%#construct the graph
%a = tf.placeholder(tf.float32, [256])
%b = tf.placeholder(tf.float32, [256])
%
%x = a+b
%\end{lstlisting}
%
%\begin{itemize}
%\item The placeholders can later be used to input data to the computation graph
%\item The operation x = a+b does not immediatly add something, it creates a graph.
%\item In fact, print(x) returns:
%\end{itemize}
%
%\begin{lstlisting}
%Tensor("add:0", shape=(256,), dtype=float32)
%\end{lstlisting}
%
%\end{frame}
%
%\begin{frame}[fragile]
%\frametitle{A session on a computation device}
%
%\begin{lstlisting}
%with tf.device('/cpu'):
%    with tf.Session() as sess:
%       x_data = sess.run(x, {a: a_data, b: b_data})
%       print(x_data)
%\end{lstlisting}
%
%\begin{itemize}
%\item This fills the inputs a and b with a\_data and b\_data (our random data), runs the computation graph and retrieves the results of x in x\_data
%\item Obviously not terrible useful as is, but you could run the operation easily on a gpu by changing tf.device('/cpu') to  tf.device('/gpu:1'). Copying data to and from the GPU is handled automatically for you.
%\end{itemize}
%
%\end{frame}
%
%\begin{frame}[fragile]
%\frametitle{Small warm up exercise!}
%
%\begin{itemize}
%	\item We change a and b to random matrices:
%\end{itemize}
%
%\begin{lstlisting}
%a = np.random.rand(256, 128)
%b = np.random.rand(128, 512)
%\end{lstlisting}
%
%\begin{itemize}
%	\item Calculate the resulting matrix of shape (256, 512) in TensorFlow.
%\end{itemize}
%
%\end{frame}

%\section{Simple Optimization}
%
%% https://medium.com/@saxenarohan97/intro-to-tensorflow-solving-a-simple-regression-problem-e87b42fd4845
%% https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/linear_regression.py
%\begin{frame}
%  \frametitle{Linear Regression}
%
%  \begin{itemize}
%  \item Given: $(x_1, y_1)$, \ldots, $(x_n,y_n)$
%  \item Goal: find $w$ and $b$ such that:
%    \begin{displaymath}
%      \argmin_{w, b} \frac{\sum^n_{i=1} (\hat{y}_i - y_i)^2}{n}
%    \end{displaymath}
%    where $\hat{y}_i = wx_i + b$.
%  \end{itemize}
%
%\end{frame}
%
%\begin{frame}[fragile]
%  \frametitle{Define model parameters}
%  Model: $\hat{y}_i = wx_i + b$
%
%\begin{lstlisting}
%w = tf.Variable(np.random.randn(), name="weight")
%b = tf.Variable(np.random.randn(), name="bias")
%\end{lstlisting}
%\end{frame}
%
%\begin{frame}[fragile]
%  \frametitle{Define the model}
%
%  \begin{displaymath}
%    \begin{pmatrix} \hat{y}_1\\\vdots\\\hat{y}_n\end{pmatrix} =
%    \begin{pmatrix} w\\\vdots\\w\end{pmatrix} *
%    \begin{pmatrix} \hat{x}_1\\\vdots\\\hat{x}_n\end{pmatrix} +
%    \begin{pmatrix} b\\\vdots\\b\end{pmatrix}
%  \end{displaymath}
%
%\begin{lstlisting}
%yhat = tf.add(tf.multiply(X, w), b)
%\end{lstlisting}
%
%{\footnotesize The scalars $w$ and $b$ are converted into vectors of the same
%  length as X (broadcast); \url{https://www.tensorflow.org/performance/xla/broadcasting}}
%
%\end{frame}
%
%\begin{frame}[fragile]
%  \frametitle{Define the loss}
%
%\begin{lstlisting}
%loss = tf.reduce_mean(tf.square(y - yhat))
%\end{lstlisting}
%
%\end{frame}
%
%
%\begin{frame}[fragile]
%  \frametitle{Optimization}
%
%\begin{lstlisting}
%epochs = 10
%optimizer = tf.train.GradientDescentOptimizer(
%    learning_rate).minimize(loss)
%
%with tf.Session() as sess:
%    ## initalize parameters
%    sess.run(tf.global_variables_initializer())
%
%    for i in list(range(epochs)):
%        ## run one epoch
%        sess.run(optimizer)
%        ## print result and loss
%        print(sess.run(yhat) + ' ' + sess.run(loss))
%\end{lstlisting}
%
%\end{frame}
%
%\begin{frame}[fragile]
%  \frametitle{Hands on: Simple optimization}
%
%  \begin{enumerate}
%  \item Do a linear regression to learn $y = 2x + 1$
%  \item Do a multiple linear regression with Boston housing prices
%  \end{enumerate}
%
%\begin{lstlisting}
%from sklearn.datasets import load_boston
%from sklearn.preprocessing import scale
%
%total_X, total_Y = load_boston(True)
%total_x = scale(total_x)
%\end{lstlisting}
%\end{frame}
%
%% https://www.tensorflow.org/tutorials/wide
%% https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/2_BasicModels/logistic_regression.py
%\begin{frame}
%  \frametitle{Logistic regression}
%
%  TODO
%  tf.sigmoid
%
%\end{frame}
%
%\begin{frame}
%  \frametitle{Hands on: Regression model}
%
%  TODO: more complex example
%\end{frame}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-engine: luatex
%%% End:


% declare document class and geometry
\documentclass[12pt]{article} % use larger type; default would be 10pt
\usepackage[margin=1in]{geometry} % handle page geometry

\input{../header2.tex}

\title{Phys 220A -- Classical Mechanics -- Lec14}
\author{UCLA, Fall 2014}
\date{\formatdate{25}{11}{2014}} % Activate to display a given date or no date (if empty),
         % otherwise the current date is printed

\begin{document}
\setlength{\unitlength}{1mm}
\maketitle

\section{More on rigid bodies}

For an angular momentum $\v \omega$ we have kinetic energy
\begin{eqn}
T = \frac{1}{2} \sum_{ij} \omega_i I_{ij} \omega_j
\end{eqn}
and angular momentum
\begin{eqn}
L_i = \sum_j I_{ij} \omega_j.
\end{eqn}
[One or both of these are independent of frame?] Recall the Euler equations
\begin{eqn}
I \vd \omega = \v \omega \times (I \v \omega),
\end{eqn}
or in the frame of the principal axes
\begin{align}
I_1 \od{\omega_1}{t} &= (I_2 - I_3) \omega_2 \omega_3, \\
I_2 \od{\omega_2}{t} &= (I_3 - I_1) \omega_3 \omega_1, \\
I_3 \od{\omega_3}{t} &= (I_1 - I_2) \omega_1 \omega_2.
\end{align}

\subsection{Stability of solutions}

Suppose without loss of generality that $I_1 < I_2 < I_3$. Then we can write our solutions as
\begin{align}
\omega_1 &= \omega + \eta_1, \\
\omega_2 &= 0 + \eta_2, \\
\omega_3 &= 0 + \eta_3.
\end{align}
For some reason we have a 0th order case
\begin{eqn}
\od{\omega}{t} = 0
\qquad \implies \qquad
\omega = \text{const}
\end{eqn}
and a 1st order case
\begin{eqn}
\od{\eta_1}{t} = 0
\end{eqn}
in which case we can set $\eta_1 = 0$ (can be absorbed into $\omega$ [what?]) Then we can write ODEs for $\eta_2, \eta_3$,
\begin{eqn}
\od{\eta_2}{t} = J_2 \eta_3, \qquad
\od{\eta_3}{t} = J_3 \eta_2
\end{eqn}
where
\begin{eqn}
J_2 = \frac{I_3 - I_1}{I_2} \, \omega, \qquad
J_3 = \frac{I_1 - I_2}{I_3} \, \omega.
\end{eqn}
So in general we can write
\begin{eqn}
\pmat{\dot \eta_2 \\ \dot \eta_3} = \pmat{0 & J_2 \\ J_3 & 0} \pmat{\eta_2 \\ \eta_3},
\end{eqn}
which has solutions
\begin{eqn}
\eta_i(t) = e^{\lambda t} \eta_i(0), \quad i = 2,3
\end{eqn}
where $\lambda$ is determined by
\begin{eqn}
0 = \det \pmat{\lambda & -J_2 \\ -J_3 & \lambda} = \lambda^2 - J_2 J_3 \qquad
\implies \qquad
\lambda = \pm \sqrt{J_2 J_3}.
\end{eqn}
So we have
\begin{eqn}
\lambda =
\begin{cases}
\pm i \sqrt{\abs{J_2 J_3}}, & J_2 J_3 < 0 \\
\pm \sqrt{\abs{J_2 J_3}}, & J_2 J_3 > 0
\end{cases}
\end{eqn}
which is stable in the first case and unstable in the second case. In our case $I_1 < I_2 < I_3$ so $J_2 > 0$ and $J_3 < 0$ so it's stable. In other cases it can be unstable.

\subsection{Geometrical (Poinsot) Picture of Conserved Quantities}

[stuff about intersections of ellipsoids and spheres]

[something about why choice of Euler angles is nice]

\subsection{Example: Heavy symmetric top}

[missed last hour]

\end{document}


\documentclass[12pt]{cdblatex}
\usepackage{eqtns}

\begin{document}

\section*{PhysRevD.67.084023 equation (20)}

\begin{cadabra}
   from shared import *
   import cdblib

   jsonfile = 'momentum.json'
   cdblib.create (jsonfile)

   defG2GBar = cdblib.get ('defG2GBar','gamma.json')

   # --------------------------------------------------------------------------
   # Momentum constraint pt.1

   Mom := D_{j}{K^{i j} - g^{i j} trK}.                                       # cdb(Mom.101,Mom)

   defDgD := D_{a}{g_{b c}} -> 0.
   defDgU := D_{a}{g^{b c}} -> 0.

   defDtrK   := D_{a}{trK} -> \partial_{a}{trK}.
   defDexp   := D_{a}{\exp(-4\phi)} -> -4\exp(-4\phi) \partial_{a}{\phi}.

   distribute   (Mom)                                                         # cdb(Mom.102,Mom)
   product_rule (Mom)                                                         # cdb(Mom.103,Mom)
   substitute   (Mom, defDgU)                                                 # cdb(Mom.104,Mom)

   defK2ABarU := K^{i j} -> \exp(-4\phi) ABar^{i j} + (1/3) g^{i j} trK.

   substitute   (Mom, defK2ABarU)                                             # cdb(Mom.105,Mom)
   distribute   (Mom)                                                         # cdb(Mom.106,Mom)
   product_rule (Mom)                                                         # cdb(Mom.107,Mom)
   substitute   (Mom, defDtrK)                                                # cdb(Mom.108,Mom)
   substitute   (Mom, defDgU)                                                 # cdb(Mom.109,Mom)
   substitute   (Mom, defDexp)                                                # cdb(Mom.110,Mom)

\end{cadabra}

\clearpage

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      {\cal D}^{j}
         = \Cdb*{Mom.101}
         = \Cdb*{Mom.102}
         = \Cdb*{Mom.103}
         = \Cdb*{Mom.104}
         = \Cdb*{Mom.105}
         = \Cdb*{Mom.106}
         = \Cdb*{Mom.107}
         = \Cdb*{Mom.108}
         = \Cdb*{Mom.109}
         = \Cdb*{Mom.110}
   \end{dmath*}
\end{dgroup*}

\clearpage

\begin{cadabra}
   # --------------------------------------------------------------------------
   # Momentum constraint pt.2

   confMom := \exp(4\phi) @(Mom).

   defG2GBarU := g^{i j} -> \exp(-4\phi) gBar^{i j}.

   distribute   (confMom)                                                     # cdb(confMom.101,confMom)
   substitute   (confMom, defG2GBarU)                                         # cdb(confMom.102,confMom)
   map_sympy    (confMom, "simplify")                                         # cdb(confMom.103,confMom)

   defDAabU  := D_{a}{ABar^{b c}} ->  \partial_{a}{ABar^{b c}}
                                    + \Gamma^{b}_{i a} ABar^{i c}
                                    + \Gamma^{c}_{i a} ABar^{b i}.

   substitute     (confMom, defDAabU)                                         # cdb(confMom.104,confMom)
   substitute     (confMom, defG2GBar)                                        # cdb(confMom.105,confMom)
   distribute     (confMom)                                                   # cdb(confMom.106,confMom)
   confMom = product_sort (confMom)                                           # cdb(confMom.107,confMom)
   rename_dummies (confMom)                                                   # cdb(confMom.108,confMom)
   canonicalise   (confMom)                                                   # cdb(confMom.109,confMom)
   substitute     (confMom, $gBar^{i}_{i} -> 3$)                              # cdb(confMom.110,confMom)
   substitute     (confMom, $gBar_{i j} ABar^{i j} -> 0$)                     # cdb(confMom.111,confMom)
   substitute     (confMom, $gBar_{a i} gBar^{i b} -> gBar_{a}^{b}$)          # cdb(confMom.112,confMom)
   substitute     (confMom, $GammaBar^{b}_{a b} -> 0$)                        # cdb(confMom.113,confMom) # follows from det gBar = 1
   eliminate_kronecker (confMom)                                              # cdb(confMom.114,confMom)
   rename_dummies (confMom)
   canonicalise   (confMom)                                                   # cdb(confMom.115,confMom)

   cdblib.put ('confMom',confMom,jsonfile)
\end{cadabra}

\clearpage

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \exp(4\phi) {\cal D}^{j}
         = \Cdb*{confMom.101}
         = \Cdb*{confMom.102}
         = \Cdb*{confMom.103}
         = \Cdb*{confMom.104}
         = \Cdb*[\hskip3.0cm\hfill]{confMom.105}
         = \Cdb*[\hskip2.0cm\hfill]{confMom.106}
         = \Cdb*[\hskip2.0cm\hfill]{confMom.107}
         = \Cdb*[\hskip2.0cm\hfill]{confMom.108}
         = \Cdb*[\hskip2.5cm\hfill]{confMom.109}
         = \Cdb*{confMom.110}
         = \Cdb*{confMom.111}
   \end{dmath*}
\end{dgroup*}

\clearpage

\begin{dgroup*}[spread=5pt]
   \begin{dmath*}
      \exp(4\phi) {\cal D}^{j}
         = \Cdb*{confMom.112}
         = \Cdb*{confMom.113}
         = \Cdb*{confMom.114}
         = \Cdb*{confMom.115}
   \end{dmath*}
\end{dgroup*}

\clearpage

\begin{cadabra}
   tmpA := @(confMom).                                              # cdb(confMom.201,tmpA)
   tmpB := @(confMom).

   X^{b c}_{a}::Weight(label=numX).

   Xbca := \partial_{a}{ABar^{b c}}.                                # cdb(confMom.202,Xbca)

   foo := \partial_{a}{ABar^{b c}} -> X^{b c}_{a}.
   bah := X^{b c}_{a} -> \partial_{a}{ABar^{b c}}.

   substitute  (tmpA, foo)                                          # cdb(confMom.203,tmpA)
   substitute  (tmpB, foo)                                          # cdb(confMom.204,tmpB)
   drop_weight (tmpA, $numX=1$)                                     # cdb(confMom.205,tmpA)
   keep_weight (tmpB, $numX=1$)                                     # cdb(confMom.206,tmpB)
   substitute  (tmpB, bah)                                          # cdb(confMom.207,tmpB)

   tmpC := - @(tmpA).                                               # cdb(confMom.208,tmpC)

   defMomSub := @(tmpB) -> @(tmpC).                                 # cdb(confMom.209,defMomSub)

   cdblib.put ('defMomSub',defMomSub,jsonfile)
\end{cadabra}

\clearpage

\begin{dgroup*}
   \begin{dmath*} 0 =  \Cdb*{confMom.201} \end{dmath*}
   \begin{dmath*} 0 =  \Cdb*{confMom.203} \end{dmath*}
   \begin{dmath*}
      \cdb{confMom.202} = \Cdb*{confMom.206}
                        = \Cdb*{confMom.208}
   \end{dmath*}
   \begin{dmath*} \Cdb*{confMom.209} \end{dmath*}
\end{dgroup*}

\clearpage

\begin{cadabra}
   # --------------------------------------------------------------------------
   # Check against prd67.

   foo := @(confMom).                                   # cdb(prd67.eq20.lcb,foo)
   bah  = cdblib.get('prd67.eq20.rhs','prd67.json')     # cdb(prd67.eq20.prd,bah)

   diff := @(foo) - @(bah).

   distribute     (diff)
   diff = product_sort (diff)
   rename_dummies (diff)
   map_sympy      (diff, "simplify")
   canonicalise   (diff)                                # cdb(prd67.eq20.chk,diff)
\end{cadabra}

% \clearpage

\begin{dgroup*}
   \begin{dmath*} \cdb*{prd67.eq20.lcb} \end{dmath*}
   \begin{dmath*} \cdb*{prd67.eq20.prd} \end{dmath*}
   \begin{dmath*} \cdb*{prd67.eq20.chk} \end{dmath*}
\end{dgroup*}

\end{document}


\section{Broadcast time}\label{sec:startnodetime}

Files \code{\{low,high\}-density-time.ipynb} contain the analysis for the
broadcast time needed to reach the 98th percentile of the coverage. The analysis
has been done fixing all the factors (also \(R\)) and by just varying the
position of the starting user. Configurations used are
``StartNodePositionLowDensityTime'' and ``StartNodePositionHighDensityTime''.

We get the results shown in \tableref{table:startnodetimeresults}.

\begin{table}[hbt]
	\centering
	\begin{tabular}{lcccc}
		\multicolumn{5}{c}{Low density (98th percentile broadcast time)}\\
		\toprule
		Start Node Pos\@. & Mean & Std\@. Dev\@. & Min\@. & Max\@. \\
		\midrule
		Center & \(35.166667s\) & \(1.533158s\) & \(31s\) & \(40s\) \\[16pt]
		Border & \(53.7s\) & \(1.914554s\) & \(50s\) & \(58s\) \\[16pt]
		Corner & \(65.1s\) & \(2.186952s\) & \(61s\) & \(70s\) \\
		\bottomrule
	\end{tabular}\\[16pt]
	\begin{tabular}{lcccc}
		\multicolumn{5}{c}{High density (98th percentile broadcast time)}\\
		\toprule
		Start Node Pos\@. & Mean & Std\@. Dev\@. & Min\@. & Max\@. \\
		\midrule
		Center & \(32.533333s\) & \(2.161311s\) & \(30s\) & \(39s\) \\[16pt]
		Border & \(48.666667s\) & \(2.039833s\) & \(45s\) & \(53s\) \\[16pt]
		Corner & \(57.7s\) & \(1.803254s\) & \(55s\) & \(62s\) \\
		\bottomrule
	\end{tabular}
	\caption{With the starting node at the border or the corner, we get an
	higher broadcast time compared to the case where the starting node is at
	the center}\label{table:startnodetimeresults}
\end{table}

As we can see, as the starting node moves near to the border/corner, we get
higher values for both the mean broadcast time. This can be explained by the
fact that, when the starting node is in the corner, the message need to traverse
the entire length of the floorplan in both its dimensions.  When, instead, it
starts from the center, since it is sent out radially by the user, the message
just needs to cover half of the side of the floorplan in each dimension. The
standard deviation is higher in the corner for the low density scenario but it
is lower in the high density scenario: a higher number of users leads to a lower
variance.

So, if possibile, also for the broadcast time, is better to set the starting
node near the center or, at least, avoid borders and corners of the floorplan.


\chapter{NAND, NOR gates}
%\ref{sec:background}.

\section{Aim}
%\label{sec:objectives}
	To verify and interpret the logic and truth table for NAND, NOR gates using Resistor Transistor Logic (RTL)

\section{Apparatus}
%\label{sec:objectives}
	\begin{itemize}
		\tightlist
		\item Kit for realization of gates
		\item Connecting Leads
	\end{itemize}

\section{Circuits}

\section{Theory}
	Logic gates are the basic building blocks of any digital system. Logic gates are electronic circuits having one or more than one input and only one output. The relationship between the input and the output is based on a certain logic. Based on this, logic gates are named as:
	\begin{enumerate}
		\tightlist
		\item AND gate
		\item OR gate
		\item NOT gate
		\item NAND gate
		\item NOT gate
		\item Ex-OR gate
		\item Ex-NOR gate
	\end{enumerate}

	\subsection{NAND gate}
	This is a NOT-AND gate which is equal to an AND gate followed by a NOT gate. The outputs of all NAND gates are high if any of the inputs are low. The symbol is an AND gate with a small circle on the output. The small circle represents inversion.
	\begin{align*}
		Y &= \overline{A . B}
	\end{align*}
	A simple 2-input logic NAND gate can be constructed using RTL (Resistor-transistor-logic) switches connected together as shown in Figure \ref{fig:nand_circuit} with the inputs connected directly to the transistor bases. Either transistor must be cut-off or “OFF” for an output at Q.
	\begin{figure}[ht]
		\centering
		\subfloat[Symbol]
		{
			\begin{circuitikz} \draw
				(0,0) node[nand port] (myand1) {}
				(myand1.in 1) node[anchor=east] {A}
				(myand1.in 2) node[anchor=east] {B}
				(myand1.out) node[anchor=west] {Y}
				;
			\end{circuitikz}
			\label{fig:nand_symbol}
		}
		\hfill
		\subfloat[Truth Table]
		{
			\begin{tabular}{|c|c|c|}
				\hline
				\multicolumn{2}{|c|}{Input} & Output \\
				\hline
				$A$ & $B$ & $Y=\overline{A.B}$ \\
				\hline
				0 & 0 & 1 \\
				\hline
				0 & 1 & 1 \\
				\hline
				1 & 0 & 1 \\
				\hline
				1 & 1 & 0 \\
				\hline
			\end{tabular}
			\label{fig:nand_table}
		}
		\hfill
		\subfloat[RTL Design]{
			\includegraphics[width=0.25\textwidth,valign=c]{img/exp2/fig1}
			\label{fig:nand_circuit}
		}
		\caption{\textit{NAND gate}}
	\end{figure}

	\subsection{NOR gate}
		This is a NOT-OR gate which is equal to an OR gate followed by a NOT gate. The outputs of all NOR gates are low if any of the inputs are high. The symbol is an OR gate with a small circle on the output. The small circle represents inversion.
		\begin{align*}
			Y &= \overline{A + B}
		\end{align*}
		A simple 2-input logic NOR gate can be constructed using RTL (Resistor-transistor-logic) switches connected together as shown in Figure \ref{fig:nor_circuit} with the inputs connected directly to the transistor bases. Both transistors must be cut-off or “OFF” for an output at Q.
		\begin{figure}[ht]
			\centering
			\subfloat[Symbol]
			{
				\begin{circuitikz} \draw
					(0,0) node[nor port] (myand1) {}
					(myand1.in 1) node[anchor=east] {A}
					(myand1.in 2) node[anchor=east] {B}
					(myand1.out) node[anchor=west] {Y}
					;
				\end{circuitikz}
				\label{fig:nor_symbol}
			}
			\hfill
			\subfloat[Truth Table]
			{
				\begin{tabular}{|c|c|c|}
					\hline
					\multicolumn{2}{|c|}{Input} & Output \\
					\hline
					$A$ & $B$ & $Y=\overline{A+B}$ \\
					\hline
					0 & 0 & 1 \\
					\hline
					0 & 1 & 0 \\
					\hline
					1 & 0 & 0 \\
					\hline
					1 & 1 & 0 \\
					\hline
				\end{tabular}
				\label{fig:nor_table}
			}
			\hfill
			\subfloat[RTL Design]{
				\includegraphics[width=0.25\textwidth,valign=c]{img/exp2/fig2}
				\label{fig:nor_circuit}
			}
			\caption{\textit{NOR gate}}
		\end{figure}

\section{Procedure}
	\subsection{NAND gate}
		\begin{figure}[ht]
			\centering
			\subfloat[Simulator 1]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig3}
				\label{fig:nand_sim:1}}
			\hfill
			\subfloat[Simulator 2]{\includegraphics[width=0.4\textwidth,valign=c]{img/exp2/fig4}
				\label{fig:nand_sim:2}}
			\caption{\textit{Simulator for realizing circuit for NAND gate}}
		\end{figure}
		\subsubsection{Simulator 1}
			\begin{enumerate}
				\tightlist
				\item Connect the supply(+5V) to the circuit.
				\item Press the switches for inputs "A" and "B".
				\item The bulb glows if any one or both the switches are OFF else it won't glow.
				\item Repeat step-2 and step-3 for all state of inputs.
			\end{enumerate}
		\subsubsection{Simulator 2}
			\begin{enumerate}
				\tightlist
				\item Enter the Boolean input "A" and "B".
				\item Enter the Boolean output for your corresponding inputs.
				\item Click on "Check" Button to verify your output.
			\end{enumerate}

	\subsection{NOR gate}
		\begin{figure}[ht]
			\centering
			\subfloat[Simulator 1]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig5}
				\label{fig:nor_sim:1}}
			\hfill
			\subfloat[Simulator 2]{\includegraphics[width=0.4\textwidth,valign=c]{img/exp2/fig6}
				\label{fig:nor_sim:2}}
			\caption{\textit{Simulator for realizing circuit for NOR gate}}
		\end{figure}
		\subsubsection{Simulator 1}
			\begin{enumerate}
				\tightlist
				\item Connect the supply(+5V) to the circuit.
				\item Press the switches for inputs "A" and "B".
				\item The bulb glows if both the switches are OFF else it won't glow.
				\item Repeat step-2 and step-3 for all state of inputs.
			\end{enumerate}
		\subsubsection{Simulator 2}
			\begin{enumerate}
				\tightlist
				\item Enter the Boolean input "A" and "B".
				\item Enter the Boolean output for your corresponding inputs.
				\item Click on "Check" Button to verify your output.
			\end{enumerate}

\section{Observations}
	\subsection{NAND gate}
			\begin{figure}[ht]
				\centering
				\subfloat[Either of the Inputs OFF, LED is ON]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig7}
					\label{fig:nand_obs:1}}
				\hfill
				\subfloat[Both Inputs ON, LED is OFF]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig8}
					\label{fig:nand_obs:2}}
				\caption{\textit{Observations for different Input Values}}
			\end{figure}
			\begin{figure}[h]
				\centering
				\includegraphics[width=0.85\linewidth]{img/exp2/fig9}
				\caption{\textit{Observations for verification of Truth Table of the NAND gate}}
				\label{fig:nand_obs_2}
			\end{figure}

\pagebreak
	\subsection{NOR gate}
			\begin{figure}[ht]
				\centering
				\subfloat[Either of the Inputs ON, LED is OFF]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig10}
					\label{fig:nor_obs:1}}
				\hfill
				\subfloat[Both Inputs OFF, LED is ON]{\includegraphics[width=0.45\textwidth,valign=c]{img/exp2/fig11}
					\label{fig:nor_obs:2}}
				\caption{\textit{Observations for different Input Values}}
			\end{figure}
			\begin{figure}[h]
				\centering
				\includegraphics[width=0.85\linewidth]{img/exp2/fig12}
				\caption{\textit{Observations for verification of Truth Table of the NOR gate}}
				\label{fig:nor_obs_2}
			\end{figure}

\section{Precautions}
	\begin{enumerate}
		\tightlist
		\item Make the connections when power supply is OFF.
		\item Ensure that the connections are tight.
		\item Change the status of inputs only when power supply is OFF.
	\end{enumerate}