\documentclass[twocolumn]{article}
\usepackage{booktabs}
\usepackage{marginnote}
\usepackage{textcomp}
\usepackage[T1]{fontenc}

\begin{document}

\title{PsimagLite Feature Proposal:
Input System}
\author{G.A.}
\maketitle

\section{Lexical}
ASCII.
Define EOF, whitespace, and EOL.
Comments: \verb!#! until EOL

Tokens + Strings Literals + Other literals

Keywords: Types: integer, real, complex, char, string, function, vector, matrix.
Attributes: require const. Properties: defined, typeof.
\begin{verbatim}
require integer a;
# declare pi, make pi value constant
const real pi = 3.1415927;
pi = 3.14; # ERROR, pi constant
#
integer b = 3;
b = 4; # OK
b = 3.5; # ERROR, not integer
#
c = 3.5; # ERROR c undefined
\end{verbatim}

\subsection{Strings and Characters}
\begin{verbatim}
string s="Hello, ";
char c = 'W';
s = s + c + "orld!";
string d = "'";
char e = '"';
\end{verbatim}
Properties: \verb!s.length! \verb!s[0]!

\subsection{Scalars}
\begin{verbatim}
integer a=1; # defines int scalar
complex b=3*i + 1.2; # defines complex scalar
a = a + 1.5; # OK, a promoted to real
a = a + 1.5*i; # ERROR
b = b + 1.5*i; # OK
\end{verbatim}

\subsection{Vectors}
\begin{verbatim}
vector.integer v=[ 1, 2, 3 ];
vector w=[ 1, a, 3*a ];
vector x=v+w;
vector y = [];
y.size = 4;
# equivalent to y = [ 1, 1 ,1, 1]
y = [1, ... ];
\end{verbatim}
Properties: \verb!v.size! \verb!v[0]!
Vectors values: scalars, strings, and characters.

\subsection{Matrices}
\begin{verbatim}
matrix m = [[0, 1], [1, 0]];
# As vector of vectors
vector v1 = [0,1];
vector v2 = [1,0];
matrix m2 = [v1, v2];
# As a hash
hash m2 = ({ rows : 4, cols : 2, data :
[0, 1, 1, 0, 0, 1, 1, 0] });
matrix m3 = 3.5*m;
\end{verbatim}
Properties: \verb!rows! \verb!cols! \verb!data!.
Matrix values: scalars.

\subsection{Groups}
\begin{verbatim}
groupName={ name="chain";
Connections=[ 1, 1 ];
string a = "my local value"; };
#The above would be lowered to
groupName.name="chain";
groupName.Connections=[ 1, 1 ];
string groupName.a = "my local value";
# Groups can be nested
# Unnamed groups?
\end{verbatim}

\section{Operators and Functions}
\begin{tabular}{lll}\toprule
Op. & Type & Description\\\midrule
$+,-$ & 3, [], [[]] & scalar, vector, matrix\\
$+$ & ``''& str. + str. or str. + char\\
$*,/$ & 3*3, 3/3 & scalar $*$ and $/$\\
$*$ & various & 3*[], 3*[[]], [[]]*[]\\
$*$ & [[]]*[[]] & matrix*matrix = matrix\\
$*$ & []*[] & vector*vector = scalar\\
$**$ & 3$**$2 & $=9$\\
$\%$ & 3\%2 & modulus\\
$[]$ & vectors & entry\\
{[}\textquotedbl\textquotedbl{]} & hashes & value given key\\
(,) & matrices & matrix element\\
\bottomrule\\
\end{tabular}

Operators for binary numbers? User defined operators are not allowed.
User defined functions must be entered as a string in prefix notation.
\begin{verbatim}
# f(t) = 3.5*cos(t)
function f = "*,cos,%t,3.5";
real a = f(0.25); # = 3.3911...
\end{verbatim}
Mathematical functions for scalars:
\texttt{abs, conj, sqrt, exp, log, sin, cos, atan}.
Functions for complex numbers \verb!abs! \verb!real! \verb!imag! \verb!arg!
Example \texttt{real a = 3.5*sin(b);}

Inputs 52 from the TestSuite follows.
\begin{tiny}
\begin{verbatim}
##1.0
TotalNumberOfSites=8;
NumberOfTerms=1;
DegreesOfFreedom=1;
GeometryKind="ladder";
GeometryOptions="ConstantValues";
dir0:Connectors = [1.0];
dir1:Connectors = [0.5];
LadderLeg=2;
Model="HubbardOneBand";
hubbardU = [10, ... ];
potentialV = [0, ...];
SolverOptions="TimeStepTargetting";
Version="4.52";
OutputFile="data52.txt";
InfiniteLoopKeptStates=200;
FiniteLoops = [
[ 3, 400, 0],
[-6, 400, 0],
[ 6, 200, 0]
];
RepeatFiniteLoopsFrom=1;
RepeatFiniteLoopsTimes=5;

TargetElectronsUp=4;
TargetElectronsDown=4;
GsWeight=0.1;
TSPTau=0.1;
TSPTimeSteps=5;
TSPAdvanceEach=6;
TSPAlgorithm="Krylov";
TSPSites = [4, 2];
TSPLoops = [1, 1];
TSPProductOrSum="product";

TSPOperator="expression";
OperatorExpression="c+(-1.0)*c*c?1*c?1'+c?1+(-1.0)*c?1*c*c'";

TSPOperator="expression";
OperatorExpression="c'*c?1*c?1'+c?1'*c*c'";

#ci getTimeObservablesInSitu 2 '<P0|nup|P0>'
#ci getTimeObservablesInSitu 4 '<P0|nup|P0>'
#ci getTimeObservablesInSitu 5 '<P0|nup|P0>'
#ci getTimeObservablesInSitu 2 '<P0|doubleOcc|P0>'
#ci getTimeObservablesInSitu 4 '<P0|doubleOcc|P0>'
#ci getTimeObservablesInSitu 5 '<P0|doubleOcc|P0>'
\end{verbatim}
\end{tiny}

Inputs 340 from the TestSuite follows.
\begin{tiny}
\begin{verbatim}
##1.0
TotalNumberOfSites=8;
Orbitals=2;
NumberOfTerms=3;

gt0={ # gt means geometry term
  DegreesOfFreedom=Orbitals;
  GeometryKind="ladderx";
  GeometryOptions="ConstantValues";
  LadderLeg=2;
  dir0:Connectors = [ # x-dir
    [-0.058, 0],
    [0, -0.2196]];
  dir1:Connectors = [ # y-dir
    [-0.2196, 0],
    [0, -0.058]];
  dir2:Connectors = [ # x+y
    [0.20828, 0.079],
    [0.079, 0.20828]];
  dir3:Connectors = [ # x-y
    [0.20828, -0.079]
    [-0.079, 0.20828]];
};

gt1={
  DegreesOfFreedom=1;
  GeometryKind="ladderx";
  GeometryOptions="ConstantValues";
  LadderLeg=2;
  dir0:Connectors = [0.1];
  dir1:Connectors = [0.1];
  dir2:Connectors = [0.02];
  dir3:Connectors = [0.02];
};

gt2={ # gt means geometry term
  DegreesOfFreedom=Orbitals;
  GeometryKind="ladderx";
  GeometryOptions="ConstantValues";
  LadderLeg=2;
  dir0:Connectors = [ # x-dir
    [-0.058, 0],
    [0, -0.2196]];
  dir1:Connectors = [ # y-dir
    [-0.2196, 0],
    [0, -0.058]];
  dir2:Connectors = [ # x+y
    [0.20828, 0.079],
    [0.079, 0.20828]];
  dir3:Connectors = [ # x-y
    [0.20828, -0.079]
    [-0.079, 0.20828]];
};

hubbardU	 = [0.0, 0.0, 0.0, 0.0];
potentialV = [0, ...32];

Model="FeAsBasedScExtended";
FeAsMode="INT_PAPER33";
SolverOptions="none";
Version="61289987cdd32b8485213ac0415baf4e9cd16432";
OutputFile="data340.txt";
InfiniteLoopKeptStates=60;
FiniteLoops = [
[ 3, 100, 0],
[-3, 100, 0],
[-3, 100, 0],
[ 3, 100, 0]]; # note the abbreviation here
TargetElectronsUp=8;
TargetElectronsDown=TargetElectronsUp;
\end{verbatim}
\end{tiny}
\end{document}


%% LyX 2.2.1 created this file.  For more info, see http://www.lyx.org/.
%% Do not edit unless you really know what you are doing.
\documentclass[sigconf,anonymous=false,authordraft=false]{acmart}
\usepackage[latin9]{inputenc}
\usepackage{verbatim}
\usepackage{float}
\usepackage{url}
\usepackage{enumitem}
\usepackage{graphicx}

\makeatletter
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% Textclass specific LaTeX commands.
\newlength{\lyxlabelwidth}      % auxiliary length

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% User specified LaTeX commands.
%\renewcommand\footnotetextcopyrightpermission[1]{} % removes footnote with conference information in first column
%\pagestyle{plain} % removes running headers

%\makeatletter
%\renewcommand\@formatdoi[1]{\ignorespaces}
%\makeatother

\usepackage{url}
\usepackage{enumitem}
\usepackage{amstext}
\usepackage{graphicx}
\usepackage{booktabs} % For formal tables

\usepackage{subfigure}
\usepackage[ruled,vlined,linesnumbered]{algorithm2e}

\newcommand\varlist{,\makebox[0.8em][c]{.\hfil.\hfil.},}

% DOI
\acmDOI{10.475/123_4}

% ISBN
\acmISBN{123-4567-24-567/08/06}

%Conference
\acmConference[WOODSTOCK'97]{ACM Woodstock conference}{July 1997}{El
  Paso, Texas USA}
\acmYear{1997}
\copyrightyear{2016}

\acmPrice{15.00}

\fancyhead{}
\settopmatter{printacmref=false, printfolios=false}

\newcommand{\I}{\mathbb{I}}
\newcommand{\N}{\mathcal{N}}
\newcommand{\E}{\mathrm{E}}
\newcommand{\Ind}{\mathrm{\#}}
\renewcommand{\vec}[1]{\mathbf{#1}}
\newcommand{\Sim}{\mathrm{Sim}}
\newcommand{\ExpOneCall}{\text{Exp-}1\text{-Call@}k}
\newcommand{\PLAR}{\text{PLAR}}

\newcommand{\heq}{\hspace{-.5mm} = \hspace{-.5mm}}
\newcommand{\seq}{\hspace{-1mm} = \hspace{-1mm}}

\newcommand{\ExpNCall}[1]{\text{Exp-}#1\text{-Call@}k}
\newcommand{\TlessK}{T_{\!\!\:k\text{-}1}}
\def\argmax{\operatornamewithlimits{\!arg \;\! max\,}}

\makeatother

\begin{document}
%\title{VTF: A Filter Selection-Based Search Tool for Social Network Visualization}
%\title{VTF: A Visual Filter Selection-Based Search Tool for Social Network}
%\title{Viz-TSF: A Visual Twitter Search Tool based on the Filtering of Geo-Temporally Coherent Content}
\title{Viz-TSF: A Visual Twitter Search Tool based on Query-driven Filter Optimization}
\author{Mohamed Reda Bouadjenek}
%\orcid{1234-5678-9012}
\affiliation{%
  \institution{The University of Toronto}
  \streetaddress{Department of Mechanical and\\ Industrial Engineering}
  \city{Toronto}
  \state{Ontario}
   \postcode{M5S 3G8}
  \country{Canada}
}
\email{mrb@mie.utoronto.ca}

\author{Scott Sanner}
\affiliation{%
  \institution{The University of Toronto}
  \streetaddress{Department of Mechanical and\\ Industrial Engineering}
  \city{Toronto}
  \state{Ontario}
   \postcode{M5S 3G8}
  \country{Canada}
}
\email{ssanner@mie.utoronto.ca }

\newcommand{\subfour}[1]{\vspace*{3mm}{\noindent\bf #1}}
\newcommand{\subsubfour}[1]{\vspace*{1mm}{\noindent\bf #1}}
\begin{abstract}
Geo-temporal visualization of Twitter search results is a challenging task since the simultaneous display of all matching tweets would result in a saturated and unreadable display. Thus, the development of novel intuitive visual interfaces is necessary to help focus a user's attention by filtering information relevant to their query. In this work, we propose a Visual Tweet Search Filter (Viz-TSF) tool for query-driven optimization and ranking of spatial, temporal, and content-based filters to help focus a user's exploration of search results.  We leverage a fast greedy optimization algorithm to optimize an approximation of expected F1-Score metric to generate these filters and demonstrate its application to search 2 years of Twitter content, specifically for a user's information need related to natural disasters occurring in the US.  Our demo shows that Viz-TSF is capable of extracting geo-temporally coherent filters given search queries, thus aiding the user in visually searching and browsing social network content and enabling new opportunities for the application of Information Retrieval techniques to general visual information exploration interfaces.

\noindent {\bf Keywords:} Adaptive UIs; Visual Search Interfaces;
Optimization for IR.
\end{abstract}
\maketitle

\section{Introduction}

Traditional search engines such as Google or Bing display search results
in a vertical list of textual summaries. However, this display mode
is certainly not adapted for search results over Twitter content,
since related tweets are often geographically and temporally localized. Moreover, given
the massive volume of available information in Twitter, displaying
all relevant tweets for a given query prevents the visual extraction
of relevant information as it would result in a saturated and unreadable
display \cite{Liu2014}. Thus, adaptive user interfaces (AUIs) are
necessary to filter information to provide localized and relevant
tweets for each query. Such AUIs would allow the user to efficiently explore
the content of the matching tweets and to identify key properties
(author, timestamp, geolocation) relevant to the fulfillment of the user's information need.
To this end, we propose a Visual Tweet Search Filter (Viz-TSF)\footnote{\url{http://130.220.208.198:8080/AUI4IRSearch/}}
tool to optimize and rank spatial, temporal, and content-based
filters intended to focus a user's exploration of Twitter search results.

%The key difference is that while information retrieval is ubiquitous in the lives of most humans in the form of web search, it has gone virtually unrecognize as an appropriate model for the filtering problem in AUIs \textcolor{red}{[REFERENCE IS NEEDED]}.
%Instead, machine learning methods have been widely used to address this problem.

\begin{figure}[t]
% \subfigure[Global display showing anomalies.]{\includegraphics[width=4.25cm]{imgs/twitter_example}\label{Fig:GlobalDisplay}}\subfigure[Filtered and focused display.]{\includegraphics[width=4.25cm]{imgs/twitter_example_filter}\label{Fig:FilteredDisplay}}

\begin{centering}
\subfigure[Global display showing search results.]{\includegraphics[width=7cm]{imgs/map}\label{Fig:GlobalDisplay}}
\par\end{centering}
\begin{centering}
\subfigure[Filtered and focused search interface.]{\includegraphics[width=7cm]{imgs/filtered_map}\label{Fig:FilteredDisplay}}
\par\end{centering}
\caption{(a) An unfiltered user search interface with geolocated tweets that
best match a query related to natural disaster highlighted in red. (b) A filter-optimized version
of the interface showing the top three filtered subsets of data identified
to have high coverage of relevant content related to the query: a
bounding box near New York state limited to data in Feb 2015 and having
keyword ``Blizzard'' and other time and keyword restricted bounding
boxes centered on Florida and Washington state. }
\label{Fig:UseCase}
\end{figure}

% \begin{table*}[t]
% \caption{Comparison between conventional Web Search and Filtering for AUIs.}
% \label{tbl:Comparaison2IR}
% \centering{}%
% \begin{tabular}{|c|>{\centering}p{6cm}|>{\centering}p{6cm}|}
% \hline
%  & \textbf{Web Search} & \textbf{Filtering for AUIs}\tabularnewline
% \hline
% \hline
% \textbf{Information Need} & Realized by query  & Selection of an event or alert\tabularnewline
% \hline
% \textbf{Form of Results} & Ranked list of documents & Filter settings (bounding box in visual display, time ranges, property
% filters) that select a subset of information elements to display\tabularnewline
% \hline
% \textbf{Relevance Scoring} & Comparison of query and document content (and other relevant information)
% to generate a relevance score, e.g., via TF-IDF or BM25 & Third party application-specific tool (e.g., a machine learning approach)
% that predicts the probability that each information element is relevant
% to the selected alert\tabularnewline
% \hline
% \textbf{Test Data for Benchmarking} & A set of queries; human-labeled judgments of document relevance to
% each query for a test corpus (i.e., ground truth relevance) & A set of alerts or events; human-labeled judgments of information element relevance
% to a selected alert (i.e., ground truth relevance)\tabularnewline
% \hline
% \textbf{Evaluation} & Ranking metrics (P@k, AP, etc.) over documents given ground truth
% relevance  & Boolean metrics (e.g., F1-score) and Ranking metrics (P@k, AP, etc.)
% over information elements selected by the filter given ground truth
% relevance \tabularnewline
% \hline
% \end{tabular}
% \end{table*}

To make the task of AUI filtering
more concrete, we introduce an example search filtering use case
for AUIs. Consider the case of searching events related to natural
disasters that are discussed on Twitter. Typically, as shown in Figure
\ref{Fig:GlobalDisplay}, there would be some visual display of all
tweets that match the query along with highly relevant Tweets shown as red nodes.  %
%in Figure \ref{Fig:GlobalDisplay}.
%(such as user IP address, Tweet content, Tweet time, geographical coordinates, recent activity details)
%We denote all displayed tweets
%as an information tweet that could be optionally shown or not.
% Above -- huh?
Displaying
all tweets simultaneously would result in a saturated and an unreadable
display for many queries that return a high volume of matching content.
To ease the investigation
and search task, users could restrict the tweets displayed through
a variety of filter settings to get a clear overview of individual
events as shown in Figure \ref{Fig:FilteredDisplay} \textendash{}
by panning and zooming in the graph display, by restricting upper
and lower bounds on a time filter, and/or by selecting properties
(e.g., in a drop-down selection or fielded keyword search in fields
such as Tweet content, author, or hashtags).
%\textcolor{red}{Such filter settings are expected to help a monitoring agent to efficiently investigate the cause and implications of the triggered alarm, to identify common properties of the elements involved in that alert, and to initiate appropriate corrective actions in a timely manner.}

While the user would typically find it hard to manually set these
filters to optimally reveal information about specific relevant events
related to his/her query, an AUI that is aware of a probability
estimate that each tweet is relevant to the query could automatically optimize and suggest a ranking of
filter settings to visually display relevant tweet content with the
least amount of irrelevant clutter.  The user could then more efficiently browse through the subset of tweets provided by
the ranked filter settings to carry out investigation w.r.t.\ the original information need while covering a large fraction of the relevant content.
%search and investigation by identifying
%common properties of the tweets related to the query that are in the
%same cluster.

\begin{comment}
We assume the prediction of information element relevance to an alert
to be provided by a third-party since this prediction is highly specific
to each particular application setting and not the focus of this work
(see Related Work in Section 6 for further discussion).
\end{comment}

%In this work, we intend to
%focus on the broader problem of AUI filtering in a range of visual
%display applications assuming this relevance predictor is given. However,
%in addition to scoring information elements of direct causal interest
%to the alert, it is likely that the AUI will also contain other related material requiring
%the concurrent display of information.  For example, such a material could be messages exchanged, the time at which these messages were exchanged, etc.
%For example, such a rule could
%be \textquotedblleft if a connection is shown, the IP addresses of
%the two connected machines must also be shown\textquotedblright ,
%which would trigger additional (soft) constraints for the filter optimization.

Since ``retrieved\textquotedblright{} tweets are not individually
selected, but rather chosen through a filter setting (that the user
can further modify), the problem is clearly an optimization problem
of how to restrict filter settings to show the user the most relevant
tweets to his/her query. While this diverges from the standard information
retrieval setting where ranked documents are chosen individually \cite{Baeza-Yates2010},
these additional constraints do not change the overall information
retrieval objective to select relevant information given the user's
information need and constraints of the user interface. %Furthermore, standard information retrieval evaluation benchmarking
%methodology (e.g., the Cranfield paradigm) and metrics such as precision@k
%or average precision are also relevant here (assuming that visual
%saliency can serve as a surrogate method for achieving a visual rank
%ordering of selected information elements).  Unlike information retrieval
%where Boolean metrics such as F-Score are not often used since ranked
%lists are usually limited to the top-10, graphical displays can present
%substantially more information of varying sizes that make precision-recall
%trade-off metrics like F-Score also relevant for evaluation.\textcolor{red}{ [THIS SENCTENCE IS UNCLEAR]}

%With the aim of assisting network monitoring agents in carrying out their daily tasks, we present in this paper a new visual search paradigm for AUIs.
%% VISUAL SEARCH RESULTS -- VISUAL SEARCH FILTER
%% VISUAL PRESENTATION OF SEARCH RESULTS -- Reda says terminology may be confused

To the best of our knowledge, Viz-TSF is the first tool to address
a visual information retrieval problem for social media as the explicit optimization of spatial, temporal, and content-based filter selection and ranking w.r.t.\ surrogates of F1-Score well-suited to this task.

% The contributions of this paper are summarized as follows:
%  \begin{enumerate}
% \item We propose a new theory of IR for adaptive visual user interfaces.
% \item We propose new expected metrics for optimization based on conventional IR metrics, i.e., precision, recall, and F1-Score.
% We argue that F1-Score and hence expected F1-Score is a good metric for this domain.
% \item We propose new filter setting search algorithms based on greedy strategies and optimal solutions. These algorithms are based on the optimization of new proposed metrics.
% %\item  We propose new MILP-based search method, which can also used about justification of new search algorithm.
% \item We present a thorough evaluation on three different datasets to show that the greedy algorithms we propose are a good approximation of the optimal method, and perform well in the presence of noise (i.e., corruption of the ground truth labels).
% %can even perform better with a noisy classifier \textendash{} up to 42\% improvement for a classifier with an accuracy of 60\%. On the other hand,
% We experimentally demonstrate that the expected F1-Score metric is a good surrogate of F1-Score.
% %specifically with a highly accurate classifier \textendash{} for a classifier with 90\% accuracy, the RMSE is roughly 0.0111).
% \end{enumerate}

\section{V\lowercase{iz}-TSF architecture and features}

The main components of the architecture of Viz-TSF are:
\begin{enumerate}
\item \emph{Crawler and Indexer.} To find and organize tweets for further
retrieval.
\item \emph{Query Analyzer}. To understand the user's information need.
\item \emph{Scoring Function}. To score the tweets based on their relevance probability w.r.t.\ the query.
\item \emph{Filter Optimization}. To optimize and rank filter settings w.r.t. the scoring function and filtering objective.
\item \emph{Visual Adaptive User Interface}. To present the information to the user in an accessible and interactive form using filters.
\end{enumerate}
While the two first components are common to many IR systems, we briefly
describe in the following the remaining components which are at the
heart of Viz-TSF.

\subsection{Scoring Function}

The filter optimization component of Viz-TSF relies on the optimization
of an approximation of \emph{expected} F1-Score metric. This expectation
%is computed based on the conditional probability $p(R|t,q)$, i.e.,
is critically based on an estimated conditional probability $p(q|t)$
%the probability
that a tweet $t$ is relevant to the query $q$.  To obtain this probability estimate,
the ranking function component of Viz-TSF uses a language model scoring approach, which assumes that the terms of a query $q$ are ``generated'' by a
probabilistic model based on the content of tweet $t$, thus directly estimating
$p(q|t)$.
%. Hence, given a query
%$q$ and a document $d$, the language model estimates
%the conditional likelihood $p(a|d)$, i.e., the posterior probability that $d$ generates the
%observed $q$.
Specifically for this demonstration, we use a Bayesian smoothing
language model using Dirichlet priors as defined in Zhai and Lafferty~\cite{Zhai2001}.

\subsection{Filter Optimization}

The filter optimization component of Viz-TSF relies on the configuration of three sub-filters
to select a subset of relevant tweets for the data visualization interface -- specifically (i) a keyword content sub-filter, (ii) a temporal sub-filter, and (iii) a spatial (location) sub-filter.
A global filter that is generated by \emph{conjoining} these three
sub-filters is used to select the overall subset of relevant tweets for a filter. Each
sub-filter is constructed following a greedy top-down approach while
seeking to optimize an approximation of the expected F1-Score metric.
For efficiency, the algorithm implemented in this module uses a greedy binary
partitioning search, where the algorithm operates by halving the search space of one sub-filter in each iteration. Details
of this algorithm are further provided in~\cite{Bouadjenek2018b}.

In practice, a single conjoint filter setting chosen by the previous algorithm will
narrow the user on a single ``event''.  However, there are likely multiple
events and so the user should have a ranked choice of multiple filters to choose from.  Hence,
after the first filter is produced, all selected tweets by the filter
have their probability scores zeroed out (since they are already covered by one filter). The filtering algorithm is then run
again, where it will inherently focus on a different content set.
This procedure is repeated until the desired number of filters is
reached. The user should then be able to choose among a ranking of  the filters produced as provided in our demonstration interface.

\begin{comment}
\subfour{Keywords greedy algorithm:} The idea behind the keywords-based
greedy top-down algorithm is the following: given a set of tweets
and an evaluation measure, the algorithm aims to select a set of keywords
to form a negation query in order to exclude a subset of elements
containing these keywords for the purpose of maximizing the evaluation
measure.

Formally, given a keyword query representation $Q_{k}$, the algorithm
aims to select an optimal subset of $k$ terms$T_{k}^{*}\subset E$
(where $|T_{k}^{*}|=k$ and $E$ is the initial set of elements) to
build a negation keyword query $Q_{k}$, i.e. $Q_{k}=\{\neg t_{1}^{*},\dots\neg t_{k}^{*}\}$,
in order to optimize a given evaluation measure, e.g. expected precision,
expected recall, expected F1-Score. This is achieved by building $T_{k}^{*}$
in a greedy manner by choosing the next optimal term $t_{k}^{*}$
given the previous set of optimal term selections $T_{k-1}^{*}=\{t_{1}^{*},\ldots,t_{k-1}^{*}\}$
(assuming $T_{0}^{*}=\emptyset$) using the following selection criterion:

\begin{equation}
t_{k}^{*}=\argmax_{t_{k}\notin T_{k-1}^{*}}\hspace{-0.3mm}[EF1(E^{*}\textrm{ that satisfies }Q_{k}=\{\neg t_{1}^{*},\dots\neg t_{k}^{*}\})]
\end{equation}

\noindent assuming we use expected F1-Score to assess $E^{*}$, which
is a subset of the initial element set $E$ that satisfy the negation
keyword query $Q_{k}$. In order to reduce the keyword search space,
we propose to use the top 100 terms ranked using mutual information.
\end{comment}

\subsection{Visual Adaptive User Interface}

As shown in Figure~\ref{fig:MainInterface}, we designed an interactive visual interface providing a multi-perspective view consisting of: (i) an interactive map, which presents search results for each filter setting that permits further user-driven adaption of the filters (e.g., spatial refinement through panning and zooming) and pop-up information summaries by clicking on filters or tweets, (ii) a query panel allowing the user to specify a query, refine the time window filter, and tune two search parameters (the number of filters produced and the value
of $\beta$ for the F$_{\beta}$-Measure that is optimized), and (iii) a ranked list of the filters produced with respect to a given query, which can be used to select and (un)hide some filters for more clarity.

\begin{figure*}[t]
\includegraphics[width=18cm]{imgs/main}

\caption{Main interface of Viz-TSF.}
\label{fig:MainInterface}
\end{figure*}

Viz-TSF is built on the top of the Lucene IR system\footnote{\url{https://lucene.apache.org/core/}}
\cite{McCandless2010} for the indexing, query processing and retrieval
part, and on the Google Maps API for data visualization.

\section{What will be demonstrated?}

This demonstration will be illustrated using Twitter data crawled
using the Twitter Streaming API for two years spanning 2013 and 2014
\cite{Iman2017}. This dataset contains more than 2.5 TB of compressed
data, with a total of 829,026,458 English tweets.

In our demonstration, a user can perform interactive searches such as the following example search scenario:
\begin{itemize}
\item A user begins by searching for information on weather-related natural
disaster events. Using
the querying panel, the user enters a set of keywords ``flood, hurricane, typhoon, drought, storm'', selects a search time window (e.g., January 2013 to June 2014),
selects the number of filters to be returned, and sets an initial value for
$\beta$. A low value of $\beta$ constrains the algorithm to focus
on the optimization of the precision, thus obtaining filters
with small clusters, whereas a high value of $\beta$ leads to high recall filter settings with large clusters.
%constrains the
%algorithm to focus on the optimization of the recall, thus obtaining
%filters with large clusters.
The value of $\beta$ can be interactive and iteratively tuned by the user until the desired granularity or coarseness of filter results are obtained.
\item As shown in Figure \ref{fig:main2}, a list of filters is suggested
to the user and displayed on the main map (in this case restricted to the US) along with relevant tweets for the search query, where
red nodes indicate highly relevant tweets, and blue nodes indicate lower relevance tweets.
\end{itemize}
\begin{figure}[H]
\begin{centering}
\includegraphics[width=7.2cm]{imgs/main2}
\par\end{centering}
\caption{Results showing all the filters that match the query.}
\label{fig:main2}
\end{figure}

\begin{itemize}
\item Next, as shown in Figure \ref{fig:main3}, using the side panel, the
user may deselect some filters that he/she thinks are unrelated to their information need
in order to focus on a subset of filters that are of interest.
\end{itemize}
\begin{figure}[H]
\begin{centering}
\includegraphics[width=8cm]{imgs/main3}
\par\end{centering}
\caption{Sub-set of the filters that match the query.}
\label{fig:main3}
\end{figure}

\begin{itemize}
\item Then, as shown in Figure \ref{fig:main4}, the user may zoom in a
specific filter for investigation by looking for its top terms, its
associated tweets, its time window, etc. In the case of Figure \ref{fig:main4},
the user investigates a filter that contains tweets related to some
flood disasters that happened on the northeast coast of the US during
the time period ranging from December 2013 to February 2014.
\end{itemize}
\begin{figure}
\begin{centering}
\includegraphics[width=8cm]{imgs/main4}
\par\end{centering}
\caption{Investigating filter 1.}

\label{fig:main4}
\end{figure}

\begin{itemize}
\item Similarly, the user may investigate a second filter in the ranked list by looking at
its content. For example, in Figure \ref{fig:main5}, the filter investigated
is related to a series of storms and hurricanes that happened in the
southeast coast of the US (mainly around Florida) during the time period ranging from April 2013
to June 2013. Clearly, the two investigated filters are of two different
events related to two different natural disasters that happened during
two different periods. Thus, this suggests that in this search case,
Viz-TSF succeeded in finding two different filters that focus on different (distinct) events related to the initial search query.
\end{itemize}
\begin{figure}[H]
\begin{centering}
\includegraphics[width=8cm]{imgs/main5}
\par\end{centering}
\caption{Investigating filter 2.}

\label{fig:main5}
\end{figure}

\begin{itemize}
\item Finally, the user may decide the re-run his/her query based on a specific
local geographical area. In Figure \ref{fig:main6}, the query is
re-executed on the US east coast, where new local filters are produced,
which may be investigated by the user. For example, the bottom filter
shown in Figure \ref{fig:main6} with a time window from June to July
2013 contains a set of tweets that warn people about the importance
of preparing houses for the new upcoming hurricane season.
\end{itemize}
\begin{figure}[t]
\begin{centering}
\includegraphics[width=8cm]{imgs/main6}
\par\end{centering}
\caption{Filtering east US coast.}

\label{fig:main6}
\end{figure}

In summary, the content of this demonstration aims to provide users with a novel way to visually and interactively browse geo-temporally filtered content, specifically in this case for tweets relating to natural disasters in the US over a two year time period.

\begin{comment}
Conclusion: provide a quick way to visually browse through a large
set of content to get a visual interactive summary view of localized
events and occurrences related to information need.
\end{comment}

\section{Summary and Future Work}

In this paper we have introduced Viz-TSF, a tool to search Twitter
content through spatial, temporal, and content-based filters to help
focus the user's exploration of search results with geo-temporally
coherent content. We leveraged a fast greedy optimization algorithm
to optimize an approximation of expected F1-Score metric to generate
filters and demonstrate its application to search 2 years of Twitter
content to better understand tweets that match a given search query.
We have described a search scenario of queries related to an information of natural
disasters occurring in the US during a two year period and showed the
capabilities of
using Viz-TSF to search large-scale data in social networks.

Important areas of future work include consideration of the role of
(pseudo-)relevance and other explicit or implicit feedback methods
to create a tighter and more responsive user interaction loop. Furthermore,
in combination with user studies and consideration of human factors,
future work should also consider novel application-specific objectives,
e.g., in specific visualization frameworks or based on a ranking theory
of results presentation (e.g., using size or colour for visual ranking
emphasis).

Overall, we aim for this work to bring a formal IR perspective to
the important research area of AUIs while opening new research directions
for the application of optimization techniques for filter selection
in AUIs. As information retrieval has transformed our web experience,
we believe there is a similar opportunity to transform the present
nascent state of AUIs through optimization and information retrieval
principles to make them more ubiquitous in our daily lives.  We hope this demonstration serves as one example of such a novel visual information retrieval interface.

\bibliographystyle{abbrv}
\bibliography{biblio}

\end{document}


\chapter{Community Detection using Node Attributes: A Non-Negative Matrix Factorization Approach.}

\section{Introduction}
% no \IEEEPARstart
Exploratory Data Analysis [E.D.A.] is a domain on the
intersection of fields such as machine learning, pattern
recognition and information retrieval. The key goal of this
domain is to generate effective summarizations, visualizations,
information discovery and retrievals from data with a goal of reducing the exponential
costs involved in its storage. The main task performed in E.D.A. is
clustering analysis. Cluster analysis is a type of unsupervised
learning as cluster labels are not provided apriori or rather
are implicit in the data itself. The term "Cluster" doesn't have a
standard definition and hence there is subjectivity in the
deciding what forms a "Cluster". This has led to proliferation in the literature of clustering algorithms. Distance based definition of clusters have been explored
and have created a family of techniques such as partition
based clustering \cite{aps:3} , hierarchical clustering \cite{aps:4}, mixture
model based clustering \cite{aps:4}, fuzzy clustering \cite{aps:8} amongst
others. In contrast to a line of previous work a second
definition of clusters was proposed based on density, this
created popular algorithms such as DBSCAN, OPTICS \cite{aps:46}.\\

A related field is Community Detection which involves
identification of latent groups of entities in data. These groups
correspond to autonomous regions in the network which are known to have a higher
degree of homogeneity within its members than with members
of other groupings in the same network. In Network sciences such sub
groups are called communities and these are identified using
network topology. A vast area of literature has uncovered
several state of the art community detection algorithms that
aim to find such communities in un-directed as well as
directed graphs \cite{aps:2} . This literature is based
on concepts related to Information Theory or the trajectory of Random walks
on graphs or the Map Equation \cite{aps:10} \cite{aps:9}. Apart from
this, community detection also developed a concept called
modularity and a new family of algorithms were developed
that detected communities in graphs by optimizing modularity in a greedy manner \cite{aps:2} \cite{aps:8}.\\

Latent Dirichlet Allocation was another concept
that emerged and led to a new line of research on detecting
communities by utilizing meta-data that is associated with
the entities (nodes). This led to novel techniques that utilized the
information about network topology along with meta data
for obtaining generative models of networks \cite{aps:52} \cite{aps:51}. However even with such methods there
were drawbacks such as the limited applicability, as they
could not detect overlapping communities. A second drawback
is that they assumed soft node-community memberships, which
was not appropriate for modeling communities because they
did not allow a node to have high membership strength to
multiple communities simultaneously. Finally, such methods
had a large time complexity and couldn't be scaled to graphs
having more than 1000 nodes.\\

Non negative matrix factorization of the
term document matrix was found to be effective in document
clustering \cite{aps:56}. NMF was later extended to clustering by
aiming to learn the adjacency matrix of a graph. NMF
research did not pay attention to the interpretation of latent
factors which are used to find out the matrix. This led to
development of BIGCLAM which aimed to learn latent
factors which the authors argued represented strengths of
community affiliations of nodes. BIGCLAM and NMF both
used community affiliation knowledge of the nodes so that their membership strengths could be
estimated. While most existing literature \cite{aps:53}
focuses either on using the meta data or entity annotation for
improving the quality of community detection. \\

The paper is organized as follows. Section II briefly surveys
related work. In Section III, the statistical model of the
approach is defined, and in Section IV, the parameter fitting
procedure is provided in detail. This is followed by describing
experimental evaluation in Section V and the conclusion.\\

\section{Related Work}

Clustering approaches have their own biases in identifying
clusters in data but none is considered a universal
best fit. For example, the objective function to be minimized
in the k-partitioning algorithms is variance or $SSE$ i.e. Sum of Squared Distance. $  SSE = \sum_{k=1}^{k} \sum_{x_i \epsilon c_k} \left \| x_i - c_k \right \| ^2 $ where $c_k$ = centroid of the cluster. In this case the clusters are convex
but the optimization function converges to a local optima as the objective
function is non convex \cite{aps:2}. Hierarchical
Clustering algorithms are a category of algorithms
having a completely unsupervised approach to clustering.
They do not require the users to specify the number of clusters
in advance and are broadly of two types: Agglomerative
and divisive. To measure the dissimilarity between clusters
obtained in hierarchical clustering, linkage methods were
developed with several popular techniques being listed in the
literature \cite{aps:2} \cite{aps:8} \cite{aps:4}. But it difficult to decide on a suitable linkage method and at times the selection of a distance measure too isn't clear.\\

Fuzzy clustering (FCM) minimizes the objective function given as
$\sum_{j=1}^{k}\sum_{x_i \in C_j} u_{ij}^m (x_i - u_j)^2 $
 with $\mu$ being the fuzzifier and
$m$ defining the level of cluster fuzziness. The drawbacks seen in k-means such as non convex objective function, difficulty in detecting non linearly separable clusters etc. are also seen in FCM. The MST clustering
algorithm discussed in \cite{aps:54} is known to be capable of
detecting clusters with irregular boundaries. Unlike traditional
clustering algorithms, the MST clustering algorithm does
not assume a spherical shaped clustering structure of the
underlying data. If the
number of clusters $k$ is given in advance, the simplest way to
obtain $k$ clusters is to sort the edges of the minimum spanning
tree in descending order of their weights, and remove the
edges with the first $(k-1)$ heaviest weights. Undesired clustering
structures and an unnecessarily large number of clusters are
problems commonly faced by MST based clustering. \\

In \cite{aps:45} Mixture Model based clustering is discussed which
unlike the traditional clustering algorithms doesn't rely on
heuristics but assumes that the data has been generated from
a mixture of multiple probability distributions (Gaussian or
multinomial) whose parameters have
to be estimated. This is done using a technique called Expectation Maximization. Subspace clustering \cite{aps:2} \cite{aps:4} is based on key principle which
is to discretize the data-space into grids and estimate the
density by counting the number of points in a grid cell. Other
methods in the literature are Affinity propagation \cite{aps:3} which is based on concept of message passing, Spectral clustering \cite{aps:2}
in which the ﬁrst $k$ eigenvectors $u_1,u_2,...,u_k$ corresponding to the $k$ smallest eigenvalues are computed to get matrix $U \in R^{n*k}$ which has $u_1,u_2,...,u_k$ as columns. Then for $y_i \in R^k$ which is the $i^{th}$ row of $U$, all rows are treated as points and clustered by k-means to get $k$ clusters. DB-SCAN,
OPTICS are based on the concept of density and treat clusters
are dense regions connected by less dense regions. However, none of this literature is applicable for clustering in networks.\\

Community detection is a field that deals with
obtaining coarse grained descriptions of large networks as real
world graphs are too large to be analyzed efficiently.
This is done by utilizing network topology to detect communities
of nodes while ignoring node attributes. Topic Link LDA and Block LDA were the first to cluster graphs by jointly modeling links and node attributes. Topic
link LDA aims to quantify the effect of topic similarity
and community similarity to the formation of a link \cite{aps:52} \cite{aps:51}.
Block LDA is a joint model of two components, with one
that models links between pairs of entities represented as
edges in a graph with a block structure, and the second that
models text documents, through shared latent topics. There
has also been limited work on combining graph and content
information for community discovery leading to techniques
such as CESNA and BIGCLAM. CESNA was for statistically
modeling the interaction between the network structure
and the node attributes. The authors argued that this
would lead to more accurate community detection as well as
improved robustness in the presence of noise in the network
structure \cite{aps:53}. BIGCLAM is another approach that detects
both 2-mode as well as cohesive communities which may
overlap or be hierarchically nested and is based on affiliation
graph models.\\

To the best
our knowledge, in the research no mention could be found
of using the attributes of a data point for calculating the
latent features on the basis of which communities shall be
detected. The work in this paper is based on BIGCLAM framework but
the critical difference with existing techniques such as CESNA, AGMFIT and BIGCLAM is that attributes shall be used instead of community affiliations. The intuition here is that attributes are useful in determining the cluster affiliations and as such this intuition is also consistent with the phenomenon of "homophily" that is seen in human networks.\\

\section{Mathematical Model}
The stochastic generative model for generating communities
is presented in this section in which the probability of two
entities in data being present in the same community is
dependent on the attributes or the annotated text data
associated with these nodes. An efficient model fitting
procedure is the presented which allows for detecting
communities in the network. The current work is based on
the assumption that attributes of the data are categorical. The aim is to build upon BIGCLAM, an affiliation model for
community detection, however the objective is using attribute
information in place of affiliation information for building a bipartite graph which will be partitioned.\\

\textbf{Directed Attribute Affiliation Model:} BIGCLAM and AGMFIT are build on the Affiliation Graph Model based algorithms which use Maximum Likelihood Estimation to create a AGM from the network. Both however ignore importance of attributed of the nodes being responsible for community
creation. In Social networks, Homophily is the tendency to be associated with
others who share similar preferences and therefore attribute
associated with the entity in a network play an important
role in deciding communities. The hypothesis of correlation
between attributes and communities is reasonable as its
presence is also seen in empirical evidence provided in
the literature \cite{aps:53}. Based on this reasoning, a simple
conceptual model called Directed Attribute Affiliation Model
is formulated. This builds on the family of affiliation network
models, but in this work affiliation models are extended to
consider attributes.\\

To represent node and attribute affiliation a bipartite
affiliation graph is created where nodes are the bottom layer
and attributes to which they belong are shown as the top
layer as seen in Fig.\ref{fig 1}. A directed edge is created between an attribute and a node if the node has the attribute present in it. Such a bipartite graph can be constructed easily if attributes are binary valued. In case the attributes are continuous or categorical, a different mechanism might be needed. In this paper only binary attributes are considered. Cluster affiliation can then be modeled using such a
bipartite graph where directed edges are formed between
nodes and attributes to denote that those nodes contain
that attribute. \\

\begin{figure}[H]
\centering
\fbox{\includegraphics[scale=0.5]{bap.PNG}}
\caption{Bipartite Attribute Affiliation Graph}
\label{fig 1}
\end{figure}

A Bipartite Attribute Affiliation Graph is denoted as $B(X,C,M)$,
with $X$ as the nodes, $C$ as the attribute value and $M$ denotes
the directed edge from $X$ to $C$ if node $X$ has attribute
value $C$. The problem now is to create a set of communities
$S = S_1, S_2, ..., S_k$ given $B(X,C,M)$. A parameter $p_c$ is
assigned to an attribute value $c \in C$. This is for calculating
the probability that a node $x_i$ has the attribute value $c$. This
can also be called the probability that a node $x_i$ belongs to
the same community as another $x_j$ having the value of a
particular attribute as $c$. The $P_A(i, j)$ denotes that the nodes
$i, j$ belong to the same community $A$. This can be shown by
the below equation.

\begin{equation}
P_A(i,j) = 1 - \prod_{c \in M_i \cap M_j} (1 - p_c)
\end{equation}

Where,

\begin{itemize}
\item $M_i$ = node $i$ has membership to attribute value $c$.
\item $M_j$ = node $j$ has membership to attribute value $c$.\\
\end{itemize}

In Eqn. 1, the value of $P_A(i, j)$ is set to $\varepsilon$, following
the BIGCLAM procedure the value of $\varepsilon$ can be set as
$2|E|/|V|(|V| - 1)$ [21].

\subsection{Calculate the latent weights of the attributes}

Every attribute has its own importance or strength in
determining the cluster to which the node should belong to,
this is denoted here formally as $F_{uC}$. This is the strength
that attribute $C$ has for node $u$ in determining its cluster.
Considering this membership strength the Eqn. 1 can be
modified as follows:

\begin{equation}
P_A(i,j) = 1 - exp(-F_{uC}.F_{vC}^T)
\end{equation}

$F_{uC}$ is the membership strength of a single attribute,
similarly it is assumed that every node $i$ has a attribute
membership vector $F_i$ which contains the membership
strengths to all attributes in the data. The modified probability
that nodes $i, j$ now share a cluster is Eqn 2.\\

The intuition behind the above formula is simple, Consider
a node having attribute values same as the attribute values
of another node, in such a case the likelihood of both
nodes belonging to a particular community increases. This
means that for each attribute a pair of nodes shares we
get an independent chance of grouping the nodes. Thus,
naturally, the more attributes a pair of nodes shares, the
higher the probability of sharing the same community and
being connected.\\

If $M_u \cap M_v = 0$ then $P(u,v) = \varepsilon$ this is done to consider
cases where nodes might not share attributes but still are
connected. $F_u$ is the vector that denotes the strengths of
association of a node $u$ with each attribute community in the network.
The task is to find the matrix of memberships $F$ that
maximizes the likelihood of generating the graph $G(V,E)$.
The log-likelihood of this is Eqn. 5. The Gradient update
algorithm is used to find the value of $F$ as shown in Eqn. 6 \\

\begin{equation}
l(F) = \sum_{u,v \in E} log(1 - \exp(-F_u.F_v^T)) - \sum_{u,v \notin E}(F_u.F_v^T)
\end{equation}
\begin{equation}
\bigtriangledown l(F_u) = \sum_{v \in N(u)} F_v \frac{\exp(-F_u.F_v^T)}{1 - \exp(-F_u.F_v^T)} - \sum_{v \notin N(u)} F_v
\end{equation}

\textbf{Decide Community Affiliation:} The membership strengths
matrix $F$ is computed from above and the next step is to
determine a suitable threshold above which it is possible to
determine whether the node $i$ belongs to a community. This
threshold is $\delta$ set at $\sqrt{−log(1 - \varepsilon)}$ [21]. The initialization
isn't done using locally minimal neighborhoods approach of
BIG-CLAM \cite{aps:53} as entity annotated attributes are used to get
initial values of the membership strengths $F_i$. The value of
$F_{i,k}$ is 0 if attribute $k$ is present and 0 if absent.\\

$\textbf{Choosing the number of communities:}$ This is done by
procedure specified in \cite{aps:53} where the model is trained using
an initial value of K. Then we detect K communities on the
80\% of node pairs and then evaluate the likelihood on the hold
out set. The K with the best hold out likelihood is used.\\

\section{Experiments}
In this section, evaluation of the performance of the variant of BIG-CLAM and other state-of-the-art community detection methods is done. The data-set used in this work consists of an artificially generated network with node attributes created using the tool described in the work of  Christine Largeron \textit{et. al.} \cite{aps:55}. Community detection methods such as Louvain and Fastgreedy technique were not applied as the graph was directed.

\subsection{Dataset and Evaluation Criteria}
The Artificially generated data-set is described below. The comparison metrics used are Variation of information (VI), Normalized mutual information, Split-join distance, Rand index (RI) and Adjusted Rand index (ARI). NMI, RI and ARI are in the range (0 - 1) with higher value indicating better clustering. The split-join distance between partitions A and B is the sum of the projection distance of A from B and the projection distance of B from A and should be low. VI should also ideally have a low value.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Description of the dataset}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Node Attributes &  3 \\
   \hline
  2 & Nodes &  200 \\
   \hline
  3 & Communities &  4 \\
   \hline
  4 & Observed homophily &  0.74 \\
   \hline
  5 & Modularity &  0.51 \\
   \hline
  6 & Avg. Clustering Coeff &  0.33 \\
   \hline
  7 & Avg. Degree &  5 \\
   \hline
  8 & Edges &  500 \\
   \hline
  9 & Network Type &  Directed-Unweighted \\
  \hline
\end{tabular}
\end{table}

\begin{figure}[H]
\centering
\fbox{\includegraphics[scale=0.5]{dataset.png}}
\caption{Artificially generated Network dataset}
\end{figure}

\subsection{Experimental Results}

\subsubsection{InfoMap}
"infomap.community" detected multiple communities in the network with few nodes leading to the conclusion that it split large clusters. The clustering is of poor quality as seen in Fig.\ref{InfoMap} as observed in the performance metrics.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  0.6 secs \\
   \hline
  2 & Modularity &  0.567 \\
   \hline
  3 & Variation of information &  1.72 \\
   \hline
  4 & Normalized mutual information &  0.565 \\
   \hline
  5 & Split-join distance &  115 \\
   \hline
  6 & Rand index &  0.717 \\
   \hline
  7 & Adjusted Rand index &  0.231 \\
  \hline
   8 & Detected Communities &  22 \\
  \hline
\end{tabular}
\end{table}

\subsubsection{Leading Eigenvector}
"leading.eigenvector.community" has uncovered a community structure with less communities as seen in Fig.\ref{leadg} and its performance is good on the metrics.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  0.53 secs \\
   \hline
  2 & Modularity &  0.551 \\
   \hline
  3 & Variation of information &  0.88 \\
   \hline
  4 & Normalized mutual information &  0.67 \\
   \hline
  5 & Split-join distance &  74 \\
   \hline
  6 & Rand index &  0.795 \\
   \hline
  7 & Adjusted Rand index &  0.510 \\
  \hline
   8 & Detected Communities &  6 \\
  \hline
\end{tabular}
\end{table}

\subsubsection{Label Propagation}
"label.propagation.community" has detected low number of communities as seen in Fig.\ref{labelp}. Performance of this technique is better than other approaches.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  0.06 secs \\
   \hline
  2 & Modularity &  0.545 \\
   \hline
  3 & Variation of information &  0.597 \\
   \hline
  4 & Normalized mutual information &  0.778 \\
   \hline
  5 & Split-join distance &  35 \\
   \hline
  6 & Rand index &  0.91 \\
   \hline
  7 & Adjusted Rand index &  0.81 \\
  \hline
   8 & Detected Communities &  8 \\
  \hline
\end{tabular}
\end{table}

\subsubsection{Walktrap}
"walktrap.community" uncovers a community structure with higher modularity as seen in Fig.\ref{walkt}. Large number of small communities have been created and so this technique has low values on the performance metrics.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  0.11 secs \\
   \hline
  2 & Modularity &  0.579 \\
   \hline
  3 & Variation of information &  1.32 \\
   \hline
  4 & Normalized mutual information &  0.548 \\
   \hline
  5 & Split-join distance &  76 \\
   \hline
  6 & Rand index &  0.777 \\
   \hline
  7 & Adjusted Rand index &  0.446 \\
  \hline
   8 & Detected Communities &  15 \\
  \hline
\end{tabular}
\end{table}

\subsubsection{Spinglass and Clique Percolation}
"spinglass.community" detects high number of communities and has low values on VI, Split join and ARI metrics. Hence clustering quality is low as seen in Fig.\ref{sping}. Clique percolation with $k=3$ detected community structure given in as seen in Fig.\ref{cliquep} but 10\% of the nodes were unclassified.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  13.91 secs \\
   \hline
  2 & Modularity &  0.35 \\
   \hline
  3 & Variation of information &  1.12 \\
   \hline
  4 & Normalized mutual information &  0.638 \\
   \hline
  5 & Split-join distance &  94 \\
   \hline
  6 & Rand index &  0.772 \\
   \hline
  7 & Adjusted Rand index &  0.419 \\
  \hline
   8 & Detected Communities &  8 \\
  \hline
\end{tabular}
\end{table}

\subsection{Community Structures}
The figures below show the community structures uncovered by different approaches.

\begin{figure}[H]
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{infomap.png}
    \caption{infomap}
    \label{InfoMap}
  \end{subfigure}
  \hfill %%
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{leadg.png}
    \caption{Leading Eigenvector}
    \label{leadg}
  \end{subfigure}
\caption{Community structures in Network}
\end{figure}

\begin{figure}[H]
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{labelp.png}
    \caption{Label propagation}
    \label{labelp}
  \end{subfigure}
  \hfill %%
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{walkt.png}
    \caption{Walktrap}
    \label{walkt}
  \end{subfigure}
 \caption{Community structures in Network}
\end{figure}

\begin{figure}[H]
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{sping.png}
    \caption{Spinglass}
    \label{sping}
  \end{subfigure}
  \hfill %%
  \begin{subfigure}[b]{0.3\linewidth}
    \includegraphics[width=\linewidth]{cliquep.png}
    \caption{Clique Percolation}
    \label{cliquep}
  \end{subfigure}
  \caption{Community structures in Network}
\end{figure}

\subsection{Variation of BIGCLAM}
With the above experimental results as baselines, the variation of BIGCLAM approach suggested in the paper was applied. The attributes were used to create a Bipartite graph as given by Jure Lescovec \textit{et. al} \cite{aps:53}, however the key difference was that the nodes were now having edges with the attributes. Network topology was thus ignored and BIGCLAM now used the Non negative matrix factorization to calculate the strength of memberships of the nodes to the attributes. These would be used to decide memberships of nodes to communities. As seen in Fig.\ref{bigc} the hold out set has highest likelihood at $k=6$. One advantage of this approach was large communities were formed and so it was inferred that splitting into smaller communities was avoided. As BIGCLAM is highly scalable, the network of $10^5$ nodes can be processed efficiently.

\begin{table}[H]
\renewcommand{\arraystretch}{1.3}
\caption{Results}
\label{table}
\centering
\begin{tabular}{|c|c|c|}
  \hline
\multicolumn{1}{|c|}{\textbf{Sr. No}} & \multicolumn{1}{c|}{\textbf{Parameter}} & \multicolumn{1}{c|}{\textbf{Value}} \\
  \hline
  1 & Execution Time &  8.51 secs \\
   \hline
  2 & Maximum likelihood Estimate &  -1070 \\
  \hline
   3 & Detected Communities &  6 \\
  \hline
\end{tabular}
\end{table}

\begin{figure}[H]
\centering
\fbox{\includegraphics[scale=0.3]{bigc.png}}
\caption{Clustering Structure using Variant of BIGCLAM}
\label{bigc}
\end{figure}

\section{Conclusion}
Community detection using a joint model of node attributes and network topology is a challenging task as additional information has to be factored while maintaining efficiency criteria. The model implemented in this paper uses Non negative matrix factorization on Bipartite Attribute Affiliation model for community detection. This approach allows for nodes to have high membership strengths simultaneously to various attribute communities. This allows for creating nested, overlapping and hierarchical communities in networks. The intuition behind this technique is that if the number of attributes shared by two nodes are high then the nodes have a higher probability of belonging to a single community. As this approach is relying on the optimization principle of BIGCLAM hence it is possible to state that this method can also be scaled to large networks efficiently.


%!TEX root = ../TTT4150-Summary.tex
\section{Introduction}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{A brief history of navigation}

You want to know where you are. Latitude is easy to determine with a sextant, with e.g. the North star or the sun at its highest point. By keeping the home port time with a clock onboard, and measuring time where you are, you can estimate longitude. But clocks (or ``marine chronometers'' if you're a huge nerd) were too inaccurate until John Harrison made a really nice one in the 18th century. Some guys also computed time by observing the moons of Saturn.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Radionavigation}

Ground waves are below HF and hug the terrain. Sky waves are in the same frequency range and reflect back down from the ionosphere. Can communicate past the horizon, but hard to determine propagation time, which is necessary for navigation. Space waves are VHF and up, and go straight.

\subsubsection{Trilateration}
Use propagation time to estimate distance from source. Compute position from several distances. You need three sources for 2D positioning (or two sometimes). For 3D positioning, you need at least one beacon with a large $\Delta h$, which you have with satellites but not ground beacons.

\subsubsection{Hyperbolic positioning}
You get signals from two transmitters, and measure the difference in propagation time. If the difference is zero, your position is equally far from both, and therefore somewhere on a straight line. If the difference is nonzero, your position is a bit closer to one transmitter, and will be along a hyperbolic line. If you have another transmitter pair, each pair gives a line of possible positions, and your location should be at their intersection. Figure \ref{fig:hyperbolic-navigation} illustrates this, with time differences of $0.3$ units between antenna 1 and 2, and $-0.1$ units between 1 and 3. Since you only worry about differences, your clock offset cancels, assuming all transmitter clocks are synced.

\begin{figure}[htbp]
    \centering
    \includegraphics[width=.8\linewidth]{img/hyperbolic-navigation}
    \caption{The principle of hyperbolic navigation}
    \label{fig:hyperbolic-navigation}
\end{figure}

\subsubsection{Doppler positioning}
Observe the doppler shift of a satellite. If you know the orbit, you can determine your position.


\documentclass[a4paper,11pt]{article}
\usepackage{amsmath, amsthm, amssymb, commath, graphicx, bbold, endnotes, graphicx, subfigure, multirow, setspace}
\usepackage[font=footnotesize]{caption}
\usepackage[top=1.25in, bottom=1.25in, left=1.25in, right=1.25in]{geometry}
\linespread{1}
\setlength{\parindent}{30pt}

\title{Cross Phase Calculation Memo}
\author{Ruby Byrne}
\date{April 2018}

\begin{document}

\maketitle

\section{Background}

One degree of freedom in polarized calibration is the overall phase offset between the $x$- and $y$-dipoles. This phase offset, called $\phi$ for the purposes of this memo, corresponds to a mixing between the Stokes U and V modes. As we do not expect sky emission to be circularly polarized, we can constrain $\phi$ by minimizing Stokes V emission.

We use pseudo-Stokes parameters to simplify the calculation, allowing us to work in visibility space without gridding. Pseudo-Stokes parameters are calculated from the visibilities as follows:

\begin{equation}
\begin{bmatrix}
	I^{\text{pseudo}} \\
	Q^{\text{pseudo}} \\
	U^{\text{pseudo}} \\
	V^{\text{pseudo}} \\
\end{bmatrix}
=
\begin{bmatrix}
	1 & 1 & 0 & 0 \\
	1 & -1 & 0 & 0 \\
	0 & 0 & 1 & 1 \\
	0 & 0 & i & -i \\
\end{bmatrix}
\begin{bmatrix}
	xx^* \\
	yy^* \\
	xy^* \\
	yx^* \\
\end{bmatrix}
\end{equation}

It is clear that the transformation $x \rightarrow x e^{-i \phi/2}$ and $y \rightarrow y e^{i \phi/2}$ leaves pseudo-I and pseudo-Q unchanged, but affects the pseudo-U and pseudo-V parameters.

\section{Calculating the Cross Phase}

To calculate $\phi$, we minimize the quantity $\sum_n |V^{\text{pseudo}}_n|^2$ where $n$ denotes the visibility index. We write the visibilities in terms of an amplitude and phase, such that $(xy^*)_n = A_n e^{i\delta_n} \rightarrow A_n e^{i\delta_n - i\phi}$ and $(yx^*)_n = B_n e^{i\gamma_n} \rightarrow B_n e^{i\gamma_n + i\phi}$. Now
\begin{equation}
V^{\text{pseudo}}_n = A_n e^{i(\delta_n - \phi + \pi/2)} + B_n e^{i(\gamma_n + \phi - \pi/2)}
\end{equation}
It follows that
\begin{equation}
|V^{\text{pseudo}}_n|^2 = A_n^2 + B_n^2 - 2 A_n B_n \left[ \cos(\delta_n - \gamma_n) \cos(2 \phi) + \sin(\delta_n - \gamma_n) \sin(2 \phi) \right]
\end{equation}
Now to find $\phi$ we set $\frac{\partial}{\partial \phi} \sum_n |V^{\text{pseudo}}_n|^2 = 0$. This gives
\begin{equation}
\tan(2 \phi) = \frac{\sum_n A_n B_n \sin(\delta_n - \gamma_n)}{\sum_n A_n B_n \cos(\delta_n - \gamma_n)}
\end{equation}
Rewriting this in terms of the visibilities, we get
\begin{equation}
\tan(2 \phi) = \frac{\sum_n \text{Im} \left[ (xy^*)_n (yx^*)_n{}^* \right]}{\sum_n \text{Re} \left[ (xy^*)_n (yx^*)_n{}^* \right]}
\end{equation}

\section{Implementation in FHD}

This calculation is implemented in FHD in the full-pol branch with the function \linebreak[4] \texttt{vis\_calibrate\_crosspol\_phase}.

It is important to note that FHD convention stores visibilities in the \texttt{vis\_ptr} object, where counterintuitively \texttt{vis\_ptr[0]} corresponds to $yy^*$, \texttt{vis\_ptr[1]} corresponds to $xx^*$, \texttt{vis\_ptr[2]} corresponds to $yx^*$, and \texttt{vis\_ptr[3]} corresponds to $xy^*$.

The calculated value of $\phi$ is stored in the \texttt{cal.cross\_phase} object and the correct transformation is applied to the antenna gains. Since the raw visibilities are divided by the gains, the transformation $x \rightarrow x e^{-i \phi/2}$ is equivalent to $g_x \rightarrow g_x e^{i \phi/2}$ and $y \rightarrow y e^{i \phi/2}$ is equivalent to $g_y \rightarrow g_y e^{-i \phi/2}$. Here $g_x$ is the $x$-dipole gain (\texttt{cal.gain[1]}) and $g_y$ is the $y$-dipole gain (\texttt{cal.gain[0]}).

\end{document}


Now we've pinned down the mechanics of how we will incorporate our signal functions into real working code, we can write our first example. Save a copy of \hask{YampaUtils.hs} in your working directory, for your use in these examples. Our first example, Listing \ref{lst:helloYampa}, is the Hello, World of Yampa.

\lstinputlisting[caption={HelloYampa.hs}, label=lst:helloYampa]{./src/HelloYampa.hs}

Our discussion in the introduction about \hask{reactimate}, and our tidied version \yampaMain, should give you a rough idea of what is going on here. \yampaMain first calls \hask{initialise}, which prints some text to the screen, but provides no input (or, more precisely, provides as input \hask{()}). The input function passed to \yampaMain is just \hask{return Nothing}, which again provides no input to the system.

\hask{waitTwo} is a signal function, and is the new code in this listing. For the moment, lets just observe that it converts a signal of values of type \hask{()} into a signal of values of type \hask{Bool}, which is what its type signature \hask{SF () Bool} is meant to indicate (`` a signal function from signals of type \hask{()} to signals of type \hask{Bool}'').

The output function, \hask{output}, takes this boolean value, and if its false, just instructs \yampaMain to continue by passing out a \hask{False} value in the \hask{IO} monad. However, if the boolean value is true, the output function first prints a message to the screen, and then returns an \hask{IO} value of \hask{True}, which instructs \yampaMain to stop executing, and the program finishes.

Precisely when the value \hask{True} emerges from \yampaMain is determined in the signal function \hask{waitTwo}. So lets take a look at this object.

\begin{lstlisting}
waitTwo :: SF () Bool
waitTwo = time >>> arr (>=2)
\end{lstlisting}

The specification for \hask{waitTwo} is that it should take the input signal (which here is constantly taking value \hask{()}), and if \hask{waitTwo} has been transforming input values for less than 2 seconds, it should transform the input into a value \hask{False}, and otherwise, it should transform the input signal into a value \hask{True}. This boils down to saying, \yampaMain should receive a value \hask{True} to give to \hask{output}, precisely when \yampaMain has been running for 2 seconds. Compiling and running this code seems to confirm this.

So how has \hask{waitTwo} been defined? It consists of two parts combined together with the operator \hask{>>>}. First lets take a look at \hask{time}.

\begin{lstlisting}
time :: SF a Time
\end{lstlisting}

\noindent \hask{time} takes a signal of any value, and replaces it with a value of type \hask{Time}, a synonym for \hask{Double}. The value is the time the particular instantiation of \hask{time} has been in use, starting from $0$. So in our case, it takes the value \hask{()}, and replaces it with the time that \yampaMain has been using \hask{waitTwo}, which is the length of time \yampaMain has been running.

Secondly, we have \hask{arr (>=2)}. Now inspecting the type of this in ghci reveals \hask{arr} is a function which makes use of the \hask{Arrow} type class. \hask{SF} is an instance of this type class, and specialising its type signature to \hask{SF}, we have:

\begin{lstlisting}
arr :: (a -> b) -> SF a b
\end{lstlisting}

\noindent \hask{arr} is a way of lifting pure functions into \hask{SF}, analogous to how \hask{return} lifts values in a monad, and \hask{pure} lifts values for applicative functors. More precisely, if \hask{f :: a -> b}, then \hask{arr f} takes a signal of values of type \hask{a}, and transforms it into a signal of values of type \hask{b}, by applying \hask{f} to each value of type \hask{a}. In some sense, it is the continuous signal analogue of \hask{fmap} for the functor typeclass.

In our example, we have the signal function \hask{arr (>=2)}, which by specialising to types \hask{SF} again has type:

\begin{lstlisting}
arr (>=2) :: (Num a, Ord a) => SF a Bool
\end{lstlisting}

\noindent \hask{arr (>=2)} takes a signal with numerical and orderable values, and applies \hask{(>=2)} to these values. So our transformed signal will output \hask{True} precisely when the input signal takes on values greater than or equal to 2.

So we have two signal functions, with an operator which combines them (a \emph{combinator}). Lets inspect the type of \hask{>>>}, specialised to \hask{SF} again.

\begin{lstlisting}
(>>>) :: SF a b -> SF b c -> SF a c
\end{lstlisting}

\noindent The type signature (and the fact that the more general version is for objects from the typeclass \hask{Category}), is a giveaway for what \hask{(>>>)} does. \hask{(>>>)} is the \hask{SF} analogue of composition. Given a way of transforming signals of value type \hask{a} into signals of value type \hask{b}, and a way of transforming signals of value type \hask{b} into signals of value type \hask{c}, by performing each transformation in order, I can transform signals of value type \hask{a} into signals of value type \hask{c}. This is the function of the \hask{(>>>)} combinator. More precisely, \hask{f >>> g} first transforms with \hask{f}, and then with \hask{g} (recall \hask{f.g} first applies g \emph{then} applies \hask{f}).

So now we can analyse what \hask{waitTwo} actually does. \hask{waitTwo} takes an input signal of values \hask{()}, and replaces these values with the time which \yampaMain has been running. It then takes this time, and if it is less than 2 seconds, replaces this time value with a \hask{False}. Otherwise, it replaces this time value with \hask{True}. In total, \hask{waitTwo} transforms \hask{()} into \hask{False} if \yampaMain has been running less than 2 seconds, and \hask{True} once its been running at least two seconds.

\begin{observation}
Observe that \hask{arr (f.g) = arr g >>> arr f}. Observe that \hask{time} is not the \hask{arr} of a pure function.
\end{observation}

\section{Building signal functions}

Our first example is intentionally not the most intricate or complicated of examples. We build a very simple signal function, with the intention of showing how a Yampa program fits together.

The bulk of the Yampa library is concerned with building signal functions. Rather than building signal functions explicitly, Yampa supplies a basic set of signal functions (we saw for instance, \hask{time}, above), and a set of \emph{combinators} for combining various signal functions (for instance, signal function composition \hask{>>>}).

\section{Some basic signal functions and combinators}

Lets begin with some of the basic signal functions which Yampa provides. As one might expect from a library based around transformations, one has analogues for the identity map, and constant maps.

\begin{lstlisting}
identity :: SF a a
constant :: b -> SF a b
\end{lstlisting}

\noindent Of course, \hask{identity} preserves the value of a signal, and \hask{constant v} takes a signal, and replaces its value by the value \hask{v}.

\begin{exercise}
Write definitions for \hask{identity} and \hask{constant} in terms of \hask{arr}.
\end{exercise}

In our first example, we saw the signal function \hask{time}. \hask{time} takes a signals value, and replaces it with the \emph{local} time, that is the time a signal function has been processing a signal. For the moment, this would appear to be the same time as \yampaMain has been running, and with what has been covered so far, it is, but later we will see that signal functions can be switched in and out in response to events, at which point the time counter starts over from zero. For the sake of completeness (recall \hask{Time} is a synonym for Double):

\begin{lstlisting}
time :: SF a Time
\end{lstlisting}

An interesting built in function is \hask{integral}. This performs numerical integratation of a signal over time. Of course this is useful for describing dynamical systems!

\begin{lstlisting}
integral :: SF Double Double
\end{lstlisting}

\hask{integral} actually has a more general type than the above, but this suffices for now. Yampa provides an similar signal function for derivatives also.

\begin{observation}
\hask{time = constant 1.0 >>> integral}.
\end{observation}

We also have, as we have seen already, a way of lifting pure functions \hask{a -> b} to values of type \hask{SF a b}:

\begin{lstlisting}
arr :: (a -> b) -> SF a b
\end{lstlisting}

\noindent \hask{arr f} will take a signal of type \hask{a}, and process its value with \hask{f}, to create a signal of type \hask{b}.

Now that we have a basic collection of arrows, we woud like some ways to combine them, to form more interesting arrows. The type constructor \hask{SF} is an instance of the \hask{Arrow} typeclass, a generalisation of the \hask{Monad} type class, so we have all the combinators from \hask{Arrow} available for use. We have already seen \hask{arr} above, which is part of the \hask{Arrow} interface, and earlier we saw \hask{(>>>)}.

\hask{(>>>)} is the composition operator for signal functions. \hask{(f >>> g)} applies \hask{f} to a signal, then \hask{g} (note the order is reverse of \hask{.}). Similarly, \hask{(<<<)} is the composition operator the other way around, so \hask{(f <<< g)} performs \hask{g} then \hask{f}.

\begin{lstlisting}
(>>>) :: SF a b -> SF b c -> SF a c
(<<<) :: SF b c -> SF a b -> SF a c
\end{lstlisting}

Sometimes we want to process a signal in two different ways. The combinator \hask{(\&\&\&)} is designed to allow us to do exactly that.

\begin{lstlisting}
(&&&) :: SF a b -> SF a c -> SF a (b,c)
\end{lstlisting}

\noindent Here \hask{(f \&\&\& g)} is intended to mean take the value of a signal, and apply \hask{f} to it, placing the output in the first component of a tuple. Also apply \hask{g} to the value of the signal, and place the output in the second component of the tuple.

Since we can produce a signal which takes a tuple for values, (which we can think of as being two signals captured in one), we have combinators for processing them.

First we have \hask{(***)}, which allows us to take two signal functions and make each act on a particular element of a tuple.

\begin{lstlisting}
(***) :: SF a1 b1 -> SF a2 b2 -> SF (a1, a2) (b1, b2)
\end{lstlisting}

\noindent \hask{(f *** g)} acts on a signal which takes a tuple for values, by applying \hask{f} to the first component, and \hask{g} to the second component.

Sometimes, we only want to transform one of the elements of a signal of tuple type. While something of the form \hask{(f *** identity)} would do, Yampa defines such combinators for us:

\begin{lstlisting}
first :: SF a b -> SF (a, c) (b, c)
second :: SF b c -> SF (a, b) (a, c)
\end{lstlisting}

\noindent \hask{first f} transforms only the first component of a signal of tuple values with \hask{f}, and \hask{second g} transforms only the second component of a signal with tuple values with \hask{g}.

\begin{exercise}
Write \hask{second} in terms of \hask{first} and \hask{arr}.
\end{exercise}

\begin{observation}
\hask{ (f *** g) = (first f) >>> (second g) }.
\end{observation}

Our last combinator is \hask{loop}. \hask{loop} allows us to build recursive signal functions. It has type

\begin{lstlisting}
loop :: SF (a, c) (b, c) -> SF a b
\end{lstlisting}

\noindent \hask{(loop f)} is obtained from \hask{f} by looping its values' second component back in to \hask{f} for the next sample.

\section{Arrow notation}

The above primitives and combinators allow us to form more complex signal functions. In fact, any combination of basic signal functions can be formed. However, using the combinators directly can often lead to code which is difficult to read. For this reason, the arrow syntax was developed. This is the arrow analogue of monadic do notation. It allows us to express more clearly how we wish to process the values which signals take. A preprocessor transforms this notation into a description using the combinators we described above. When using arrow notation in ghci, load ghci with the -XArrows option, e.g. \hask{ghci -XArrows}. Similarly, when compiling with ghc, use the -XArrows option, e.g. \hask{ghc -XArrows main.hs}.

Let us first inspect the general form of a block in arrow syntax. Also, take a look at the example that follows the discussion, since this will make it clearer. The basic form of arrow syntax is given in the following.

\begin{lstlisting}
sigFun :: SF a b
sigFun = proc input -> do
  processedSample1 <- sigFun1 -< sample1
  processedSample2 <- sigFun2 -< sample2
  ...
  returnA -< finalSample
\end{lstlisting}

\hask{proc} is a keyword to indicate that the following is a block written in arrow syntax. \hask{input} represents a sample of type \hask{a}, and can be pattern matched against. A line of the form

\begin{lstlisting}
processedSample <- sigFun -< sample
\end{lstlisting}

\noindent takes the sample (from a signal) named \hask{sample}, processes it with \hask{sigFun}, and binds the resulting value to \hask{processedSample}. We can pattern match in the position of \hask{processedSample}. The line

\begin{lstlisting}
returnA -< finalSample
\end{lstlisting}

\noindent yields \hask{finalSample} as the final transformed sample. Note there is no binding, just like (and for the same reason as) a monadic \hask{do} block. As with \hask{do} blocks, \hask{let} bindings are also allowed in arrow syntax. As a word of warning, recursive definitions in arrow syntax must be preceeded by the keyword \hask{rec}. We will investigate this fully later.

By way of example, lets describe a signal function which receives a signal of type \hask{Double}, which is intended to represent the acceleration of an object, and is intended to output a signal describing its position (assuming it starts at $0$, lets say). Using the normal combinators we could write the following:

\begin{lstlisting}
accToPos :: SF Double Double
accToPos = integral >>> integral
\end{lstlisting}

\noindent which to be fair, is clear enough. In arrow syntax however, we can write this as

\begin{lstlisting}
accToPos :: SF Double Double
accToPos = proc acc -> do
  vel <- integral -< acc
  pos <- integral -< vel
  returnA -< pos
\end{lstlisting}

\noindent So we obtain \hask{vel} (velocity) by integrating the input \hask{acc} (acceleration), and we obtain \hask{pos} (position) by integrating the velocity \hask{vel}. \hask{pos} is the value we want, so we return that. It is not too hard to imagine signal functions where using the arrow syntax can greatly aid clarity. Observe that the wiring of samples through signal functions is represented in an intuitive manner.

The above example would potentially be used in an animation program, or a simulation of some kind. We will build these kinds of programs later, but for the moment it would be nice to have some working code. We will develop our ``Hello, Yampa!'' program a little further. Here, we start by asking the user to specify a time to delay finishing the greeting, and feed this value in to \yampaMain via the initialisation value.

\lstinputlisting[caption={askAndPause.hs}, label=lst:askingAndPausing]{./src/askAndPause.hs}


\chapter{Algorithm for Herbrand Analysis}
\label{chap:chapter4}

This chapter presents an algorithm for Herbrand analysis. The pseudocode
mentioned here is an updated version of the algorithm mentioned in \cite{Babu}
for Herbrand equivalence analysis. The corresponding implementation done for
LLVM compiler framework and a toy language can be found on
\href{https://www.github.com/himanshu520/HerbrandEquivalence}{GitHub}.
Also, see \autoref{chap:chapter7} and \autoref{chap:chapter8} for more details.

One important distinction must be clear between the general Herbrand analysis problem
and the one that the algorithm in this chapter addresses. The universe for the Herbrand
equivalence problem is the set of all expressions that can be formed using constants,
variables and operators used in the program. But as already mentioned, the algorithm
here is concerned with a restricted universe - set of expressions of length atmost two
formed using constants, variables and operators in the program.

\section{Notation}
\label{sec:NotationPseudocode}

Let $\mathcal C$ and $\mathcal X$ be the set of constants and variables used in the program;
$\mathcal W$ be our working set, which is the set of all expressions of length at most two
that can be formed using ($\mathcal C \cup \mathcal X$). Also, $V$ be the set of all program
points, with a special point \texttt{START} denoting the beginning of the program.
\bigbreak \noindent Three global variables are maintained -
\begin{itemize} \tightlist
    \item \texttt{Partitions} : It is two-dimensional integer array indexed by the
                                elements of sets $V$ and $\mathcal W$ respectively. It
                                helps to keep track of partitions at some $v \in V$ by
                                holding same integer at \texttt{Partitions[$v$][$e$]} and
                                \texttt{Partitions[$v$][$e'$]}, for $e, e' \in \mathcal W$
                                if they belongs to the same equivalence class at $v$.
                                Basically, the array maintains set identifiers which helps
                                to identify whether two expressions belong to the same sets
                                (equivalence classes).
    \item \texttt{SetCnt} : It helps to keep track of the next set identifier (an integer)
                            to be used. Whenever, a set identifier is needed the current
                            value of \texttt{SetCnt} is used and at the same time it is
                            incremented so that the same number is never used again.
    \item \texttt{Parent} : It is \texttt{map} indexed by a tuple of three elements - an
                            operator and two set identifiers. Whenever an expression
                            ($x + y$) is assigned a set identifier $c$, $c$ is stored in
                            \texttt{Parent} with $\{+, a, b\}$ as key, where $a$ and $b$ are
                            the set identifiers of $x$ and $y$ respectively. Next time
                            when a set identifier for an expression ($x' + y'$) is needed,
                            where $x'$ and $y'$ have identifiers $a$ and $b$ respectively,
                            $c$ is used instead of using a new set identifier - the \texttt
                            {Parent} of $\{+, a, b\}$ that was earlier stored in the map.
\end{itemize}

\bigbreak \noindent There are a few functions whose definitions are not required explicitly -
\begin{itemize} \tightlist
    \item \texttt{OPERATOR($e$)} : Returns operator used in expression $e \in (\mathcal W \setminus (\mathcal C \cup \mathcal X))$
    \item \texttt{LEFT($e$)} : Returns left operand of expression $e \in (\mathcal W \setminus (\mathcal C \cup \mathcal X))$
    \item \texttt{RIGHT($e$)} : Returns right operand of expression $e \in (\mathcal W \setminus (\mathcal C \cup \mathcal X))$
    \item \texttt{PREDECESSORS($v$)} : Returns the set of predecessors of program point $v \in V$
\end{itemize}

\bigbreak \noindent \textbf{NOTE} - For implementation, the identifiers corresponding to
$\top$ partition should be such that they don't occur in normal partitions and are also
easily distinguishable from them. An easy choice for consistency is to use non-negative
integers as normal set identifiers and an array of -1 to refer $\top$ partition.

\section{Pseudocode}
\label{sec:Pseudocode}

\begin{algorithm}
    \caption{Main Herbrand Equivalence Analysis Function}
    \label{alg:HerbrandEquivalenceAnalysis}
    \begin{algorithmic}
        \Procedure{HerbrandAnalysis}{$ $}
            \State \Comment{\% Initialise \texttt{Partitions} for all program points \%}
            \For{$v \in V$}
                \State \texttt{Partitions[$v$]} $\gets \top$
            \EndFor
            \State \Comment{\% Update \texttt{Partitions} for \texttt{START} point \%}
            \State \Call{findInitialPartition}{$ $}
            \State \Comment{\% Process all program points till convergence \%}
            \State \textbf{converged} $\gets$ \texttt{false}
            \While{\texttt{converged} \textit{is False}}
                \State \texttt{converged $\gets$ true}
                \For{$v \in (V \setminus \{\texttt{START}\})$}
                    \State \textbf{oldPartition} $\gets$ \texttt{Partitions[$v$]}
                    \State \Comment{\% Update \texttt{Partitions} at $v$ \%}
                    \If{$v$ \textit{is a Transfer Point}}
                        \State \Call{TransferFunction}{$v$}
                    \Else
                        \State \Call{ConfluenceFunction}{$v$}
                    \EndIf
                    \State \Comment{\$ Update \texttt{convergence} flag \%}
                    \If{\textit{not} \Call{SamePartition}{\texttt{oldPartition, Partitions[$v$]}}}
                        \State \texttt{converged $\gets$ false}
                    \EndIf
                \EndFor
            \EndWhile
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Transfer Function}
    \label{alg:TransferFunction}
    \begin{algorithmic}
        \Procedure{TransferFunction}{$v : x \gets e$}
            \State \textbf{u} $\gets$ \Call{Predecessors}{$v$}
            \State \texttt{Partitions[$v$] $\gets$ Partitions[$u$]}
            \State \Comment{\% Update set identifier for $x$ \%}
            \If{$e$ \textit{is Deterministic}}
                \State \texttt{Partitions[$v$][$x$] $\gets$ Partitions[$v$][$e$]}
            \Else
                \State \texttt{Partitions[$v$][$x$] $\gets$ SetCtr++}
            \EndIf
            \State \Comment{\% Update set identifiers for expressions containing $x$ \%}
            \For{$\{e' \in (\mathcal W \setminus (\mathcal C \cup \mathcal X)) \mid x \in e'\}$}
                \State \texttt{Partitions[v][$e'$]} $\gets$ \Call{GetSetId}{$v$, $e'$}
            \EndFor
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Confluence Function}
    \label{alg:ConfluenceFunction}
    \begin{algorithmic}
        \Procedure{ConfluenceFunction}{$v$}
            \State \Comment{If all predecessor partitions are $\top$ then current partition will also be $\top$}
            \State \textbf{continueFlag} $\gets$ false
            \For{$u \in$ \Call{Predecessors}{$v$}}
                \If{\texttt{Partitions[$u$] $\neq \top$}}
                    \State \texttt{continueFlag $\gets$ true}
                \EndIf
            \EndFor
            \State
            \If{\texttt{continueFlag} \textit{is False}}
                \State \texttt{Partitions[$v$] $\gets \top$}
                \State \Return{}
            \EndIf
            \State \Comment{\texttt{accessFlag} keeps track of processed expressions}
            \For{$e \in \mathcal W$}
                \State \textbf{accessFlag}[$e$] $\gets$ \texttt{false}
            \EndFor
            \State \Comment{Process all expressions if they are still unprocessed}
            \For{$e \in \mathcal W$}
                \If{\texttt{accessFlag[$e$]} \textit{is False}}
                    \State \Comment{\texttt{PredIDs} is the set of set identifiers of $e$ at its predecessors}
                    \State \textbf{PredIDs} $\gets \phi$
                    \For{$u \in$ \Call{Predecessors}{$v$}}
                        \If{\texttt{Partitions[u] $\neq \top$}}
                            \State \texttt{PredIDs $\gets$ (PredIDs $\cup$ Partitions[$u$][$e$])}
                        \EndIf
                    \EndFor
                    \State
                    \If{\texttt{PredIDs} \textit{is Singleton}}
                        \State \texttt{Partitions[$v$][$e$] $\gets$ PredIDs}
                        \State \texttt{accessFlag[$e$] $\gets$ true}
                    \Else
                        \State \Comment{\texttt{expClass} holds $e' \in \mathcal W$ that are equivalent to $e$ at all predecessors of $v$}
                        \State \textbf{expClass} $\gets \mathcal W$
                        \For{$u \in$ \Call{Predecessors}{$v$}}
                            \If{\texttt{Partitions[u] $\neq \top$}}
                                \State \texttt{expClass $\gets$ (expClass} $\cap$ \Call{GetClass}{$u$, $e$})
                            \EndIf
                        \EndFor
                        \State \Comment{Update \texttt{Partitions} map}
                        \State \textbf{newSetID} $\gets$ \texttt{SetCnt++}
                        \For{\texttt{$e' \in$ expClass}}
                            \State \texttt{Partitions[$v$][$e'$] $\gets$ newSetID}
                            \State \texttt{acessFlag[$e'$] $\gets$ true}
                        \EndFor
                    \EndIf
                \EndIf
            \EndFor
            \State
            \State \Comment{Update \texttt{Parent} map}
            \For{$e \in (\mathcal W \setminus (\mathcal C \cup \mathcal X))$}
                \State \textbf{op} $\gets$ \Call{Operator}{$e$}
                \State \textbf{leftSetID} $\gets$ \texttt{Partitions}[\Call{Left}{$e$}]
                \State \textbf{rightSetID} $\gets$ \texttt{Partitions}[\Call{Right}{$e$}]
                \State \texttt{Parent[\{op, leftSetID, rightSetID\}] $\gets$ Partitions[v][$e$]}
            \EndFor
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Checks whether two partitions are same or not}
    \label{alg:SamePartition}
    \begin{algorithmic}
        \Procedure{SamePartition}{\texttt{first, second}}
            \For{$e \in \mathcal W$}
                \If{\Call{GetClass}{\texttt{first}, $e$} $\neq$ \Call{GetClass}{\texttt{second}, $e$}}
                    \State \Return{\texttt{false}}
                \EndIf
            \EndFor
            \State \Return{\texttt{true}}
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Finds equivalence class of an expression in a partition}
    \label{alg:GetClass}
    \begin{algorithmic}
        \Procedure{GetClass}{\texttt{partition}, $e$}
            \State \textbf{expClass} $\gets \phi$
            \For{$e' \in \mathcal W$}
                \If{\texttt{partition[$e'$] == partition[$e$]}}
                    \State \texttt{expClass $\gets$ (expClass $\cup$ $\{e'\}$)}
                \EndIf
            \EndFor
            \State \Return{\texttt{expClass}}
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Initialises partition for START point}
    \label{alg:FindInitialPartition}
    \begin{algorithmic}
        \Procedure{FindInitialPartition}{$ $}
            \For{$x \in (\mathcal C \cup \mathcal X)$}
                \State \texttt{Partitions[START][$x$] $\gets$ SetCtr++}
            \EndFor
            \For{$e \in (\mathcal W \setminus (\mathcal C \cup \mathcal X))$}
                \State \texttt{Partitions[START][$e$]} $\gets$ \Call{GetSetID}{\texttt{Partitions[START], $e$}}
            \EndFor
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\begin{algorithm}
    \caption{Finds and returns set identifier for a two length expression in a partition, by
             looking at its operands and operator}
    \label{alg:GetSetID}
    \begin{algorithmic}
        \Procedure{GetSetID}{\texttt{partition, $e$}}
            \State \textbf{op} $\gets$ \Call{Operator}{$e$}
            \State \textbf{leftSetID} $\gets$ \texttt{partition}[\Call{Left}{$e$}]
            \State \textbf{rightSetID} $\gets$ \texttt{partition}[\Call{Right}{$e$}]
            \State
            \If{\textit{not defined} \texttt{Parent[\{op, leftSetID, rightSetID\}]}}
                \State \texttt{Parent[\{op, leftSetID, rightSetID\}] $\gets$ SetCtr++}
            \EndIf
            \State
            \State \Return{\texttt{Parent[\{op, leftSetID, rightSetID\}]}}
        \EndProcedure
    \end{algorithmic}
\end{algorithm}

\section{Updates Over Original Algorithm}
\label{sec:UpdatesOverOriginalAlgorithm}
Following major improvements/corrections have been made to the original algorithm
mentioned in \cite{Babu}.
\begin{itemize} \tightlist
    \item \textbf{Representing Partitions}\\
    The original algorithm uses \texttt{ID structure} to maintain equivalence information. A two-dimensional array \texttt{Partitions} having an entry for each program point and each expression, stores a pointer to an \texttt{ID object}. At a program point, two expressions are equivalent if they contain pointers to the same object. Though this approach is correct, it adds a lot of overhead in the implementation both in terms of time and space - the \texttt{ID object}s have to be created and destroyed during runtime, their data fields have to be maintained properly etc.\\
    The updated algorithm solves this problem by using just integer set identifiers instead of pointers to some dynamically created objects and the \texttt{Parent map}. Again same as before, two expressions are equivalent at a program point if \texttt{Partitions} array stores same set identifiers for the expressions at that program point. \texttt{Parent} map captures the relation between the set identifiers. This modification makes the actual implementation to be more simpler, efficient and intuitive.
    \item \textbf{Confluence Function}\\
    The \texttt{Confluence} function in the original algorithm processes only the set $\mathcal C \cup \mathcal V$ in its main loop. This is wrong and instead the whole working set $\mathcal W$ should be considered.\\
    As an example, consider the test case in \autoref{sec:tc15}. At the confluence point, $y$ and $x + 2$ must be equivalent, but the original algorithm assigns them pointers to different \texttt{ID objects} indicating they are not equivalent.
    \item \textbf{Non-deterministic Assignment}\\
    Non-deterministic assignment need not be handled separately and a single transfer function is sufficent (see \autoref{alg:TransferFunction}).
\end{itemize}