\documentclass{beamer}
% Class options include: notes, notesonly, handout, trans,
%                        hidesubsections, shadesubsections,
%                        inrow, blue, red, grey, brown

% Theme for beamer presentation. 
%\usetheme[pageofpages=of,% String used between the current page and the
 %                        % total page count.
 %         alternativetitlepage=true,% Use the fancy title page.
%          ]{Torino}
%\usecolortheme{nouvelle}
\usepackage[latin1]{inputenc}
\usetheme{Warsaw}
\usecolortheme{beaver}
\usepackage{array}
\usepackage{wrapfig}
\usepackage{multirow}
\usepackage{alltt}
\newcommand{\bi}{\begin{itemize}}
\newcommand{\ei}{\end{itemize}}
\newcounter{over}
\newcounter{max}
\newcommand{\fred}[2]{\setcounter{over}{#2}\addtocounter{over}{#1}}
\newcommand{\boxplot}[5]{%
\scalebox{0.5}{\textcolor{white}{\setcounter{max}{#4}\addtocounter{max}{#5}\fred{#4}{-#2}}%
\begin{picture}(100,10)%
\put(0,0){\line(0,1){8}}%
\put(100,0){\line(0,1){8}}%
%\put(#1,4){\circle*{1}}%
%\put(\themax,4){\circle*{1}}%
\put(#2,4){\line(1,0){\theover}}%
\put(#3,4){\circle*{8}}%
%\put(#4,4){\line(1,0){#5}}%
\put(50,0){\line(0,1){8}}%
\end{picture}}
} 

\setcounter{tocdepth}{1}

\AtBeginSection[]
{
   \begin{frame}
       \frametitle{Outline}
       \tableofcontents[current]
   \end{frame}
}

% Other themes include: beamerthemebars, beamerthemelined, 
%                       beamerthemetree, beamerthemetreebars  

\title{Trade-offs of Heuristic Vs. Rigorous Algorithms in Text Mining}    % Enter your title between curly braces
\author{Andrew Matheny}           % Enter your name between curly braces
\institute{Masters Defense\newline\newline
{\bf Committee} \newline
Tim Menzies, Ph.D (chair) \newline
Tim McGraw, Ph.D \newline
James Mooney, Ph.D\newline\newline
West Virginia University, LCSEE}      % Enter your institute name between curly braces
\date{July 21, 2010}                    % Enter the date or \today between curly braces
\usepackage{graphicx}
%\includegraphics[width=1in]{logo}
\begin{document}

% Creates title page of slide show using above information
\begin{frame}
  \titlepage
\end{frame}
                           % typeset with the notes or notesonly class options

% Creates table of contents slide incorporating
% all \section and \subsection commands

%\begin{frame}
%  \tableofcontents
%\end{frame}


%%%%%%%%%%%%% PROBLEM SLIDES %%%%%%%%%%%%%%%%%

\section{Overview}

\subsection{Abstract}

\begin{frame}
	\frametitle{Abstract}
	\begin{block}{Abstract}
	Within the field of text mining, rigorous methods for dimensionality reduction and clustering don't scale.
\newline \newline
	We examine the trade-offs between standard rigorous approaches and a few heuristic alternatives.  
\newline \newline
	The findings indicate that heuristic approximations aren't so bad.
	\end{block}
\end{frame}

\subsection{Motivations}

\begin{frame}
	\frametitle{National Archival Needs}


	\begin{center}
		\includegraphics[width=0.2\paperwidth]{img/nara.png}
		\includegraphics[width=0.2\paperwidth]{img/netl-c.pdf}
	\end{center}

	\bi
		\item Research arose out of NETL-funded look into archiving 
				needs of NARA (National Archives and Records Administration)
		\item Vast amounts of textual documents with no infrastructure for exploring them
	\ei
\end{frame}

\begin{frame}
	\frametitle{Navigating the Technical Document Space}
	Specifically, NARA needs to explore thousands of projects worth of technical documents.

	To name a few...
	\bi
		\item STEP/EXPRESS documents
		\item Source code
		\item Engineering artifacts (requirements, use cases, etc.)
	\ei

	\begin{block}{\$1M Question}
		What other designs are similar to this one?
	\end{block}

\end{frame}

\begin{frame}
	\frametitle{But why?}
	
	\bi
		\item Estimated that over 80\% of potentially usable business information in
			an unstructured form \cite{grimes80rule08}
		\item
			Heaps' Law tells us that the growth rate of the term space, or vocabulary, is approximately
			equivalent to the square root of the total number of words in the document set \cite{grootjenHeapsLaw03}.
	\ei

\end{frame}

\section{Background}

\subsection{Text Mining}

\begin{frame}
	\frametitle{What is Text Mining?}
	
		\begin{center}
			\includegraphics[width=0.3\paperwidth]{img/text_mining.png}
		\end{center}

	\begin{block}{Definition}
		The exploration of large libraries of natural language with
		the goal extracting information not previously known.
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Example Applications}
	
	\bi
		\item Spam filtering
		\item Identifying similar news items
		\item Targeted advertising
		\item Program comprehension
		\item Search indexing
		\item etc...
	\ei
\end{frame}

\begin{frame}
	\frametitle{Basic Process}
	
	The basic operation involved in text mining are:
	\bi
		\item Defining the scope of a document
		\item Processing the documents into manageable data structures
		\item Computational analysis of document collection as a whole
		\item Output of new information
	\ei
\end{frame}

\begin{frame}
	\frametitle{Explicit Process}
	A more descriptive explanation of these steps and how they are 
	used in this study are as follows:
	\bi
		\item {\bf Preprocess: } \newline
			Translate documents into a vector space model \cite{salton.SMART.91}
		\item {\bf Reduce Dimensionality:} \newline
			Remove non-descriptive terms and extract valuable terms
		\item {\bf Cluster:} \newline 
			Group similar documents 
	\ei
	
\end{frame}


\subsection{Preprocessing}

\begin{frame}
	\frametitle{Preprocessing Techniques}
	
	A few of the preprocessing techniques used in this study are:
	\bi
		\item Tokenization
		\item Stop Lists
		\item Stemming
		\item Conversion to TF*IDF Space
	\ei		

\end{frame}

\begin{frame}
	\frametitle{Tokenization}
	\begin{block}{Definition}
	The processes of dividing written text into meaningful units, such as words, sentences, or topics
	\end{block}
	In addition to dividing the text into words, some house keeping is often done in this step such as
	removing punctuation and removing any variation in case (i.e. sending all upper case to lower) \newline
\end{frame}

\begin{frame}
	\frametitle{Stop Lists}
	\begin{block}{Definition}
	A collection of words that provide no lexical context and can be removed to aid in classification
	\end{block}
	\begin{center}
		\begin{figure}[!h]
		\begin{alltt}\scriptsize
		a           about    across   again      against
		almost      alone    along    already    also
		although    always   am       among      amongst
		amongst     amount   an       and        another
		any         anyhow   anyone   anything   anyway
		anywhere    are      around   as         at
		...         ...      ...      ...        ...  
		\end{alltt}
		\caption[Stop words]{24 of the 262 stop words used in this study.}
		\end{figure}
	\end{center}
\end{frame}

\begin{frame}
	\frametitle{Stemming}
	\begin{block}{Definition}
	 The process for reducing inflected (or sometimes derived) words to their stem, base or root form. 
		\cite{Porter80}
	\end{block}
	
	\begin{center}
		\begin{figure}[!h]
		\begin{alltt}\scriptsize
			\bi
				\item CONNECT
				\item CONNECTED
				\item CONNECTING
				\item CONNECTION
				\item CONNECTIONS
			\ei
		\end{alltt}
		\caption[Stemming]{Stemming example for the word {\em connect}}
		\end{figure}
	\end{center}

\end{frame}

\begin{frame}
	\frametitle{TF*IDF}
	A weight assigned to a given word in a given document 
	\cite{jones93} \newline
	\begin{block}{Definition}
	Term frequency * inverse document frequency
	\end{block}
\end{frame}

\begin{frame}
	\frametitle{Term Frequency}

	Count of occurrence of term $t_i$ in document $d_j$

	\begin{center}
	\bi
	\item
		$n_{i,j}$ = number of occurrences of term $t_i$ in document $d_j$
	\item
		$\displaystyle\sum_k n_{k,j}$ = sum of the number of occurrences of all terms in document $d_j$
	\ei
	\end{center}

	\begin{block}{Term Frequency Equation}
		TF$\displaystyle _{i,j} = \frac{n_{i,j}}{\sum_k n_{k,j}}$
	\end{block}

\end{frame}

\begin{frame}
\frametitle{Inverse Document Frequency}

	Normalized count of occurrence of term $t_i$ in all documents

	\begin{center}
	\bi
	\item
		$D$ = set of all documents
	\item
		$\{d : t_i \epsilon d\}$ = set of documents that contain term $t_i$
	\ei
	\end{center}

	\begin{block}{Inverse Document Frequency Equation}
	IDF$_i = log \frac{|D|}{\{|d : t_i \epsilon d\}|}$
	\end{block}

\end{frame}

\begin{frame}
	\frametitle{TF*IDF}
	Term-Frequency Inverse-Document-Frequency\newline

	\bi
	\item
		TF$\displaystyle _{i,j} = \frac{n_{i,j}}{\sum_k n_{k,j}}$
	\item
		IDF$_i = log \frac{|D|}{\{|d : t_i \epsilon d\}|}$	
	\ei


	\begin{block}{TF*IDF Equation}
		TF*IDF$_{t_i,d_j} = TF_{i,j} x IDF_i$
	\end{block}

\end{frame}

\begin{frame}
	\frametitle{Preprocessing Example}
	An example of applying preprocessing to documents and achieving a vector space model of terms
	\begin{center}
		\tiny
		\begin{tabular}{|p{50pt}|c|c|c|c|c|c|}\hline
			\multirow{2}{*}{\bf Document$_j$} & \multicolumn{6}{|c|} {\bf TFIDF($t_i$, $d_j$)}\\\cline{2-7}
			& $t_1$: woodchucks & $t_2$: chuck & $t_3$: lumberjacks & $t_4$: wood & $t_5$: chop & $t_6$: norris\\\hline\cline{1-7}
			{\em Woodchucks can chuck wood }						& 1 & 0.6 & 0 & 0.3 & 0 & 0 \\\hline
			{\em Lumberjacks can chop woodchucks }					& 1 & 0 & 0.6 & 0 & 0.6 & 0 \\\hline
			{\em Chuck Norris can chop lumberjacks and woodchucks }	& 1 & 0.6 & 0.6 & 0 & 0.6 & 0.3\\\hline
		\end{tabular}
		.
	\end{center}

\end{frame}

\subsection{Dimensionality Reduction}


\begin{frame}
	\frametitle{What is Dimensionality Reduction?}
	\begin{block}{Definition}
	The mapping of a feature space $N$, to a subspace $N_\prime$,  with $|N_{\prime}|$ $<$ $|N|$
	\end{block}

	\bi
	\item
		The preprocessing steps mentioned above a preliminary form of dimensionality reduction
	\item 
		Given the large amount of different terms in most document collections, almost any type of analysis is computationally impossible
	\ei
	
\end{frame}



\subsection{Clustering}
\begin{frame}
	\frametitle{What is Clustering?}
	\begin{block}{Definition}
	Assignment of items in a set into subsets, where the items of each subset
	have similar attributes, based on a given criteria
	\end{block}

	\begin{center}
		\begin{figure}
			\includegraphics[height=0.25\paperheight]{img/clusters.jpg}
		\caption[clusters]{Clustering of points based on two dimensional distance}
		\end{figure}
	\end{center}
	
	
\end{frame}


\section{Algorithm Review}


%%%%%%%%%%% BEGIN REDUCTION SLIDES

\subsection{Reduction: PCA}

\begin{frame}
	\frametitle{Principal Components Analysis}
Involves a mathematical procedure which includes computing the eigenvalue decomposition
of the covariance matrix (SVD)  \cite{jolliffe}.  The new space ranks the variation of
projections and places them in the new coordinates in decending order.
	\begin{figure}[!b]
		\begin{center}
			\includegraphics[angle=270,width=0.4\paperwidth]{img/pca.pdf}
		\end{center}
	\caption{The two features in the left plot can be transferred to the right plot via one latent feature.}
	\end{figure}
\end{frame}

\begin{frame}
	\frametitle{Principal Components Analysis}
\begin{block}{Assumptions}
	\scriptsize
	\bi
		\item
			The new coordinate system can be defined by a linear combination
			of existing features
		\item
			That mean and variance are stastically important
		\item
			That large variances have important dynamics
	\ei
\end{block}

	\bi
		\item
			Recently shown to provide the relaxed solution for K-means clustering \cite{ding04}
		\item
			Run-time of $O(n^2)$ where $n$ is the number of initial dimensions
		\item
			Does {\em not} scale
	\ei

\end{frame}

\subsection{Reduction: Fastmap}
\begin{frame}
	\frametitle{Fastmap}
Performs a linear transformation of points based on the euclidian geometry of the initial space.
\cite{fastmap95}

\begin{block}{Algorithm}
\scriptsize
	\bi
		\item
			$n$ = $D$, the number of dimenions in the original space
		\item
			While $n$ $>$ $d$ (new dimensionality)
		\bi
			\item
				Uses heuristic to find an aproximation of the two points which lie the furthest
				from each other in constant time.  These are the pivot elements
			\item
				Uses the consine law to project all other points onto the plane that lies perpendicular
				to the line between the two pivot elements
			\item
				Decrement $n$
		\ei
	\ei
\end{block}

\end{frame}

\begin{frame}
\frametitle{Fastmap}
	
\begin{minipage}[t]{0.4\paperwidth}
\begin{figure}
\includegraphics[height=80pt]{img/fastmap1.png}
\caption{\tiny Example of using the cosine law to find the position of $Oi$ in
the dimension $k$}
\end{figure}
\end{minipage}\hfill
\begin{minipage}[t]{0.4\paperwidth}
\begin{figure}
\includegraphics[height=80pt]{img/fastmap2.png}
\caption{\tiny Projects of points $O_i$ and $O_j$ onto the hyper-plane perpendicular to the line $O_a$$O_b$}
\end{figure}
\end{minipage}
\end{frame}


\subsection{Reduction: TF*IDF Sort}
\begin{frame}
	\frametitle{TF*IDF Sort}
A simple linear time method of determining the most useful terms in a document collection.  First
reported by \cite{Ramos_usingtf-idf}

\begin{block}{Algorithm}
	\bi
		\item for each term, $t_i$
			\bi
				\item Compute tfidfSum = $\displaystyle \sum_{d \epsilon D} TFIDF(t_i, d)$
			\ei
		\item sort terms by tfidfSum in decending order
		\item return top $n$ terms
	\ei
\end{block}

\end{frame}

\begin{frame}
	\frametitle{TF*IDF Sort: Example}
	\begin{center}
		\tiny
		\begin{tabular}{|p{50pt}|c|c|c|c|c|c|}\hline
			\multirow{2}{*}{\bf Document$_j$} & \multicolumn{6}{|c|} {\bf TFIDF($t_i$, $d_j$)}\\\cline{2-7}
			& $t_1$: woodchucks & $t_2$: chuck & $t_3$: lumberjacks & $t_4$: wood & $t_5$: chop & $t_6$: norris\\\hline\cline{1-7}
			{\em Woodchucks can chuck wood }						& 1 & 0.6 & 0 & 0.3 & 0 & 0 \\\hline
			{\em Lumberjacks can chop woodchucks }					& 1 & 0 & 0.6 & 0 & 0.6 & 0 \\\hline
			{\em Chuck Norris can chop lumberjacks and woodchucks }	& 1 & 0.6 & 0.6 & 0 & 0.6 & 0.3\\\hline
		\end{tabular}	.
	\end{center}
	
\scriptsize
	Top 4 terms as determined by TF*IDF Sort:
	\bi
		\item woodchucks
		\item chuck
		\item lumberjacks
		\item chop
	\ei

\end{frame}



%%%%%%% BEGIN CLUSTERING SLIDES


\subsection{Clustering: K-means}
\begin{frame}
	\frametitle{K-means}
\scriptsize
\begin{block}{Algorithm}
\tiny
\bi
	\item
		initialize i to 0
	\item
		randomly select $k$ initial cluster centroids
	\item 
		while ($i \le maxIterations$ or no point changes set membership)
	\bi
		\tiny
		\item {\bf assignment:} assign each point to the cluster with the nearest centroid
		\item {\bf update:} recompute centroids using the mean of all items in the cluster
		\item  increment $i$
	\ei
\ei
\end{block}

\bi
	\item 
		An iterative refinement clustering algorithm
	\item
		With each iteration of the algorithm, cluster centroids are updated according
		to the mean of the items in the cluster
	\item
		Iteration continues until certain criteria is reached
	\item
		Variation on expectation-maximimization algorithm
	\item
		Closely related to PCA as recently show by \cite{ding04}
\ei
\cite{simpleKmeans00}

\end{frame}

\begin{frame}
	\frametitle{K-means: Illustrated}

		\begin{minipage}[t]{0.2\paperwidth}
				\includegraphics[width=0.2\paperwidth]{img/kmeans1.png}\newline
				\tiny {\bf (a)} Randomly select cluster centroids
		\end{minipage}\hfill
		\begin{minipage}[t]{0.2\paperwidth}
				\includegraphics[width=0.2\paperwidth]{img/kmeans2.png}\newline
				\tiny {\bf (b)} Assign each object to it's nearest centroid
		\end{minipage}\hfill
		\begin{minipage}[t]{0.2\paperwidth}
				\includegraphics[width=0.2\paperwidth]{img/kmeans3.png}\newline
				\tiny {\bf (c)} Compute new centroids by using the mean value from 
					the centroids picked in step
		\end{minipage}\hfill
		\begin{minipage}[t]{0.2\paperwidth}
				\includegraphics[width=0.2\paperwidth]{img/kmeans4.png}\newline
				\tiny {\bf (d)} Repeat steps \em{b} and \em{c} until the convergence 
					criteria has been reached
		\end{minipage}
\end{frame}

\subsection{Clustering: Canopy}
\begin{frame}
	\frametitle{Canopy}

\bi
	\item Often used as a preprocessor for K-means \cite{canopies00}
	\item Uses a cheap distance metric to build canopies
	\item Canopies are then used in clustering to reduce the
			number of times the expensive distance metric is used
	\item Cheap distance metric is highly dependant on the domain
\ei

\includegraphics[height=0.3\paperheight]{img/Canopy-logo.png}

\end{frame}

\begin{frame}
\frametitle{Canopy}
	\begin{minipage}[l]{0.4\paperwidth}
	\includegraphics[height=0.5\paperheight]{img/canopies.png}\newline
	\tiny {\bf Canopy Clustering:} The darker circle represents all 
		points in a given canopy ($distance(p, p_\prime) < T1$).  Points in the smaller circles cannot 
		be used as a new canopy center ($distance(p, p_\prime) < T2$).
	\end{minipage}\hfill
	\begin{minipage}[l]{0.4\paperwidth}
	\begin{block}{Canopy Creation Algorithm}
		\tiny
		\bi
			\item
			while $|P| > 0$
				\bi
				\tiny
					\item
					pick a point, $p$, at random and create a cluster here
					\item
					remove $p$ from the set $P$
					\item foreach $p_\prime \epsilon P$
					\bi
					\tiny
						\item
						if $distance(p, p_\prime) < T1$
						\item
						then; add $p_\prime$ to the cluster
						\item
						if $distance(p, p_\prime) < T2$
						\item
						then; remove $p_\prime$ from the set $P$
					\ei
				\ei
		\ei
	\end{block}
	\end{minipage}
\end{frame}

\subsection{Clustering: Genic}
\begin{frame}
	\frametitle{Genic}
	\bi
		\item Generalized incremental clustering algorithm developed by \cite{Gupta04}
		\item Only requires single pass
		\item Highly scalable
	\ei

	\tiny
	\begin{block}{Genic Parameters}
	\tiny
	\bi
		\item
		$k$: number of initial 
		\item
		$m$: number of initial candidate centers
		\item
		$n$: size of a generation
	\ei
	\end{block}
\end{frame}

\begin{frame}
	\tiny
	\frametitle{Genic: Algorithm}
	\begin{block}{Algorithm}
		\bi
		\tiny
		\item \textbf{Initialization}
		\bi
		\tiny
			\item Select $m$ points, $c_1, ..., c_m$ to be the initial candidate centers.
			\item Assign a weight of $w_i$ = 1 to each of these candidate centers.
		\ei

		\item \textbf{Incremental Clustering} For each subsequent data point $p$ in the stream: do
		\bi
		\tiny
			\item $Count$ = $Count$ + 1
			\item Find the nearest candidate center $c_i$ to the point $p$
			\item Move the nearest candidate center using $c_i = \frac{(w_i * c_i + p)}{w_i + 1}$
			\item Increment the corresponding weight $w_i = w_i + 1$
			\item When $Count$ $mod$ $n$ = 0, go to last bullet
		\ei

		\item \textbf{Generational Update of Candidate Centers} \\
		When $Count$ equals $n, 2n, 3n$, ..., for every center $c_i$ in the list L of centers, do:
		\bi
		\tiny
			\item Calculate its probability of survival, $p_i = \frac{w_i}{\sum_{i=1}^n w_i}$
			\item Select a random number $\delta$ uniformly from [0,1]. 
			\item If $p_i$ $>$ $\delta$, retain the center $c_i$ in the list L of centers
			\item Set the weight $w_i$ = 1 back to one.
			\item Go back to second bullet and continue processing the input stream
		\ei

		\item \textbf{Calculate Final Clusters} The list L contains the $m$ centers. These $m$ centers
		can be grouped into the final $k$ centers based on their Euclidean distances.

		\ei
	\end{block}	
\end{frame}



% \section{Hamlet}
% \begin{frame}
% 	\frametitle{"To do or not to do?"}
% 
% \end{frame}
% 
% 
% \subsection{User Interface}
% \begin{frame}
% 	\frametitle{Visually Navigating the Space}
% 
% \end{frame}
% 
% 
% \subsection{Parsing Framework}
% \begin{frame}
% 	\frametitle{Extending Hamlets Parsing Capabilities}
% 
% \end{frame}
% 
% 
% \subsection{API}
% \begin{frame}
% 	\frametitle{Reuse of Core Functionality}
% 
% \end{frame}


\section{Experiments}

\subsection{Design}

\begin{frame}
\frametitle{Experimental Design}
Factorial or fully-crossed design (a 8x3x3x6x6 factorial design) with
a total of 2,592 (8x3x3x6x6) different treatments
\scriptsize
\begin{block}{Experimental Factors}
\begin{tabular}{l|m{5.5cm}|r}
\scriptsize
{\bf Factor } & {\bf Levels } & {\bf Level count } \\\hline
dataset     &  ap203, ap214, BBC, BBCSport, ngBias3, ngBias8, ngBal3, ngBal8  & 8\\
 & & \\
clusterer     & KMeans, Canopy, Genic & 3\\
 & & \\
reduction method             & PCA, FastMap, TF-IDF Ranking & 3\\
 & & \\
$k$          & 3, 5, 8, 15, 40, 75 & 6 \\
 & & \\
$d$			 & 3, 15, 25, 50, 100, 200 & 6 \\

\end{tabular}\newline
\end{block}

\end{frame}

\begin{frame}
	\frametitle{Classification of Algorithms}
\small
\begin{center}
\begin{tabular}{l|r|r}
                & {\bf Clustering } & {\bf Dimensionality Reduction} \\\hline
{\bf Exhaustive}& K-Means & PCA \\\hline
\multirow{2}{*}{\bf Heuristic} & Canopy & FastMap\\
               & Genic & TF-IDF Ranking \\
\end{tabular}
\end{center}

\end{frame}


\subsection{Datasets}

\begin{frame}
\frametitle{Dataset Statistics}
\tiny
\begin{table}[!h]{
\begin{center}
\begin{tabular}{|p{2.5cm}||r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r@{~}|r|}\hline
Documents                & AP203 & AP214 & BBC  & BBCSport  & ngBias3 & ngBias8 & ngBal3 & ngBal8 \\\hline
$D$ documents            & 484   & 1373  & 2224 & 737        & 1500    & 2395    & 1499   & 3999   \\
$T$ terms                & 1103  & 3050  & 9635 & 4613       & 8631    & 9826    & 8158   & 14984  \\
Mean document length     & 143   & 164   &  130 & 104        & 116     & 87      & 91     & 90     \\
Natural Classes			 & N/A   & N/A   & 5    & 5          & 3       & 8       & 3      & 8 \\
Stemming used            & -   & -   &  x & x        & x     & x      & x     & x     \\
Stop-words removed       & -   & -   &  x & x        & x     & x      & x     & x     \\\hline
\end{tabular}
\end{center}}
\caption{Statistics on our datasets.}
\end{table}
\end{frame}

\begin{frame}
\frametitle{Natural Classes of Supervised Datasets}
\tiny
\begin{table}[!h]{
\begin{center}
\begin{tabular}{|l|p{7cm}|}\hline
{\bf Dataset}                & {\bf Natural classes} \\\hline
BBC            & business(509), entertainment(386), politics(417), sport(511), tech(302) \\\hline
BBCSport       & athletics(100), cricket(124), football(265), rugby(147), tennis(57) \\\hline
ngBias3      & graphics(136), hockey(591), windows(587) \\\hline
ngBias8		 & atheism(704), autos(202), crypt(204), hockey(205), mac(202), mideast(202), space(204), xwindows(205) \\\hline
ngBal3       &  graphics(463), hockey(459), mideast(453)\\\hline
ngBal8       &  atheism(471), autos(463), crypt(467), hockey(461), mac(466), mideast(470), space(461), xwindows(469)\\\hline
\end{tabular}
\end{center}}
\caption{Supervised datasets along with class values.}
\end{table}

\scriptsize
BBCSports \& BBC datasets found at \cite{mlgUcd} \newline
ngBias3, ngBias8, ngBal3, \& ngBal8 datasets found at \cite{Lang95}, \cite{bay00uci}

\end{frame}

\subsection{Assessment Criteria}
\begin{frame}
\frametitle{Assessment Criteria}
MWU tests, AUC plots, and relative AUC sums of the following data items.
	\bi
		\item
		Run-time
		\item
		Cluster Purity
		\item
		Internal Cluster Similarity
		\item
		External Cluster Similarity
		\item
		Similarity Loss		
	\ei
\end{frame}

\begin{frame}
\frametitle{Cluster Purity}
\bi
	\item
	A method for ranking cluster heterogeneity
	\item Answers the question: How dominant is the the most frequently occurring class in a cluster?
	\item Can only be used on supervised datasets (No STEP/EXPRESS)
\ei

\begin{block}{Purity Equations}
	$purity(C_j) = \frac{1}{|C_j|}max_i(|C_j|_{class=i})$ \newline
	$purity = \sum^k_{j=1}\frac{|C_j|}{|D|}purity(C_j)$
\end{block}

\end{frame}

\begin{frame}
\frametitle{Internal Similarity}
\bi
	\item
	Measure of how similar the things within the same class are in terms
	of geometric distance
	\item Can be used with supervised and unsupervised datasets
	\item Closely related to external similarity
\ei

\begin{block}{Internal Similarity Equation}
{\em $iSim_i = \displaystyle\sum_{d \in C_i} \displaystyle\sum_{d'\in C_i} \frac{cos(d, d')}{n_i^2} $}\newline
{\em $iSim = \displaystyle\sum_i \frac{n_i}{N} iSim_i $}
\end{block}


\end{frame}

\begin{frame}
\frametitle{External Similarity}
\bi
	\item
	Measure of how different the things within the same class are in
	terms of geometric distance
	\item Can be used with supervised and unsupervised datasets
	\item Closely related to internal similarity
\ei

\begin{block}{External Similarity Equation}
\em{$eSim_{ij} =  \displaystyle\sum_{d \in C_i} \displaystyle\sum_{d' \in C_j} \frac{cos(d, d')}{n_i n_j}$}\newline
\em{$eSim = \displaystyle\sum_i \frac{n_i}{N} eSim_{ij}$}
\end{block}

\end{frame}

\begin{frame}
\frametitle{Similarity Loss}
Measure of difference between internal and external similarity

\begin{block}{Similarity Loss Equation}
${\bf Similarity Loss} = eSim - iSim$
\end{block}
\end{frame}

\begin{frame}
\frametitle{AUC Plots}
\bi
	\scriptsize
	\item Area under the curve (AUC)
	\item Used to display the total scope of values 
		received for a given treatment.  
	\item Plotted by sorting all results for a treatment and plotting this
		as the $y$ value while the $x$ axis is incremented by one for each value
	\item Useful for helping to explain disparities between MWU and other results
\ei

\center
\scriptsize
\begin{minipage}[b]{0.4\paperwidth}
\begin{block}{Sample AUC Plot}
\center
\includegraphics[width=0.3\paperwidth]{img/clusterer_runtime_auc.pdf}
\end{block}
\end{minipage}
		  
\end{frame}

\subsection{Results: Reduction}

\begin{frame}
\frametitle{Runtimes}
		\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Run-time MWU results}\\\cline{2-5}
			{\bf Reduction Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
		   	tfidf &    0 &   2 &    0 &        2 \\
		 	FastMap &    0 &   1 &    1 &        0 \\
			PCA &    0 &   0 &    2 &       -2 \\
		\end{tabular} \newline\newline
		\scriptsize
		As expected, the quick linear time method $tfidf$ 
			performs better than both $PCA$ and $FastMap$ while $PCA$ performs the worst.\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Reduction Method} & {\bf Run-time AUC Relative to PCA} \\\hline
		 	tfidf&  $<$1 \\
		 	FastMap&     3 \\
			PCA&     100 \\
		\end{tabular}\newline\newline
		\scriptsize
		Notice the massive computational requirements of 
			PCA compared to FastMap and tfidf.  tfidf runs in less than
			1\% of the time of PCA.
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{Runtime AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/reducer_runtime_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{Purity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Purity MWU results}\\\cline{2-5}
			{\bf Reduction Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
			 PCA&    0&   2&    0&        2 \\
		 TfIdf-Sort&    0&   1&    1&        0 \\
		   Fastmap&    0&   0&    2 & -2 \\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Reduction Method} & {\bf Purity AUC Relative to PCA} \\\hline
     pca& 100\\
 fastmap&  83\\
   tfidf&  83\\
		\end{tabular}\newline\newline
		\scriptsize

		As expected, $PCA$ wins with purity though $FastMap$ and $tfidf$ are not far behind.
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{External Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf External Sim MWU results}\\\cline{2-5}
			{\bf Reduction Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
     pca&    0&   2&    0&        2 \\
   tfidf-sort&    1&   0&    1&       -1 \\
 fastmap&    1&   0&    1&       -1 \\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Reduction Method} & {\bf External Sim AUC Relative to PCA} \\\hline
 fastmap&  102  \\
     pca&  100  \\
   tfidf&   81  \\
		\end{tabular}\newline\newline
		\scriptsize

		For external similarity, lower is better.  Surprisingly, tfidf is coming out on top of PCA.
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{Internal Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Internal Sim MWU results}\\\cline{2-5}
			{\bf Reduction Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
  			 pca&		0&	2&	0&	2 \\
			 fastmap&	0&	1&	1&	 0 \\
			 tfidf-sort&		0&	0&	2&	 -2 \\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Reduction Method} & {\bf Internal Sim AUC Relative to PCA} \\\hline
			 fastmap& 118 \\
				 pca& 100 \\
			   tfidf&  88 \\
		\end{tabular}\newline\newline
		\scriptsize

		Again with internal similarity, we have an example of a heuristic algorithm, $fastmap$, outperforming our rigorous baseline, $PCA$
		\end{center}		

\end{frame}


\begin{frame}
\frametitle{Similarity Loss}
	\scriptsize
		\begin{center}
			\begin{tabular}{|r|c|}
				\hline
				\textbf{Reduction Method} & \textbf{Similarity Loss} \\ \hline \hline
				tfidf	& 6 \\ \hline
				fastmap		& 30 \\ \hline
			\end{tabular}
		\end{center}
\end{frame}

\begin{frame}
\frametitle{Visualizing the Reduction: PCA}

\begin{center}
\begin{tabular}{p{1in}|c|c|}\hline
PCA ($D-T^2$)& \includegraphics[width=0.3\paperwidth]{img/ap203_pca3.pdf} &
\includegraphics[width=0.3\paperwidth]{img/ap214_pca3.pdf}\\\hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Visualizing the Reduction: Fastmap}

\begin{center}
\begin{tabular}{p{1in}|c|c|}\hline
FASTMAP ($2N$) & \includegraphics[width=0.3\paperwidth]{img/ap203_fastmap3.pdf} &
\includegraphics[width=0.3\paperwidth]{img/ap214_fastmap3.pdf}\\\hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Visualizing the Reduction: TF*IDF}

\begin{center}
\begin{tabular}{p{1in}|c|c|}\hline
Tf-IDF ($t-N$) & \includegraphics[width=0.3\paperwidth]{img/ap203_tfidf3.pdf} &
\includegraphics[width=0.3\paperwidth]{img/ap214_tfidf3.pdf}\\\hline
\end{tabular}
\end{center}
\end{frame}


\subsection{Results: Clustering}

\begin{frame}
\frametitle{Runtimes}
		\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Run-time MWU results}\\\cline{2-5}
			{\bf Clustering Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
  genic&    0&   2&    0&        2\\
 kmeans&    1&   0&    1&       -1\\
 canopy&    1&   0&    1&       -1\\
		\end{tabular} \newline\newline
		\scriptsize.\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Clustering Method} & {\bf Run-time AUC Relative to K-means} \\\hline
  genic&   6\\
 canopy&  52\\
 kmeans& 100\\
		\end{tabular}\newline\newline
		\scriptsize
		\end{center}		

	Not as drastic of increases as with reduction, though still much faster.  Genic is only 6\% of K-means
\end{frame}

\begin{frame}
\frametitle{Runtime AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/clusterer_runtime_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{Purity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Purity MWU results}\\\cline{2-5}
			{\bf Clustering Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
 kmeans&    0&   2&    0&        2 \\
  genic&    0&   1&    1&        0 \\
 canopy&    0&   0&    2&        -2 \\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Clustering Method} & {\bf Purity AUC Relative to K-means} \\\hline
 kmeans& 100 \\
 canopy&  72 \\
  genic&  64 \\
		\end{tabular}\newline\newline
		\scriptsize

		As expected, $K-means$ wins with purity.  
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{External Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf External Sim MWU results}\\\cline{2-5}
			{\bf Clustering Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
 genic&    0&   2&    0&       2\\
 kmeans&    0&   1&    1&        0\\
  canopy&    0&   0&    2&        -2\\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Clustering Method} & {\bf External Sim AUC Relative to K-means} \\\hline
 genic&  91 \\
 kmeans& 100 \\
  canopy& 113 \\
		\end{tabular}\newline\newline
		\scriptsize

		\end{center}		

\end{frame}

\begin{frame}
\frametitle{Internal Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Internal Sim MWU results}\\\cline{2-5}
			{\bf Clustering Method} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
 kmeans&    0&   2&    0&        2 \\
  genic&    0&   1&    1&        0 \\
 canopy&    0&   0&    2&       -2 \\
		\end{tabular} \newline\newline
		\scriptsize\newline\newline

	\tiny
		\begin{tabular}{l|c}
			{\bf Clustering Method} & {\bf Internal Sim AUC Relative to K-means} \\\hline
 canopy&  83 \\
  genic&  91 \\
 kmeans& 100 \\
		\end{tabular}\newline\newline
		\scriptsize

		\end{center}		

\end{frame}


\begin{frame}
\frametitle{Similarity Loss}
	\scriptsize
		\begin{center}
			\begin{tabular}{|r|c|}
				\hline
				\textbf{Clustering Method} & \textbf{Similarity Loss} \\ \hline \hline
canopy		& 26 \\ \hline
				genic		& 22 \\ \hline
			\end{tabular}
		\end{center}
\end{frame}

\subsection{Results: Combined}


\begin{frame}
\frametitle{Runtimes}
		\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Run-time MWU results}\\\cline{2-5}
			{\bf Reducer-Clusterer} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
   tfidf-genic&    0&   8&    0&        8 \\
   tfidf-kmeans&    0&   7&    1&        6 \\
  fastmap-genic&    0&   6&    2&        4 \\
 fastmap-kmeans&    0&   5&    3&        2 \\ 
   tfidf-canopy&    0&   4&    4&        0 \\
 fastmap-canopy&    0&   3&    5&       -2 \\ 
      pca-genic&    0&   2&    6&       -4 \\
     pca-canopy&    0&   1&    7&       -6 \\
     pca-kmeans&    0&   0&    8&       -8 \\
		\end{tabular} \newline\newline
		\scriptsize

	\tiny
		\begin{tabular}{l|c}
			{\bf Reducer-Clusterer} & {\bf Run-time AUC Relative to PCA-Kmeans} \\\hline
tfidf-genic	&   1 \\
fastmap-genic		&   3 \\
tfidf-canopy	&   5 \\
fastmap-canopy		&   7 \\
tfidf-kmeans	&  12 \\
fastmap-kmeans		&  20 \\
pca-genic			&  71 \\
pca-canopy			&  76 \\
pca-kmeans			& 100 \\
		\end{tabular}\newline\newline
		\scriptsize
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{Runtime AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/clusterer_reducer_runtime_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{Purity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Purity MWU results}\\\cline{2-5}
			{\bf Reducer-Clusterer} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
     pca-kmeans&    0&   8&    0&        8 \\
      pca-genic&    0&   7&    1&        6 \\
  fastmap-genic&    1&   5&    2&        3 \\
 fastmap-canopy&    1&   5&    2&        3 \\ 
   tfidf-kmeans&    1&   3&    4&       -1 \\
 fastmap-kmeans&    1&   3&    4&       -1 \\
   tfidf-canopy&    0&   2&    6&       -4 \\
    tfidf-genic&    0&   1&    7&       -6 \\
     pca-canopy&    0&   0&    8&       -8 \\
		\end{tabular} \newline\newline
		\scriptsize

	\tiny
		\begin{tabular}{l|c}
			{\bf Reducer-Clusterer} & {\bf Purity AUC Relative to PCA-Kmeans} \\\hline
     pca-kmeans& 100 \\
      pca-genic&  95 \\
   tfidf-kmeans&  73 \\
 fastmap-kmeans&  73 \\
    tfidf-genic&  67 \\
  fastmap-genic&  68 \\
   tfidf-canopy&  50 \\
 fastmap-canopy&  50 \\
     pca-canopy&  38 \\
		\end{tabular}\newline\newline
		\scriptsize

		As expected, $K-means$ wins with purity.  
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{Purity AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/clusterer_reducer_purity_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{External Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf External Sim MWU results}\\\cline{2-5}
			{\bf Reducer-Clusterer} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
     pca-kmeans&    0&   8&    0&        8\\
    tfidf-genic&    0&   7&    1&        6\\
      pca-genic&    0&   6&    2&        4\\
   tfidf-kmeans&    1&   4&    3&        1\\
     pca-canopy&    0&   4&    4&        0\\
   tfidf-canopy&    1&   3&    4&       -1\\
  fastmap-genic&    0&   2&    6&       -4\\
 fastmap-kmeans&    0&   1&    7&       -6\\
 fastmap-canopy&    0&   0&    8&       -8\\\hline
		\end{tabular} \newline\newline
		\scriptsize

	\tiny
		\begin{tabular}{l|c}
			{\bf Reducer-Clusterer} & {\bf External Sim AUC Relative to PCA-Kmeans} \\\hline
tfidf-canopy	&	78\\
fastmap-canopy	&	79\\
tfidf-kmeans	&	80\\
pca-canopy		&	81\\
fastmap-kmeans	&	87\\
pca-kmeans		&	100 \\
pca-genic		&	102\\
tfidf-genic		&	103\\
fastmap-genic	&	107\\
\hline
		\end{tabular}\newline\newline
		\scriptsize

		Lower means better performance and Genic, one of the heuristic approaches, is defnitely lower that K-means
		\end{center}		

\end{frame}

\begin{frame}
\frametitle{External Similarity AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/clusterer_reducer_eSim_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{Internal Similarity}
			\begin{center}
	\tiny
		\begin{tabular}{l|ccc|r}
			& \multicolumn{4}{|c}{\bf Internal Sim MWU results}\\\cline{2-5}
			{\bf Reducer-Clusterer} & {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
     pca-kmeans&    0&   8&    0&        8 \\
      pca-genic&    0&   7&    1&        6 \\
  fastmap-genic&    1&   5&    2&        3 \\
 fastmap-canopy&    1&   5&    2&        3 \\
   tfidf-kmeans&    1&   3&    4&       -1 \\
 fastmap-kmeans&    1&   3&    4&       -1 \\
   tfidf-canopy&    0&   2&    6&       -4 \\
    tfidf-genic&    0&   1&    7&       -6 \\
     pca-canopy&    0&   0&    8&       -8 \\
		\end{tabular} \newline\newline
		\scriptsize

	\tiny
		\begin{tabular}{l|c}
			{\bf Reducer-Clusterer} & {\bf Internal Sim AUC Relative to PCA-Kmeans} \\\hline
pca-kmeans		&	100\\
fastmap-kmeans	&	90\\
pca-genic		& 	88\\
fastmap-genic	&	82\\
fastmap-canopy	&	81\\
tfidf-kmeans	& 	79\\
tfidf-genic		& 	75 \\
tfidf-canopy	& 	74\\
pca-canopy		& 	71\\\hline
		\end{tabular}\newline\newline
		\scriptsize

		\end{center}		

\end{frame}


\begin{frame}
\frametitle{Internal Similarity AUC Plot}
\center
		\includegraphics[width=0.65\paperwidth]{img/clusterer_reducer_iSim_auc.pdf}
\end{frame}

\begin{frame}
\frametitle{Similarity Loss}
\tiny
		\begin{center}
		\begin{tabular}{l|l}
			{\bf Reducer-Clusterer}& {\bf Similarity Loss Relative To pca-kmeans}\\\hline
fastmap-kmeans	&	-3\\
fastmap-canopy	& 	-2\\
tfidf-kmeans	&	1\\
tfidf-canopy	&	4\\
pca-canopy		&	10\\
pca-genic		&	14\\
fastmap-genic	&	25\\
tfidf-genic		& 	28\\\hline
		\end{tabular}\newline\newline\newline

		\begin{tabular}{l|ccc|r}
			{\bf Reducer-Clusterer}& {\bf ties} & {\bf wins} & {\bf losses} & {\bf wins-losses} \\\hline
     pca-kmeans&    0&   8&    0&        8\\
   tfidf-kmeans&    1&   6&    1&        5\\
 fastmap-kmeans&    1&   6&    1&        5\\
      pca-genic&    0&   5&    3&        2\\
   tfidf-canopy&    2&   2&    4&       -2\\
  fastmap-genic&    2&   2&    4&       -2\\
 fastmap-canopy&    2&   2&    4&       -2\\
    tfidf-genic&    0&   1&    7&       -6\\
     pca-canopy&    0&   0&    8&       -8\\\hline
		\end{tabular}\newline\newline
	Reducer-Clusterer Similarity Difference MWU test results
\end{center}
\end{frame}


\section{Conclusions}
\begin{frame}
\frametitle{Conclusions}
	\bi
		\item
		{\bf Dimensionality Reduction:}
			\bi
				\item
				PCA wins at all validity contents, but is also the largest potential performance bottleneck
				\item
				Fastmap can do the job if run-time contraints exist
				\item
				TFIDF-sort is your best bet if even larger run-time contraints exist
			\ei
		\item
		{\bf Clustering:}
			\bi
				\item
				As expected, K-means wins at most validity contests
				\item
				Genic outperforms Canopy in every validity and even beats K-means on external similarity
				\item
				Canopy performance leaves much to be desired
			\ei
	\ei
\end{frame}

\begin{frame}
\frametitle{Conclusions}
\bi
		\item
		{\bf Combined:}
		\bi
			\item
			Fastmap-Genic provide a highly scalable solution without sacraficing too much in the way of validity
			\item
			Simply replacing PCA can vastly reduce run-times and still maintain the validity of K-means
			\item
             TfIdf and KMeans combination shows astounding performance in validity
				at only 12\% of the base line run-time
		\ei
\ei

\end{frame}


\begin{frame}
\frametitle{Summary}
	Heuristic methods are worth it
\end{frame}

\subsection{Future Work}

\begin{frame}
\frametitle{Paths Going forward}

\bi
	\item Closer examination of in-use applications
	\item More in-depth analysis on the effect of the various algorithms parameters
	\item Use more current algorithms (i.e. LSI)
	\item Find out what Genic is capable of
	\item Explore heuristic ways of finding the optimal/inherent dimensionality
	\item Explore heuristic ways of determining the number of clusters
\ei
\end{frame}
%%%%%%%%%%%%% QUESTIONS SLIDES %%%%%%%%%%%%%%%%%
\section{}


\begin{frame}
\frametitle{Questions?}

\begin{center}
	\includegraphics[height=0.7\paperheight]{img/questions.jpg}
\end{center}

\end{frame}

%%%%%%%%%%%%% QUESTIONS SLIDES %%%%%%%%%%%%%%%%%

%%%%%%%%%%%%% REFS SLIDES %%%%%%%%%%%%%%%%%


%\begin{frame}
%	\frametitle{References}
	\bibliographystyle{apalike}
	{\tiny
	\bibliography{myrefs}
	}
%\end{frame}



\end{document}
