ie

text_processing/ie_introduction.tex

@@ -90,7 +90,7 @@ Why doing \myemph{information extraction}?
 The \textbf{structured form} can be implemented either as a database or form (slot filling) or by using XML tags (tagging)
 \vfill
 
-\todo{ADD A GRAPHIC WITH OBAMA, events = president start, end... WIFE = michelle etc...}
+%\todo{ADD A GRAPHIC WITH OBAMA, events = president start, end... WIFE = michelle etc...}
 
 \end{frame}
 
@@ -330,7 +330,7 @@ Identify all assertions of relations between entities %usually binary
 May be divided into two subtasks:
 \begin{itemize}
 \item \textbf{Relation detection}: find pairs of entities between which a relation holds
-\item \textbf{Relation classification}: determine the type of a previously extracted relation
+\item \textbf{Relation classification}: determine the type of a previously detected relation
 \end{itemize}
 
 \vfill
@@ -385,7 +385,7 @@ Example:
 \begin{frame}
 \frametitle{Information Extraction: Event extraction}
 
-\begin{block}{\textbf{Relation Extraction}}
+\begin{block}{\textbf{Event Extraction}}
 Identify all reports of event instances, typically of a small set of classes
 \end{block}
 

text_processing/ie_ner.tex

@@ -910,6 +910,31 @@ the \myemph{Ashoka} mentioned is likely to be in Sheffield, while the Sheffield
 
 \end{frame}
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: NER: Conclusion}
+
+\textbf{Named Entity Recognition} (NER) is a core IE technology 
+\begin{itemize}
+\item detect and classify all mentions of NE of a given set of entity types within a given text
+\item now relatively mature and at “usable” performance levels
+\end{itemize}
+
+\textbf{Techniques used have included:}
+\begin{itemize}
+\item knowledge engineering approaches
+\item supervised learning approaches, e.g. BIO sequence labelling
+\end{itemize}
+
+\textbf{Open challenges include:}
+\begin{itemize}
+\item reducing the amount of training data needed via, e.g. bootstrapping techniques
+\item exploiting existing structured data sources to generate “weakly labelled” training data (aka distant supervision)
+\item expanding the classes of entities addressed
+\item developing NERs for languages other than English
+\end{itemize}
+
+\end{frame}
 
 
 

text_processing/ie_relation_extraction.tex

+% !TEX root = text_processing.tex
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{}
+
+\vfill
+\centering
+\Huge{\edinred{[Information Extraction]\\Relation Extraction}}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Overview}
+
+\begin{itemize}
+\item \gray{Introduction to Information Extraction}
+\begin{itemize}
+	\item \gray{Definition + contrast with IR}
+	\item \gray{Example Applications}
+	\item \gray{Overview of Tasks}
+	\item \gray{Overview of Approaches}
+	\item \gray{Evaluation + Shared Task Challenges}
+	\item \gray{Brief(est) history of IE}
+\end{itemize}
+
+\item \gray{Named Entity Recognition}
+\begin{itemize}
+	\item \gray{Task}
+	\item \gray{Approaches: Rule-based, Supervised Learning}
+	\item \gray{Entity Linking}
+\end{itemize}
+
+\item \textbf{Relation Extraction}
+\begin{itemize}
+	\item \textbf{Task}
+	\item \textbf{Approaches: Rule-based, Supervised Learning, Bootstrapping, Distant Supervision}
+\end{itemize}
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Overview}
+
+\textbf{Relation Extraction}
+\begin{itemize}
+\item Task definition
+\end{itemize}
+
+\textbf{Approaches to Relation Extraction}
+\begin{itemize}
+\item Knowledge-engineering approaches to RE
+\item Supervised learning approaches to RE
+\item Bootstrapping Approaches to RE
+\item Distant Supervision Approaches to RE
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction - recap}
+
+\begin{block}{\textbf{Relation Extraction (RE)}}
+Task: Identify all \myemph{assertions of relations} holding between \myemph{entities} in a text $T$\\
+\begin{itemize}
+\item the entities are identified in previous entity extraction step
+\item The set of possible relations $\mathbf{R}$ is determined in advance
+\end{itemize}
+\end{block}
+
+\textbf{Note:}
+\begin{itemize}
+\item relations in $\mathbf{R}$ are usually binary
+\item the entity classes involved in a relation in $\mathbf{R}$ are assumed to be a subset of those identified in the entity extraction process
+\end{itemize}
+
+\textbf{Generally divided into two subtasks:}
+\begin{enumerate}
+\item \myemph{Relation detection}: find pairs of entities between which a relation holds
+\item \myemph{Relation classification}: determine the type of a previously detected relation 
+\end{enumerate}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Examples}
+
+\textbf{Some examples}
+
+\begin{itemize}
+\item \textsc{location\_of} holding between:
+\begin{itemize}
+	\item \textsc{organisation} and \textsc{geopolitical\_location}
+	\item medical \textsc{investigation} and \textsc{ body\_part}
+	\item \textsc{gene} and \textsc{chromosome\_location}
+\end{itemize}
+
+\item \textsc{employee\_of} holding between \textsc{person} and \textsc{organisation}\\
+\item \textsc{product\_of} holding between \textsc{artifact} and \textsc{organisation}\\
+\item \textsc{is\_exposed\_to} holding between \textsc{organisation} and \textsc{risk}\\
+\item \textsc{is\_associated\_with} holding between \textsc{drug} and \textsc{side\_effect}\\
+\item \textsc{interaction} holding between \textsc{protein} and \textsc{protein}\\
+\end{itemize}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Challenges}
+
+\textbf{Challenges for Relation Extraction}
+\begin{enumerate}
+\item<1-> Many different ways to express the same relation
+\begin{itemize}
+\item \myemph{Canonical}: \annot{Microsoft}{ORG} \textbf{\underline{is located in}} \annot{Redmond}{LOC}
+\item \myemph{Synonyms}: \annot{Microsoft}{ORG} \textbf{\underline{is located/based/headquartered in}}  \annot{Redmond}{LOC}
+\item \myemph{Syntactic variations}:
+\begin{itemize}
+	\item \annot{Microsoft}{ORG}, the software giant and ... , \textbf{\underline{is based in}} \annot{Redmond}{LOC}
+	\item \annot{Redmond}{LOC}\textbf{\underline{-based}} \annot{Microsoft}{ORG} ...
+	\item \annot{Redmond}{LOC}\textbf{\underline{'s}} \annot{Microsoft}{ORG} ...
+	\item \annot{Redmond}{LOC} software giant \annot{Microsoft}{ORG} ...
+\end{itemize}
+\end{itemize}
+
+\item<2-> Relations often involve coreference links\\
+\small{
+\myemph{\annot{Bill Gates}{PER}} \textbf{\underline{is co-founder, technology advisor and board member}} of \annot{Microsoft}{ORG}. \myemph{\annot{He}{PER}} \textbf{\underline{served as chairman of the board}} until Feb. 4, 2014. On June 27, 2008, \myemph{\annot{Gates}{PER}} \textbf{\underline{transitioned out}} of a day-to-day role in the company to spend more time on his global health and education work at the \annot{Bill \& Melinda Gates Foundation}{ORG}.
+}
+\end{enumerate}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Challenges}
+
+\textbf{Challenges for Relation Extraction}
+\begin{enumerate}
+\setcounter{enumi}{2}
+\item The information required may be spread across multiple sentences
+\item The information may be implied by the text rather than explicitly asserted \ra\ \myemphb{inference}
+\end{enumerate}
+
+\textbf{Ex1.:}
+\myemph{\annot{Dirk Ruthless}{PER}} of \annot{MegaCorp}{ORG} made a stunning announcement today. In September \myemph{\annot{he}{PER}} will be stepping down as \annot{Chief Executive Officer}{POS} to spend more time with his pet piranhas.
+
+\begin{itemize}
+\item[\Ra] resolve \textbf{pronominal anaphor}  \myemph{Dirk Ruthless} \Lra\ \myemph{he} to determine the corporate \textsc{position}
+\item[\Ra] In Ex1.: no explicit statement that \myemph{Dirk Ruthless} \textbf{is} CEO of MegaCorp
+\begin{itemize}
+	\item "\myemph{\annot{Dirk Ruthless}{PER}} of \annot{MegaCorp}{ORG}"  + "will be stepping down as \annot{Chief Executive Officer}{POS}" = \myemph{\annot{Dirk Ruthless}{PER}} is CEO of \annot{MegaCorp}{ORG}
+\end{itemize}
+\item Solving RE may imply solving \myemphb{textual entailment}
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Overview}
+
+\gray{Relation Extraction}
+\begin{itemize}
+\item \gray{Task definition}
+\end{itemize}
+
+\textbf{Approaches to Relation Extraction}
+\begin{itemize}
+\item \textbf{Knowledge-engineering approaches to RE}
+\item Supervised learning approaches to RE
+\item Bootstrapping Approaches to RE
+\item Distant Supervision Approaches to RE
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Knowledge-engineering}
+
+\colorbox{liumlightgray}{
+\parbox{.99\textwidth}{
+
+\annot{Mr. \tikzmark{a} Wright}{PER}, \textbf{\underline{\annot{executive vice president}{POSITION}}} of \annot{Merrill Lynch \tikzmark{b} Canada Inc.}{ORG}\\
+
+\begin{tikzpicture}[overlay,remember picture]
+ \draw [very thick, color=carminered] ($({pic cs:a})+(0ex,-1ex)$) -- ($({pic cs:a})+(0ex,-3ex)$);
+ \draw [very thick, color=carminered] ($({pic cs:a})+(0ex,-3ex)$) -- ($({pic cs:b})+(0ex,-3ex)$)  node [midway, below, color=carminered] {is-employed-by};
+ \draw [very thick, color=carminered] ($({pic cs:b})+(0ex,-3ex)$) -- ($({pic cs:b})+(0ex,-1ex)$);
+\end{tikzpicture}
+
+}}
+
+\vfill
+
+Such systems use manually authored rules and can be divided into
+\begin{itemize}
+\item<1-> “\myemph{shallow}”: systems engineered to the IE task, typically using \myemphb{pattern-action} rules
+\begin{center}
+\begin{tabular}{ll}
+Pattern: & \textbf{"\$Person, \$Position of \$Organization"} \\
+Action: & \textbf{add-relation(is-employed-by(\$Person,\$Organization))} \\
+\end{tabular}
+\end{center}
+\vspace{.5cm}
+\item<2-> “\myemph{deep}”: linguistically inspired language understanding systems
+\end{itemize}
+
+\vfill
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Knowledge-engineering}
+
+\begin{itemize}
+\item “\myemph{deep}”: linguistically inspired language understanding systems
+\begin{itemize}
+	\item parse input to identify key grammatical relations (e.g. subject, object)
+	\item use transduction rules on parser output to extract relations 
+	\item[\ra] more powerful than regex on NE tags alone
+\end{itemize}
+\end{itemize}
+\vspace{3.5cm}
+
+\only<2->{
+\begin{itemize}
+\item Ex. parse trees:
+\begin{itemize}
+\item Multiple surface forms share underlying syntactic structure
+\item \textsc{subject} = PER, \textsc{object} = ORG and \textsc{verb} = \textbf{works for}
+\end{itemize}
+\end{itemize}
+
+\begin{textblock*}{50mm}[0,0](115mm,25mm)
+\includegraphics[height=.45\textheight]{syntaxtree1}
+\end{textblock*}
+
+\begin{textblock*}{50mm}[0,0](55mm,25mm)
+\includegraphics[height=.45\textheight]{syntaxtree2}
+\end{textblock*}
+}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Knowledge-engineering}
+
+\textbf{Strengths}
+\begin{itemize}
+\item High precision
+\item Interpretable results \ra\ system behaviour is human-comprehensible
+\end{itemize}
+
+\textbf{Weaknesses}
+\begin{itemize}
+\item The writing of rules has no end
+\item New rules needed for every new domain 
+\begin{itemize}
+	\item pattern action rules for shallow approaches
+	\item transduction rules for deep approaches
+\end{itemize}
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Overview}
+
+\gray{Relation Extraction}
+\begin{itemize}
+\item \gray{Task definition}
+\end{itemize}
+
+\textbf{Approaches to Relation Extraction}
+\begin{itemize}
+\item \gray{Knowledge-engineering approaches to RE}
+\item \textbf{Supervised learning approaches to RE}
+\item Bootstrapping Approaches to RE
+\item Distant Supervision Approaches to RE
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Supervised learning}
+
+\textbf{Question: what is to be learned?}
+
+Answer 1: \myemph{rules}
+\begin{itemize}
+\item to match \textbf{relation bearing sentences}
+\item capture the \textbf{relation arguments} in the matched text
+\end{itemize}
+
+Answer 2: \myemph{binary classifier}
+\begin{itemize}
+\item Classifies a sentence as to whether it bears a specific relation between some entity types
+\item Specialized binary classifier
+\item Can be divided in two stages: 
+\begin{itemize}
+\item \myemph{relation detection}: determines whether a sentence expresses a relation (binary)
+\item \myemph{relation classification}: determines the relation (multi-way)
+\end{itemize}
+
+\item Rule learning popular until early 2000's. Then classifier approach.
+\item[\ra] Details on classifier approach only
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Supervised Learning}
+
+Classification approaches to relation extraction:
+\begin{itemize}
+\item Assume entities  are already tagged \ra\ output of NER
+\item Use an algorithm to learn binary classifiers to distinguish instances where
+\begin{itemize}
+\item entities co-occur and relation holds (positive instances)
+\item entities co-occur and relation does not hold (negative instances)
+\end{itemize}
+\end{itemize}
+
+Key issue: what \textbf{features} to use to represent instances?
+\begin{itemize}
+\item 3 broad classes:
+\begin{itemize}
+\item Features of the Named Entities
+\item Features from the words in the text
+\begin{itemize}
+	\item words between the two NE 
+	\item words surrounding the candidates (left of 1st word and right of 2nd word)
+\end{itemize}
+\item Features about the entity pair within the sentence, e.g.
+\begin{itemize}
+	\item distance between the entities (in words or constituents)
+	\item is there a NE in between them?
+	\item clues from the syntactic structure of the sentence (parse tree)
+\end{itemize}
+\end{itemize}
+
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Supervised Learning: Example}
+
+Ex.: \annot{American Airlines}{ORG}, a unit of \annot{AMR Inc.}{ORG}, immediately matched the move, spokesman \annot{Tim Wagner}{PER} said. \source{Jurafsky and Martin, 2nd ed., p. 730}
+
+Features extracted from this example:
+\scriptsize{
+\begin{center}
+\setlength\extrarowheight{-3pt}
+\begin{tabular}{ll}
+\toprule
+\textbf{Features} & \textbf{Value}  \\ 
+\midrule
+\MC{2}{l}{\textbf{Entity-based features} }\\
+~~~~~~~~Entity$_1$ type			& ORG\\
+~~~~~~~~Entity$_1$ head 	& \textit{airlines}\\
+~~~~~~~~Entity$_2$ type		& PERS \\
+~~~~~~~~Entity$_2$ head 		& \textit{Wagner}\\
+~~~~~~~~Concatenated types		& ORGPERS \\ %\midrule
+\MC{2}{l}{\textbf{Word-based features} }\\
+~~~~~~~~Between-entity BOW 	& \{ \textit{a, unit, of, AMR, Inc., immediately, matched, the, move, spokesman} \}\\
+~~~~~~~~Word(s) before Entity$_1$ 	& NONE\\
+~~~~~~~~Word(s) before Entity$_2$		& \textit{said}\\ %\midrule
+\MC{2}{l}{\textbf{Syntactic features} }\\
+~~~~~~~~Constituent path	& NP \ua\ NP \ua\ S \ua\ S \da\ NP\\
+~~~~~~~~Base syntactic chunk path & NP \ra\ NP \ra\ PP \ra\ NP \ra\ VP \ra\ NP \ra\ NP \\
+~~~~~~~~Typed-dependency path & \textit{Airlines} \la\ $_{subj}$ \textit{matched} \la\ $_{comp}$ \textit{said} \ra\ $_{subj}$ \textit{Wagner} \\
+\bottomrule
+\end{tabular}
+
+\source{Jurafsky and Martin, 2nd ed., p. 730}
+\end{center}
+}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: RE: Supervised Learning}
+
+\textbf{Strengths}
+\begin{itemize}
+\item No need to write extensive/complex rule sets for each domain
+\item System can adapt to any new domain \ra\ provided that training data is supplied
+\item[\ra] beware of data sparsity!
+\end{itemize}
+
+\textbf{Weaknesses}
+\begin{itemize}
+\item Quality of relation extraction dependent on quality and quantity of training data
+\item[\ra] can be difficult, costly and time consuming to generate
+\item Feature extractors can be noisy (e.g. parsers) \ra\ reduce overall performance
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Overview}
+
+\gray{Relation Extraction}
+\begin{itemize}
+\item \gray{Task definition}
+\end{itemize}
+
+\textbf{Approaches to Relation Extraction}
+\begin{itemize}
+\item \gray{Knowledge-engineering approaches to RE}
+\item \gray{Supervised learning approaches to RE}
+\item \textbf{Bootstrapping Approaches to RE}
+\item Distant Supervision Approaches to RE
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Bootstrapping}
+
+\textbf{Motivation}: reduce the number of manually labelled examples needed to build a system
+
+\textbf{Requirements}: 
+\begin{enumerate}
+\item a document collection $\mathcal{D}$
+\item set of trusted tuples \textbf{T}, also called \myemph{seed tuples}
+\item set of trusted patterns \textbf{P}, also called \myemph{seed patterns}
+\end{enumerate}
+
+\textbf{Principle}:
+\begin{enumerate}
+%\setcounter{enumi}{3}
+\item Find tuples from \textbf{T} in $\mathcal{D}$ \Ra\ extract patterns, add them to \textbf{P}
+\item Match patterns \textbf{P} in $\mathcal{D}$ \Ra\ extract tuples, add them to \textbf{T}
+\item[\ra] Rinse, repeat
+\end{enumerate}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Bootstrapping}
+
+\begin{center}
+\includegraphics[width=.55\textwidth]{pattern_based_RE}
+
+\source{Jurafsky and Martin, 2nd ed., p. 740}
+\end{center}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Bootstrapping - DIPRE}
+
+\myemphb{DIPRE} Dual Iterative Pattern Relation Expansion – proposed by Sergie Brin (1999)
+
+\vfill
+
+\textbf{Aim}: to extract useful relational tuples from the Web, of the form (\textsc{person}, \textsc{book\_title})
+Ex.: (\textsc{Leo Tolstoy}, \textsc{War and Peace})
+
+\vfill
+
+\textbf{Method}:
+\begin{itemize}
+\item Exploit duality of patterns and relations
+\begin{itemize}
+\item Good tuples help find good patterns
+\item Good patterns help find good tuples
+\end{itemize}
+
+\item Use the bootstrapping method starting with user-supplied tuples
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Information Extraction: Relation Extraction: Bootstrapping - DIPRE}
+
+Main loop in DIPRE is the following:
+\begin{enumerate}
+\item $\mathcal{R}'$ \la\ Sample
+\begin{itemize}
+\item[] $\mathcal{R}'$ is a set of tuples \ra\ approximation of target relation
+\item[] Sample is the seed tuples (e.g. 5 author-title pairs)
+\end{itemize}
+
+\item $\mathcal{O}$ \la\ FindOccurrences($\mathcal{R}'$, $\mathcal{D}$)
+\begin{itemize}
+\item[] $\mathcal{O}$ contains all occurrences of tuples in $\mathcal{R}'$ appearing in $\mathcal{D}$
+\end{itemize}
+
+\item $\mathcal{P}$ \la\ GenPatterns($\mathcal{O}$)
+\begin{itemize}
+\item[] $\mathcal{P}$ contains the patterns generated based on the occurrences
+\item[] Seek for low error rate patterns, ideally having high coverage
+\end{itemize}
+
+\item $\mathcal{R}'$ \la\ $M_{\mathcal{D}}(\mathcal{P})$
+\begin{itemize}
+\item[] Update $\mathcal{R}'$ with the tuples from $\mathcal{D}$ matching patterns in $\mathcal{P}$
+\end{itemize}
+
+\item Stop if $\mathcal{R}'$ is large enough, otherwise go to \circled{2}{black}
+
+\end{enumerate}
+
+\end{frame}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+