end of Rel Extract + start of Deep learning

macros.tex

@@ -112,6 +112,8 @@
 \newcommand{\annot}[2]{[#1]$_{#2}$}
 %\newcommand{\annot}[2]{#1\_#2}
 
+\newcommand{\tuple}[1]{\textlangle#1\textrangle}
+
 \newcommand{\B}[1]{B$_{#1}$}
 \newcommand{\I}[1]{I$_{#1}$}
 

text_processing/ie_introduction.tex

@@ -480,12 +480,12 @@ Approaches to IE may be placed into four categories:
 \colorbox{liumlightgray}{
 \parbox{.99\textwidth}{
 
-\myhl{brown!90}{\annot{Mr. \tikzmark{a} Wright}{PER}}, \textbf{\underline{\annot{executive vice president}{POSITION}}} of \myhl{cyan!40}{\annot{Merrill Lynch \tikzmark{b} Canada Inc.}{ORG}}\\
+\myhl{brown!90}{\annot{Mr. \tikzmark{ie_a} Wright}{PER}}, \textbf{\underline{\annot{executive vice president}{POSITION}}} of \myhl{cyan!40}{\annot{Merrill Lynch \tikzmark{ie_b} Canada Inc.}{ORG}}\\
 
 \begin{tikzpicture}[overlay,remember picture]
- \draw [very thick, color=carminered] ($({pic cs:a})+(0ex,-1ex)$) -- ($({pic cs:a})+(0ex,-3ex)$);
- \draw [very thick, color=carminered] ($({pic cs:a})+(0ex,-3ex)$) -- ($({pic cs:b})+(0ex,-3ex)$)  node [midway, below, color=carminered] {is-employed-by};
- \draw [very thick, color=carminered] ($({pic cs:b})+(0ex,-3ex)$) -- ($({pic cs:b})+(0ex,-1ex)$);
+ \draw [very thick, color=carminered] ($({pic cs:ie_a})+(0ex,-1ex)$) -- ($({pic cs:ie_a})+(0ex,-3ex)$);
+ \draw [very thick, color=carminered] ($({pic cs:ie_a})+(0ex,-3ex)$) -- ($({pic cs:ie_b})+(0ex,-3ex)$)  node [midway, below, color=carminered] {is-employed-by};
+ \draw [very thick, color=carminered] ($({pic cs:ie_b})+(0ex,-3ex)$) -- ($({pic cs:ie_b})+(0ex,-1ex)$);
 \end{tikzpicture}
 
 }}
@@ -519,12 +519,12 @@ Action: & \textbf{add-relation(is-employed-by(\$Person,\$Organization))} \\
 \colorbox{liumlightgray}{
 \parbox{.99\textwidth}{
 
-\myhl{brown!90}{\annot{Mr. \tikzmark{a2} Wright}{PER}}, \textbf{\underline{\annot{executive vice president}{POSITION}}} of \myhl{cyan!40}{\annot{Merrill Lynch \tikzmark{b2} Canada Inc.}{ORG}}\\
+\myhl{brown!90}{\annot{Mr. \tikzmark{ie_a2} Wright}{PER}}, \textbf{\underline{\annot{executive vice president}{POSITION}}} of \myhl{cyan!40}{\annot{Merrill Lynch \tikzmark{ie_b2} Canada Inc.}{ORG}}\\
 
 \begin{tikzpicture}[overlay,remember picture]
- \draw [very thick, color=carminered] ($({pic cs:a2})+(0ex,-1ex)$) -- ($({pic cs:a2})+(0ex,-3ex)$);
- \draw [very thick, color=carminered] ($({pic cs:a2})+(0ex,-3ex)$) -- ($({pic cs:b2})+(0ex,-3ex)$)  node [midway, below, color=carminered] {is-employed-by};
- \draw [very thick, color=carminered] ($({pic cs:b2})+(0ex,-3ex)$) -- ($({pic cs:b2})+(0ex,-1ex)$);
+ \draw [very thick, color=carminered] ($({pic cs:ie_a2})+(0ex,-1ex)$) -- ($({pic cs:ie_a2})+(0ex,-3ex)$);
+ \draw [very thick, color=carminered] ($({pic cs:ie_a2})+(0ex,-3ex)$) -- ($({pic cs:ie_b2})+(0ex,-3ex)$)  node [midway, below, color=carminered] {is-employed-by};
+ \draw [very thick, color=carminered] ($({pic cs:ie_b2})+(0ex,-3ex)$) -- ($({pic cs:ie_b2})+(0ex,-1ex)$);
 \end{tikzpicture}
 
 }}

text_processing/refs.bib

+
+
+@book{Goodfellow-et-al-2016,
+    title={Deep Learning},
+    author={Ian Goodfellow and Yoshua Bengio and Aaron Courville},
+    publisher={MIT Press},
+    note={\url{http://www.deeplearningbook.org}},
+    year={2016}
+}
+

text_processing/refs_sa.bib

@@ -111,3 +111,83 @@ title = {{A Simple Named Entity Extractor Using AdaBoost}},
 url = {https://doi.org/10.3115/1119176.1119197},
 year = {2003}
 }
+
+@inproceedings{Mintz2009,
+ author = {Mintz, Mike and Bills, Steven and Snow, Rion and Jurafsky, Dan},
+ title = {Distant Supervision for Relation Extraction Without Labeled Data},
+ booktitle = {Proceedings of the Joint Conference of the 47th Annual Meeting of the ACL and the 4th International Joint Conference on Natural Language Processing of the AFNLP: Volume 2},
+ series = {ACL '09},
+ year = {2009},
+ isbn = {978-1-932432-46-6},
+ location = {Suntec, Singapore},
+ pages = {1003--1011},
+ numpages = {9},
+ url = {http://dl.acm.org/citation.cfm?id=1690219.1690287},
+ acmid = {1690287},
+ publisher = {Association for Computational Linguistics},
+ address = {Stroudsburg, PA, USA},
+} 
+
+@inproceedings{Agichtein:2000,
+ author = {Agichtein, Eugene and Gravano, Luis},
+ title = {Snowball: Extracting Relations from Large Plain-text Collections},
+ booktitle = {Proceedings of the Fifth ACM Conference on Digital Libraries},
+ series = {DL '00},
+ year = {2000},
+ isbn = {1-58113-231-X},
+ location = {San Antonio, Texas, USA},
+ pages = {85--94},
+ numpages = {10},
+ url = {http://doi.acm.org/10.1145/336597.336644},
+ doi = {10.1145/336597.336644},
+ acmid = {336644},
+ publisher = {ACM},
+ address = {New York, NY, USA},
+} 
+
+
+@inproceedings{Brin:1998,
+ author = {Brin, Sergey},
+ title = {Extracting Patterns and Relations from the World Wide Web},
+ booktitle = {Selected Papers from the International Workshop on The World Wide Web and Databases},
+ series = {WebDB '98},
+ year = {1999},
+ isbn = {3-540-65890-4},
+ pages = {172--183},
+ numpages = {12},
+ url = {http://dl.acm.org/citation.cfm?id=646543.696220},
+ acmid = {696220},
+ publisher = {Springer-Verlag},
+ address = {London, UK},
+} 
+
+@book{Jurafsky:2009,
+ author = {Jurafsky, Daniel and Martin, James H.},
+ title = {Speech and Language Processing, see chapter 18.2 Relation Extraction},
+ year = {2009},
+ isbn = {0131873210},
+ publisher = {Prentice-Hall, Inc.},
+ address = {Upper Saddle River, NJ, USA},
+ url = {https://web.stanford.edu/~jurafsky/slp3/},
+ comment = {},
+} 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

text_processing/sa_definition.tex

@@ -553,7 +553,7 @@ I returned the phone yesterday.
 \only<2->{
 On \myhl{orange}{4th Nov. 2018}, \myhl{brown!90}{John Doe} wrote:  \\
 %\begin{spacing}{0.5}
-This past Saturday, \myhl{brown!90}{\tikzmark{a}I} bought a \myhl{cyan!40}{\tikzmark{a1}Nokia} phone and \myhl{brown!90}{\tikzmark{b}my girlfriend} bought a \myhl{cyan!40}{\tikzmark{b1}Motorola} phone with \myhl{cyan!40}{Bluetooth}. 
+This past Saturday, \myhl{brown!90}{\tikzmark{a0}I} bought a \myhl{cyan!40}{\tikzmark{a1}Nokia} phone and \myhl{brown!90}{\tikzmark{b0}my girlfriend} bought a \myhl{cyan!40}{\tikzmark{b1}Motorola} phone with \myhl{cyan!40}{Bluetooth}. 
 We called each other when we got home.\\
 The \myhl{blue!20}{voice\tikzmark{a2}} on \myhl{brown!90}{my} phone \myhl{red!40}{was not so clear}, \myhl{red!40}{worse than my previous phone.}\\
 The \myhl{blue!20}{battery life\tikzmark{a3}} \myhl{red!40}{was short too.} \\
@@ -572,13 +572,13 @@ I returned the phone yesterday. \\
  \textbf{\ra\  Identify the relations}
 
 \begin{tikzpicture}[overlay,remember picture]
-    \draw[very thick, -Stealth, carminered]         ($({pic cs:a})+(1ex,2ex)$)  to [bend left, sloped, ""]  ($({pic cs:a1})+(1ex,+2ex)$);
+    \draw[very thick, -Stealth, carminered]         ($({pic cs:a0})+(1ex,2ex)$)  to [bend left, sloped, ""]  ($({pic cs:a1})+(1ex,+2ex)$);
     \draw[very thick, -Stealth, carminered]         ($({pic cs:a1})+(1ex,-1ex)$)  to [bend left, sloped, ""]  ($({pic cs:a2})+(1ex,+1ex)$);
     \draw[very thick, -Stealth, carminered]         ($({pic cs:a1})+(1ex,-1ex)$)  to [bend left, sloped, ""]  ($({pic cs:a3})+(1ex,+1ex)$);
     \draw[very thick, -Stealth, carminered]         ($({pic cs:a1})+(1ex,-1ex)$)  to [bend left, sloped, ""]  ($({pic cs:a4})+(3ex,+2ex)$);
     \draw[very thick, -Stealth, carminered]         ($({pic cs:a4})+(3ex,2ex)$)  to [bend left, sloped, ""]  ($({pic cs:a5})+(2ex,+2ex)$);
         
-    \draw[very thick, -Stealth, carnationpink]         ($({pic cs:b})+(2ex,2ex)$)  to [bend left, sloped, ""]  ($({pic cs:b1})+(1ex,+1em)$);
+    \draw[very thick, -Stealth, carnationpink]         ($({pic cs:b0})+(2ex,2ex)$)  to [bend left, sloped, ""]  ($({pic cs:b1})+(1ex,+1em)$);
     \draw[very thick, -Stealth, carnationpink]         ($({pic cs:b1})+(3em,-1ex)$)  to [bend left, sloped, ""]  ($({pic cs:b2})+(1ex,+1ex)$);
 \end{tikzpicture}
 }

text_processing/sa_nn.tex

-% !TEX root = text_processing.tex
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{frame}
-\frametitle{}
-
-\vfill
-\centering
-\Huge{\edinred{Sentiment Analysis\\Corpus-based / Machine Learning}}
-
-\end{frame}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{frame}
-\frametitle{Sentiment Analysis: 2 main approaches}
-
-\begin{itemize}
-\item {\bf \color{lightgray} Lexicon based}
-\begin{itemize}
-	\item {\bf \color{lightgray} Binary}
-	\item {\bf \color{lightgray} Gradable}
-\end{itemize}
-\item \textbf{Corpus based}
-\begin{itemize}
-	\item {\bf \color{lightgray} Naive Bayes}
-	\item \textbf{Deep Learning}
-\end{itemize}
-\end{itemize}
-
-\end{frame}
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{frame}
-\frametitle{Deep Learning for Sentiment Analysis}
-
-\begin{block}{Principle}
-\myemph{Project} or represent the \textbf{text} into a \myemph{continuous space} and train an estimator operating into this space to compute the probability of the sentiment.
-\end{block}
-
-Basically it is like:
-
-\includegraphics[width=0.75\textwidth]{sa_nn}
-
-
-
-\end{frame}
\ No newline at end of file

text_processing/text_processing.tex

@@ -84,17 +84,19 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \input{textproc_introduction}
 
-%\input{sa_plan.tex}
+%\input{sa_intro.tex}
 %\input{sa_definition.tex} % 1st lecture
 %\input{sa_lexicon.tex} % 2nd lecture
 %\input{sa_bayes.tex} % 3rd lecture
-%\input{sa_nn.tex}
-
+%
 %\input{ie_plan.tex}
-%\input{ie_introduction.tex} % 4th lecture
-\input{ie_ner.tex} % 5th lecture
-\input{ie_relation_extraction.tex} % 6th lecture
-%\input{extra_reading}
+%\input{ie_introduction.tex} % 4th lecture + start 5th lecture
+%\input{ie_ner.tex} % end of 5th lecture + 6th lecture
+%\input{ie_relation_extraction.tex} % 7th lecture
+
+\input{tp_nn.tex}
+
+\input{sa_extra_reading}
 
 % and kill the abominable icon
 \setbeamertemplate{bibliography item}{}
@@ -104,7 +106,7 @@
 %        \bibliographystyle{amsalpha}
 %        \bibliographystyle{apalike}
         \bibliographystyle{IEEEtran}
-        \bibliography{refs,refs_sa}
+{\footnotesize        \bibliography{refs,refs_sa} }
 \end{frame}
 
 

text_processing/text_processing_L6.tex

+
+\documentclass[aspectratio=169,t,xcolor=table]{beamer}
+%\documentclass[t]{beamer}
+%\documentclass[handout,t]{beamer}
+
+\mode<presentation>
+{
+  %\usetheme{PaloAlto}
+%  \usetheme{Hannover}
+  \usetheme{informatics}
+  \useoutertheme{infolines}
+ % \setbeamercovered{transparent} % or whatever (possibly just delete it)
+}
+
+\setbeamertemplate{navigation symbols}{}
+
+\setlength{\extrarowheight}{3pt}
+
+\input{../LatexColors.incl.tex}
+\input ../mycolors.tex
+\input ../macros.tex
+\input ../macros_en.tex
+\input ../macros_beamer.tex
+
+\usepackage{setspace}
+
+%\setbeamercovered{transparent}
+
+\usepackage[absolute,showboxes,overlay]{textpos}
+%\TPshowboxestrue % commenter une fois fini
+\TPshowboxesfalse % décommenter pour faire disparaitre les boites
+\textblockorigin{10mm}{10mm} % origine des positions
+
+
+% This is only inserted into the PDF information catalog. Can be left out. 
+\subject{COM3110/Text Processing}
+\title[]{[Text Processing]\\ Information Extraction: Relation Extraction}
+
+\author[L. Barrault]{Loïc Barrault}
+
+\institute[University of Sheffield]
+{
+  l.barrault@sheffield.ac.uk \\
+  Department of Computer Science\\
+}
+
+\date{November 26, 2019}
+
+\begin{document}
+
+\begin{frame}
+  \titlepage
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\input{textproc_introduction}
+\input{ie_relation_extraction.tex}
+%\input{extra_reading}
+
+
+% and kill the abominable icon
+\setbeamertemplate{bibliography item}{}
+
+\begin{frame}[allowframebreaks]
+        \frametitle{References}
+%        \bibliographystyle{amsalpha}
+%        \bibliographystyle{apalike}
+        \bibliographystyle{IEEEtran}
+{\footnotesize        \bibliography{refs,refs_sa} }
+\end{frame}
+
+
+\end{document}
+
+

text_processing/tp_nn.tex

+% !TEX root = text_processing.tex
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{}
+
+\vfill
+\centering
+\Huge{\edinred{Text processing\\Deep Learning}}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Text Processing: Deep Learning: Overview}
+
+\begin{itemize}
+\item Shortest introduction to Neural networks
+\item Representing words
+\item Representing sentences
+\item Classifying
+\item Deep Learning for Sentiment Analysis
+\item Deep Learning for Information Extraction (NER)
+\end{itemize}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Biological neuron / nerve cell}
+
+\begin{center}
+\includegraphics[width=0.95\textwidth]{figures/neuron_en}
+\end{center}
+
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Hebb principle}
+
+\begin{center}
+\includegraphics[width=0.5\textwidth]{figures/neuron_en}
+\vfill
+Hebb: \myemph{``Neurons that fire together, wire together''}
+\end{center}
+
+
+%\vspace{1cm}
+\begin{itemize}
+  \item Cells are active together \ra\ reinforce their connection
+  \item Cells are not active together \ra\ diminish their connection
+  \item[] \Ra\ \myemph{local process} there is no global supervision
+\end{itemize}
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{The perceptron}
+
+\textbf{Perceptron}: computing unit loosely inspired by the biological neuron
+\vfill
+
+\centerline{
+\begin{tabular}[c]{cc}
+  \begin{tabular}[c]{c}
+  \includegraphics[width=0.4\textwidth]{figures/BpNeurone}
+  \end{tabular}
+  \hspace*{-1cm}
+  &
+  \begin{tabular}[c]{rl}
+    input: & $x_i$ \\
+    weights: & $w_i$ \\
+    threshold: & $s$ \\
+    activity: & $\displaystyle a = \sum_i w_i x_i + s$ \\
+    output: & $y=f(a)$ \\
+  activation function: & $f=threshold(a)$
+  \end{tabular}
+\end{tabular}
+}
+\vfill
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{The Perceptron and the logical functions}
+
+\begin{tabular}[t]{c}
+  y = a OR b \\[10pt]
+  \includegraphics[height=0.7\textheight]{figures/or}
+\end{tabular}
+\hfill
+\begin{tabular}[t]{c}
+  y = a AND b \\[10pt]
+  \includegraphics[height=0.7\textheight]{figures/and}
+\end{tabular}
+\hfill
+\begin{tabular}[t]{c}
+  y = a XOR b \\[10pt]
+  \includegraphics[height=0.7\textheight]{figures/xor}
+\end{tabular}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Multilayer perceptron}
+
+\begin{columns}
+\begin{column}{.5\textwidth}
+
+\begin{center}
+\includegraphics[width=0.95\textwidth]{figures/mlp}
+\end{center}
+
+\end{column}
+\begin{column}{.5\textwidth}
+\begin{eqnarray*}
+y_i^{2} & = & f\left(\sum_j w^{1}_{ij} ~ x_j^{1}\right) \\
+y_i^{3} & = & f\left(\sum_j w^{2}_{ij} ~ y_j^{2}\right) \\
+       & \vdots & \\
+y_i^{c} & = & f \left(\sum_j w^{c-1}_{ij} ~ y_j^{c-1}\right) \\
+\end{eqnarray*}
+\end{column}
+\end{columns}
+\Ra\ \myemph{propagation} of the input $\vx$ towards the output $\vy$
+\end{frame}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Deep Learning for Sentiment Analysis}
+
+\begin{block}{Principle}
+\myemph{Project} or represent the \textbf{text} into a \myemph{continuous space} and train an estimator operating into this space to compute the probability of the sentiment.
+\end{block}
+
+Basically it is like:
+
+%\includegraphics[width=0.75\textwidth]{sa_nn}
+
+
+
+\end{frame}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}
+\frametitle{Text Processing: Deep Learning: Resources}
+
+
+Deep Learning book: \url{https://www.deeplearningbook.org/}
+\cite{Goodfellow-et-al-2016}
+
+\end{frame}
\ No newline at end of file