textproc_nn.tex 4.61 KB
Newer Older
1
2
3
4
5
6
7
% !TEX root = text_processing.tex
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{}

\vfill
\centering
Loïc Barrault's avatar
Loïc Barrault committed
8
\Huge{\edinred{[Text processing]\\Deep Learning}}
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Text Processing: Deep Learning: Overview}

\begin{itemize}
\item Shortest introduction to Neural networks
\item Representing words
\item Representing sentences
\item Classifying
\item Deep Learning for Sentiment Analysis
\item Deep Learning for Information Extraction (NER)
\end{itemize}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Biological neuron / nerve cell}

\begin{center}
\includegraphics[width=0.95\textwidth]{figures/neuron_en}
\end{center}

\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Hebb principle}

\begin{center}
\includegraphics[width=0.5\textwidth]{figures/neuron_en}
\vfill
Hebb: \myemph{``Neurons that fire together, wire together''}
\end{center}


%\vspace{1cm}
\begin{itemize}
  \item Cells are active together \ra\ reinforce their connection
  \item Cells are not active together \ra\ diminish their connection
  \item[] \Ra\ \myemph{local process} there is no global supervision
\end{itemize}
\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{The perceptron}

\textbf{Perceptron}: computing unit loosely inspired by the biological neuron

Loïc Barrault's avatar
Loïc Barrault committed
63
64
65
66
67
68
69
70
\begin{columns}
\begin{column}{.5\textwidth}
\begin{center}
	\includegraphics[width=0.6\textwidth]{figures/BpNeurone}
\end{center}
\end{column}
\begin{column}{.5\textwidth}
\begin{center}
71
  \begin{tabular}[c]{rl}
Loïc Barrault's avatar
Loïc Barrault committed
72
73
    \textbf{input}: & $\vx = \{x_i\}$ \\
    \myemph{weights}: & $\vw = \{w_i\}$ \\
74
75
    threshold: & $s$ \\
    activity: & $\displaystyle a = \sum_i w_i x_i + s$ \\
Loïc Barrault's avatar
Loïc Barrault committed
76
77
    \myemph{activation function}: & $f=\text{threshold function}$ \\
    \textbf{output}: & $\hat{y}=f(a)$ \\
78
  \end{tabular}
Loïc Barrault's avatar
Loïc Barrault committed
79
80
81
82
83
84
85
86
87
\end{center}
\end{column}
\end{columns}
\vspace{.5cm}
Training method: change the weights $\vw$ if a training example $\vx$ is misclassified as follows:
\begin{itemize}
	\item[]   $\hat{\vw}^{new} = \hat{\vw}^{cur} + \hat{\vx} . y $ ~~~ with ~~~ $ y \in \{+1, -1\}$
\end{itemize}
	
88
89
90
91
92
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{The Perceptron and the logical functions}
Loïc Barrault's avatar
Loïc Barrault committed
93
\hspace{1cm}
94
\begin{tabular}[t]{c}
Loïc Barrault's avatar
Loïc Barrault committed
95
  y = a OR b \\[5pt]
96
97
  \includegraphics[height=0.7\textheight]{figures/or}
\end{tabular}
Loïc Barrault's avatar
Loïc Barrault committed
98
\hspace{1cm}%
99
\begin{tabular}[t]{c}
Loïc Barrault's avatar
Loïc Barrault committed
100
101
  y = a AND b \\[5pt]
  \includegraphics[height=0.69\textheight]{figures/and}
102
\end{tabular}
Loïc Barrault's avatar
Loïc Barrault committed
103
\hspace{1cm}%
104
\begin{tabular}[t]{c}
Loïc Barrault's avatar
Loïc Barrault committed
105
106
  y = a XOR b \\[5pt]
  \includegraphics[height=0.73\textheight]{figures/xor}
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
\end{tabular}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Multilayer perceptron}

\begin{columns}
\begin{column}{.5\textwidth}

\begin{center}
\includegraphics[width=0.95\textwidth]{figures/mlp}
\end{center}

\end{column}
\begin{column}{.5\textwidth}
\begin{eqnarray*}
y_i^{2} & = & f\left(\sum_j w^{1}_{ij} ~ x_j^{1}\right) \\
y_i^{3} & = & f\left(\sum_j w^{2}_{ij} ~ y_j^{2}\right) \\
       & \vdots & \\
y_i^{c} & = & f \left(\sum_j w^{c-1}_{ij} ~ y_j^{c-1}\right) \\
\end{eqnarray*}
\end{column}
\end{columns}
\Ra\ \myemph{propagation} of the input $\vx$ towards the output $\vy$
\end{frame}

Loïc Barrault's avatar
Loïc Barrault committed
134
135
136
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{How to train a multilayer perceptron?}
137
138


Loïc Barrault's avatar
Loïc Barrault committed
139
140
141
142
143
\begin{block}{\center \textbf{Backpropagation}}
\begin{center}
\Large{Backward propagation of errors}
\end{center}
\end{block}
144

Loïc Barrault's avatar
Loïc Barrault committed
145
146
147
148
\begin{itemize}
\item What error? \Ra\ Error function depending on the task
\item Estimating a real value: \ra\ 
\end{itemize}
149
150
151
152




Loïc Barrault's avatar
Loïc Barrault committed
153
154
\end{frame}

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173



















Loïc Barrault's avatar
Loïc Barrault committed
174
175
176
177
178
179
180
181
182



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{TITLE}

\end{frame}

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Deep Learning for Sentiment Analysis}

\begin{block}{Principle}
\myemph{Project} or represent the \textbf{text} into a \myemph{continuous space} and train an estimator operating into this space to compute the probability of the sentiment.
\end{block}

Basically it is like:

%\includegraphics[width=0.75\textwidth]{sa_nn}



\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}
\frametitle{Text Processing: Deep Learning: Resources}


Deep Learning book: \url{https://www.deeplearningbook.org/}
\cite{Goodfellow-et-al-2016}

\end{frame}