Commit 6ea31fce by Loïc Barrault

### more printing friendly

parent cd08954c
 ... @@ -16,8 +16,9 @@ ... @@ -16,8 +16,9 @@ \begin{frame} \begin{frame} \frametitle{Reminder: RNNLM} \frametitle{Reminder: RNNLM} \centering \centerline{ \includegraphics[width=0.55\textwidth]{figures_en/rnn_unrolled_all} \includegraphics[width=0.55\textwidth]{figures_en/rnn_unrolled_all} } \begin{itemize} \begin{itemize} \item<+-> Probability of a word sequence $\vw = (w_1, w_2, ..., w_\ell)$ \item<+-> Probability of a word sequence $\vw = (w_1, w_2, ..., w_\ell)$ \item[]{ \small{ $p(\vw) = p(w_1) \times p(w_2|w_1) \times p(w_3 | w_1, w_2) \times \dots \times p(w_l | w_1, ..., w_{\ell-1}) = \ds \prod_{t=1}^{\ell} p(w_t|w_1, ..., w_{t-1})$ } } \item[]{ \small{ $p(\vw) = p(w_1) \times p(w_2|w_1) \times p(w_3 | w_1, w_2) \times \dots \times p(w_l | w_1, ..., w_{\ell-1}) = \ds \prod_{t=1}^{\ell} p(w_t|w_1, ..., w_{t-1})$ } } ... @@ -125,52 +126,55 @@ A document & A summary \\ ... @@ -125,52 +126,55 @@ A document & A summary \\ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Back to the encoder} \frametitle{Back to the encoder} \begin{block}{How to represent the source sequence with a fixed size vector $\edinred{\vm{x}}$ ? } \textbf{How to represent the source sequence with a fixed size vector $\edinred{\vm{x}}$ ? } \begin{itemize} \begin{itemize} \item Previous part: RNN, GRU, LSTM \item Previous part: RNN, GRU, LSTM \item What about this architecture? \\ \item What about this architecture? \\ {\centering \centerline{ \includegraphics[width=0.35\textwidth]{figures_en/bow} \includegraphics[width=0.35\textwidth]{figures_en/bow} \item[]} } \item<2> \textbf{Bag of words} representation \item<2> \textbf{Bag of words} representation \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Back to the encoder} \frametitle{Back to the encoder} \begin{block}{\cite{kalchbrenner2013} } \textbf{\cite{kalchbrenner2013}} \vfill \begin{itemize} \begin{itemize} \item[] \item[] { \centering \centerline{ \includegraphics[width=0.35\textwidth]{figures_en/conv_sent_encoder} \includegraphics[width=0.35\textwidth]{figures_en/conv_sent_encoder} \item[] } } \vfill \item<+-> \edinred{Convolutional} encoder \item<+-> \edinred{Convolutional} encoder \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{How to condition on $\vm{x}$ ?} \frametitle{How to condition on $\vm{x}$ ?} \begin{block}{\cite{kalchbrenner2013} } \centering \textbf{\cite{kalchbrenner2013}} \includegraphics[width=0.95\textwidth]<+>{figures_en/rnn_unrolled_4} \includegraphics[width=0.95\textwidth]<+>{figures_en/cond_rnn_unrolled_1} \centerline{ \includegraphics[width=0.95\textwidth]<+>{figures_en/cond_rnn_unrolled_2} \includegraphics[width=0.95\textwidth]{figures_en/rnn_unrolled_4}<+> \includegraphics[width=0.95\textwidth]<+>{figures_en/cond_rnn_unrolled_3} \includegraphics[width=0.95\textwidth]{figures_en/cond_rnn_unrolled_1}<+> \includegraphics[width=0.95\textwidth]<+>{figures_en/cond_rnn_unrolled_all} \includegraphics[width=0.95\textwidth]{figures_en/cond_rnn_unrolled_2}<+> \includegraphics[width=0.55\textwidth]<+>{figures_en/cond_rnn_unrolled_all} \includegraphics[width=0.95\textwidth]{figures_en/cond_rnn_unrolled_3}<+> \includegraphics[width=0.95\textwidth]{figures_en/cond_rnn_unrolled_all}<+> \includegraphics[width=0.55\textwidth]{figures_en/cond_rnn_unrolled_all}<+> } \begin{itemize} \begin{itemize} \item[]<.-> $\vm{h}_t = \phi(\vm{M}[\vm{h}_{t-1}; \vm{w}_{t-1}] \edinred{+ \vm{x}} + \vm{b})$ \item[]<.-> $\vm{h}_t = \phi(\vm{M}[\vm{h}_{t-1}; \vm{w}_{t-1}] \edinred{+ \vm{x}} + \vm{b})$ \item[]<.-> $\vm{z}_t = \vm{S}~\vm{h}_{t} + \vm{b'}$ \item[]<.-> $\vm{z}_t = \vm{S}~\vm{h}_{t} + \vm{b'}$ \item[]<.-> $p(\vm{w}_t | \edinred{\vm{x}}, \vm{w}_{$ p(\vm{w}_t | \edinred{\vm{x}}, \vm{w}_{{ {\color{edinred} [1.]} ~Word encoded into \emph{1-hot} vector } \item<2->{ {\color{edinred} [1.]} ~Word encoded into \emph{1-hot} vector } \item<3->{ {\color{cyan} [2.]} Projection into an \textbf{\textit{embedding}} } \item<3->{ {\color{cyan} [2.]} Projection into an \textbf{\textit{embedding}} } ... @@ -219,18 +223,16 @@ A document & A summary \\ ... @@ -219,18 +223,16 @@ A document & A summary \\ \item<8->{ {\color{orange} [7.]} Next word (most probable) } \item<8->{ {\color{orange} [7.]} Next word (most probable) } \end{itemize} \end{itemize} \end{block} \column{0.5\textwidth} \\ \column{0.5\textwidth} \\ \centering{ \centering{ \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_1} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_1}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_2} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_2}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_3} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_3}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_4} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_4}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_5} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_5}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_6} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_6}<+> \includegraphics[height=0.8\textwidth]<+>{figures_en/enc_dec_all_7} \includegraphics[height=0.8\textwidth]{figures_en/enc_dec_all_7}<+> }%centering }%centering \end{columns} \end{columns} ... @@ -240,7 +242,6 @@ A document & A summary \\ ... @@ -240,7 +242,6 @@ A document & A summary \\ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Results} \frametitle{Results} \begin{block}{} \centerline{ \centerline{ \includegraphics[width=0.8\textwidth]{figures_en/nmt_sentence_length} \includegraphics[width=0.8\textwidth]{figures_en/nmt_sentence_length} } } ... @@ -260,7 +261,6 @@ A document & A summary \\ ... @@ -260,7 +261,6 @@ A document & A summary \\ \end{enumerate} \end{enumerate} } } \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ... @@ -306,7 +306,7 @@ A document & A summary \\ ... @@ -306,7 +306,7 @@ A document & A summary \\ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{From vector to matrix representation} \frametitle{From vector to matrix representation} \begin{block}{} \begin{itemize} \begin{itemize} \item Represent input sequence with a matrix \item Represent input sequence with a matrix \item Generate output sequence using the matrix \item Generate output sequence using the matrix ... @@ -317,13 +317,13 @@ A document & A summary \\ ... @@ -317,13 +317,13 @@ A document & A summary \\ \item[\ra] Solve the problem of gradient stream \item[\ra] Solve the problem of gradient stream \item[] \item[] \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Representing sentences with a matrix} \frametitle{Representing sentences with a matrix} \begin{block}{} \begin{itemize} \begin{itemize} \item Fixed size vector: regardless the input sequence size \item Fixed size vector: regardless the input sequence size \item[] \item[] ... @@ -334,13 +334,13 @@ A document & A summary \\ ... @@ -334,13 +334,13 @@ A document & A summary \\ \item[\ra] How to build this matrix? \item[\ra] How to build this matrix? \item[] \item[] \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Concatenation} \frametitle{Concatenation} \begin{block}{} \begin{itemize} \begin{itemize} \item Concatenation of word embeddings \item Concatenation of word embeddings \item simplest possible model \item simplest possible model ... @@ -352,14 +352,14 @@ A document & A summary \\ ... @@ -352,14 +352,14 @@ A document & A summary \\ \item Using bidirectional RNNs \cite{bahdanau2014} \item Using bidirectional RNNs \cite{bahdanau2014} \item[\ra] most used method \item[\ra] most used method \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Bidirectional Encoder} \frametitle{Bidirectional Encoder} \begin{block}{} \centering{ \centering{ \only<1>{ \includegraphics[height=0.5\textheight]{figures_en/bidir_enc_1} \\ \only<1>{ \includegraphics[height=0.5\textheight]{figures_en/bidir_enc_1} \\ {\color{gray} [1.]} ~\emph{1-hot} vector + projection + update \alert{forward} hidden unit } {\color{gray} [1.]} ~\emph{1-hot} vector + projection + update \alert{forward} hidden unit } ... @@ -369,14 +369,14 @@ A document & A summary \\ ... @@ -369,14 +369,14 @@ A document & A summary \\ {\color{brown} [2.]} \alert{Annotation} = concatenation of \alert{forward} and \alert{backward} vectors \\ {\color{brown} [2.]} \alert{Annotation} = concatenation of \alert{forward} and \alert{backward} vectors \\ {\small Every $\vm{h}_i$ encodes the full sentence with a focus on the \ith\ word} } {\small Every $\vm{h}_i$ encodes the full sentence with a focus on the \ith\ word} } } } \end{block} \vspace{.2cm} \vspace{.2cm} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Attention Mechanism} \frametitle{Attention Mechanism} \begin{block}{} \begin{itemize} \begin{itemize} \item How to process this matrix into the decoder? \item How to process this matrix into the decoder? \item Reminder: decoder is made of one (or several) recurrent units \item Reminder: decoder is made of one (or several) recurrent units ... @@ -389,28 +389,26 @@ A document & A summary \\ ... @@ -389,28 +389,26 @@ A document & A summary \\ \item[\ra] \textbf{Attention mechanism} \item[\ra] \textbf{Attention mechanism} \item[] \item[] \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Attention mechanism} \frametitle{Attention mechanism} \begin{block}{Before: sentence represented by a vector } \textbf{Before: sentence represented by a vector } \centering{ \centerline{ \includegraphics[height=0.7\textheight]{figures_en/enc_dec_all} \includegraphics[height=0.7\textheight]{figures_en/enc_dec_all} } } \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Attention mechanism} \frametitle{Attention mechanism} \begin{block}{After: sentence represented by a matrix} \textbf{After: sentence represented by a matrix} \centering{ \centerline{ \includegraphics[height=0.7\textheight]{figures_en/dec_attention_0} \includegraphics[height=0.7\textheight]{figures_en/dec_attention_0} } } \end{block} \end{frame} \end{frame} ... @@ -425,7 +423,7 @@ A document & A summary \\ ... @@ -425,7 +423,7 @@ A document & A summary \\ % % \begin{columns} \begin{columns} \column{0.5\textwidth} \column{0.5\textwidth} \begin{block}{} \begin{itemize} \begin{itemize} \item<+-> {\color{brown} [2.]} ~Decoder gets the \alert{annotations} from encoder. \item<+-> {\color{brown} [2.]} ~Decoder gets the \alert{annotations} from encoder. \item<+-> {\color{cyan} [3.]} ~ \alert{Attention weights} calculated with feedforward NN. \\ \item<+-> {\color{cyan} [3.]} ~ \alert{Attention weights} calculated with feedforward NN. \\ ... @@ -435,7 +433,7 @@ A document & A summary \\ ... @@ -435,7 +433,7 @@ A document & A summary \\ \item<+-> {\color{purple} [5.]} Calculate probability distribution for \alert{all} words \item<+-> {\color{purple} [5.]} Calculate probability distribution for \alert{all} words \item<+-> {\color{orange} [6.]} Generate next word (most probable) \item<+-> {\color{orange} [6.]} Generate next word (most probable) \end{itemize} \end{itemize} \end{block} \column{0.5\textwidth} \column{0.5\textwidth} \end{columns} \end{columns} ... @@ -474,35 +472,33 @@ A document & A summary \\ ... @@ -474,35 +472,33 @@ A document & A summary \\ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{A word on gradients} \frametitle{A word on gradients} \begin{block}{Without attention mechanism: } \textbf{Without attention mechanism: } \centering{ \centerline{ \only<1>{ \includegraphics[height=0.6\textheight]{figures_en/enc_dec_all} } \only<1>{ \includegraphics[height=0.6\textheight]{figures_en/enc_dec_all} } \only<2>{ \includegraphics[height=0.6\textheight]{figures_en/dec_backprop} } \only<2>{ \includegraphics[height=0.6\textheight]{figures_en/dec_backprop} } } } \begin{itemize} \begin{itemize} \item<2-> Gradients go through the last encoder hidden state. \item<2-> Gradients go through the last encoder hidden state. \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{A word on gradients} \frametitle{A word on gradients} \begin{block}{With attention mechanism: } \textbf{With attention mechanism: } \centering{ \centerline{ \only<1>{ \includegraphics[height=0.6\textheight]{figures_en/dec_attention_backprop} } \only<1>{ \includegraphics[height=0.6\textheight]{figures_en/dec_attention_backprop} } } } \begin{itemize} \begin{itemize} \item Attention mechanism facilitate gradients propagation towards the encoder \item Attention mechanism facilitate gradients propagation towards the encoder \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Attention and translation} \frametitle{Attention and translation} \begin{block}{ Some considerations/remarks :} \textbf{Some considerations/remarks:} \begin{itemize} \begin{itemize} \item Does a human translator memorise the whole source sentence and then proceed to translate? \item Does a human translator memorise the whole source sentence and then proceed to translate? \begin{itemize} \begin{itemize} ... @@ -516,7 +512,6 @@ A document & A summary \\ ... @@ -516,7 +512,6 @@ A document & A summary \\ \item Should humans be a model for machines? that's another story... \item Should humans be a model for machines? that's another story... \item[] \item[] \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% ... @@ -524,14 +519,13 @@ A document & A summary \\ ... @@ -524,14 +519,13 @@ A document & A summary \\ \frametitle{Attention and translation} \frametitle{Attention and translation} \begin{columns} \begin{columns} \column{0.5\textwidth} \column{0.5\textwidth} \begin{block}{Attention Mechanism \Ra\ alignment } \textbf{Attention Mechanism \Ra\ alignment} \begin{itemize} \begin{itemize} \item For each produced word, a set of attention weights is created (set length is size of source sequence) \item For each produced word, a set of attention weights is created (set length is size of source sequence) \item \textbf{Alignment} and translation models jointly trained! \item \textbf{Alignment} and translation models jointly trained! \item[\ra] \cite{bahdanau2014} \item[\ra] \cite{bahdanau2014} \item[] \item[] \end{itemize} \end{itemize} \end{block} \column{0.5\textwidth} \column{0.5\textwidth} \centering{ \centering{ ... @@ -543,7 +537,6 @@ A document & A summary \\ ... @@ -543,7 +537,6 @@ A document & A summary \\ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Summary} \frametitle{Summary} \begin{block}{ } \begin{itemize} \begin{itemize} \item Attention \item Attention \begin{itemize} \begin{itemize} ... @@ -556,13 +549,12 @@ A document & A summary \\ ... @@ -556,13 +549,12 @@ A document & A summary \\ \end{itemize} \end{itemize} \item[] \item[] \end{itemize} \end{itemize} \end{block} \end{frame} \end{frame} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{frame} \begin{frame} \frametitle{Algorithm} \frametitle{Algorithm} \centering{ \centerline{ \includegraphics[height=0.8\textheight]{figures_en/dec_algo} \includegraphics[height=0.8\textheight]{figures_en/dec_algo} } } \begin{itemize} \begin{itemize} ... ...