text: {
   "cells": [
    {
     "cell_type": "code",
     "execution_count": 67,
     "metadata": {},
     "outputs": [],
     "source": [
      "import torch\n",
      "from torch.autograd import Variable\n",
      "from fastai.rnn_reg import *\n",
      "import numpy as np"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 34,
     "metadata": {
      "collapsed": true
     },
     "outputs": [],
     "source": [
      "def embedded_dropout(embed, words, dropout=0.1, scale=None):\n",
      "    \"\"\" Applies dropout in the embedding layer by zeroing out some elements of the embedding vector.\n",
      "    Uses the dropout_mask custom layer to achieve this.\n",
      "\n",
      "    Args:\n",
      "        embed (torch.nn.Embedding): An embedding torch layer\n",
      "        words (torch.nn.Variable): A torch variable\n",
      "        dropout (float): dropout fraction to apply to the embedding weights\n",
      "        scale (float): additional scaling to apply to the modified embedding weights\n",
      "\n",
      "    Returns:\n",
      "        tensor of size: (batch_size x seq_length x embedding_size)\n",
      "\n",
      "    Example:\n",
      "\n",
      "    >> embed = torch.nn.Embedding(10,3)\n",
      "    >> words = Variable(torch.LongTensor([[1,2,4,5] ,[4,3,2,9]]))\n",
      "    >> words.size()\n",
      "        (2,4)\n",
      "\n",
      "    >> dropout_out_ = embedded_dropout(embed, words, dropout=0.40)\n",
      "    >> dropout_out_\n",
      "        Variable containing:\n",
      "        (0 ,.,.) =\n",
      "          1.2549  1.8230  1.9367\n",
      "          0.0000 -0.0000  0.0000\n",
      "          2.2540 -0.1299  1.5448\n",
      "          0.0000 -0.0000 -0.0000\n",
      "\n",
      "        (1 ,.,.) =\n",
      "          2.2540 -0.1299  1.5448\n",
      "         -4.0457  2.4815 -0.2897\n",
      "          0.0000 -0.0000  0.0000\n",
      "          1.8796 -0.4022  3.8773\n",
      "        [torch.FloatTensor of size 2x4x3]\n",
      "    \"\"\"\n",
      "\n",
      "    if dropout:\n",
      "        mask = Variable(dropout_mask(embed.weight.data, (embed.weight.size(0), 1), dropout))\n",
      "        masked_embed_weight = mask * embed.weight\n",
      "    else: masked_embed_weight = embed.weight\n",
      "    if scale: masked_embed_weight = scale * masked_embed_weight\n",
      "\n",
      "    padding_idx = embed.padding_idx\n",
      "    if padding_idx is None: padding_idx = -1\n",
      "    X = embed._backend.Embedding.apply(words, masked_embed_weight,\n",
      "        padding_idx, embed.max_norm, embed.norm_type,\n",
      "        embed.scale_grad_by_freq, embed.sparse\n",
      "    )\n",
      "    return X\n",
      "\n"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Test 1"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### Initialize embedding matrix and input"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 35,
     "metadata": {},
     "outputs": [],
     "source": [
      "embed = torch.nn.Embedding(10,3)\n",
      "words = torch.autograd.Variable(torch.LongTensor([[1,2,4,5] ,[4,3,2,9]]))"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### propagate the input via the old method (embedded_dropout)"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 53,
     "metadata": {},
     "outputs": [
      {
       "data": {
        "text/plain": [
         "Variable containing:\n",
         "(0 ,.,.) = \n",
         " -0.0000 -0.0000  0.0000\n",
         " -0.0000 -0.0000  0.0000\n",
         " -0.8727 -1.3131 -1.2833\n",
         " -0.0000 -0.0000 -0.0000\n",
         "\n",
         "(1 ,.,.) = \n",
         " -0.8727 -1.3131 -1.2833\n",
         " -0.7425 -4.4689  0.9931\n",
         " -0.0000 -0.0000  0.0000\n",
         " -1.3907 -0.6359 -3.4528\n",
         "[torch.FloatTensor of size 2x4x3]"
        ]
       },
       "execution_count": 53,
       "metadata": {},
       "output_type": "execute_result"
      }
     ],
     "source": [
      "torch.manual_seed(88123)\n",
      "dropout_out_old = embedded_dropout(embed, words, dropout=0.40)\n",
      "dropout_out_old"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### propagate the input via the forward method in the new layer (EmbeddingDropout)"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 56,
     "metadata": {},
     "outputs": [
      {
       "data": {
        "text/plain": [
         "Variable containing:\n",
         "(0 ,.,.) = \n",
         " -0.0000 -0.0000  0.0000\n",
         " -0.0000 -0.0000  0.0000\n",
         " -0.8727 -1.3131 -1.2833\n",
         " -0.0000 -0.0000 -0.0000\n",
         "\n",
         "(1 ,.,.) = \n",
         " -0.8727 -1.3131 -1.2833\n",
         " -0.7425 -4.4689  0.9931\n",
         " -0.0000 -0.0000  0.0000\n",
         " -1.3907 -0.6359 -3.4528\n",
         "[torch.FloatTensor of size 2x4x3]"
        ]
       },
       "execution_count": 56,
       "metadata": {},
       "output_type": "execute_result"
      }
     ],
     "source": [
      "torch.manual_seed(88123)\n",
      "embed_dropout_layer = EmbeddingDropout(embed)\n",
      "dropout_out_new = embed_dropout_layer(words, dropout=0.4)\n",
      "dropout_out_new"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 70,
     "metadata": {},
     "outputs": [
      {
       "name": "stdout",
       "output_type": "stream",
       "text": [
        "None\n"
       ]
      }
     ],
     "source": [
      "print(np.testing.assert_array_equal(to_np(dropout_out_old),to_np(dropout_out_new)))"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Test 2"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### Initialize embedding and matrix"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 71,
     "metadata": {
      "collapsed": true
     },
     "outputs": [],
     "source": [
      "embed = torch.nn.Embedding(10,7)\n",
      "words = torch.autograd.Variable(torch.LongTensor([[1,2,4,5,2,8] ,[4,3,2,9,7,6]]))"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### get the input by propagating via the old method"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 72,
     "metadata": {},
     "outputs": [
      {
       "data": {
        "text/plain": [
         "Variable containing:\n",
         "(0 ,.,.) = \n",
         " -0.1285  0.7315  0.1410 -1.6623 -3.7160 -1.1500  1.9748\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978  2.7126  4.9408\n",
         "  0.0000 -0.0000 -0.0000 -0.0000 -0.0000 -0.0000  0.0000\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -0.0000  0.0000  0.0000 -0.0000 -0.0000  0.0000  0.0000\n",
         "\n",
         "(1 ,.,.) = \n",
         " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978  2.7126  4.9408\n",
         "  0.0000 -0.0000  0.0000  0.0000 -0.0000  0.0000 -0.0000\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -6.4958  2.6005 -2.2331 -7.8277 -0.9653  3.3815  1.9252\n",
         " -0.0000 -0.0000  0.0000  0.0000 -0.0000  0.0000  0.0000\n",
         "  0.0000  0.0000  0.0000  0.0000 -0.0000 -0.0000  0.0000\n",
         "[torch.FloatTensor of size 2x6x7]"
        ]
       },
       "execution_count": 72,
       "metadata": {},
       "output_type": "execute_result"
      }
     ],
     "source": [
      "torch.manual_seed(7123)\n",
      "dropout_out_old = embedded_dropout(embed, words, dropout=0.64)\n",
      "dropout_out_old"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#### get the input by propagating input via the embedding layer"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 73,
     "metadata": {},
     "outputs": [
      {
       "data": {
        "text/plain": [
         "Variable containing:\n",
         "(0 ,.,.) = \n",
         " -0.1285  0.7315  0.1410 -1.6623 -3.7160 -1.1500  1.9748\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978  2.7126  4.9408\n",
         "  0.0000 -0.0000 -0.0000 -0.0000 -0.0000 -0.0000  0.0000\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -0.0000  0.0000  0.0000 -0.0000 -0.0000  0.0000  0.0000\n",
         "\n",
         "(1 ,.,.) = \n",
         " -1.4062 -0.2985 -1.2622 -1.2019 -3.9978  2.7126  4.9408\n",
         "  0.0000 -0.0000  0.0000  0.0000 -0.0000  0.0000 -0.0000\n",
         " -0.0000 -0.0000 -0.0000 -0.0000  0.0000 -0.0000  0.0000\n",
         " -6.4958  2.6005 -2.2331 -7.8277 -0.9653  3.3815  1.9252\n",
         " -0.0000 -0.0000  0.0000  0.0000 -0.0000  0.0000  0.0000\n",
         "  0.0000  0.0000  0.0000  0.0000 -0.0000 -0.0000  0.0000\n",
         "[torch.FloatTensor of size 2x6x7]"
        ]
       },
       "execution_count": 73,
       "metadata": {},
       "output_type": "execute_result"
      }
     ],
     "source": [
      "torch.manual_seed(7123)\n",
      "embed_dropout_layer = EmbeddingDropout(embed)\n",
      "dropout_out_new = embed_dropout_layer(words, dropout=0.64)\n",
      "dropout_out_new"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": 74,
     "metadata": {},
     "outputs": [
      {
       "name": "stdout",
       "output_type": "stream",
       "text": [
        "None\n"
       ]
      }
     ],
     "source": [
      "print(np.testing.assert_array_equal(to_np(dropout_out_old),to_np(dropout_out_new)))"
     ]
    },
    {
     "cell_type": "code",
     "execution_count": null,
     "metadata": {
      "collapsed": true
     },
     "outputs": [],
     "source": []
    }
   ],
   "metadata": {
    "kernelspec": {
     "display_name": "Python 3",
     "language": "python",
     "name": "python3"
    },
    "language_info": {
     "codemirror_mode": {
      "name": "ipython",
      "version": 3
     },
     "file_extension": ".py",
     "mimetype": "text/x-python",
     "name": "python",
     "nbconvert_exporter": "python",
     "pygments_lexer": "ipython3",
     "version": "3.6.2"
    }
   },
   "nbformat": 4,
   "nbformat_minor": 2
  }
  