Feed Forward Sarcasm Detector.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feed Forward Neural Network for Sarcasm Detection\n",
    "Created by Paul K. Mandal"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a very simple Feed Forward Neural Network that I wrote in a few hours. As Andrew Ng said, the best thing to do is to immediately create a simple model and then add to it. I will be attempting to create a more advanced model later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "data = []\n",
    "for line in open('Sarcasm_Headlines_Dataset.json', 'r'):\n",
    "    data.append(json.loads(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = []\n",
    "y_vals = []\n",
    "\n",
    "for i in range(0,len(data)):\n",
    "    titles.append(data[i]['headline'])\n",
    "    y_vals.append(data[i]['is_sarcastic'])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Obdyg\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "#Let's now do some preprocessing\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "from nltk import word_tokenize\n",
    "\n",
    "titles_tokenized = []\n",
    "for title in titles:\n",
    "    titles_tokenized.append(word_tokenize(title))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles_an = [] #alphanumeric\n",
    "for title in titles_tokenized:\n",
    "    words = [word for word in title if word.isalpha()]\n",
    "    titles_an.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['former',\n",
       " 'versace',\n",
       " 'store',\n",
       " 'clerk',\n",
       " 'sues',\n",
       " 'over',\n",
       " 'secret',\n",
       " 'code',\n",
       " 'for',\n",
       " 'minority',\n",
       " 'shoppers']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles_an[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's now stem the words\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "porter = PorterStemmer()\n",
    "titles_preprocessed = []\n",
    "for title in titles_an:\n",
    "    stemmed = [porter.stem(word) for word in title]\n",
    "    titles_preprocessed.append(stemmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['former',\n",
       " 'versac',\n",
       " 'store',\n",
       " 'clerk',\n",
       " 'sue',\n",
       " 'over',\n",
       " 'secret',\n",
       " 'code',\n",
       " 'for',\n",
       " 'minor',\n",
       " 'shopper']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles_preprocessed[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Brilliant. Now, let's create a large list of all of the words and find the 10,000 most frequent ones\n",
    "word_list = []\n",
    "\n",
    "for title in titles_preprocessed:\n",
    "    for word in title:\n",
    "        word_list.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter #Here, we create a counter\n",
    "freq_list = Counter(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = freq_list.most_common(10000) #Get the 10,000 most common words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = list(zip(*dictionary))[0] #Remove the counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We now have a list with the 10000 most common words. Let us convert our sentences to lists of these words in order to\n",
    "#       feed it into the Neural Network\n",
    "nums = range(0,10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_int = dict(zip(dictionary, nums))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_vals = []\n",
    "\n",
    "for title in titles_preprocessed:\n",
    "    x_vals.append([word_int[x] for x in title if x in word_int.keys()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now, let's format the data for the Neural Network and divide the training, validation, and test sets\n",
    "import numpy as np\n",
    "\n",
    "x = np.array(x_vals)\n",
    "test_data = x[:5000]\n",
    "train_data = x[5000:]\n",
    "\n",
    "def vectorize_sequences(sequences, dimension=10000):\n",
    "    results = np.zeros((len(sequences), dimension))\n",
    "    for i, sequence in enumerate(sequences):\n",
    "        results[i, sequence] = 1.\n",
    "    return results\n",
    "\n",
    "x_train = vectorize_sequences(train_data)\n",
    "x_test = vectorize_sequences(test_data)\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "y = np.asarray(y_vals).astype('float32')\n",
    "y_test = y[:5000]\n",
    "y_train = y[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the validation set\n",
    "x_val = x_train[:5000]\n",
    "x_partial_train = x_train[5000:]\n",
    "\n",
    "y_val = y_train[:5000]\n",
    "y_partial_train = y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prevent Tensorflow from allocating my entire GPU\n",
    "import keras\n",
    "\n",
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Now, let us define our model\n",
    "from keras import models\n",
    "from keras import layers\n",
    "\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(32, activation='relu', input_shape = (10000,)))\n",
    "model.add(layers.Dense(4, activation='relu'))\n",
    "model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "model.compile(optimizer = 'rmsprop', loss = 'binary_crossentropy', metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 16709 samples, validate on 5000 samples\n",
      "Epoch 1/20\n",
      "16709/16709 [==============================] - 3s 165us/step - loss: 0.6461 - acc: 0.7451 - val_loss: 0.5784 - val_acc: 0.8332\n",
      "Epoch 2/20\n",
      "16709/16709 [==============================] - 2s 133us/step - loss: 0.5110 - acc: 0.8606 - val_loss: 0.4758 - val_acc: 0.8420\n",
      "Epoch 3/20\n",
      "16709/16709 [==============================] - 2s 133us/step - loss: 0.4030 - acc: 0.8796 - val_loss: 0.4096 - val_acc: 0.8456\n",
      "Epoch 4/20\n",
      "16709/16709 [==============================] - 2s 137us/step - loss: 0.3290 - acc: 0.8945 - val_loss: 0.3744 - val_acc: 0.8490\n",
      "Epoch 5/20\n",
      "16709/16709 [==============================] - 2s 133us/step - loss: 0.2786 - acc: 0.9056 - val_loss: 0.3587 - val_acc: 0.8482\n",
      "Epoch 6/20\n",
      "16709/16709 [==============================] - 2s 134us/step - loss: 0.2428 - acc: 0.9145 - val_loss: 0.3573 - val_acc: 0.8446\n",
      "Epoch 7/20\n",
      "16709/16709 [==============================] - 2s 134us/step - loss: 0.2156 - acc: 0.9215 - val_loss: 0.3616 - val_acc: 0.8452\n",
      "Epoch 8/20\n",
      "16709/16709 [==============================] - 2s 134us/step - loss: 0.1937 - acc: 0.9297 - val_loss: 0.3696 - val_acc: 0.8428\n",
      "Epoch 9/20\n",
      "16709/16709 [==============================] - 2s 137us/step - loss: 0.1756 - acc: 0.9354 - val_loss: 0.3807 - val_acc: 0.8412\n",
      "Epoch 10/20\n",
      "16709/16709 [==============================] - 2s 134us/step - loss: 0.1590 - acc: 0.9417 - val_loss: 0.3947 - val_acc: 0.8390\n",
      "Epoch 11/20\n",
      "16709/16709 [==============================] - 2s 135us/step - loss: 0.1442 - acc: 0.9481 - val_loss: 0.4101 - val_acc: 0.8360\n",
      "Epoch 12/20\n",
      "16709/16709 [==============================] - 2s 139us/step - loss: 0.1307 - acc: 0.9554 - val_loss: 0.4269 - val_acc: 0.8360\n",
      "Epoch 13/20\n",
      "16709/16709 [==============================] - 2s 133us/step - loss: 0.1180 - acc: 0.9595 - val_loss: 0.4458 - val_acc: 0.8344\n",
      "Epoch 14/20\n",
      "16709/16709 [==============================] - 2s 135us/step - loss: 0.1067 - acc: 0.9643 - val_loss: 0.4674 - val_acc: 0.8308\n",
      "Epoch 15/20\n",
      "16709/16709 [==============================] - 2s 137us/step - loss: 0.0953 - acc: 0.9683 - val_loss: 0.4892 - val_acc: 0.8300\n",
      "Epoch 16/20\n",
      "16709/16709 [==============================] - 2s 133us/step - loss: 0.0856 - acc: 0.9724 - val_loss: 0.5121 - val_acc: 0.8262\n",
      "Epoch 17/20\n",
      "16709/16709 [==============================] - 2s 138us/step - loss: 0.0760 - acc: 0.9767 - val_loss: 0.5349 - val_acc: 0.8270\n",
      "Epoch 18/20\n",
      "16709/16709 [==============================] - 2s 143us/step - loss: 0.0673 - acc: 0.9797 - val_loss: 0.5606 - val_acc: 0.8264\n",
      "Epoch 19/20\n",
      "16709/16709 [==============================] - 2s 137us/step - loss: 0.0592 - acc: 0.9831 - val_loss: 0.5856 - val_acc: 0.8236\n",
      "Epoch 20/20\n",
      "16709/16709 [==============================] - 3s 151us/step - loss: 0.0518 - acc: 0.9854 - val_loss: 0.6122 - val_acc: 0.8248\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x_partial_train, y_partial_train, epochs = 20, batch_size = 512, validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "21709/21709 [==============================] - 3s 138us/step - loss: 0.5834 - acc: 0.6943\n",
      "Epoch 2/4\n",
      "21709/21709 [==============================] - 2s 110us/step - loss: 0.4217 - acc: 0.8541\n",
      "Epoch 3/4\n",
      "21709/21709 [==============================] - 2s 115us/step - loss: 0.3348 - acc: 0.8797\n",
      "Epoch 4/4\n",
      "21709/21709 [==============================] - 2s 111us/step - loss: 0.2844 - acc: 0.8937\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x287aba25940>"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Let us train the model with 6 epochs.\n",
    "model = models.Sequential()\n",
    "model.add(layers.Dense(32, activation='relu', input_shape = (10000,)))\n",
    "model.add(layers.Dense(4, activation='relu'))\n",
    "model.add(layers.Dense(1, activation = 'sigmoid'))\n",
    "\n",
    "model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])\n",
    "model.fit(x_train, y_train, epochs = 4, batch_size = 512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000/5000 [==============================] - 1s 174us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.35768395495414734, 0.8456]"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = model.evaluate(x_test, y_test)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}