Sentiment Analysis using pytorch-cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random \n",
    "import time\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2 µs, sys: 1e+03 ns, total: 3 µs\n",
      "Wall time: 5.72 µs\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "def clean_data(reviews):\n",
    "    reviews.text = reviews.text.apply(lambda x: x.lower())\n",
    "    \n",
    "    def remove_punctuation(s):\n",
    "    #s = s.split()\n",
    "    \n",
    "        s = [ch for ch in s if ch not in string.punctuation]\n",
    "\n",
    "        \n",
    "        s = \"\".join(s)\n",
    "        \n",
    "        s = [c for c in s if c == \" \" or c.isalnum()]\n",
    "        \n",
    "        s = \"\".join(s)\n",
    "        \n",
    "        return s\n",
    "    \n",
    "    \n",
    "    def remove_stopwords(s):\n",
    "        stop_words = set(stopwords.words(\"english\"))\n",
    "\n",
    "        s = s.split(\" \")\n",
    "\n",
    "        s = [word for word in s if word not in stop_words]\n",
    "        \n",
    "        \n",
    "        s = \" \".join(s)\n",
    "\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        return s\n",
    "    \n",
    "    \n",
    "    reviews.text = reviews.text.apply(lambda x:remove_punctuation(x))\n",
    "    reviews.text = reviews.text.apply(lambda x:remove_stopwords(x))\n",
    "    \n",
    "    return reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 28s, sys: 50.7 s, total: 2min 19s\n",
      "Wall time: 3min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "reviews = pd.read_json(\"/home/bh_parijat/data/yelp/review.json\",lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12 µs, sys: 7 µs, total: 19 µs\n",
      "Wall time: 22.4 µs\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6685900, 9)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "reviews.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ind = random.sample(list(reviews.index),1000000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_filtered = reviews[reviews.index.isin(ind) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000000, 9)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_filtered.to_csv(\"model/reviews_filtered.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 428 µs, sys: 41 µs, total: 469 µs\n",
      "Wall time: 409 µs\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>business_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>useful</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "      <th>text_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1sbwvVQXV2734tPgoKj4Q</td>\n",
       "      <td>hG7b0MtEbXx5QzbzE6C_VA</td>\n",
       "      <td>ujmEBvifdJM6h6RLv4wQIg</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Total bill for this horrible service? Over $8G...</td>\n",
       "      <td>2013-05-07 04:34:36</td>\n",
       "      <td>204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GJXCdrto3ASJOqKeVWPi6Q</td>\n",
       "      <td>yXQM5uF2jS6es16SJzNHfg</td>\n",
       "      <td>NZnhc2sEQy3RmzKTZnqtwQ</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I *adore* Travis at the Hard Rock's new Kelly ...</td>\n",
       "      <td>2017-01-14 21:30:33</td>\n",
       "      <td>1561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2TzJjDVDEuAW6MR5Vuc1ug</td>\n",
       "      <td>n6-Gk65cPZL6Uz8qRm3NYw</td>\n",
       "      <td>WTqjgwHlXbSFevF32_DJVw</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>I have to say that this office really has it t...</td>\n",
       "      <td>2016-11-09 20:09:03</td>\n",
       "      <td>615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>yi0R0Ugj_xUx_Nek0-_Qig</td>\n",
       "      <td>dacAIZ6fTM6mqwW5uxkskg</td>\n",
       "      <td>ikCg8xy5JIg_NGPx-MSIDA</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Went in for a lunch. Steak sandwich was delici...</td>\n",
       "      <td>2018-01-09 20:56:38</td>\n",
       "      <td>407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11a8sVPMUFtaC7_ABRkmtw</td>\n",
       "      <td>ssoyf2_x0EQMed6fgHeMyQ</td>\n",
       "      <td>b1b1eb3uo-w561D0ZfCEiQ</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Today was my second out of three sessions I ha...</td>\n",
       "      <td>2018-01-30 23:07:38</td>\n",
       "      <td>3509</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                review_id                 user_id             business_id  \\\n",
       "0  Q1sbwvVQXV2734tPgoKj4Q  hG7b0MtEbXx5QzbzE6C_VA  ujmEBvifdJM6h6RLv4wQIg   \n",
       "1  GJXCdrto3ASJOqKeVWPi6Q  yXQM5uF2jS6es16SJzNHfg  NZnhc2sEQy3RmzKTZnqtwQ   \n",
       "2  2TzJjDVDEuAW6MR5Vuc1ug  n6-Gk65cPZL6Uz8qRm3NYw  WTqjgwHlXbSFevF32_DJVw   \n",
       "3  yi0R0Ugj_xUx_Nek0-_Qig  dacAIZ6fTM6mqwW5uxkskg  ikCg8xy5JIg_NGPx-MSIDA   \n",
       "4  11a8sVPMUFtaC7_ABRkmtw  ssoyf2_x0EQMed6fgHeMyQ  b1b1eb3uo-w561D0ZfCEiQ   \n",
       "\n",
       "   stars  useful  funny  cool  \\\n",
       "0      1       6      1     0   \n",
       "1      5       0      0     0   \n",
       "2      5       3      0     0   \n",
       "3      5       0      0     0   \n",
       "4      1       7      0     0   \n",
       "\n",
       "                                                text                date  \\\n",
       "0  Total bill for this horrible service? Over $8G... 2013-05-07 04:34:36   \n",
       "1  I *adore* Travis at the Hard Rock's new Kelly ... 2017-01-14 21:30:33   \n",
       "2  I have to say that this office really has it t... 2016-11-09 20:09:03   \n",
       "3  Went in for a lunch. Steak sandwich was delici... 2018-01-09 20:56:38   \n",
       "4  Today was my second out of three sessions I ha... 2018-01-30 23:07:38   \n",
       "\n",
       "   text_length  \n",
       "0          204  \n",
       "1         1561  \n",
       "2          615  \n",
       "3          407  \n",
       "4         3509  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "reviews.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 26min 23s, sys: 54.9 s, total: 27min 18s\n",
      "Wall time: 27min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "reviews_cleaned = reviews.copy()\n",
    "reviews_cleaned = clean_data(reviews_cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_null_row_cond = reviews_cleaned.text == \"null\"\n",
    "reviews_cleaned = reviews_cleaned[~filter_null_row_cond]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews['text_length'] = reviews.text.apply(lambda x:len(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['review_id', 'user_id', 'business_id', 'stars', 'useful', 'funny',\n",
       "       'cool', 'text', 'date', 'text_length', 'label'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_cleaned.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>business_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>useful</th>\n",
       "      <th>funny</th>\n",
       "      <th>cool</th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "      <th>text_length</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Q1sbwvVQXV2734tPgoKj4Q</td>\n",
       "      <td>hG7b0MtEbXx5QzbzE6C_VA</td>\n",
       "      <td>ujmEBvifdJM6h6RLv4wQIg</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>total bill horrible service 8gs crooks actuall...</td>\n",
       "      <td>2013-05-07 04:34:36</td>\n",
       "      <td>129</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GJXCdrto3ASJOqKeVWPi6Q</td>\n",
       "      <td>yXQM5uF2jS6es16SJzNHfg</td>\n",
       "      <td>NZnhc2sEQy3RmzKTZnqtwQ</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>adore travis hard rocks new kelly cardenas sal...</td>\n",
       "      <td>2017-01-14 21:30:33</td>\n",
       "      <td>1064</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2TzJjDVDEuAW6MR5Vuc1ug</td>\n",
       "      <td>n6-Gk65cPZL6Uz8qRm3NYw</td>\n",
       "      <td>WTqjgwHlXbSFevF32_DJVw</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>say office really together organized friendly ...</td>\n",
       "      <td>2016-11-09 20:09:03</td>\n",
       "      <td>369</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>yi0R0Ugj_xUx_Nek0-_Qig</td>\n",
       "      <td>dacAIZ6fTM6mqwW5uxkskg</td>\n",
       "      <td>ikCg8xy5JIg_NGPx-MSIDA</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>went lunch steak sandwich delicious caesar sal...</td>\n",
       "      <td>2018-01-09 20:56:38</td>\n",
       "      <td>279</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11a8sVPMUFtaC7_ABRkmtw</td>\n",
       "      <td>ssoyf2_x0EQMed6fgHeMyQ</td>\n",
       "      <td>b1b1eb3uo-w561D0ZfCEiQ</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>today second three sessions paid although firs...</td>\n",
       "      <td>2018-01-30 23:07:38</td>\n",
       "      <td>2021</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                review_id                 user_id             business_id  \\\n",
       "0  Q1sbwvVQXV2734tPgoKj4Q  hG7b0MtEbXx5QzbzE6C_VA  ujmEBvifdJM6h6RLv4wQIg   \n",
       "1  GJXCdrto3ASJOqKeVWPi6Q  yXQM5uF2jS6es16SJzNHfg  NZnhc2sEQy3RmzKTZnqtwQ   \n",
       "2  2TzJjDVDEuAW6MR5Vuc1ug  n6-Gk65cPZL6Uz8qRm3NYw  WTqjgwHlXbSFevF32_DJVw   \n",
       "3  yi0R0Ugj_xUx_Nek0-_Qig  dacAIZ6fTM6mqwW5uxkskg  ikCg8xy5JIg_NGPx-MSIDA   \n",
       "4  11a8sVPMUFtaC7_ABRkmtw  ssoyf2_x0EQMed6fgHeMyQ  b1b1eb3uo-w561D0ZfCEiQ   \n",
       "\n",
       "   stars  useful  funny  cool  \\\n",
       "0      1       6      1     0   \n",
       "1      5       0      0     0   \n",
       "2      5       3      0     0   \n",
       "3      5       0      0     0   \n",
       "4      1       7      0     0   \n",
       "\n",
       "                                                text                date  \\\n",
       "0  total bill horrible service 8gs crooks actuall... 2013-05-07 04:34:36   \n",
       "1  adore travis hard rocks new kelly cardenas sal... 2017-01-14 21:30:33   \n",
       "2  say office really together organized friendly ... 2016-11-09 20:09:03   \n",
       "3  went lunch steak sandwich delicious caesar sal... 2018-01-09 20:56:38   \n",
       "4  today second three sessions paid although firs... 2018-01-30 23:07:38   \n",
       "\n",
       "   text_length  label  \n",
       "0          129      0  \n",
       "1         1064      1  \n",
       "2          369      1  \n",
       "3          279      1  \n",
       "4         2021      0  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_cleaned.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((6685900, 10), (6685899, 11))"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.shape,reviews_cleaned.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviews_cleaned['label']= reviews.stars.apply(lambda x: 1 if x>=3 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 56s, sys: 3.23 s, total: 1min 59s\n",
      "Wall time: 2min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "reviews_cleaned.to_csv(\"model/master_reviews.csv\",index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}