Data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import spacy, re, nltk\n",
    "#!python -m spacy download en\n",
    "\n",
    "# # For cleaning and processing\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk import sent_tokenize\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer, WordNetLemmatizer\n",
    "# # nltk.download('stopwords')\n",
    "# # nltk.download('punkt')\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "comprehensive_stopwords = [\"a\",\"about\",\"above\",\"after\",\"again\",\"against\",\"ain\",\"all\",\"am\",\"an\",\"and\",\"any\",\"are\",\"aren\",\"aren't\",\"as\",\"at\",\"be\",\"because\",\"been\",\"before\",\"being\",\"below\",\"between\",\"both\",\"but\",\"by\",\"can\",\"couldn\",\"couldn't\",\"d\",\"did\",\"didn\",\"didn't\",\"do\",\"does\",\"doesn\",\"doesn't\",\"doing\",\"don\",\"don't\",\"down\",\"during\",\"each\",\"few\",\"for\",\"from\",\"further\",\"had\",\"hadn\",\"hadn't\",\"has\",\"hasn\",\"hasn't\",\"have\",\"haven\",\"haven't\",\"having\",\"he\",\"her\",\"here\",\"hers\",\"herself\",\"him\",\"himself\",\"his\",\"how\",\"i\",\"if\",\"in\",\"into\",\"is\",\"isn\",\"isn't\",\"it\",\"it's\",\"its\",\"itself\",\"just\",\"ll\",\"m\",\"ma\",\"me\",\"mightn\",\"mightn't\",\"more\",\"most\",\"mustn\",\"mustn't\",\"my\",\"myself\",\"needn\",\"needn't\",\"no\",\"nor\",\"not\",\"now\",\"o\",\"of\",\"off\",\"on\",\"once\",\"only\",\"or\",\"other\",\"our\",\"ours\",\"ourselves\",\"out\",\"over\",\"own\",\"re\",\"s\",\"same\",\"shan\",\"shan't\",\"she\",\"she's\",\"should\",\"should've\",\"shouldn\",\"shouldn't\",\"so\",\"some\",\"such\",\"t\",\"than\",\"that\",\"that'll\",\"the\",\"their\",\"theirs\",\"them\",\"themselves\",\"then\",\"there\",\"these\",\"they\",\"this\",\"those\",\"through\",\"to\",\"too\",\"under\",\"until\",\"up\",\"ve\",\"very\",\"was\",\"wasn\",\"wasn't\",\"we\",\"were\",\"weren\",\"weren't\",\"what\",\"when\",\"where\",\"which\",\"while\",\"who\",\"whom\",\"why\",\"will\",\"with\",\"won\",\"won't\",\"wouldn\",\"wouldn't\",\"y\",\"you\",\"you'd\",\"you'll\",\"you're\",\"you've\",\"your\",\"yours\",\"yourself\",\"yourselves\",\"could\",\"he'd\",\"he'll\",\"he's\",\"here's\",\"how's\",\"i'd\",\"i'll\",\"i'm\",\"i've\",\"let's\",\"ought\",\"she'd\",\"she'll\",\"that's\",\"there's\",\"they'd\",\"they'll\",\"they're\",\"they've\",\"we'd\",\"we'll\",\"we're\",\"we've\",\"what's\",\"when's\",\"where's\",\"who's\",\"why's\",\"would\",\"able\",\"abst\",\"accordance\",\"according\",\"accordingly\",\"across\",\"act\",\"actually\",\"added\",\"adj\",\"affected\",\"affecting\",\"affects\",\"afterwards\",\"ah\",\"almost\",\"alone\",\"along\",\"already\",\"also\",\"although\",\"always\",\"among\",\"amongst\",\"announce\",\"another\",\"anybody\",\"anyhow\",\"anymore\",\"anyone\",\"anything\",\"anyway\",\"anyways\",\"anywhere\",\"apparently\",\"approximately\",\"arent\",\"arise\",\"around\",\"aside\",\"ask\",\"asking\",\"auth\",\"available\",\"away\",\"awfully\",\"b\",\"back\",\"became\",\"become\",\"becomes\",\"becoming\",\"beforehand\",\"begin\",\"beginning\",\"beginnings\",\"begins\",\"behind\",\"believe\",\"beside\",\"besides\",\"beyond\",\"biol\",\"brief\",\"briefly\",\"c\",\"ca\",\"came\",\"cannot\",\"can't\",\"cause\",\"causes\",\"certain\",\"certainly\",\"co\",\"com\",\"come\",\"comes\",\"contain\",\"containing\",\"contains\",\"couldnt\",\"date\",\"different\",\"done\",\"downwards\",\"due\",\"e\",\"ed\",\"edu\",\"effect\",\"eg\",\"eight\",\"eighty\",\"either\",\"else\",\"elsewhere\",\"end\",\"ending\",\"enough\",\"especially\",\"et\",\"etc\",\"even\",\"ever\",\"every\",\"everybody\",\"everyone\",\"everything\",\"everywhere\",\"ex\",\"except\",\"f\",\"far\",\"ff\",\"fifth\",\"first\",\"five\",\"fix\",\"followed\",\"following\",\"follows\",\"former\",\"formerly\",\"forth\",\"found\",\"four\",\"furthermore\",\"g\",\"gave\",\"get\",\"gets\",\"getting\",\"give\",\"given\",\"gives\",\"giving\",\"go\",\"goes\",\"gone\",\"got\",\"gotten\",\"h\",\"happens\",\"hardly\",\"hed\",\"hence\",\"hereafter\",\"hereby\",\"herein\",\"heres\",\"hereupon\",\"hes\",\"hi\",\"hid\",\"hither\",\"home\",\"howbeit\",\"however\",\"hundred\",\"id\",\"ie\",\"im\",\"immediate\",\"immediately\",\"importance\",\"important\",\"inc\",\"indeed\",\"index\",\"information\",\"instead\",\"invention\",\"inward\",\"itd\",\"it'll\",\"j\",\"k\",\"keep\",\"keeps\",\"kept\",\"kg\",\"km\",\"know\",\"known\",\"knows\",\"l\",\"largely\",\"last\",\"lately\",\"later\",\"latter\",\"latterly\",\"least\",\"less\",\"lest\",\"let\",\"lets\",\"like\",\"liked\",\"likely\",\"line\",\"little\",\"'ll\",\"look\",\"looking\",\"looks\",\"ltd\",\"made\",\"mainly\",\"make\",\"makes\",\"many\",\"may\",\"maybe\",\"mean\",\"means\",\"meantime\",\"meanwhile\",\"merely\",\"mg\",\"might\",\"million\",\"miss\",\"ml\",\"moreover\",\"mostly\",\"mr\",\"mrs\",\"much\",\"mug\",\"must\",\"n\",\"na\",\"name\",\"namely\",\"nay\",\"nd\",\"near\",\"nearly\",\"necessarily\",\"necessary\",\"need\",\"needs\",\"neither\",\"never\",\"nevertheless\",\"new\",\"next\",\"nine\",\"ninety\",\"nobody\",\"non\",\"none\",\"nonetheless\",\"noone\",\"normally\",\"nos\",\"noted\",\"nothing\",\"nowhere\",\"obtain\",\"obtained\",\"obviously\",\"often\",\"oh\",\"ok\",\"okay\",\"old\",\"omitted\",\"one\",\"ones\",\"onto\",\"ord\",\"others\",\"otherwise\",\"outside\",\"overall\",\"owing\",\"p\",\"page\",\"pages\",\"part\",\"particular\",\"particularly\",\"past\",\"per\",\"perhaps\",\"placed\",\"please\",\"plus\",\"poorly\",\"possible\",\"possibly\",\"potentially\",\"pp\",\"predominantly\",\"present\",\"previously\",\"primarily\",\"probably\",\"promptly\",\"proud\",\"provides\",\"put\",\"q\",\"que\",\"quickly\",\"quite\",\"qv\",\"r\",\"ran\",\"rather\",\"rd\",\"readily\",\"really\",\"recent\",\"recently\",\"ref\",\"refs\",\"regarding\",\"regardless\",\"regards\",\"related\",\"relatively\",\"research\",\"respectively\",\"resulted\",\"resulting\",\"results\",\"right\",\"run\",\"said\",\"saw\",\"say\",\"saying\",\"says\",\"sec\",\"section\",\"see\",\"seeing\",\"seem\",\"seemed\",\"seeming\",\"seems\",\"seen\",\"self\",\"selves\",\"sent\",\"seven\",\"several\",\"shall\",\"shed\",\"shes\",\"show\",\"showed\",\"shown\",\"showns\",\"shows\",\"significant\",\"significantly\",\"similar\",\"similarly\",\"since\",\"six\",\"slightly\",\"somebody\",\"somehow\",\"someone\",\"somethan\",\"something\",\"sometime\",\"sometimes\",\"somewhat\",\"somewhere\",\"soon\",\"sorry\",\"specifically\",\"specified\",\"specify\",\"specifying\",\"still\",\"stop\",\"strongly\",\"sub\",\"substantially\",\"successfully\",\"sufficiently\",\"suggest\",\"sup\",\"sure\",\"take\",\"taken\",\"taking\",\"tell\",\"tends\",\"th\",\"thank\",\"thanks\",\"thanx\",\"thats\",\"that've\",\"thence\",\"thereafter\",\"thereby\",\"thered\",\"therefore\",\"therein\",\"there'll\",\"thereof\",\"therere\",\"theres\",\"thereto\",\"thereupon\",\"there've\",\"theyd\",\"theyre\",\"think\",\"thou\",\"though\",\"thoughh\",\"thousand\",\"throug\",\"throughout\",\"thru\",\"thus\",\"til\",\"tip\",\"together\",\"took\",\"toward\",\"towards\",\"tried\",\"tries\",\"truly\",\"try\",\"trying\",\"ts\",\"twice\",\"two\",\"u\",\"un\",\"unfortunately\",\"unless\",\"unlike\",\"unlikely\",\"unto\",\"upon\",\"ups\",\"us\",\"use\",\"used\",\"useful\",\"usefully\",\"usefulness\",\"uses\",\"using\",\"usually\",\"v\",\"value\",\"various\",\"'ve\",\"via\",\"viz\",\"vol\",\"vols\",\"vs\",\"w\",\"want\",\"wants\",\"wasnt\",\"way\",\"wed\",\"welcome\",\"went\",\"werent\",\"whatever\",\"what'll\",\"whats\",\"whence\",\"whenever\",\"whereafter\",\"whereas\",\"whereby\",\"wherein\",\"wheres\",\"whereupon\",\"wherever\",\"whether\",\"whim\",\"whither\",\"whod\",\"whoever\",\"whole\",\"who'll\",\"whomever\",\"whos\",\"whose\",\"widely\",\"willing\",\"wish\",\"within\",\"without\",\"wont\",\"words\",\"world\",\"wouldnt\",\"www\",\"x\",\"yes\",\"yet\",\"youd\",\"youre\",\"z\",\"zero\",\"a's\",\"ain't\",\"allow\",\"allows\",\"apart\",\"appear\",\"appreciate\",\"appropriate\",\"associated\",\"best\",\"better\",\"c'mon\",\"c's\",\"cant\",\"changes\",\"clearly\",\"concerning\",\"consequently\",\"consider\",\"considering\",\"corresponding\",\"course\",\"currently\",\"definitely\",\"described\",\"despite\",\"entirely\",\"exactly\",\"example\",\"going\",\"greetings\",\"hello\",\"help\",\"hopefully\",\"ignored\",\"inasmuch\",\"indicate\",\"indicated\",\"indicates\",\"inner\",\"insofar\",\"it'd\",\"keep\",\"keeps\",\"novel\",\"presumably\",\"reasonably\",\"second\",\"secondly\",\"sensible\",\"serious\",\"seriously\",\"sure\",\"t's\",\"third\",\"thorough\",\"thoroughly\",\"three\",\"well\",\"wonder\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>category_id</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Polarized Sunglasses for Men and Women Semi-R...</td>\n",
       "      <td>Plastic frame anti-reflective lens Polarized ...</td>\n",
       "      <td>https://www.amazon.ca/Polarized-Sunglasses-Fin...</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-08-20 20:59:28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>SORVINO Vintage Sunglasses Retro Cateye Sungl...</td>\n",
       "      <td>HIGH QUALITY MATERIAL:This gradient cat eye s...</td>\n",
       "      <td>https://www.amazon.ca/SORVINO-Vintage-Sunglass...</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-08-20 20:59:31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Under Armour Womens Launch Run Visor</td>\n",
       "      <td>Under Armour's mission is to make all athlete...</td>\n",
       "      <td>https://www.amazon.ca/Under-Armour-Womens-Laun...</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-08-20 20:59:33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>GRFISIA Square Oversized Sunglasses for Women...</td>\n",
       "      <td>Plastic frame anti-reflective lens Non-Polari...</td>\n",
       "      <td>https://www.amazon.ca/GRFISIA-Square-Oversized...</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-08-20 20:59:36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>FEISEDY Retro Oversized Square Polarized Sung...</td>\n",
       "      <td>FEISEDY RETRO Design--Square Oversized Frame ...</td>\n",
       "      <td>https://www.amazon.ca/FEISEDY-Oversized-Polari...</td>\n",
       "      <td>44</td>\n",
       "      <td>2021-08-20 20:59:38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                              title  \\\n",
       "0   1   Polarized Sunglasses for Men and Women Semi-R...   \n",
       "1   2   SORVINO Vintage Sunglasses Retro Cateye Sungl...   \n",
       "2   3              Under Armour Womens Launch Run Visor    \n",
       "3   4   GRFISIA Square Oversized Sunglasses for Women...   \n",
       "4   5   FEISEDY Retro Oversized Square Polarized Sung...   \n",
       "\n",
       "                                         description  \\\n",
       "0   Plastic frame anti-reflective lens Polarized ...   \n",
       "1   HIGH QUALITY MATERIAL:This gradient cat eye s...   \n",
       "2   Under Armour's mission is to make all athlete...   \n",
       "3   Plastic frame anti-reflective lens Non-Polari...   \n",
       "4   FEISEDY RETRO Design--Square Oversized Frame ...   \n",
       "\n",
       "                                                 url  category_id  \\\n",
       "0  https://www.amazon.ca/Polarized-Sunglasses-Fin...           44   \n",
       "1  https://www.amazon.ca/SORVINO-Vintage-Sunglass...           44   \n",
       "2  https://www.amazon.ca/Under-Armour-Womens-Laun...           44   \n",
       "3  https://www.amazon.ca/GRFISIA-Square-Oversized...           44   \n",
       "4  https://www.amazon.ca/FEISEDY-Oversized-Polari...           44   \n",
       "\n",
       "            created_at  \n",
       "0  2021-08-20 20:59:28  \n",
       "1  2021-08-20 20:59:31  \n",
       "2  2021-08-20 20:59:33  \n",
       "3  2021-08-20 20:59:36  \n",
       "4  2021-08-20 20:59:38  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original = pd.read_csv('amazon-product.csv', delimiter = ';')\n",
    "df_original.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Polarized Sunglasses for Men and Women Semi-R...</td>\n",
       "      <td>Plastic frame anti-reflective lens Polarized ...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SORVINO Vintage Sunglasses Retro Cateye Sungl...</td>\n",
       "      <td>HIGH QUALITY MATERIAL:This gradient cat eye s...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Under Armour Womens Launch Run Visor</td>\n",
       "      <td>Under Armour's mission is to make all athlete...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GRFISIA Square Oversized Sunglasses for Women...</td>\n",
       "      <td>Plastic frame anti-reflective lens Non-Polari...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEISEDY Retro Oversized Square Polarized Sung...</td>\n",
       "      <td>FEISEDY RETRO Design--Square Oversized Frame ...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0   Polarized Sunglasses for Men and Women Semi-R...   \n",
       "1   SORVINO Vintage Sunglasses Retro Cateye Sungl...   \n",
       "2              Under Armour Womens Launch Run Visor    \n",
       "3   GRFISIA Square Oversized Sunglasses for Women...   \n",
       "4   FEISEDY Retro Oversized Square Polarized Sung...   \n",
       "\n",
       "                                         description  category_id  \n",
       "0   Plastic frame anti-reflective lens Polarized ...           44  \n",
       "1   HIGH QUALITY MATERIAL:This gradient cat eye s...           44  \n",
       "2   Under Armour's mission is to make all athlete...           44  \n",
       "3   Plastic frame anti-reflective lens Non-Polari...           44  \n",
       "4   FEISEDY RETRO Design--Square Oversized Frame ...           44  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Seems like we can drop id, url and created_at column from the original data\n",
    "df = df_original[['title', 'description', 'category_id']]\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text summarization to keep just the top 2 sentences in the description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_summarization(ind):\n",
    "    if len(df.loc[ind,'title'].strip()) > 0: #Title should be present to perform text summarization\n",
    "        sentences = sent_tokenize(df.loc[ind,'description'])\n",
    "        if len(sentences) > 0: #Text summarization needs description\n",
    "            # append title to description\n",
    "            sentences.append(df.loc[ind,'title'])\n",
    "\n",
    "            # vectorize sentences based on tf-idf\n",
    "            vectorizer = TfidfVectorizer()\n",
    "            try:\n",
    "                X = vectorizer.fit_transform(sentences)\n",
    "            except ValueError as ve:\n",
    "                print('ValueError: {0} at index {1}'.format(ve, ind))\n",
    "                return(\" \")\n",
    "\n",
    "            # measure similarity between title and each sentence\n",
    "            if X.shape[0] == 2:\n",
    "                similarity = cosine_similarity(X[1], X[0]) \n",
    "            else:\n",
    "                similarity = cosine_similarity(X[-1], X[:-1])\n",
    "\n",
    "            # find top 2 sentences with greatest similarity\n",
    "            sorted_index = np.argsort(similarity)[0][::-1]\n",
    "            top_sentences = []\n",
    "            for i in range(0,min(2, len(sorted_index))):\n",
    "                top_sentences.append(sentences[sorted_index[i]])\n",
    "            return(\" \".join(top_sentences))\n",
    "   \n",
    "        else: #Description not present\n",
    "#             print('Description not present at index: {}'.format(ind))\n",
    "            return(\" \")\n",
    "    else: #Title not present\n",
    "#         print('Title not present at index: {}'.format(ind))\n",
    "        return(df.loc[ind,'description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ValueError: empty vocabulary; perhaps the documents only contain stop words at index 199897\n"
     ]
    }
   ],
   "source": [
    "important_sentences = [text_summarization(i) for i in range(len(df))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining a function to clean the data\n",
    "\n",
    "nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])\n",
    "\n",
    "def clean_data(new):\n",
    "\n",
    "    table = str.maketrans(string.punctuation, ' '*len(string.punctuation)) #map punctuation to space\n",
    "    stripped = new.translate(table)\n",
    "    tokens = stripped.lower().split()\n",
    "    \n",
    "    words = [word for word in tokens if word.isalpha()]\n",
    "    words = list(filter(lambda x: x not in comprehensive_stopwords, words))\n",
    "\n",
    "#     nlp = spacy.load('en_core_web_sm', disable=['parser', 'ner'])\n",
    "    doc = nlp(\" \".join(words))\n",
    "    lemmatized = \" \".join([token.lemma_ for token in doc])\n",
    "    \n",
    "    return(lemmatized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cleaning the description\n",
    "clean_description = [clean_data(important_sentences[i]) for i in range(len(df))]\n",
    "# clean_data(important_sentences[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-9-135c805f1cd6>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['clean_description'] = clean_description\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category_id</th>\n",
       "      <th>clean_description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Polarized Sunglasses for Men and Women Semi-R...</td>\n",
       "      <td>Plastic frame anti-reflective lens Polarized ...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglass polarize lense resistant impact scrat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SORVINO Vintage Sunglasses Retro Cateye Sungl...</td>\n",
       "      <td>HIGH QUALITY MATERIAL:This gradient cat eye s...</td>\n",
       "      <td>44</td>\n",
       "      <td>high quality material gradient cat eye sunglas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Under Armour Womens Launch Run Visor</td>\n",
       "      <td>Under Armour's mission is to make all athlete...</td>\n",
       "      <td>44</td>\n",
       "      <td>mission athlete passion design relentless purs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GRFISIA Square Oversized Sunglasses for Women...</td>\n",
       "      <td>Plastic frame anti-reflective lens Non-Polari...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglasse soft microfiber pouch soft glass clo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEISEDY Retro Oversized Square Polarized Sung...</td>\n",
       "      <td>FEISEDY RETRO Design--Square Oversized Frame ...</td>\n",
       "      <td>44</td>\n",
       "      <td>feisedy retro design square oversized frame de...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0   Polarized Sunglasses for Men and Women Semi-R...   \n",
       "1   SORVINO Vintage Sunglasses Retro Cateye Sungl...   \n",
       "2              Under Armour Womens Launch Run Visor    \n",
       "3   GRFISIA Square Oversized Sunglasses for Women...   \n",
       "4   FEISEDY Retro Oversized Square Polarized Sung...   \n",
       "\n",
       "                                         description  category_id  \\\n",
       "0   Plastic frame anti-reflective lens Polarized ...           44   \n",
       "1   HIGH QUALITY MATERIAL:This gradient cat eye s...           44   \n",
       "2   Under Armour's mission is to make all athlete...           44   \n",
       "3   Plastic frame anti-reflective lens Non-Polari...           44   \n",
       "4   FEISEDY RETRO Design--Square Oversized Frame ...           44   \n",
       "\n",
       "                                   clean_description  \n",
       "0  sunglass polarize lense resistant impact scrat...  \n",
       "1  high quality material gradient cat eye sunglas...  \n",
       "2  mission athlete passion design relentless purs...  \n",
       "3  sunglasse soft microfiber pouch soft glass clo...  \n",
       "4  feisedy retro design square oversized frame de...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_description'] = clean_description\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cleaning the title\n",
    "clean_title = [clean_data(df.loc[i,'title']) for i in range(len(df))]\n",
    "# clean_data(df.loc[1,'title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-11-de41336d7b6c>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['clean_title'] = clean_title\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category_id</th>\n",
       "      <th>clean_description</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Polarized Sunglasses for Men and Women Semi-R...</td>\n",
       "      <td>Plastic frame anti-reflective lens Polarized ...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglass polarize lense resistant impact scrat...</td>\n",
       "      <td>polarize sunglass man woman semi rimless frame...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SORVINO Vintage Sunglasses Retro Cateye Sungl...</td>\n",
       "      <td>HIGH QUALITY MATERIAL:This gradient cat eye s...</td>\n",
       "      <td>44</td>\n",
       "      <td>high quality material gradient cat eye sunglas...</td>\n",
       "      <td>sorvino vintage sunglass retro cateye sunglass...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Under Armour Womens Launch Run Visor</td>\n",
       "      <td>Under Armour's mission is to make all athlete...</td>\n",
       "      <td>44</td>\n",
       "      <td>mission athlete passion design relentless purs...</td>\n",
       "      <td>armour women launch visor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GRFISIA Square Oversized Sunglasses for Women...</td>\n",
       "      <td>Plastic frame anti-reflective lens Non-Polari...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglasse soft microfiber pouch soft glass clo...</td>\n",
       "      <td>grfisia square oversized sunglass woman man fl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEISEDY Retro Oversized Square Polarized Sung...</td>\n",
       "      <td>FEISEDY RETRO Design--Square Oversized Frame ...</td>\n",
       "      <td>44</td>\n",
       "      <td>feisedy retro design square oversized frame de...</td>\n",
       "      <td>feisedy retro oversized square polarize sungla...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0   Polarized Sunglasses for Men and Women Semi-R...   \n",
       "1   SORVINO Vintage Sunglasses Retro Cateye Sungl...   \n",
       "2              Under Armour Womens Launch Run Visor    \n",
       "3   GRFISIA Square Oversized Sunglasses for Women...   \n",
       "4   FEISEDY Retro Oversized Square Polarized Sung...   \n",
       "\n",
       "                                         description  category_id  \\\n",
       "0   Plastic frame anti-reflective lens Polarized ...           44   \n",
       "1   HIGH QUALITY MATERIAL:This gradient cat eye s...           44   \n",
       "2   Under Armour's mission is to make all athlete...           44   \n",
       "3   Plastic frame anti-reflective lens Non-Polari...           44   \n",
       "4   FEISEDY RETRO Design--Square Oversized Frame ...           44   \n",
       "\n",
       "                                   clean_description  \\\n",
       "0  sunglass polarize lense resistant impact scrat...   \n",
       "1  high quality material gradient cat eye sunglas...   \n",
       "2  mission athlete passion design relentless purs...   \n",
       "3  sunglasse soft microfiber pouch soft glass clo...   \n",
       "4  feisedy retro design square oversized frame de...   \n",
       "\n",
       "                                         clean_title  \n",
       "0  polarize sunglass man woman semi rimless frame...  \n",
       "1  sorvino vintage sunglass retro cateye sunglass...  \n",
       "2                          armour women launch visor  \n",
       "3  grfisia square oversized sunglass woman man fl...  \n",
       "4  feisedy retro oversized square polarize sungla...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['clean_title'] = clean_title\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title                         0 \n",
       "description           0 0 0 0 0 \n",
       "category_id                  251\n",
       "clean_description               \n",
       "clean_title                     \n",
       "Name: 199897, dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[199897]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can drop this record as it does not convey any meaningfull information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>category_id</th>\n",
       "      <th>clean_description</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Polarized Sunglasses for Men and Women Semi-R...</td>\n",
       "      <td>Plastic frame anti-reflective lens Polarized ...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglass polarize lense resistant impact scrat...</td>\n",
       "      <td>polarize sunglass man woman semi rimless frame...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SORVINO Vintage Sunglasses Retro Cateye Sungl...</td>\n",
       "      <td>HIGH QUALITY MATERIAL:This gradient cat eye s...</td>\n",
       "      <td>44</td>\n",
       "      <td>high quality material gradient cat eye sunglas...</td>\n",
       "      <td>sorvino vintage sunglass retro cateye sunglass...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Under Armour Womens Launch Run Visor</td>\n",
       "      <td>Under Armour's mission is to make all athlete...</td>\n",
       "      <td>44</td>\n",
       "      <td>mission athlete passion design relentless purs...</td>\n",
       "      <td>armour women launch visor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GRFISIA Square Oversized Sunglasses for Women...</td>\n",
       "      <td>Plastic frame anti-reflective lens Non-Polari...</td>\n",
       "      <td>44</td>\n",
       "      <td>sunglasse soft microfiber pouch soft glass clo...</td>\n",
       "      <td>grfisia square oversized sunglass woman man fl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FEISEDY Retro Oversized Square Polarized Sung...</td>\n",
       "      <td>FEISEDY RETRO Design--Square Oversized Frame ...</td>\n",
       "      <td>44</td>\n",
       "      <td>feisedy retro design square oversized frame de...</td>\n",
       "      <td>feisedy retro oversized square polarize sungla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253923</th>\n",
       "      <td>AW BRIDAL Cotton Waffle Robe Couples Spa Robe...</td>\n",
       "      <td>Machine Wash 【MATERIAL】 High quality waffle b...</td>\n",
       "      <td>310</td>\n",
       "      <td>aw cotton couple robe set amp amp king amp que...</td>\n",
       "      <td>aw bridal cotton waffle robe couple spa robe k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253924</th>\n",
       "      <td>Avidlove Women Lace Kimono Robe Babydoll Sexy...</td>\n",
       "      <td>Please going a size up if you wanted a looser...</td>\n",
       "      <td>310</td>\n",
       "      <td>size want looser long fit kimono small woman s...</td>\n",
       "      <td>avidlove woman lace kimono robe babydoll sexy ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253925</th>\n",
       "      <td>PAVILIA Womens Fleece Housecoat Zipper Robe, ...</td>\n",
       "      <td>Fleece Polyester Zipper closure ZIP UP FLEECE...</td>\n",
       "      <td>310</td>\n",
       "      <td>product description nbsp pavilia fleece house ...</td>\n",
       "      <td>pavilia women fleece housecoat zipper robe plu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253926</th>\n",
       "      <td>Silky Satin Robes For Women Pure Lightweight ...</td>\n",
       "      <td>100% Silk Tie closure Machine Wash 100% Polye...</td>\n",
       "      <td>310</td>\n",
       "      <td>silk tie closure machine wash polyester tie cl...</td>\n",
       "      <td>silky satin robe woman pure lightweight short ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253927</th>\n",
       "      <td>Women's Bride Bridesmaid Robes Silky Dusty Bl...</td>\n",
       "      <td>Tie closure Machine Wash Material:Satain-Poly...</td>\n",
       "      <td>310</td>\n",
       "      <td>sleepwear lounge wear bathrobe bridal robe won...</td>\n",
       "      <td>woman bride bridesmaid robe silky dusty blue k...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>253927 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "0        Polarized Sunglasses for Men and Women Semi-R...   \n",
       "1        SORVINO Vintage Sunglasses Retro Cateye Sungl...   \n",
       "2                   Under Armour Womens Launch Run Visor    \n",
       "3        GRFISIA Square Oversized Sunglasses for Women...   \n",
       "4        FEISEDY Retro Oversized Square Polarized Sung...   \n",
       "...                                                   ...   \n",
       "253923   AW BRIDAL Cotton Waffle Robe Couples Spa Robe...   \n",
       "253924   Avidlove Women Lace Kimono Robe Babydoll Sexy...   \n",
       "253925   PAVILIA Womens Fleece Housecoat Zipper Robe, ...   \n",
       "253926   Silky Satin Robes For Women Pure Lightweight ...   \n",
       "253927   Women's Bride Bridesmaid Robes Silky Dusty Bl...   \n",
       "\n",
       "                                              description  category_id  \\\n",
       "0        Plastic frame anti-reflective lens Polarized ...           44   \n",
       "1        HIGH QUALITY MATERIAL:This gradient cat eye s...           44   \n",
       "2        Under Armour's mission is to make all athlete...           44   \n",
       "3        Plastic frame anti-reflective lens Non-Polari...           44   \n",
       "4        FEISEDY RETRO Design--Square Oversized Frame ...           44   \n",
       "...                                                   ...          ...   \n",
       "253923   Machine Wash 【MATERIAL】 High quality waffle b...          310   \n",
       "253924   Please going a size up if you wanted a looser...          310   \n",
       "253925   Fleece Polyester Zipper closure ZIP UP FLEECE...          310   \n",
       "253926   100% Silk Tie closure Machine Wash 100% Polye...          310   \n",
       "253927   Tie closure Machine Wash Material:Satain-Poly...          310   \n",
       "\n",
       "                                        clean_description  \\\n",
       "0       sunglass polarize lense resistant impact scrat...   \n",
       "1       high quality material gradient cat eye sunglas...   \n",
       "2       mission athlete passion design relentless purs...   \n",
       "3       sunglasse soft microfiber pouch soft glass clo...   \n",
       "4       feisedy retro design square oversized frame de...   \n",
       "...                                                   ...   \n",
       "253923  aw cotton couple robe set amp amp king amp que...   \n",
       "253924  size want looser long fit kimono small woman s...   \n",
       "253925  product description nbsp pavilia fleece house ...   \n",
       "253926  silk tie closure machine wash polyester tie cl...   \n",
       "253927  sleepwear lounge wear bathrobe bridal robe won...   \n",
       "\n",
       "                                              clean_title  \n",
       "0       polarize sunglass man woman semi rimless frame...  \n",
       "1       sorvino vintage sunglass retro cateye sunglass...  \n",
       "2                               armour women launch visor  \n",
       "3       grfisia square oversized sunglass woman man fl...  \n",
       "4       feisedy retro oversized square polarize sungla...  \n",
       "...                                                   ...  \n",
       "253923  aw bridal cotton waffle robe couple spa robe k...  \n",
       "253924  avidlove woman lace kimono robe babydoll sexy ...  \n",
       "253925  pavilia women fleece housecoat zipper robe plu...  \n",
       "253926  silky satin robe woman pure lightweight short ...  \n",
       "253927  woman bride bridesmaid robe silky dusty blue k...  \n",
       "\n",
       "[253927 rows x 5 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new = df.drop(199897, axis = 0)\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_title</th>\n",
       "      <th>clean_description</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>polarize sunglass man woman semi rimless frame...</td>\n",
       "      <td>sunglass polarize lense resistant impact scrat...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sorvino vintage sunglass retro cateye sunglass...</td>\n",
       "      <td>high quality material gradient cat eye sunglas...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>armour women launch visor</td>\n",
       "      <td>mission athlete passion design relentless purs...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>grfisia square oversized sunglass woman man fl...</td>\n",
       "      <td>sunglasse soft microfiber pouch soft glass clo...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feisedy retro oversized square polarize sungla...</td>\n",
       "      <td>feisedy retro design square oversized frame de...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253923</th>\n",
       "      <td>aw bridal cotton waffle robe couple spa robe k...</td>\n",
       "      <td>aw cotton couple robe set amp amp king amp que...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253924</th>\n",
       "      <td>avidlove woman lace kimono robe babydoll sexy ...</td>\n",
       "      <td>size want looser long fit kimono small woman s...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253925</th>\n",
       "      <td>pavilia women fleece housecoat zipper robe plu...</td>\n",
       "      <td>product description nbsp pavilia fleece house ...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253926</th>\n",
       "      <td>silky satin robe woman pure lightweight short ...</td>\n",
       "      <td>silk tie closure machine wash polyester tie cl...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253927</th>\n",
       "      <td>woman bride bridesmaid robe silky dusty blue k...</td>\n",
       "      <td>sleepwear lounge wear bathrobe bridal robe won...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252937 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              clean_title  \\\n",
       "0       polarize sunglass man woman semi rimless frame...   \n",
       "1       sorvino vintage sunglass retro cateye sunglass...   \n",
       "2                               armour women launch visor   \n",
       "3       grfisia square oversized sunglass woman man fl...   \n",
       "4       feisedy retro oversized square polarize sungla...   \n",
       "...                                                   ...   \n",
       "253923  aw bridal cotton waffle robe couple spa robe k...   \n",
       "253924  avidlove woman lace kimono robe babydoll sexy ...   \n",
       "253925  pavilia women fleece housecoat zipper robe plu...   \n",
       "253926  silky satin robe woman pure lightweight short ...   \n",
       "253927  woman bride bridesmaid robe silky dusty blue k...   \n",
       "\n",
       "                                        clean_description  category_id  \n",
       "0       sunglass polarize lense resistant impact scrat...           44  \n",
       "1       high quality material gradient cat eye sunglas...           44  \n",
       "2       mission athlete passion design relentless purs...           44  \n",
       "3       sunglasse soft microfiber pouch soft glass clo...           44  \n",
       "4       feisedy retro design square oversized frame de...           44  \n",
       "...                                                   ...          ...  \n",
       "253923  aw cotton couple robe set amp amp king amp que...          310  \n",
       "253924  size want looser long fit kimono small woman s...          310  \n",
       "253925  product description nbsp pavilia fleece house ...          310  \n",
       "253926  silk tie closure machine wash polyester tie cl...          310  \n",
       "253927  sleepwear lounge wear bathrobe bridal robe won...          310  \n",
       "\n",
       "[252937 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#There is a chance that after data cleaning we don't have any title or description left (string of len = 0) so let's drop those records\n",
    "df_new = df_new[['clean_title', 'clean_description', 'category_id']]\n",
    "df_new = df_new[df_new['clean_title'].apply(lambda x: len(x) > 0)]\n",
    "df_new = df_new[df_new['clean_description'].apply(lambda x: len(x) > 0)]\n",
    "df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doing this dropped 990 records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the data and using this for feature engineering and modeling in the next python file\n",
    "df_new.to_csv('Clean_product_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Nouns should provide better information than keeping the entire sentences, performing keyword extraction and keeping just the nouns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title_keywords</th>\n",
       "      <th>description_keywords</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sunglass man woman frame drive sun glass uv block</td>\n",
       "      <td>impact scratch day time walk park drive fishin...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>vintage sunglass cateye sunglass woman man fra...</td>\n",
       "      <td>quality material gradient cat eye sunglass pc ...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>armour women visor</td>\n",
       "      <td>mission athlete passion design pursuit innovat...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sunglass woman man fashion shade</td>\n",
       "      <td>microfiber pouch glass cloth grfisia nbsp grfi...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feisedy sunglass woman style</td>\n",
       "      <td>feisedy retro design frame design minimalist s...</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253923</th>\n",
       "      <td>cotton waffle robe couple spa robe bathrobe dr...</td>\n",
       "      <td>cotton couple robe amp king amp hubby amp embr...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253924</th>\n",
       "      <td>woman lace kimono robe lingerie mesh chemise n...</td>\n",
       "      <td>size fit woman kimono lingerie swimsuit polyes...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253925</th>\n",
       "      <td>pavilia women zipper robe zip front lounger</td>\n",
       "      <td>product description nbsp pavilia fleece house ...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253926</th>\n",
       "      <td>satin robe woman robe wedding gift</td>\n",
       "      <td>silk tie closure machine wash polyester tie cl...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253927</th>\n",
       "      <td>woman bride robe robe woman emerald</td>\n",
       "      <td>lounge wear bathrobe bridal robe gift woman pr...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252937 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           title_keywords  \\\n",
       "0       sunglass man woman frame drive sun glass uv block   \n",
       "1       vintage sunglass cateye sunglass woman man fra...   \n",
       "2                                      armour women visor   \n",
       "3                        sunglass woman man fashion shade   \n",
       "4                            feisedy sunglass woman style   \n",
       "...                                                   ...   \n",
       "253923  cotton waffle robe couple spa robe bathrobe dr...   \n",
       "253924  woman lace kimono robe lingerie mesh chemise n...   \n",
       "253925        pavilia women zipper robe zip front lounger   \n",
       "253926                 satin robe woman robe wedding gift   \n",
       "253927                woman bride robe robe woman emerald   \n",
       "\n",
       "                                     description_keywords  category_id  \n",
       "0       impact scratch day time walk park drive fishin...           44  \n",
       "1       quality material gradient cat eye sunglass pc ...           44  \n",
       "2       mission athlete passion design pursuit innovat...           44  \n",
       "3       microfiber pouch glass cloth grfisia nbsp grfi...           44  \n",
       "4       feisedy retro design frame design minimalist s...           44  \n",
       "...                                                   ...          ...  \n",
       "253923  cotton couple robe amp king amp hubby amp embr...          310  \n",
       "253924  size fit woman kimono lingerie swimsuit polyes...          310  \n",
       "253925  product description nbsp pavilia fleece house ...          310  \n",
       "253926  silk tie closure machine wash polyester tie cl...          310  \n",
       "253927  lounge wear bathrobe bridal robe gift woman pr...          310  \n",
       "\n",
       "[252937 rows x 3 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp = spacy.load('en_core_web_sm')\n",
    "df_new1 = pd.DataFrame()\n",
    "df_new1['title_keywords'] = df_new['clean_title'].apply(lambda x: \" \".join([ent.text for ent in nlp(x) if ent.pos_ == 'NOUN']))\n",
    "df_new1['description_keywords'] = df_new['clean_description'].apply(lambda x: \" \".join([ent.text for ent in nlp(x) if ent.pos_ == 'NOUN']))\n",
    "df_new1['category_id'] = df_new['category_id']\n",
    "df_new1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the data and using this for feature engineering and modeling in the next python file\n",
    "df_new1.to_csv('Keywords_product_data.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}