naive_bayes_classification_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np \n",
    "from sklearn.datasets import load_wine\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Naive Bayes classifier with Gaussian distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NaiveBayesGauss:\n",
    "    \"\"\" Naive Bayes classification using Gaussian distribution \"\"\"\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.class_prior = {} \n",
    "        self.train_params = {} \n",
    "    \n",
    "    def _compute_params(self, feat_df):\n",
    "        \"\"\" compute training parameters: mean and std per feature \"\"\"\n",
    "        training_params = {}\n",
    "        \n",
    "        # compute mean and std per column (i.e., feature)\n",
    "        mean_vec = feat_df.mean()\n",
    "        mean_vec = mean_vec.drop(\"labels\").to_dict()\n",
    "        \n",
    "        std_vec = feat_df.std() \n",
    "        std_vec = std_vec.drop(\"labels\").to_dict() \n",
    "        \n",
    "        for key, val in mean_vec.items():\n",
    "            training_params[key] = (val, std_vec[key])\n",
    "        \n",
    "        return training_params\n",
    "    \n",
    "    def fit(self, X_train, y_train, show_samples=True):\n",
    "        \"\"\" train a Naive Bayes classifier with Gaussian distribution \n",
    "        :param X_train: training data in the format (num_samples x num_features)\n",
    "        :param y_train: a list of labels for the corresponding samples \n",
    "        \"\"\"\n",
    "        # create a pandas dataframe \n",
    "        num_samples, num_feats = X_train.shape \n",
    "        data_mat = np.column_stack((X_train, y_train))\n",
    "        self.cols = [\"feat\" + str(i+1) for i in range(num_feats)] + [\"labels\"] \n",
    "        df = pd.DataFrame(data=data_mat, columns=self.cols)\n",
    "        if show_samples:\n",
    "            print(\"training data \\n\", df.head())\n",
    "        \n",
    "        # compute class priors and training params (mean and std) per feature per class \n",
    "        for label, feats_per_class in df.groupby('labels'):\n",
    "            print(f\"number of samples per class ({label}): {len(feats_per_class)}\")\n",
    "            self.class_prior[int(label)] = round(len(feats_per_class) / num_samples, 3) \n",
    "            self.train_params[int(label)] = self._compute_params(feats_per_class)\n",
    "        \n",
    "    def _compute_prob(self, x, label, feat_id):\n",
    "        \"\"\" compute a Gaussian probability for a given feature value and for a given class label. \n",
    "            Gaussian parameters (mean and std) are computed from the training data \n",
    "        \"\"\"\n",
    "        feat_mean = self.train_params[label][self.cols[feat_id]][0]\n",
    "        feat_std = self.train_params[label][self.cols[feat_id]][1] \n",
    "        \n",
    "        exp_part = math.exp(-((x-feat_mean)**2 / (2 * feat_std**2 )))\n",
    "        gauss_prob = (1 / (math.sqrt(2 * math.pi) * feat_std)) * exp_part\n",
    "        return gauss_prob\n",
    "    \n",
    "    def _compute_likelihood_prob(self, feat_vec, label):\n",
    "        \"\"\" compute a likelihood prob which is a product of probabilities of each feature variable for a given class label \"\"\"\n",
    "        likelihood_probs = [self._compute_prob(feat, label, feat_id) for feat_id, feat in enumerate(feat_vec)]\n",
    "        return np.prod(likelihood_probs)\n",
    "    \n",
    "    def predict(self, X_test):\n",
    "        \"\"\" prediction on a given test data \n",
    "        :param X_test: numpy array containing samples of feature vectors \n",
    "        :return predict_vals: predicted label and probabilities \n",
    "        \"\"\"\n",
    "        # convert numpy array to a list of lists\n",
    "        X_test = X_test.tolist() \n",
    "        predict_vals = {\"probs\": [], \"labels\": []}\n",
    "        \n",
    "        # compute max posterior prob for each sample \n",
    "        for ind, sample in enumerate(X_test):\n",
    "            \n",
    "            # compute posterior probabilities for each class and pick the max one as final class label \n",
    "            predict_prob = -1\n",
    "            predict_label = -1\n",
    "            norm_constant = 0.0 \n",
    "            for label, prior_prob in self.class_prior.items():\n",
    "                # compute likelihood prob which is a product of probs of each feature \n",
    "                likelihood_prob = self._compute_likelihood_prob(sample, label)\n",
    "                \n",
    "                # compute posterior distrition as multiplication of prior and likelihood. Note: normalization constant \n",
    "                # since we will compute the max prob afterwards \n",
    "                post_prob = prior_prob * likelihood_prob \n",
    "                \n",
    "                # compute normalization constant \n",
    "                norm_constant += post_prob \n",
    "                if post_prob > predict_prob:\n",
    "                    predict_prob = post_prob \n",
    "                    predict_label = label \n",
    "                    \n",
    "            print(f\"predicted label for the {ind}th sample is {predict_label} with probability {predict_prob}\")\n",
    "            predict_vals[\"probs\"].append(predict_prob / norm_constant)\n",
    "            predict_vals[\"labels\"].append(predict_label) \n",
    "            \n",
    "        # return the label with the max probability \n",
    "        return predict_vals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load the wine data and split it into train and test datasets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "wine_data = load_wine()\n",
    "data = wine_data.data \n",
    "labels = wine_data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, labels, test_size=0.20, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train the naive Bayes model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training data \n",
      "    feat1  feat2  feat3  feat4  feat5  feat6  feat7  feat8  feat9  feat10  \\\n",
      "0  13.69   3.26   2.54   20.0  107.0   1.83   0.56   0.50   0.80    5.88   \n",
      "1  12.69   1.53   2.26   20.7   80.0   1.38   1.46   0.58   1.62    3.05   \n",
      "2  11.62   1.99   2.28   18.0   98.0   3.02   2.26   0.17   1.35    3.25   \n",
      "3  13.40   3.91   2.48   23.0  102.0   1.80   0.75   0.43   1.41    7.30   \n",
      "4  13.50   1.81   2.61   20.0   96.0   2.53   2.61   0.28   1.66    3.52   \n",
      "\n",
      "   feat11  feat12  feat13  labels  \n",
      "0    0.96    1.82   680.0     2.0  \n",
      "1    0.96    2.06   495.0     1.0  \n",
      "2    1.16    2.96   345.0     1.0  \n",
      "3    0.70    1.56   750.0     2.0  \n",
      "4    1.12    3.82   845.0     0.0  \n",
      "number of samples per class (0.0): 45\n",
      "number of samples per class (1.0): 55\n",
      "number of samples per class (2.0): 42\n"
     ]
    }
   ],
   "source": [
    "nb_classifier = NaiveBayesGauss()\n",
    "nb_classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print the trained parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 0.317, 1: 0.387, 2: 0.296}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_classifier.class_prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: {'feat1': (13.724666666666664, 0.44862719894848513),\n",
       "  'feat2': (2.030888888888889, 0.7027634485703312),\n",
       "  'feat3': (2.4506666666666677, 0.23376756132379173),\n",
       "  'feat4': (16.911111111111115, 2.7057365172924914),\n",
       "  'feat5': (105.13333333333334, 10.010449086284337),\n",
       "  'feat6': (2.8331111111111116, 0.3586833385783864),\n",
       "  'feat7': (3.000666666666668, 0.40596909868967757),\n",
       "  'feat8': (0.28577777777777774, 0.06790665012583916),\n",
       "  'feat9': (1.9393333333333336, 0.4165082122948969),\n",
       "  'feat10': (5.496444444444444, 1.2767486325456041),\n",
       "  'feat11': (1.0684444444444445, 0.112167803587376),\n",
       "  'feat12': (3.1768888888888904, 0.3767011247311817),\n",
       "  'feat13': (1104.6, 218.80651477753327)},\n",
       " 1: {'feat1': (12.265636363636364, 0.5725827245306799),\n",
       "  'feat2': (1.9080000000000001, 0.924336398600519),\n",
       "  'feat3': (2.2412727272727273, 0.30330924535282183),\n",
       "  'feat4': (20.314545454545453, 3.4304420502221955),\n",
       "  'feat5': (95.9090909090909, 17.976602825846967),\n",
       "  'feat6': (2.213090909090909, 0.5147738544371899),\n",
       "  'feat7': (1.9796363636363639, 0.5556509817034552),\n",
       "  'feat8': (0.35618181818181816, 0.12149517455881712),\n",
       "  'feat9': (1.6656363636363647, 0.6286545410529792),\n",
       "  'feat10': (2.9318181818181825, 0.7449088706608517),\n",
       "  'feat11': (1.059745454545454, 0.20375695563513155),\n",
       "  'feat12': (2.7965454545454542, 0.47047343445220424),\n",
       "  'feat13': (527.8, 165.67967755749515)},\n",
       " 2: {'feat1': (13.134047619047614, 0.524795242411369),\n",
       "  'feat2': (3.3471428571428574, 1.1100940098934837),\n",
       "  'feat3': (2.4416666666666664, 0.18896272269711217),\n",
       "  'feat4': (21.392857142857142, 2.344000422161668),\n",
       "  'feat5': (100.07142857142857, 10.664261851447428),\n",
       "  'feat6': (1.702857142857143, 0.34388941582082583),\n",
       "  'feat7': (0.7826190476190478, 0.2882669792668876),\n",
       "  'feat8': (0.4564285714285713, 0.12708402115158204),\n",
       "  'feat9': (1.1709523809523812, 0.4180781203917402),\n",
       "  'feat10': (7.3292856904761905, 2.3339460640726095),\n",
       "  'feat11': (0.6907142857142856, 0.11500568081711907),\n",
       "  'feat12': (1.6780952380952374, 0.25497238583373455),\n",
       "  'feat13': (635.1190476190476, 110.82555010090593)}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_classifier.train_params"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Perform prediction on test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted label for the 0th sample is 0 with probability 6.79147236643029e-07\n",
      "predicted label for the 1th sample is 2 with probability 2.4632187160557036e-09\n",
      "predicted label for the 2th sample is 1 with probability 8.452845725369398e-11\n",
      "predicted label for the 3th sample is 0 with probability 4.070009987966507e-07\n",
      "predicted label for the 4th sample is 1 with probability 1.4955915525288458e-12\n",
      "predicted label for the 5th sample is 1 with probability 2.6800560577303106e-23\n",
      "predicted label for the 6th sample is 0 with probability 1.1631681502473758e-07\n",
      "predicted label for the 7th sample is 2 with probability 1.3155434166767746e-07\n",
      "predicted label for the 8th sample is 1 with probability 1.7287921194231835e-07\n",
      "predicted label for the 9th sample is 1 with probability 2.982284376636477e-08\n",
      "predicted label for the 10th sample is 2 with probability 2.072781781472864e-09\n",
      "predicted label for the 11th sample is 2 with probability 4.660554513091282e-11\n",
      "predicted label for the 12th sample is 0 with probability 1.903391193966711e-07\n",
      "predicted label for the 13th sample is 0 with probability 6.44617931275783e-12\n",
      "predicted label for the 14th sample is 2 with probability 3.19924469626462e-07\n",
      "predicted label for the 15th sample is 1 with probability 1.2258638438952193e-08\n",
      "predicted label for the 16th sample is 0 with probability 6.321083111192501e-11\n",
      "predicted label for the 17th sample is 0 with probability 7.898405831093356e-11\n",
      "predicted label for the 18th sample is 2 with probability 4.682452633741e-12\n",
      "predicted label for the 19th sample is 0 with probability 3.997314773003913e-07\n",
      "predicted label for the 20th sample is 0 with probability 3.663158993531375e-12\n",
      "predicted label for the 21th sample is 0 with probability 4.6051450860859726e-08\n",
      "predicted label for the 22th sample is 0 with probability 6.48822806791824e-09\n",
      "predicted label for the 23th sample is 1 with probability 8.089251428874476e-08\n",
      "predicted label for the 24th sample is 1 with probability 1.7715551030162364e-11\n",
      "predicted label for the 25th sample is 1 with probability 4.1718123001813305e-08\n",
      "predicted label for the 26th sample is 1 with probability 1.8491241343238556e-09\n",
      "predicted label for the 27th sample is 1 with probability 5.6256536736422796e-08\n",
      "predicted label for the 28th sample is 1 with probability 5.069803530443511e-10\n",
      "predicted label for the 29th sample is 2 with probability 1.0038160819269315e-06\n",
      "predicted label for the 30th sample is 0 with probability 6.227783827503993e-07\n",
      "predicted label for the 31th sample is 0 with probability 6.112650439913045e-07\n",
      "predicted label for the 32th sample is 1 with probability 3.783752861153431e-09\n",
      "predicted label for the 33th sample is 0 with probability 4.081004560433599e-09\n",
      "predicted label for the 34th sample is 0 with probability 1.0811099657971065e-07\n",
      "predicted label for the 35th sample is 0 with probability 2.2099114531138426e-07\n"
     ]
    }
   ],
   "source": [
    "predict_vals = nb_classifier.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[14  0  0]\n",
      " [ 2 13  1]\n",
      " [ 0  0  6]]\n"
     ]
    }
   ],
   "source": [
    "conf_mat = confusion_matrix(y_test, predict_vals[\"labels\"])\n",
    "print(conf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      1.00      0.93        14\n",
      "           1       1.00      0.81      0.90        16\n",
      "           2       0.86      1.00      0.92         6\n",
      "\n",
      "   micro avg       0.92      0.92      0.92        36\n",
      "   macro avg       0.91      0.94      0.92        36\n",
      "weighted avg       0.93      0.92      0.92        36\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, predict_vals[\"labels\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy rate: 91.667\n"
     ]
    }
   ],
   "source": [
    "acc = 1 - (np.sum(conf_mat) - np.trace(conf_mat))/np.sum(conf_mat)\n",
    "print(f\"accuracy rate: {round(acc*100.0, 3)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}