5.1_adult.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "data_folder = os.path.join(\"data\", \"adult\")\n",
    "adult_filename = os.path.join(data_folder, \"adult.data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult = pd.read_csv(adult_filename, header=None, names=[\"Age\", \"Work-Class\", \"fnlwgt\", \"Education\",\n",
    "                                                        \"Education-Num\", \"Marital-Status\", \"Occupation\",\n",
    "                                                        \"Relationship\", \"Race\", \"Sex\", \"Capital-gain\",\n",
    "                                                        \"Capital-loss\", \"Hours-per-week\", \"Native-Country\",\n",
    "                                                        \"Earnings-Raw\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32561, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult.dropna(how='all', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32561, 15)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Age', 'Work-Class', 'fnlwgt', 'Education', 'Education-Num',\n",
       "       'Marital-Status', 'Occupation', 'Relationship', 'Race', 'Sex',\n",
       "       'Capital-gain', 'Capital-loss', 'Hours-per-week', 'Native-Country',\n",
       "       'Earnings-Raw'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    32561.000000\n",
       "mean        40.437456\n",
       "std         12.347429\n",
       "min          1.000000\n",
       "25%         40.000000\n",
       "50%         40.000000\n",
       "75%         45.000000\n",
       "max         99.000000\n",
       "Name: Hours-per-week, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult[\"Hours-per-week\"].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult[\"Education-Num\"].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([' State-gov', ' Self-emp-not-inc', ' Private', ' Federal-gov',\n",
       "       ' Local-gov', ' ?', ' Self-emp-inc', ' Without-pay',\n",
       "       ' Never-worked'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult[\"Work-Class\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.feature_selection import VarianceThreshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = adult[[\"Age\", \"Education-Num\", \"Capital-gain\", \"Capital-loss\", \"Hours-per-week\"]].values\n",
    "y = (adult[\"Earnings-Raw\"] == ' >50K').values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "transformer = SelectKBest(score_func=chi2, k=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[8.60061182e+03 2.40142178e+03 8.21924671e+07 1.37214589e+06\n",
      " 6.47640900e+03]\n"
     ]
    }
   ],
   "source": [
    "Xt_chi2 = transformer.fit_transform(X, y)\n",
    "print(transformer.scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr\n",
    "\n",
    "def multivariate_pearsonr(X, y):\n",
    "    scores, pvalues = [], []\n",
    "    for column in range(X.shape[1]):\n",
    "        cur_score, cur_p = pearsonr(X[:,column], y)\n",
    "        scores.append(abs(cur_score))\n",
    "        pvalues.append(cur_p)\n",
    "    return (np.array(scores), np.array(pvalues))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.2340371  0.33515395 0.22332882 0.15052631 0.22968907]\n"
     ]
    }
   ],
   "source": [
    "transformer = SelectKBest(score_func=multivariate_pearsonr, k=3)\n",
    "Xt_pearson = transformer.fit_transform(X, y)\n",
    "print(transformer.scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "clf = DecisionTreeClassifier(random_state=14)\n",
    "scores_chi2 = cross_val_score(clf, Xt_chi2, y, scoring='accuracy', cv=5)\n",
    "scores_pearson = cross_val_score(clf, Xt_pearson, y, scoring='accuracy', cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chi2 performance: 0.829\n",
      "Pearson performance: 0.772\n"
     ]
    }
   ],
   "source": [
    "print(\"Chi2 performance: {0:.3f}\".format(scores_chi2.mean()))\n",
    "print(\"Pearson performance: {0:.3f}\".format(scores_pearson.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_test_filename = os.path.join(data_folder, \"adult.test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_test = pd.read_csv(adult_test_filename, header=None, names=[\"Age\", \"Work-Class\", \"fnlwgt\", \"Education\",\n",
    "                                                        \"Education-Num\", \"Marital-Status\", \"Occupation\",\n",
    "                                                        \"Relationship\", \"Race\", \"Sex\", \"Capital-gain\",\n",
    "                                                        \"Capital-loss\", \"Hours-per-week\", \"Native-Country\",\n",
    "                                                        \"Earnings-Raw\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16282, 15)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Age', 'Work-Class', 'fnlwgt', 'Education', 'Education-Num',\n",
       "       'Marital-Status', 'Occupation', 'Relationship', 'Race', 'Sex',\n",
       "       'Capital-gain', 'Capital-loss', 'Hours-per-week', 'Native-Country',\n",
       "       'Earnings-Raw'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "adult_test.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Work-Class</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>Education</th>\n",
       "      <th>Education-Num</th>\n",
       "      <th>Marital-Status</th>\n",
       "      <th>Occupation</th>\n",
       "      <th>Relationship</th>\n",
       "      <th>Race</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Capital-gain</th>\n",
       "      <th>Capital-loss</th>\n",
       "      <th>Hours-per-week</th>\n",
       "      <th>Native-Country</th>\n",
       "      <th>Earnings-Raw</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>Private</td>\n",
       "      <td>226802.0</td>\n",
       "      <td>11th</td>\n",
       "      <td>7.0</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>38</td>\n",
       "      <td>Private</td>\n",
       "      <td>89814.0</td>\n",
       "      <td>HS-grad</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Farming-fishing</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>28</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>336951.0</td>\n",
       "      <td>Assoc-acdm</td>\n",
       "      <td>12.0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Protective-serv</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>44</td>\n",
       "      <td>Private</td>\n",
       "      <td>160323.0</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Machine-op-inspct</td>\n",
       "      <td>Husband</td>\n",
       "      <td>Black</td>\n",
       "      <td>Male</td>\n",
       "      <td>7688.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&gt;50K.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>18</td>\n",
       "      <td>?</td>\n",
       "      <td>103497.0</td>\n",
       "      <td>Some-college</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Never-married</td>\n",
       "      <td>?</td>\n",
       "      <td>Own-child</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Age  Work-Class    fnlwgt      Education  Education-Num  \\\n",
       "1  25     Private  226802.0           11th            7.0   \n",
       "2  38     Private   89814.0        HS-grad            9.0   \n",
       "3  28   Local-gov  336951.0     Assoc-acdm           12.0   \n",
       "4  44     Private  160323.0   Some-college           10.0   \n",
       "5  18           ?  103497.0   Some-college           10.0   \n",
       "\n",
       "        Marital-Status          Occupation Relationship    Race      Sex  \\\n",
       "1        Never-married   Machine-op-inspct    Own-child   Black     Male   \n",
       "2   Married-civ-spouse     Farming-fishing      Husband   White     Male   \n",
       "3   Married-civ-spouse     Protective-serv      Husband   White     Male   \n",
       "4   Married-civ-spouse   Machine-op-inspct      Husband   Black     Male   \n",
       "5        Never-married                   ?    Own-child   White   Female   \n",
       "\n",
       "   Capital-gain  Capital-loss  Hours-per-week  Native-Country Earnings-Raw  \n",
       "1           0.0           0.0            40.0   United-States       <=50K.  \n",
       "2           0.0           0.0            50.0   United-States       <=50K.  \n",
       "3           0.0           0.0            40.0   United-States        >50K.  \n",
       "4        7688.0           0.0            40.0   United-States        >50K.  \n",
       "5           0.0           0.0            30.0   United-States       <=50K.  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adult_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = adult_test[[\"Age\", \"Capital-gain\", \"Capital-loss\"]].values\n",
    "y_test = (adult_test[\"Earnings-Raw\"] == ' >50K').values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "                       max_features=None, max_leaf_nodes=None,\n",
       "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                       min_samples_leaf=1, min_samples_split=2,\n",
       "                       min_weight_fraction_leaf=0.0, presort=False,\n",
       "                       random_state=14, splitter='best')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.fit(Xt_chi2,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((16281,), (16281,))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92844419875929"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(predict==y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[15116,  1165],\n",
       "       [    0,     0]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "92.84441987592899"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test,predict)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mehar/.local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'              precision    recall  f1-score   support\\n\\n       False       1.00      0.93      0.96     16281\\n        True       0.00      0.00      0.00         0\\n\\n    accuracy                           0.93     16281\\n   macro avg       0.50      0.46      0.48     16281\\nweighted avg       1.00      0.93      0.96     16281\\n'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classification_report(y_test, predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}