Home credit bank visualization EDA, all plots-Copy2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'lightgbm'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-6b12c371f74c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mlightgbm\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mlgb\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpylab\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'lightgbm'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import roc_curve, make_scorer, roc_auc_score, f1_score,auc, accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import make_scorer, accuracy_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from scipy import stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('./application_train.csv/application_train.csv')\n",
    "train = train.loc[:100000,: ]\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_and_plot(df, variable):\n",
    "    grouped_by_target = df.groupby('TARGET')[variable].value_counts().reset_index(name='count')\n",
    "    grouped_by_target.pivot(\"TARGET\",variable ,\"count\").plot(kind='bar', figsize=(20,10))\n",
    "    plt.xlabel(\"TARGET\",size=20)\n",
    "    plt.ylabel(\"number\",size=20)\n",
    "    plt.xticks(rotation = 0)\n",
    "    plt.xticks(size=20)\n",
    "    plt.title(variable + \" / target\" ,size=20)\n",
    "    plt.yticks(size=20)\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = [  'FLAG_DOCUMENT_2','FLAG_DOCUMENT_3','FLAG_DOCUMENT_4','FLAG_DOCUMENT_5','FLAG_DOCUMENT_6',\n",
    "                 'FLAG_DOCUMENT_7','FLAG_DOCUMENT_8','FLAG_DOCUMENT_9','FLAG_DOCUMENT_10','FLAG_DOCUMENT_11',\n",
    "                 'FLAG_DOCUMENT_12','FLAG_DOCUMENT_13','FLAG_DOCUMENT_14','FLAG_DOCUMENT_15','FLAG_DOCUMENT_16',\n",
    "                 'FLAG_DOCUMENT_17','FLAG_DOCUMENT_18','FLAG_DOCUMENT_19','FLAG_DOCUMENT_20','FLAG_DOCUMENT_21',\n",
    "                 'REG_REGION_NOT_LIVE_REGION','REG_REGION_NOT_WORK_REGION',\n",
    "                 'LIVE_REGION_NOT_WORK_REGION','REG_CITY_NOT_LIVE_CITY',\n",
    "                 'REG_CITY_NOT_WORK_CITY','LIVE_CITY_NOT_WORK_CITY','ORGANIZATION_TYPE','WEEKDAY_APPR_PROCESS_START',\n",
    "                 'REGION_RATING_CLIENT','REGION_RATING_CLIENT_W_CITY','FLAG_MOBIL',\n",
    "                 'FLAG_EMP_PHONE','FLAG_WORK_PHONE',\n",
    "                 'FLAG_CONT_MOBILE','FLAG_PHONE','FLAG_EMAIL','OCCUPATION_TYPE','CODE_GENDER',\n",
    " 'FLAG_OWN_CAR',\n",
    " 'FLAG_OWN_REALTY','NAME_CONTRACT_TYPE',\n",
    " 'NAME_EDUCATION_TYPE',\n",
    " 'NAME_FAMILY_STATUS','TARGET',\n",
    " 'NAME_HOUSING_TYPE',\n",
    " 'NAME_INCOME_TYPE','NAME_TYPE_SUITE','FONDKAPREMONT_MODE','HOUSETYPE_MODE','WALLSMATERIAL_MODE','EMERGENCYSTATE_MODE'\n",
    "              ]\n",
    "\n",
    "numerical = ['SK_ID_CURR','CNT_CHILDREN','AMT_INCOME_TOTAL','AMT_CREDIT','AMT_ANNUITY',\n",
    "             'AMT_GOODS_PRICE','AMT_REQ_CREDIT_BUREAU_HOUR','AMT_REQ_CREDIT_BUREAU_DAY','AMT_REQ_CREDIT_BUREAU_WEEK',\n",
    "             'AMT_REQ_CREDIT_BUREAU_MON','AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR',\n",
    "            'OBS_30_CNT_SOCIAL_CIRCLE','DEF_30_CNT_SOCIAL_CIRCLE',\n",
    "            'OBS_60_CNT_SOCIAL_CIRCLE','DEF_60_CNT_SOCIAL_CIRCLE','DAYS_LAST_PHONE_CHANGE',\n",
    "            'APARTMENTS_AVG',\n",
    "            'BASEMENTAREA_AVG','YEARS_BEGINEXPLUATATION_AVG','YEARS_BUILD_AVG','COMMONAREA_AVG','ELEVATORS_AVG',\n",
    "            'ENTRANCES_AVG','FLOORSMAX_AVG','FLOORSMIN_AVG','LANDAREA_AVG','LIVINGAPARTMENTS_AVG',\n",
    "            'LIVINGAREA_AVG',\n",
    "            'NONLIVINGAPARTMENTS_AVG','NONLIVINGAREA_AVG','APARTMENTS_MODE','BASEMENTAREA_MODE','YEARS_BEGINEXPLUATATION_MODE',\n",
    "            'YEARS_BUILD_MODE','COMMONAREA_MODE','ELEVATORS_MODE',\n",
    "            'ENTRANCES_MODE','FLOORSMAX_MODE','FLOORSMIN_MODE','LANDAREA_MODE','LIVINGAPARTMENTS_MODE',\n",
    "            'LIVINGAREA_MODE',\n",
    "            'NONLIVINGAPARTMENTS_MODE','NONLIVINGAREA_MODE','APARTMENTS_MEDI',\n",
    " 'BASEMENTAREA_MEDI','YEARS_BEGINEXPLUATATION_MEDI','YEARS_BUILD_MEDI','COMMONAREA_MEDI',\n",
    " 'ELEVATORS_MEDI','ENTRANCES_MEDI','FLOORSMAX_MEDI','FLOORSMIN_MEDI','LANDAREA_MEDI',\n",
    " 'LIVINGAPARTMENTS_MEDI','LIVINGAREA_MEDI','NONLIVINGAPARTMENTS_MEDI','NONLIVINGAREA_MEDI','EXT_SOURCE_1','EXT_SOURCE_2',\n",
    " 'EXT_SOURCE_3','DAYS_BIRTH','DAYS_EMPLOYED','CNT_FAM_MEMBERS',\n",
    " 'DAYS_REGISTRATION','DAYS_ID_PUBLISH','OWN_CAR_AGE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fillna_cat(col):\n",
    "    train[col].value_counts().sort_values(ascending=False)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categorical:\n",
    "    train[col] = train[col].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in numerical:\n",
    "    try:\n",
    "        train[col] = train[col].astype(np.float64)\n",
    "    except ValueError:\n",
    "        print(col)\n",
    "    train[col] = train[col].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_categorical_nan():\n",
    "    for col in categorical:\n",
    "        if train[train[col].isna()].any().sum()>0:\n",
    "            print(\"this has NAN:\" + col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_numerical_nan():\n",
    "    for col in numerical:\n",
    "        if train[train[col].isna()].any().sum()>0:\n",
    "            print(\"this numeric col has NAN:\" + col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_cols = numerical+categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(all_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.loc[:, all_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_popular = fillna_cat('OCCUPATION_TYPE')\n",
    "train['OCCUPATION_TYPE'] = train['OCCUPATION_TYPE'].fillna(value= most_popular, method='ffill')\n",
    "\n",
    "most_popular = fillna_cat('NAME_TYPE_SUITE')\n",
    "train['NAME_TYPE_SUITE'] = train['NAME_TYPE_SUITE'].fillna(value= most_popular, method='ffill')\n",
    "\n",
    "most_popular = fillna_cat('FONDKAPREMONT_MODE')\n",
    "train['FONDKAPREMONT_MODE'] = train['FONDKAPREMONT_MODE'].fillna(value= most_popular, method='ffill')\n",
    "\n",
    "most_popular = fillna_cat('HOUSETYPE_MODE')\n",
    "train['HOUSETYPE_MODE'] = train['HOUSETYPE_MODE'].fillna(value= most_popular, method='ffill')\n",
    "\n",
    "most_popular = fillna_cat('WALLSMATERIAL_MODE')\n",
    "train['WALLSMATERIAL_MODE'] = train['WALLSMATERIAL_MODE'].fillna(value= most_popular, method='ffill')\n",
    "\n",
    "most_popular = fillna_cat('EMERGENCYSTATE_MODE')\n",
    "train['EMERGENCYSTATE_MODE'] = train['EMERGENCYSTATE_MODE'].fillna(value= most_popular, method='ffill')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_categorical_nan()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_numerical_nan()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,20))\n",
    "sns.heatmap(train.corr())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_high_corr_var(df):\n",
    "    corr_matrix = df.corr().abs()\n",
    "    high_corr_var=np.where(corr_matrix>0.8)\n",
    "\n",
    "    high_corr_var=[ (corr_matrix.columns[x],corr_matrix.columns[y]) for x,y in zip(*high_corr_var) if x!=y and x<y]\n",
    "    to_drop = list()\n",
    "    for tupl in high_corr_var:\n",
    "        to_drop.append(tupl[1])\n",
    "    #to_drop.remove(\"TARGET\")\n",
    "    print(to_drop)\n",
    "    df.drop(columns=to_drop, axis=1, inplace=True)\n",
    "\n",
    "remove_high_corr_var(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,20))\n",
    "sns.heatmap(train.corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_numerical(df):\n",
    "    numerical = list()\n",
    "    for col in df.columns.tolist():\n",
    "        if df[col].dtype == np.float64:\n",
    "            numerical.append(col)\n",
    "    return numerical\n",
    "            \n",
    "def get_categorical(df):\n",
    "    categ = list()\n",
    "    for col in df.columns.tolist():\n",
    "        if df[col].dtype != np.float64:\n",
    "            categ.append(col)\n",
    "    return categ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical = get_numerical(train)\n",
    "categorical = get_categorical(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "numericals_to_draw_against = list(['CNT_CHILDREN','AMT_INCOME_TOTAL',\n",
    " 'AMT_CREDIT','AMT_ANNUITY',\n",
    " 'AMT_REQ_CREDIT_BUREAU_HOUR','AMT_REQ_CREDIT_BUREAU_DAY',\n",
    " 'AMT_REQ_CREDIT_BUREAU_WEEK','AMT_REQ_CREDIT_BUREAU_MON',\n",
    " 'AMT_REQ_CREDIT_BUREAU_QRT','AMT_REQ_CREDIT_BUREAU_YEAR',\n",
    " 'OBS_30_CNT_SOCIAL_CIRCLE','DEF_30_CNT_SOCIAL_CIRCLE',\n",
    " 'DAYS_LAST_PHONE_CHANGE','APARTMENTS_AVG',\n",
    " 'BASEMENTAREA_AVG','YEARS_BEGINEXPLUATATION_AVG',\n",
    " 'YEARS_BUILD_AVG','COMMONAREA_AVG',\n",
    " 'ENTRANCES_AVG','FLOORSMAX_AVG',\n",
    " 'FLOORSMIN_AVG','LANDAREA_AVG',\n",
    " 'LIVINGAPARTMENTS_AVG','NONLIVINGAPARTMENTS_AVG',\n",
    " 'NONLIVINGAREA_AVG','EXT_SOURCE_1',\n",
    " 'EXT_SOURCE_2','EXT_SOURCE_3',\n",
    " 'DAYS_BIRTH','DAYS_EMPLOYED', 'DAYS_REGISTRATION',\n",
    " 'DAYS_ID_PUBLISH', 'OWN_CAR_AGE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categoricals_to_draw_against = list(['REG_REGION_NOT_LIVE_REGION','REG_REGION_NOT_WORK_REGION',\n",
    "'LIVE_REGION_NOT_WORK_REGION','REG_CITY_NOT_LIVE_CITY',\n",
    "'REG_CITY_NOT_WORK_CITY','LIVE_CITY_NOT_WORK_CITY','WEEKDAY_APPR_PROCESS_START',\n",
    "'REGION_RATING_CLIENT',\n",
    "'REGION_RATING_CLIENT_W_CITY','FLAG_MOBIL',\n",
    "'FLAG_EMP_PHONE','FLAG_WORK_PHONE',\n",
    "'FLAG_CONT_MOBILE','FLAG_PHONE','FLAG_EMAIL',\n",
    "'CODE_GENDER','FLAG_OWN_CAR','FLAG_OWN_REALTY'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "categoricals_to_draw_against_name_types = list(['NAME_CONTRACT_TYPE','OCCUPATION_TYPE','ORGANIZATION_TYPE',\n",
    "'NAME_EDUCATION_TYPE','NAME_FAMILY_STATUS','TARGET','NAME_HOUSING_TYPE',\n",
    "'NAME_INCOME_TYPE','NAME_TYPE_SUITE',\n",
    "'FONDKAPREMONT_MODE','HOUSETYPE_MODE',\n",
    "'WALLSMATERIAL_MODE','EMERGENCYSTATE_MODE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grouper(train, var):\n",
    "    \n",
    "    qq = train.groupby('TARGET')[var].value_counts().sort_values(ascending=True).reset_index(name='count')\n",
    "    rotate_x_labels=True\n",
    "    if train[var].value_counts().nunique() < 4:\n",
    "        rotate_x_labels=False\n",
    "        \n",
    "    #for visual details\n",
    "    mainfontsize = 40\n",
    "    bartextsize=30\n",
    "    rotation_for_bar_text = 0\n",
    "    format_for_bar_text = '.2f'\n",
    "    figure_size_for_big_chart = (25,10)\n",
    "    #decide how to draw many categories - select diff size, text size etc\n",
    "    if len(qq) > 20:\n",
    "        many_categories = True\n",
    "        mainfontsize = 15\n",
    "        bartextsize=10\n",
    "        rotation_for_bar_text = 30\n",
    "        format_for_bar_text = '.1f'\n",
    "        figure_size_for_big_chart = (20,10)\n",
    "        \n",
    "    #compute percent rate \n",
    "    totals = list()\n",
    "    for c in qq[var].values:\n",
    "        totals.append(np.sum(qq[qq[var] == c]['count']))\n",
    "\n",
    "    qq['totals'] = pd.Series(totals,index=qq.index)\n",
    "    qq['percent_rate'] = (qq['count']*100) / qq['totals']\n",
    "    \n",
    "    fig, ax1 = plt.subplots(figsize=figure_size_for_big_chart)\n",
    "    \n",
    "    qq['TARGET'] = qq['TARGET'].apply(lambda status: rename_target(status))\n",
    "    #df qq is sorted so bars are in descending order\n",
    "    qq = qq.sort_values(by=['percent_rate'],ascending=False)\n",
    "    barplot = sns.barplot(x=var,y='percent_rate',hue='TARGET',data=qq,\n",
    "                palette='rainbow',saturation=0.5,\n",
    "                hue_order=['YES', 'NO'],ax=ax1)\n",
    "       \n",
    "    for p in barplot.patches:\n",
    "        barplot.annotate(format(p.get_height(), format_for_bar_text), \n",
    "                               (p.get_x() + p.get_width() / 2., \n",
    "                                p.get_height()), color = 'black', fontsize=bartextsize,\n",
    "                                 rotation=rotation_for_bar_text,\n",
    "                                 ha = 'center', va = 'center', \n",
    "                                 xytext = (0, 10), \n",
    "                                 textcoords = 'offset points')\n",
    "    \n",
    "    plt.xlabel(s=var,size=40)\n",
    "    plt.ylabel(s= '%',size=40)\n",
    "    \n",
    "    if var == 'ORGANIZATION_TYPE':\n",
    "        plt.xticks(rotation=90,size=mainfontsize)    \n",
    "        \n",
    "    elif rotate_x_labels:\n",
    "        plt.xticks(rotation=45,size=mainfontsize)    \n",
    "    else:\n",
    "        plt.xticks(size=mainfontsize)\n",
    "       \n",
    "    plt.yscale('log')\n",
    "    plt.yticks(size=20)\n",
    "    plt.title(var + \" vs \" + \" % count\",size=40)\n",
    "    plt.legend(loc='center', prop={'size':30})\n",
    "    plt.show()\n",
    "    print()\n",
    "    print()\n",
    "    print()\n",
    "    return qq\n",
    "\n",
    "\n",
    "w = grouper(train, 'NAME_INCOME_TYPE')\n",
    "w = grouper(train, 'ORGANIZATION_TYPE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against:\n",
    "    grouper(train, col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against_name_types:\n",
    "    try:\n",
    "        grouper(train, col)\n",
    "    except ValueError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_target(status):\n",
    "    if status == 1:\n",
    "        return 'YES'\n",
    "    else:\n",
    "        return 'NO'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_N_kde_plots(df, feature_list):\n",
    "    cmap = sns.palplot(sns.color_palette(\"Paired\"))\n",
    "    # plot N features\n",
    "    plt.figure(figsize=(20,10))\n",
    "    for feature in feature_list:                \n",
    "        plt.yscale('log')\n",
    "        p1=sns.kdeplot(df[feature], shade=True, color=cmap)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using PyOD to outlier detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# draw pairplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.pairplot(train.iloc[:,:10], diag_kind=\"kde\", markers=\"+\",\n",
    "                  plot_kws=dict(s=50, edgecolor=\"b\", linewidth=1),\n",
    "                  diag_kws=dict(shade=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# draw box plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_box_plot(x_boxen, y_boxen, df):\n",
    "    plt.figure(figsize=(20,10))\n",
    "    ax = sns.boxplot(x=x_boxen, y=y_boxen, hue=\"TARGET\",\n",
    "                      data=df, palette=\"Set3\")\n",
    "    plt.yticks(fontsize=20)\n",
    "    plt.xticks(fontsize=10)\n",
    "    plt.xlabel(fontsize=10, s= col)\n",
    "    \n",
    "    plt.title(x_boxen + ' vs ' + y_boxen,size=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        plot_box_plot(categ, numer, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against_name_types:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        plot_box_plot(categ, numer, train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# drawing distplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_distplots(col,df, scaling=False):    \n",
    "    plt.figure(figsize=(20,10))    \n",
    "    sns.distplot(df[col],color='r')    \n",
    "    plt.yticks(fontsize=20)\n",
    "    plt.xlabel(fontsize=10, s= col)\n",
    "    if scaling:\n",
    "        plt.yscale('log')\n",
    "    plt.xticks(fontsize=10)\n",
    "    plt.title(col,size=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in numericals_to_draw_against:\n",
    "    plot_distplots(col, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# drawing all boxen plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def boxen_plot_chart(x_boxen, y_boxen,df, scaling=False, name_type = False):    \n",
    "    plt.figure(figsize=(20,10))    \n",
    "    ax = sns.boxenplot(x=x_boxen, y=y_boxen, data=df)\n",
    "    if scaling:\n",
    "        plt.yscale('log')\n",
    "    plt.xticks(fontsize=20)\n",
    "    plt.title(x_boxen + \" vs \" + y_boxen,size=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        boxen_plot_chart(categ, numer, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against_name_types:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        boxen_plot_chart(categ, numer, train, name_type=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# drawing all stripplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def strip_plot_chart(x_boxen, y_boxen,df, scaling=False, name_type=False, show_hue = False):    \n",
    "    plt.figure(figsize=(20,10))\n",
    "    plt.yticks(fontsize=20)\n",
    "    if scaling:\n",
    "        plt.yscale('log')\n",
    "    if show_hue:\n",
    "        ax = sns.stripplot(x=x_boxen, y=y_boxen, data=df,hue='TARGET', jitter=0.05)\n",
    "    else:\n",
    "        ax = sns.stripplot(x=x_boxen, y=y_boxen, data=df, jitter=0.05)\n",
    "    plt.xticks(rotation=60,fontsize=20)\n",
    "    plt.title(x_boxen + \" vs \" + y_boxen,size=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        strip_plot_chart(categ, numer, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against_name_types:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        strip_plot_chart(categ, numer, train, name_type=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for categ in categoricals_to_draw_against_name_types:\n",
    "    for numer in numericals_to_draw_against:\n",
    "        strip_plot_chart(categ, numer, train, name_type=True,show_hue=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "sns.set(style=\"ticks\", color_codes=True)\n",
    "g = sns.FacetGrid(train, col=\"TARGET\", row=\"NAME_EDUCATION_TYPE\",size=10)\n",
    "g = g.map(plt.hist, 'AMT_ANNUITY',  color = 'r')\n",
    "#plt.ylim(0,4000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "sns.set(style=\"ticks\", color_codes=True)\n",
    "g = sns.FacetGrid(train, col=\"TARGET\", row=\"OCCUPATION_TYPE\")\n",
    "g = g.map(plt.hist, \"AMT_CREDIT\",color = 'r')\n",
    "plt.ylim(0,4000)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# group and plot function plots()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against:\n",
    "    group_and_plot(train, col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against_name_types:\n",
    "    try:\n",
    "        group_and_plot(train, col)\n",
    "    except ValueError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# drawing JOINt grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def joint_dist_scatter(x_col, y_col, df):\n",
    "#     plt.figure(figsize=(20,10))    \n",
    "#     plt.yticks(fontsize=20)\n",
    "#     plt.xticks(fontsize=10)\n",
    "#     plt.xlabel(fontsize=10, s= col)\n",
    "#     plt.title(x_col + ' vs ' + y_col,size=20)\n",
    "    \n",
    "    g = sns.JointGrid(x=x_col, y=y_col, data=df,size=6)\n",
    "    g = g.plot_joint(plt.scatter, color=\".5\", edgecolor=\"black\", marker='*')    \n",
    "    g = g.plot_marginals(sns.distplot, kde=False, color=\"blue\")\n",
    "    g = g.plot_marginals(sns.kdeplot, color=\"green\")    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(numerical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for numer1 in numericals_to_draw_against:\n",
    "    for numer2 in numericals_to_draw_against:\n",
    "        if numer1 != numer2:\n",
    "            joint_dist_scatter(numer1, numer2, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against:\n",
    "    plt.figure(figsize=(20,10))\n",
    "    sns.countplot(train[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categoricals_to_draw_against_name_types:\n",
    "    plt.figure(figsize=(20,10))\n",
    "    sns.countplot(train[col])\n",
    "    plt.yticks(fontsize=20)\n",
    "    plt.xlabel(fontsize=20, s= col)\n",
    "    plt.ylabel(fontsize=20, s = 'count')\n",
    "    plt.xticks(fontsize=20)\n",
    "    \n",
    "    plt.title(col,size=20)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scatter plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_scatter(x_col, y_col, df, scale = False):\n",
    "    plt.figure(figsize=(15,10))\n",
    "    sns.set_palette(\"Paired\")\n",
    "    if scale:\n",
    "        plt.yscale('log')\n",
    "    sns.scatterplot(x=x_col, y=y_col,marker='X', hue='TARGET',linewidth=0,data=train)    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for numer1 in numericals_to_draw_against:\n",
    "    for numer2 in numericals_to_draw_against:\n",
    "        if numer1 != numer2:            \n",
    "            plot_scatter(numer1, numer2, train, scale=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.xticks(size=20)\n",
    "sns.distplot(train['AMT_CREDIT'],bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "plt.xticks(size=20)\n",
    "plt.xlim(-0.5,5)\n",
    "plt.ylim(0,4)\n",
    "sns.distplot(train['CNT_CHILDREN'],bins=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for col in train.columns.tolist():\n",
    "    fig = plt.figure(figsize=(20,10))\n",
    "    train['CODE_GENDER'].value_counts().plot(kind='bar')\n",
    "    plt.xticks(size=20)\n",
    "    plt.yticks(size=20)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,10))\n",
    "g = sns.JointGrid(x=\"AMT_INCOME_TOTAL\", y=\"TARGET\", data=train,\n",
    "                   height=5, ratio=2)\n",
    "g = g.plot_joint(sns.kdeplot, cmap=\"Reds_d\")\n",
    "g = g.plot_marginals(sns.kdeplot, color=\"r\", shade=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.JointGrid(x=\"AMT_INCOME_TOTAL\", y=\"TARGET\", data=train,\n",
    "                   height=5, ratio=2)\n",
    "g = g.plot_joint(sns.kdeplot, cmap=\"Reds_d\")\n",
    "g = g.plot_marginals(sns.kdeplot, color=\"r\", shade=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,ax = plt.subplots(1,2, figsize=(20,10))\n",
    "ax1 = train['NAME_CONTRACT_TYPE'].value_counts().plot(kind='bar',ax= ax[0])\n",
    "ax1.set_x\n",
    "\n",
    "train['NAME_EDUCATION_TYPE'].value_counts().plot(kind='bar',ax= ax[1])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}