demo/coles-catboost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a7431993",
   "metadata": {},
   "source": [
    "## Data load"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f96b188",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "587df1ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# import logging\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "# import warnings\n",
    "\n",
    "# warnings.filterwarnings('ignore')\n",
    "# logging.getLogger(\"pytorch_lightning\").setLevel(logging.ERROR)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82f989bc",
   "metadata": {},
   "source": [
    "## Data preproccessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7e06bd27",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>client_id</th>\n",
       "      <th>trans_date</th>\n",
       "      <th>small_group</th>\n",
       "      <th>amount_rur</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33172</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>71.463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33172</td>\n",
       "      <td>6</td>\n",
       "      <td>35</td>\n",
       "      <td>45.017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   client_id  trans_date  small_group  amount_rur\n",
       "0      33172           6            4      71.463\n",
       "1      33172           6           35      45.017"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "source_data = pd.read_csv('https://huggingface.co/datasets/dllllb/age-group-prediction/resolve/main/transactions_train.csv.gz?download=true', compression='gzip')\n",
    "source_data.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8615b722",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pretrained preprocessor\n",
    "import pickle\n",
    "from ptls.preprocessing import PandasDataPreprocessor\n",
    "\n",
    "with open('preprocessor.p', 'rb') as f:\n",
    "    preprocessor = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fca72f6e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 34.9 s, sys: 7.15 s, total: 42.1 s\n",
      "Wall time: 42 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "dataset = preprocessor.transform(source_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bef9a944",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = sorted(dataset, key=lambda x: x['client_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "98e7d39d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24000 6000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, test = train_test_split(dataset, test_size=0.2, random_state=42)\n",
    "\n",
    "print(len(train), len(test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d65b5e3",
   "metadata": {},
   "source": [
    "## Inference "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f7f2c6c",
   "metadata": {},
   "source": [
    "### load SequenceEncoder obtained from `coles-emb.ipynb`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "715e00f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ptls.nn import TrxEncoder, RnnSeqEncoder\n",
    "from ptls.frames.supervised import SequenceToTarget\n",
    "\n",
    "trx_encoder_params = dict(\n",
    "    embeddings_noise=0.003,\n",
    "    numeric_values={'amount_rur': 'identity'},\n",
    "    embeddings={\n",
    "        'trans_date': {'in': 800, 'out': 16},\n",
    "        'small_group': {'in': 250, 'out': 16},\n",
    "    },\n",
    ")\n",
    "\n",
    "seq_encoder = RnnSeqEncoder(\n",
    "    trx_encoder=TrxEncoder(**trx_encoder_params),\n",
    "    hidden_size=256,\n",
    "    type='gru',\n",
    ")\n",
    "\n",
    "seq_encoder.load_state_dict(torch.load('coles-emb.pt'))\n",
    "\n",
    "model = SequenceToTarget(seq_encoder)\n",
    "model.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7957dcb3",
   "metadata": {},
   "source": [
    "### embedding inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3c32741d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n",
      "/home/kireev/pipenv_envs/pytorch-lifestream-KxQJF1XF/lib/python3.8/site-packages/pytorch_lightning/trainer/data_loading.py:132: UserWarning: The dataloader, predict_dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 24 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "417fcb8238b54a6ca767b0e0b9df16d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kireev/pipenv_envs/pytorch-lifestream-KxQJF1XF/lib/python3.8/site-packages/pytorch_lightning/loops/epoch/prediction_epoch_loop.py:174: UserWarning: Lightning couldn't infer the indices fetched for your dataloader.\n",
      "  warning_cache.warn(\"Lightning couldn't infer the indices fetched for your dataloader.\")\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "245076e326d34d938041717d295769c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Predicting: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(torch.Size([24000, 256]), torch.Size([6000, 256]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ptls.data_load.datasets import inference_data_loader\n",
    "\n",
    "trainer = pl.Trainer(gpus=1 if torch.cuda.is_available() else 0)\n",
    "\n",
    "train_dl = inference_data_loader(train, num_workers=0, batch_size=256)\n",
    "train_embeds = torch.vstack(trainer.predict(model, train_dl))\n",
    "\n",
    "test_dl = inference_data_loader(test, num_workers=0, batch_size=256)\n",
    "test_embeds = torch.vstack(trainer.predict(model, test_dl))\n",
    "\n",
    "train_embeds.shape, test_embeds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "18245f84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24000, 258) (6000, 258)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>embed_0</th>\n",
       "      <th>embed_1</th>\n",
       "      <th>embed_2</th>\n",
       "      <th>embed_3</th>\n",
       "      <th>embed_4</th>\n",
       "      <th>embed_5</th>\n",
       "      <th>embed_6</th>\n",
       "      <th>embed_7</th>\n",
       "      <th>embed_8</th>\n",
       "      <th>embed_9</th>\n",
       "      <th>...</th>\n",
       "      <th>embed_248</th>\n",
       "      <th>embed_249</th>\n",
       "      <th>embed_250</th>\n",
       "      <th>embed_251</th>\n",
       "      <th>embed_252</th>\n",
       "      <th>embed_253</th>\n",
       "      <th>embed_254</th>\n",
       "      <th>embed_255</th>\n",
       "      <th>client_id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.369912</td>\n",
       "      <td>0.107698</td>\n",
       "      <td>-0.078325</td>\n",
       "      <td>0.028368</td>\n",
       "      <td>-0.463864</td>\n",
       "      <td>-0.297325</td>\n",
       "      <td>0.182668</td>\n",
       "      <td>0.282931</td>\n",
       "      <td>0.335743</td>\n",
       "      <td>-0.470002</td>\n",
       "      <td>...</td>\n",
       "      <td>0.024539</td>\n",
       "      <td>0.325287</td>\n",
       "      <td>0.660900</td>\n",
       "      <td>-0.180033</td>\n",
       "      <td>-0.276484</td>\n",
       "      <td>-0.288909</td>\n",
       "      <td>-0.240146</td>\n",
       "      <td>0.042354</td>\n",
       "      <td>36253</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.333840</td>\n",
       "      <td>-0.006155</td>\n",
       "      <td>-0.031773</td>\n",
       "      <td>-0.203356</td>\n",
       "      <td>-0.670753</td>\n",
       "      <td>-0.421903</td>\n",
       "      <td>0.141004</td>\n",
       "      <td>0.272937</td>\n",
       "      <td>0.444377</td>\n",
       "      <td>-0.550712</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.064095</td>\n",
       "      <td>0.020957</td>\n",
       "      <td>0.169336</td>\n",
       "      <td>-0.377122</td>\n",
       "      <td>-0.250504</td>\n",
       "      <td>-0.083562</td>\n",
       "      <td>-0.050570</td>\n",
       "      <td>0.012870</td>\n",
       "      <td>396</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 258 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    embed_0   embed_1   embed_2   embed_3   embed_4   embed_5   embed_6  \\\n",
       "0  0.369912  0.107698 -0.078325  0.028368 -0.463864 -0.297325  0.182668   \n",
       "1  0.333840 -0.006155 -0.031773 -0.203356 -0.670753 -0.421903  0.141004   \n",
       "\n",
       "    embed_7   embed_8   embed_9  ...  embed_248  embed_249  embed_250  \\\n",
       "0  0.282931  0.335743 -0.470002  ...   0.024539   0.325287   0.660900   \n",
       "1  0.272937  0.444377 -0.550712  ...  -0.064095   0.020957   0.169336   \n",
       "\n",
       "   embed_251  embed_252  embed_253  embed_254  embed_255  client_id  target  \n",
       "0  -0.180033  -0.276484  -0.288909  -0.240146   0.042354      36253       1  \n",
       "1  -0.377122  -0.250504  -0.083562  -0.050570   0.012870        396       2  \n",
       "\n",
       "[2 rows x 258 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join target and embeddings\n",
    "\n",
    "df_target = pd.read_csv('https://huggingface.co/datasets/dllllb/age-group-prediction/resolve/main/train_target.csv?download=true')\n",
    "df_target = df_target.set_index('client_id')\n",
    "df_target.rename(columns={\"bins\": \"target\"}, inplace=True)\n",
    "\n",
    "train_df = pd.DataFrame(data=train_embeds, columns=[f'embed_{i}' for i in range(train_embeds.shape[1])])\n",
    "train_df['client_id'] = [x['client_id'] for x in train]\n",
    "train_df = train_df.merge(df_target, how='left', on='client_id')\n",
    "\n",
    "test_df = pd.DataFrame(data=test_embeds, columns=[f'embed_{i}' for i in range(test_embeds.shape[1])])\n",
    "test_df['client_id'] = [x['client_id'] for x in test]\n",
    "test_df = test_df.merge(df_target, how='left', on='client_id')\n",
    "\n",
    "print(train_df.shape, test_df.shape)\n",
    "train_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "80fb9418",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "y_train = train_df['target'].values\n",
    "X_train = train_df.drop(['client_id', 'target'], axis=1)\n",
    "X_train_emb = pd.DataFrame(np.arange(len(X_train)))\n",
    "X_train_emb['embeddings'] = X_train.values.tolist()\n",
    "X_train_emb = X_train_emb.drop([0], axis=1)\n",
    "\n",
    "y_val = test_df['target'].values\n",
    "X_val = test_df.drop(['client_id', 'target'], axis=1)\n",
    "X_val_emb = pd.DataFrame(np.arange(len(X_val)))\n",
    "X_val_emb['embeddings'] = X_val.values.tolist()\n",
    "X_val_emb = X_val_emb.drop([0], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68bf49cf",
   "metadata": {},
   "source": [
    "## Lets use obtained embeddings for CatBoost training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4a14793b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install catboost\n",
    "\n",
    "from catboost import CatBoostClassifier, metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18ef862f",
   "metadata": {},
   "source": [
    "### With CatBoost embedding_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "56782093",
   "metadata": {},
   "outputs": [],
   "source": [
    "CatBoostModel_emb = CatBoostClassifier(\n",
    "    iterations=1000,\n",
    "    learning_rate=0.05,\n",
    "    use_best_model=True,\n",
    "    custom_metric=[metrics.Accuracy()],\n",
    "    random_seed=42,\n",
    "    logging_level='Silent',\n",
    "    embedding_features=['embeddings'],\n",
    "    depth=5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cf37fdbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51a855875c0f4a3cbbcef26b480b893d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "MetricVisualizer(layout=Layout(align_self='stretch', height='500px'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 7s, sys: 3.85 s, total: 1min 11s\n",
      "Wall time: 35 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f2e3ac79940>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "CatBoostModel_emb.fit(\n",
    "    X_train_emb, y_train,\n",
    "    eval_set=(X_val_emb, y_val),\n",
    "    plot=True\n",
    "#     logging_level='Verbose',  # you can uncomment this for text output\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e6eb8606",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'learn': {'Accuracy': 0.616875, 'MultiClass': 0.8707710171950899},\n",
       " 'validation': {'Accuracy': 0.5991666666666666,\n",
       "  'MultiClass': 0.9140885900754535}}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CatBoostModel_emb.get_best_score()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5455bb2e",
   "metadata": {},
   "source": [
    "### Without CatBoost embedding_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "edb384f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "CatBoostModel = CatBoostClassifier(\n",
    "    iterations=1000,\n",
    "    learning_rate=0.05,\n",
    "    use_best_model=True,\n",
    "    custom_metric=[metrics.Accuracy()],\n",
    "    random_seed=42,\n",
    "    logging_level='Silent',\n",
    "    depth=4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "204a9c10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a1ad352b9214fe692ba5ebcdcb85c07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "MetricVisualizer(layout=Layout(align_self='stretch', height='500px'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 30s, sys: 4.72 s, total: 2min 35s\n",
      "Wall time: 7.99 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x7f2e24311100>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "CatBoostModel.fit(\n",
    "    X_train, y_train,\n",
    "    eval_set=(X_val, y_val),\n",
    "    plot=True\n",
    "#     logging_level='Verbose',  # you can uncomment this for text output\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2b63ffdb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'learn': {'Accuracy': 0.6730833333333334, 'MultiClass': 0.7880862797825388},\n",
       " 'validation': {'Accuracy': 0.6115, 'MultiClass': 0.8752625132255518}}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CatBoostModel.get_best_score()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "24d5b0ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6093333333333333"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CatBoostModel.score(X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20cfe88e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ptls",
   "language": "python",
   "name": "ptls"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}