Boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_splits(col):\n",
    "    col=np.sort(np.unique(col))\n",
    "   \n",
    "    split_values=np.array((np.array(col[1:]) + np.array(col[:-1]) )/ 2)\n",
    "    return split_values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$Entropy_{(\\text{node})} = -p_{\\text{class1}}*\\log_2(p_{\\text{class1}}) + -p_{\\text{class2}}*\\log_2(p_{\\text{class2}})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entropy(class1_n, class2_n):\n",
    "    if (class1_n == 0) or (class2_n == 0):\n",
    "        return 0\n",
    "    total = class1_n + class2_n\n",
    "    class1_proprtion = class1_n/total\n",
    "    class2_proportion = class2_n/total\n",
    "    return  sum([-1 * prop * np.log2(prop)\n",
    "                 for prop in [class1_proprtion, class2_proportion] ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "$$Entropy_{(\\text{split})} = p_{\\text{node1}} * Entropy_{\\text{(node1})}+ p_{\\text{node2}}* Entropy_{(\\text{node2})},$$  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def node_classes(col, split_value, labels): #provided\n",
    "        le_node = labels[col <= split_value]\n",
    "        g_node = labels[col > split_value]\n",
    "\n",
    "        le_c1 = np.count_nonzero(le_node)\n",
    "        le_c2 = len(le_node) - le_c1\n",
    "\n",
    "        g_c1 = np.count_nonzero(g_node)\n",
    "        g_c2 = len(g_node) - g_c1    \n",
    "        return le_c1, le_c2, g_c1, g_c2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ent_from_split(col, split_value, labels):\n",
    "    A_1,A_0,B_1,B_0=node_classes(col, split_value, labels)\n",
    "    \n",
    "    entA = entropy(A_1,A_0)\n",
    "    entB=entropy(B_1,B_0)\n",
    "    total = len(col)\n",
    "    \n",
    "    pA=(A_1+A_0)/(total)\n",
    "    pB=(B_1+B_0)/(total)    \n",
    "    return pA*entA+pB*entB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred_from_split(X, y, col_idx, split_value): \n",
    "    le_c1,le_c2,g_c1,g_c2=node_classes(X[:,col_idx], split_value, y)\n",
    "    def pred_for_node(c11,c12):\n",
    "        if c11>c12: return True\n",
    "        elif c11<c12: return False\n",
    "        else: return None\n",
    "    left = pred_for_node(le_c1,le_c2)\n",
    "    right = pred_for_node(g_c1,g_c2)\n",
    "    if ((right==None)or left==None) and (right!=left):\n",
    "        if left ==None:\n",
    "            left = not right\n",
    "        else:\n",
    "            right = not left\n",
    "    if (right==None) and (left==None):\n",
    "        right = True\n",
    "        left = True\n",
    "\n",
    "    return (int(left),int(right))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_binary_tree_predict(X, col_idx, split_value, left_pred, right_pred):\n",
    "    col = X[:,col_idx]\n",
    "    col=np.where(col<=split_value, left_pred, right_pred)\n",
    "    return np.array(col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_binary_tree_fit(X,y): #provided\n",
    "    best_split = (-1,-1,1)    \n",
    "    for col_idx, col in enumerate(X.T):\n",
    "        splits = find_splits(col)\n",
    "        for s in splits:\n",
    "            ent = ent_from_split(col, s, y)          \n",
    "            if ent < best_split[2]:\n",
    "                best_split = (col_idx, s, ent) \n",
    "    left_pred, right_pred = pred_from_split(X, y, *best_split[:2])    \n",
    "    col_idx, split_value = best_split[:2]    \n",
    "    return col_idx, split_value, left_pred, right_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "X = np.array([[0.5, 3. ], [1.,  2. ], [3.,  0.5],\n",
    "                [2.,  3. ], [3.,  4. ]])\n",
    "col_idx = 0\n",
    "split_value = 1.5\n",
    "left_pred = 1\n",
    "right_pred = 0\n",
    "\n",
    "preds = simple_binary_tree_predict(X, col_idx, split_value, left_pred, right_pred)\n",
    "\n",
    "print(preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adaptive Boosting Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def default_weights(n):\n",
    "    return np.full(n,1/n)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\epsilon_t = \\sum_{i=1}^nw_t(i)\\mathbb{1}\\{y_i\\ne f_t(x_i)\\}$$  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_epsilon(y_true, y_pred, weights):\n",
    "    summ=0\n",
    "    for i in range(len(y_true)):\n",
    "        if y_true[i]!=y_pred[i]:\n",
    "            summ=summ+weights[i]\n",
    "    return np.float(summ)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$\\alpha_t = \\frac12ln(\\frac{1-\\epsilon_t}{\\epsilon_t})$$  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_alpha(epsilon):\n",
    "    if epsilon==0:\n",
    "        return np.inf\n",
    "    else:\n",
    "        return 0.5*np.log((1-epsilon)/epsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$w_{t+1}(i) = \\frac{\\hat{w}_{t+1}(i)}{\\sum_j\\hat{w}_{t+1}(j)}$$  \n",
    "\n",
    "\n",
    "$$\\hat{w}_{t+1}(i) = w_t(i)e^{-\\alpha_ty_if_t(x_i)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_weights(weights, alpha, y_true, y_pred):\n",
    "    def target_for_weight_update(y):\n",
    "        toRet=y.copy()\n",
    "        toRet[toRet==0]=-1\n",
    "        return toRet\n",
    "    y=target_for_weight_update(y_true)\n",
    "    pred=target_for_weight_update(y_pred)\n",
    "    weights=weights*np.exp(-alpha*y*pred)\n",
    "    weights=weights/sum(weights)\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prediction from boosted trees\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$f_{boost}(x_0) = sign(\\sum_{t=1}^T\\alpha_tf_t(X_0))$$  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X, est_dict):\n",
    "    fin_preds=np.zeros(X.shape[0])\n",
    "    for k in est_dict:\n",
    "        preds=est_dict[k][0].predict(X)\n",
    "        preds[preds<1]=-1\n",
    "        preds=preds*est_dict[k][1]\n",
    "        fin_preds +=preds\n",
    "    fin_preds[fin_preds>=0]=1\n",
    "    fin_preds[fin_preds<0]=0\n",
    "    return fin_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAEICAYAAAB2wembAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAXRUlEQVR4nO3df5wcd33f8dfb67uz8A+IsSH+AcgnGlBBqkSvKMUBLIcHsY2DaQuCJFaRHgRLQBIT04cfJX00JSmEJG2JH4/ERXHiyEYmwFEgbZ2QB06Q7Fqxz5wsWRKWWqOzHcACy1aMLazoxPHpH985a3S+u53bm9253Xk/H4996HZn9jufnZn3fmdmd79SRGBmve2Uqgsws/Zz0M1qwEE3qwEH3awGHHSzGnDQzWrAQS9A0hFJg1XXsVBIWiwpJJ06w/RHJL2l03VNU8ctkj7eoWV9TNJtnVhWKzoWdEm/KGk0C81BSV+V9DMFnxuSXtnuGmcSEWdExFiZbWav6YfZ+nhS0t9KeneZy5hl2Qt6p8yTdJakGyT9fbauvpXdP6dDy3+DpL/L/r5K0i5JT0t6IttmiztRx3x1JOiSrgNuAH4HeCnwcuC/A1d1YvmtmqnHKtE/i4gzgFcBtwB/JOk/tXmZXUNSP/C3wGuAy4CzgDcATwKvb+Ny89v9CuCvso7mM8BHgBcCF5H24R+3q45SRURbb6SVcgR41yzzvB64B3gKOAj8EdCfTbsLCOCHWTvvzh6/EtiVPefvgOW59l4H7ASeAb4IfAH4eG76+4FvAYeB/wWcn5sWwIeAh4CHc4+9Mvt7APivwN8D3wc2AYuyaecAt2c1HQb+D3DKDK/5uTZzj70T+Efgxbl1d3O2Tr4LfBxoZNPWAXdntfwD8DBwea6t87PXdjh7re/PHr8MGAeOZ+vzgQLLamTLeQIYy9ZPAKfO8NoeAT4KPJjVthk4LZu2F/j53Lx9Wbsrpmnnl7N1fMYs+85SYFu2zr8JvD037Zb5bPfs8fuz/emdwK5Z6vgYMEx6M3gmq2UoN/3fAweyaQ8C/yo3bR2wHfhD4AfAfuBnp2Ro2m1TOIcdCPplwI9m2imyef458NPAqcBiYB/w4ZlCka34x4FV2U743mznGgD6gUeBa7Od6F9nO/bHs+demu1Yr8vm/0PgrinLugM4mxMBzgf9hmwnORs4E/jfwCezaZ8kBb8vu70R0ByC3petq8uz+38B/DFwOvAS4D5gQ27nOJ7tvA3gA8Bjk8sD7iT1OKcBK4BDkztPtlPeNmXZsy1rY7bzvSx73VtpHvS9ufm359b/9cAXcvNeBeyZoZ3PA7fOst/0kYL7G9l2v5QUpFdNDXqL2/08UrAEDJLehP8AWM2UN59snf4j6Qigke0L9+amv4v05nsK8G5Sx3Veblv+CPj17DW9mxT4s5ttm4UU9F8CvjfH53wY+MosQf808J+nPOf/Am8G3jS5cXLT7s5t8JuB389NO4MUmMW5ZV06XSizDf5DYElu2r/kRM//28D/ZEqAiwY9e/x72Tp7KXBscqfLpv0CsDW3c3wrN+0FWZs/SQrYBHBmbvongVumC3qBZX0d2Jib9laaBz0//xXAgezv80lhPCu7/z+A62do5w7gd2dZh2/M1tcpucc+B3wsnh/0Vrb7+4Cbc/d/mtRrHyKF+haywGfr9G9y8/5T4Ogste8Crspty+fepLPH7gPWNts2RW/tPgeFdD51jqRTI+JH080g6aeATwFDpB32VGDHLG2+AnivpF/NPdZP2okC+G5kayTz7dzf55MOxwCIiCOSngQuIO2gU+fPOzerb4ek58onvYMD/BfSBv9aNv2miPjdWV7HSST1Zcs4nL3GPuBgblmnTKnte7nX8Ww23xnAi4HDEfFMbt5HSet3Os2Wdf6U5T5a4OVMnf/8rM7HJG0H/o2krwCXk46+pvMkqVedyfnAtyMif578KGlbTjfvXLf7FcCf555zL7AGQNK/IJ0S/gfSaQrktgfwLHDa5H4v6d8C15GOWCFtp/wFxan77OQ6K7IfNNWJi3H3kN793jHLPJ8mHRr+k4g4i3Qoplnm/zbwiYh4Ue72goj4HOk85gLl1gqph5v0GGnlASDpdFIwvpubJ7/C854AjgKvyS33hZEuqBERz0TERyJiEPh54DpJPzvL65jqKtIh3H3ZazwGnJNb1lkR8ZoC7TwGnC3pzNxjL8+9xqmvr9myDnLyOnx5gRqmzv9Y7v6twNWkw9l7IiK/7vP+Bvi5bBtN5zHgZZLy+3H+dU6dt/B2z95030w6qnieiPgG8GXgtTPU9hxJrwD+BPgV0vWXF5FObfL76NR9dnKdzWc/eE7bgx4RPwB+E7hR0jskvUBSn6TLJf1+NtuZwNPAEUmvJp1v5n2fdI406U+AjZJWKTld0tuyHfse0mHrr0g6VdJVnHyF9s+B9ZJWSBogfRIwEhGPFHgtP86W/QeSXgIg6QJJP5f9faWkV2Yb7Omsjolm7Uo6W9IvATcCvxcRT0bEQeBrwH/LPmI6RdISSW8uUOe3SRcoPynpNEnLSYehn81m+T6weDIgBZY1DPyapAsl/QTpwlIzH8rmP5v0xv2F3LS/IJ0rX0u6eDWTLaQd/UuSXp3V9WJJvyHpCmCEdCp1fbZPXUJ6g/38NG3Ndbu/EdgdEU8DSPoZSe/PbfdXA28H7m2+Kjid9CZyKHvuep7/BvES0jruk/Qu0kXGv5rPfnCSuRznz+dGOu8cJW2Y7wF/Cbwhm/YmUo9+hHSl+reBu3PP3UjqVZ4C1mSPXQZ8gxNX6r9Idk5KOkTdlbX3RdI773+c0t4B0iHy7cCFuWnTXSTLX4w7jbSTjJHCvA/4tWzar5MOA38IfCe/zGnWR3Dik4TDpAtcvzhlnheSjna+Q7o4sxN4T+687u5Z6rwwe22Hs9eaP2d+Mem6xT8A9xdY1qmki1BPkq7uz+Wq+1OkHvwFU+b50+z1z3hFPVfXDaTAH8ley6c48cnEa0gXHn/A869m38LJV90Lb3fSpwz/Lnf/taQLr9/P6ngE+D2gL06co+eveyzOryPgE9lyn8jqvxP45dy23E76tOkHwP8D3lpkPyh6m7xC29MkjQCbImJz1bVYIuk3gZ+KiKurrmU6kh4E3hkRD3ZgWetIoS/0BbJW9ORXYCW9WdJPZofu7wWWA39ddV2WZIfz7wNuqrqW6WRf1PlMJ0LeKT0ZdNI3zR4gHeZ8hPTOfLDakgxA0vtJh+FfjYi7qq5nOhExHnP4tKQb1OLQ3azuerVHN7Octnxh5pxzzonFixe3o2kzm8WOHTueiIhzpz7elqAvXryY0dHRdjRtZrOQNO23Fn3obl3pkkvSzYpx0M1qwEE3qwEH3awGHHSzGnDQzWrAQTerAQfdrAYcdLMa6MSYcWZzUuSLMHfeWXzebdvmUUy77NkDn/40bN0KY2MwPg79/TA4CKtXwwc+AMuWlbY4B92sk8bGYO1a2LULjh2DidxIY+PjsH8/PPQQ3HorrFgBW7ak8M+Tg24LTpEeeLInX5C99UyGh2H9+ucHfKqJCXj2WRgZSb365s2wZs28Fu2gm3XC8DCsWwdHjxZ/zmTg161L9+cRdl+MM2u3sbHUk88l5HlHj6bnP/xwyyU46GbttnZtOlyfj2PH4OrWx9F00M3aaffudOFttnPyIiYmYOfOdLW+BQ66WTtt2jT/3nzS+HhqrwUOulk7bd06/9580sREaq8FDrpZO42NldvegQMtPc1BN2un8fFy2zt+vKWn+XN060pd80WZ/v5yw97X19LT3KObtVMJX189yZIlLT3NQTdrp9WrodEop61GI7XXAgfdaq3tw0Zv3AgDA+W01d+f2muBg27WTsuXp1+hzbdXbzRg5cqWf7rqoJu125Yt8+/VBwbgtttafrqDbtZug4Ppp6aLFrX2/EWL0vMvuqjlEvzxmlknTP7EtMjv0Sc1GqknL+H36O7RzTplzZr0o5RVq1IvPdN5e6ORpq9aBXv3zjvk4KCbddbgIGzfnkaP2bABli5NV9Ol9O/SpenxkZE03zwO1/N86G5WhWXL4MYbO7Y49+hmNeAe3UqxEAdrrMWw0QW5RzerAffo1rN6dtjoFrhHN6sBB92sBhx0sxpw0M1qoHDQJTUk7ZR0ezsLMrPyzaVHvxbY165CzKx9Cn28JulC4G3AJ4Dr2lqRLTj+4kn3K9qj3wBcD/x4phkkXSNpVNLooUOHSinOzMrRtEeXdCXweETskHTJTPNFxE3ATQBDQ0NRWoVWuV7+4km31duqIj36xcDbJT0CfB64VFLrY9qYWcc1DXpEfDQiLoyIxcB7gK9HROv/f6uZdZw/RzergTn9qCUitgHb2lKJmbWNe3SzGnDQzWrAQTerAQfdrAY8woyVoi5fPOlW7tHNasBBN6sBB92sBhx0sxpw0M1qwEE3qwEH3awGHHSzGnDQzWrAQTerAQfdrAYcdLMacNDNasBBN6sBB92sBhx0sxpw0M1qwEE3qwEH3awGHHSzGnDQzWrAQTerAQfdrAYcdLMaqDzol1ySbrW3Zw988IOwdCkMDICU/l26ND2+Z0/VFVoX8//UUrWxMVi7FnbtgmPHYGLixLTxcdi/Hx56CG69FVasgC1bYHCwunqtK1Xeo9fa8DAsWwYjI/DssyeHPG9iIk0fGUnzDw93tk7reu7RqzI8DOvWwdGjxZ8zGfh169L9NWvaUZn1IPfoVRgbg/Xr5xbyvKNH0/MffrjcuqxnOehVWLs2nY/Px7FjcPXV5dRjPc9B77Tdu9OFt5nOx4uamICdO3013gppGnRJp0m6T9IDkr4p6bc6UVjP2rRp/r35pPHx1J5ZE0Uuxh0DLo2II5L6gLslfTUi7m1zbb1p69b59+aTJiZSe2ZNNA16RARwJLvbl92iSONFvghz553F5922rchSF7ixsXLbO3Cg3PasJxU6R5fUkLQLeBy4IyJGppnnGkmjkkYPHTpUdp29Y3y83PaOHy+3PetJSh12wZmlFwFfAX41IvbONN/Q0FCMjo4WanOyJ++J3rqIgYFyw97fX945v3U9STsiYmjq43O66h4RTwHbgMtKqqt+yv766pIl5bZnPanIVfdzs54cSYuAtwD7211Yz1q9GhqNctpqNFJ7Zk0U6dHPA7ZK2g18g3SOfnt7y+phGzemw/cy9Pen9syaKHLVfTewsgO11MPy5elXaCMj8/uYrdGAlSvTj1zMmvA346qwZcv8e/WBAbjttnLqsZ7noFdhcBA2b4ZFi1p7/qJF6fkXXVRuXdaz/DPVqkz+xHT9+ucPODGTRiP15Js3+yeqNifu0au0Zk36UcqqVamXnulqfKORpq9aBXv3OuQ2Z5X36LX5osxMBgdh+/YU+E2b0nfXDxxI33jr60ufk69ena6u+8KbtajyoFtm2TK48caqq7Ae5UN3W5g8Km6p3KPbwuJRcdvCPbotHB4Vt23co9vC4FFx28o9ulXPo+K2nYNu1fOouG3noFu1PCpuRzjoVi2PitsRDrpVy6PidoSDbtXyqLgd4aBbtTwqbkc46Fat/v5y2+vrK7e9HuGgW7U8Km5HOOhWLY+K2xEOulXLo+J2hINu1ZocFXe+vbpHxZ2Vg27V86i4beegW/U8Km7b+WeqtjB4VNy2co9uC4dHxW0bB90WlslRcUdGYMOGNEZcf38aM66/P93fsCFN377dh+sF+dDdFiaPilsq9+hmNdDTQb/kknQz65gFOky1D93NyrDAh6nu6R7drCO6YJhq9+hm89Elw1S7RzdrVRcNU+2gm7Wqi4apdtDNWtFlw1Q3Dbqkl0naKmmfpG9KuratFZl1gy4bprrIxbgfAR+JiPslnQnskHRHRDzY1srMFrIuG6a6adAj4iBwMPv7GUn7gAuASoNe5Iswd95ZfN5t2+ZRjNVPlw1TPadzdEmLgZXAyDTTrpE0Kmn00KFD5VRntlB12TDVhT9Hl3QG8CXgwxHx9NTpEXETcBPA0NBQlFbhDIr0wJM9uXtrK11/f7lhb/Mw1YV6dEl9pJB/NiK+3NaKzLpBlw1TXeSqu4CbgX0R8am2VmPWLbpsmOoiPfrFwFrgUkm7stsVba3KbKHrsmGqi1x1vxtQW6sw6zaTw1SPjMzvY7YODVPtb8aZtaqLhql20M1a1UXDVPtnqmbz0SXDVLtHN5uvLhimuqd7dH9RxjpmcpjqPXvSD1S2bk1faz1+PH0ZZsmS9BHaxo2V/P9wPR10s45boMNU+9DdrAYcdLMacNDNasBBN6sBB92sBhx0sxpw0M1qwEE3qwEH3awGHHSzGnDQzWrAQTerAQfdrAYcdLMacNDNasBBN6sBB92sBhx0sxpw0M1qwEE3qwEH3awGHHSzGnDQzWrAQTerAQfdrAYcdLMacNDNasBBN6sBB92sBhx0sxpoGnRJfybpcUl7O1GQmZWvSI9+C3BZm+swszZqGvSIuAs43IFazKxNSjtHl3SNpFFJo4cOHSqrWTMrQWlBj4ibImIoIobOPffcspo1sxL4qrtZDTjoZjVQ5OO1zwH3AK+S9B1J72t/WWZWplObzRARv9CJQsysfXzoblYDDrpZDTjoZjXgoJvVgINuVgMOulkNOOhmNeCgW+v27IEPfhCWLoWBAZDSv0uXpsf37Km6Qss0/cKM2fOMjcHatbBrFxw7BhMTJ6aNj8P+/fDQQ3DrrbBiBWzZAoOD1dVr7tFtjoaHYdkyGBmBZ589OeR5ExNp+shImn94uLN12knco1txw8Owbh0cPVr8OZOBX7cu3V+zph2VWRPu0a2YsTFYv35uIc87ejQ9/+GHy63LCnHQrZi1a9P5+HwcOwZXX11OPTYnDro1t3t3uvA20/l4URMTsHOnr8ZXwEG35jZtmn9vPml8PLVnHeWgW3Nbt86/N580MZHas45y0K25sbFy2ztwoNz2rCkH3ZobHy+3vePHy23PmnLQrbn+/nLb6+srtz1rykG35sr++uqSJeW2Z0056Nbc6tXQaJTTVqOR2rOOctCtuY0b06/SytDfn9qzjnLQrbnly9Ov0ObbqzcasHJl+pGLdZSDbsVs2TL/Xn1gAG67rZx6bE4cdCtmcBA2b4ZFi1p7/qJF6fkXXVRuXVaIf6ZqxU3+xHT9+ucPODGTRiP15Js3+yeqFXKPbnOzZk36UcqqVamXnum8vdFI01etgr17HfKKOeg2d4ODsH17Gj1mw4Y0Rlx/fxozrr8/3d+wIU3fvt2H6wuAD92tdcuWwY03Vl2FFeAe3awGHHSzGnDQzWrAQTerAUVE+Y1Kh4BHS2+4NecAT1RdRAu6se5urBl6q+5XRMS5U2dsS9AXEkmjETFUdR1z1Y11d2PNUI+6fehuVgMOulkN1CHoN1VdQIu6se5urBlqUHfPn6ObWT16dLPac9DNaqBngy7pzyQ9Lmlv1bUUJellkrZK2ifpm5KurbqmIiSdJuk+SQ9kdf9W1TXNhaSGpJ2Sbq+6lqIkPSJpj6Rdkkabzt+r5+iS3gQcAT4TEa+tup4iJJ0HnBcR90s6E9gBvCMiHqy4tFlJEnB6RByR1AfcDVwbEfdWXFohkq4DhoCzIuLKquspQtIjwFBEFPqiT8/26BFxF3C46jrmIiIORsT92d/PAPuAC6qtqrlIjmR3+7JbV/Qgki4E3gb8adW1tFPPBr3bSVoMrARGqq2kmOzwdxfwOHBHRHRF3cANwPXAj6suZI4C+JqkHZKuaTazg74ASToD+BLw4Yh4uup6ioiIiYhYAVwIvF7Sgj9dknQl8HhE7Ki6lhZcHBGvAy4HPpSdqs7IQV9gsnPcLwGfjYgvV13PXEXEU8A24LKKSyniYuDt2fnu54FLJXXFeNQR8Vj27+PAV4DXzza/g76AZBe1bgb2RcSnqq6nKEnnSnpR9vci4C3A/mqrai4iPhoRF0bEYuA9wNcj4uqKy2pK0unZxVoknQ68FZj106WeDbqkzwH3AK+S9B1J76u6pgIuBtaSepZd2e2Kqosq4Dxgq6TdwDdI5+hd81FVF3opcLekB4D7gL+MiL+e7Qk9+/GamZ3Qsz26mZ3goJvVgINuVgMOulkNOOhmNeCgm9WAg25WA/8fCoRaxko0TvsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df:\n",
      "      x    y  classification\n",
      "0  0.5  3.0               1\n",
      "1  1.0  2.0               1\n",
      "2  3.0  0.5              -1\n",
      "3  2.0  3.0              -1\n",
      "4  3.0  4.0               1\n",
      "5  3.5  2.5              -1\n",
      "6  3.6  4.7               1\n",
      "7  4.0  4.2               1\n",
      "8  4.5  2.0              -1\n",
      "9  4.7  4.5              -1 \n",
      "\n",
      "threshold: 1.5 feature: 0\n",
      "threshold: 4.25 feature: 0\n",
      "\n",
      "tree1 predictions on all elements: [1 1 0 0 0 0 0 0 0 0]\n",
      "tree2 predictions on all elements: [1 1 1 1 1 1 1 1 0 0]\n",
      "\n",
      "Entropy of different splits for observations 5-9\n",
      "Col 1, @ 3.35: 0.5509775004326937\n",
      "Col 0, # 4.25: 0.5509775004326937\n"
     ]
    }
   ],
   "source": [
    "#provided\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "def simple_tree():\n",
    "    return DecisionTreeClassifier(criterion = 'entropy', max_depth= 1)\n",
    "\n",
    "#example\n",
    "pts = [[.5, 3,1],[1,2,1],[3,.5,-1],[2,3,-1],[3,4,1],\n",
    " [3.5,2.5,-1],[3.6,4.7,1],[4,4.2,1],[4.5,2,-1],[4.7,4.5,-1]]\n",
    "\n",
    "df = pd.DataFrame(pts, columns = ['x','y','classification'])\n",
    "\n",
    "# Plotting by category\n",
    "\n",
    "b = df[df.classification ==1]\n",
    "r = df[df.classification ==-1]\n",
    "plt.figure(figsize = (4,4))\n",
    "plt.scatter(b.x, b.y, color = 'b', marker=\"+\", s = 400)\n",
    "plt.scatter(r.x, r.y, color = 'r', marker = \"o\", s = 400)\n",
    "plt.title(\"Categories Denoted by Color/Shape\")\n",
    "plt.show()\n",
    "\n",
    "\n",
    "print(\"df:\\n\",df, \"\\n\")\n",
    "\n",
    "### split out X and y\n",
    "X = df[['x','y']]\n",
    "\n",
    "# Change from -1 and 1 to 0 and 1\n",
    "y = np.array([1 if x == 1 else 0 for x in df['classification']])\n",
    "\n",
    "### Split data in half\n",
    "X1 = X.iloc[:len(X.index)//2, :]\n",
    "X2 = X.iloc[len(X.index)//2:, :]\n",
    "\n",
    "y1 = y[:len(y)//2]\n",
    "y2 = y[len(X)//2:]\n",
    "\n",
    "\n",
    "### Fit classifier to both sets of data, save to dictionary:\n",
    "\n",
    "tree_dict = {}\n",
    "\n",
    "tree1 = simple_tree()\n",
    "tree1.fit(X1,y1)\n",
    "print(\"threshold:\", tree1.tree_.threshold[0], \"feature:\", tree1.tree_.feature[0])\n",
    "\n",
    "### made up alpha, for example\n",
    "alpha1 = .6\n",
    "tree_dict[1] = (tree1, alpha1)\n",
    "\n",
    "tree2 = simple_tree()\n",
    "tree2.fit(X2,y2)\n",
    "print(\"threshold:\", tree2.tree_.threshold[0], \"feature:\" ,tree2.tree_.feature[0])\n",
    "\n",
    "### made up alpha, again.\n",
    "alpha2 = .35\n",
    "\n",
    "tree_dict[2] = (tree2, alpha2)\n",
    "\n",
    "### Create predictions using trees stored in dictionary\n",
    "print(\"\\ntree1 predictions on all elements:\", tree_dict[1][0].predict(X))\n",
    "print(\"tree2 predictions on all elements:\", tree_dict[2][0].predict(X))\n",
    "\n",
    "### Showing Ent\n",
    "print(\"\\nEntropy of different splits for observations 5-9\")\n",
    "print(\"Col 1, @ 3.35:\", ent_from_split(X2.iloc[:,1].values,3.35, y2))\n",
    "print(\"Col 0, # 4.25:\", ent_from_split(X2.iloc[:,0].values, 4.25, y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold: 4.25 feature: 0\n",
      "[1. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "alpha1 = .6\n",
    "tree_dict[1] = (tree1, alpha1)\n",
    "\n",
    "tree2 = simple_tree()\n",
    "tree2.fit(X2,y2)\n",
    "print(\"threshold:\", tree2.tree_.threshold[0], \"feature:\" ,tree2.tree_.feature[0])\n",
    "\n",
    "\n",
    "alpha2 = .35\n",
    "tree_dict[2] = (tree2, alpha2)\n",
    "\n",
    "print(predict(X, tree_dict))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}