Concrete_dropout_and_Variational_dropout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Define Concrete Dropout and Variational Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConcreteDropout(nn.Module):\n",
    "    def __init__(self, p_logit=-2.0, temp=0.01, eps=1e-8):\n",
    "        super(ConcreteDropout, self).__init__()\n",
    "        self.p_logit = nn.Parameter(torch.Tensor([p_logit]))\n",
    "        self.temp = temp\n",
    "        self.eps = eps\n",
    "        \n",
    "    @property\n",
    "    def p(self):\n",
    "        return torch.sigmoid(self.p_logit)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        if self.train():\n",
    "            unif_noise = torch.rand_like(x)\n",
    "            drop_prob = torch.log(self.p + self.eps) -\\\n",
    "            torch.log(1-self.p + self.eps)+\\\n",
    "            torch.log(unif_noise + self.eps)-\\\n",
    "            torch.log(1-unif_noise + self.eps)\n",
    "            drop_prob = torch.sigmoid(drop_prob/ self.temp)\n",
    "            random_tensor = 1. - drop_prob\n",
    "            retain_prob = 1. - self.p\n",
    "            x *= random_tensor\n",
    "            x /= retain_prob\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.1353, 0.0000, 1.1353, 1.1353, 1.1353, 1.1353, 1.1353, 1.1353, 1.0871,\n",
      "         1.1353]], grad_fn=<DivBackward0>)\n",
      "tensor([[1.1111, 1.1111, 1.1111, 1.1111, 0.0000, 1.1111, 1.1111, 1.1111, 1.1111,\n",
      "         1.1111]])\n"
     ]
    }
   ],
   "source": [
    "cdrop = ConcreteDropout()\n",
    "drop = nn.Dropout(p=0.1)\n",
    "input = torch.ones([1,10])\n",
    "output1 = cdrop(input)\n",
    "input = torch.ones([1,10])\n",
    "output2 = drop(input)\n",
    "print(output1)\n",
    "print(output2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VariationalDropout(nn.Module):\n",
    "    def __init__(self, log_alpha=-3.):\n",
    "        super(VariationalDropout, self).__init__()\n",
    "        self.max_log_alpha = 0.0\n",
    "        self.log_alpha = nn.Parameter(torch.Tensor([log_alpha]))\n",
    "        \n",
    "    @property\n",
    "    def alpha(self):\n",
    "        return torch.exp(self.log_alpha)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        if self.train():\n",
    "            normal_noise = torch.randn_like(x)\n",
    "            self.log_alpha.data = torch.clamp(self.log_alpha.data, max=self.max_log_alpha)\n",
    "            random_tensor = 1. + normal_noise * torch.sqrt(self.alpha)\n",
    "            x *= random_tensor\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Loading and normalizing CIFAR-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "train set size: 50000\n",
      "log freq: 50\n",
      "test set size: 10000\n"
     ]
    }
   ],
   "source": [
    "batch_size = 1000\n",
    "\n",
    "train_transform = transforms.Compose(\n",
    "    [transforms.RandomCrop(32, padding=4),\n",
    "     transforms.RandomHorizontalFlip(),\n",
    "     transforms.ToTensor(),\n",
    "     transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
    "\n",
    "test_transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=train_transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size,\n",
    "                                          shuffle=True, num_workers=4)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=test_transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=4)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "print('train set size: {}'.format(len(trainset)))\n",
    "log_freq = len(trainset)//batch_size\n",
    "print('log freq: {}'.format(log_freq))\n",
    "print('test set size: {}'.format(len(testset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imshow(img):\n",
    "    img = img / 2 + 0.5     # unnormalize\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1000, 3, 32, 32])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAB6CAYAAACvHqiXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnX18XVWV93/7Xq433ElIk8bUkBJTOoEaKC01pRSYQiloQbEIjBZ8qY/41PGFYWaYhweHQYoyKuowvqNVkaICasFSh4LUWqiFUhr6QktoG9rG2EzamDamiZfEy717/lhrn72SnLynSXNd388nn3Oyzz7n7H3OueesvdbaaxlrLRRFUZTxT2SsG6AoiqKMDPpCVxRFyRL0ha4oipIl6AtdURQlS9AXuqIoSpagL3RFUZQsQV/oiqIoWcKwXujGmIXGmD3GmNeMMbeNVKMURVGUwWOGOrHIGBMFsBfA5QAOAtgC4Hprbc3INU9RFEUZKCcNY9/zALxmrd0PAMaYRwAsAtDrCz2RSNgJEyYM45SKoih/fTQ2NjZba9/cX73hvNBLAfxB/H8QwJy+dpgwYQKWLl06jFMqiqL89XHXXXf9fiD1jrtR1Biz1BhTbYypTiaTx/t0iqIof7UM54XeAOA08f9kLuuCtXa5tbbKWluVSCSGcTpFURSlL4bzQt8CoMIYM8UY8yYAiwGsHplmKYqiKINlyDp0a+0bxphPA/g1gCiA+621rwz2OG9dtmxA9aKDPXAfuK9Yf8dMh9Q7nZc5vKwNqR9GKqQsE7LfR43pp1XEnXfe2eX/u+66q0edonl+vYqtGwWFvjfJojgAoGYvqcKmFvn6mx+lZcuLviw6g9sr6uWUcLuXTAEALJg/N9j21S88RMc46uu7QVqjGMstmM9lB2i57huiEy20mHqeL9q1+Q4AwJr69UHZtW/diN6Y+jlavnPhGUHZC4/tBQBcFb+zR/1lIc/kY/v+DAB4cfvOoKymhuz/6TTdyelnVQbbFl40CwCw6dmng7JINAYAKC0tCcqmTaN9qgrf1Gv7T0RMyHM6kGdS6Z/u13EwDMcoCmvtGgBrhnMMRVEUZWQY1gt9JOhLSk6HrLv6Q9HGD0S/lBHrrn4sZHtHyDHDjp8ZQNnxskxXeIEUST5pIt+XPf4ISeZVXG/Do6L+flrmFPuyjmCjKOP1GWdNAwCURUuDba2HaXnh/Dy/Q7QNANDc4os2baGluw4Ll/m7Gz9KJ9ixxde/9ZHPAwA+ufhdQdnq/dcCAN7zd9yJXF//yhsmUdvKy4Ky3/5iL3rjnqdfAAB85/vfD8rqa7h+RFzAKI3T8kupz1OnVQSbnnp2HQBgzarHg7L8oonUNHETmhp/BAD49Y+/DgAQgx+4scD0XluqKF3Rqf+KoihZgr7QFUVRsoQxV7nk97FNfm2cmiIWsq17Hbk9GlKGkG1hODVPXJSVd9tXnjMVUtb9WGHb+zKmDoeGP/r1BbNI3bB5X31Qds21dDWrV1PLnZpF0iFuUNXVtMyf4suqn6flr1eRgXLKB73KpYj3bWxsC8p2sRW5/jfiJPt4yRemwms6cPlV7wAATLt4R1AWyyVdzgMbnwjKKqZeRivO3ijUQlVlzXx8r2bZ/T1e+Wf04PEtZAkuOserUBo7WwEABbG3BGXHDlODExl6skoLvaooyp1557VXiXbTBTnS2BqU7TvQCADYcZiMrvn4m2Dbbf/5CwDAh+f7m1A1kyzTO+oDBRiSzTS/L5Ki+5if8PqmZBtdiP21/r4nuB3nzfHzAKdNI7XUU0+Ruime7/syo4qUPpmwB3sYrHqJwo7EYr3XiYgfrasXFfUj/EMMex/kuB9ufz903jkujpvDXg8JLpPvALcuFIkjSphRfqCohK4oipIljLmE7uQAKaW6D6X8sHZvaJg0LomGbOt+XPnVdccLGxXIeiXoSptYD3NN7H4sietzX/sNh/xT/fpPHiAJrWymL3vv+99D50+SIbH+sN9Wcg4tc6f6slK2zjU0+rJWnnnw2JMkMT7wCS/1nTudJOgjnX6HYhYeiy/2x4jxejtf/F3bvXi9bTu5/VUKA++57I75jPBUXPM0ifw3/TPdoc4D/pzXxehKVye9lDrd21N7sOkJdjU8XC9K6Q421W0WReSPmVhI4SxiKX8nq3dup3bEvKTbnKShyIvPVgdlHTxi2s0ukM89+pRvx1YeFbR70XHzOhoSdeZ6A2+qlXxAGw/QsWLiKW5vbadzt3grdAW7SjYK39E7tlCb6uvpGNNnnR1s2/DsZABAS4sfWYwErXw4KaFHuv2Yo+Il4OpFRP1YpPd6YfPSux9f7is3OQl9vKESuqIoSpagL3RFUZQsYcxVLs7NWapc2rstAeAUXjo/XemH7kZgUq3hjtcZUu8gLzvEtmkDaWwIUgUz2JmijhG2NQXsfFX8s5UW6cm+KNlE+o8rr1kCAKh7ckWwLc1zfqOig3WkRcAOXy24wNNYlZOHqmBT1ZwLAAC/Xr/Kn7OFrlKO0FVtcpMpnS1PxvBkWvwEVMzgU8wSDtqXzCFL7fV5C7nkvh7HmCcemuceIsPgfy7reS5sYQPsG0IHlUtGQ2QO9qgeYWf8fTt3B2UP3/4FWpkqdFZpfgoamn1ZJ603NJKqozjP6xOiKZK39v+PUOXsID3T1BletVWQR9f0GDv3R8QDlZtLF7U96S/4kaNNAIBEnlcmbljronZQe5Ipr15paaY+tLaPrMrFXY4uUmUf07idukSqXFw9WdbMM5Ob+TLLEFIFhbQsEg7/8XjPevl8PPdISrXrSM5aH2lUQlcURckSxlxCL+ClNGBs46WQY9DQTFLIgiL6PorQHoHk3RxStluUOTPZob3kmnX6GecH2/oM5N4HJ3L8yFJhAG1gm1jdY77shflk4Dv3stkAgLlzRG/a6Y6sF1GYa9keGBXVTmEJ/jALrh1i3POd5WRs3STcEOFsecKw6uK1BBbmEEt2h7iRzSxAT6/wYlbN0xwI5tomLpklDuKeLi+65uEi9EqUx1pvCFms/Wh4XQBl7J/ZUC+MqBl+Gmube+5wshgmxUlkjHKfYykvSbc1HwIA7IucHJS11pHxMiVCUVdOJwk6nSQJurXdb0smSbZsa/NG0XiOszDLcWPXdjbVHQjWp5SRK2rz4UaMJCk3pBX3u4dzQj8ip7tusl4Hd6s9+Reu5OPkFHG9XDGTOJ/9D6ULdV/u1CcyKqEriqJkCfpCVxRFyRLGXOXi5v890/hyUNaw5WEAQKrdB6dtZqNO7kX30rZpZwXbnNrGz1H0YW237XvIl23+CQAg3Ug6g4651wTbWi+4AcD4HWqF0fBQz7KpvsvYvZX0GNcvIafsWtH5dh77zrvUO+QWZkidcjrPKgSAxmYyHJax8ilHmIzy2ShbLo7rZtcl/SFQx7uUs395k9AEXHk+VTw911u9NtWTficnKWZcPkvLmotoxmrlJGmidvXk3D4XevfD6EHn4FQL511MKqvnnn++z3o559Fs1s9+9t+Csom5ZO5/18VvBwDc87QIXlpHlulWCBUNq7QamrYGJQ3rtqJ/ZN9JHZVKeZeBqnk0J6Gxkfo+ucRHZYtF6WEoFpbExpBZxYMl020JABm+bc6/PCa0Xs5fvMvsb/4nI56ZfH7e8vJ6hiQ+k70f8sRBnPblRFafDhSV0BVFUbKEfiV0Y8z9AN4NoMlaezaXFQL4GSi0SR2A91lrW3o7Rl9s2P4fAICVq/49KCtgYaxxn69XWEiFDYmvAQAySe9oePqsmwAA5fBf5OZmkvgbtvtsCUfrSfroPEyWueYnvAi7MnEmAODGmW8fVPulqewZXsoL4dydPjioo44QwvBz3WJaVosYKpvYQLruMnKFO3f27GDb2o0k8iZbvaS7+We8hHfnK+GcDiu+xYFeWr318smP0KgHHeKKJOmKbdjoZ1wuoQiyiNXR8pIrfPW5FSQpNjd5qdnNEH2m3otlpXyh1zxBx6/86NX+IIFpXBpK12OkmDGbZlWu+c3Tfda7/1v/CQC4fvY5vdZJJeXcYzaypguH1T4AQMTfxzT7Cyb/7B2D37/4OgBAPE4jstrdPldN9WaKbZOfN7IyLOcF6eJm6UTMsPAuzjUxLpvB9aU9uq6eDLqun9GodzpMpd4KAJgrHoVskMwdA5HQHwCwsFvZbQDWWWsrAKzj/xVFUZQxpF8J3Vq7wRhT3q14EYBLeH0FSDj9/0NpwKEdDwIAWkS+gTTrV0UYDOx+nr62sSRJ1TlpH9xj12F2sZriZ5rEOig2RqrFS3GZJH3i21MkMaYPeFetnU99FwDQcNbyoKw01n86uC9vvD1Yb20jSaa9xUsErc10zuTVtwRlV5WRy1yYa+WIIvSch1jNet3f+ZlCP+WgLN/8MknLL2+5O9j23CukD96/1V8/p3aU0tMHeBRQ5gJovCICrJTxxlYxRSxNutx5V1wWFB2YTdJjY5zErJLZXnxqa6V79b0NPvXbRez6eIowmhxhNfnmIDxJuWilO79MHyH10sOjrIxdD1PtPbbNuHxxsP6+EMncyc3O0XDbAenAS89nLM/rs0sr6BlPCftBin8oebl0beMiOEqUQw4WF/v7XjqZLlxunterp9KkT4+xorp0ko8Vk38B/xBj/lnYsG746YOdvlw+UDlOCnfRDsU7wAWRlLGEnnrqSVo++augrKW+jlbcqCTPj3AK8qnv06f7d0XVrHNpOcffnwt5sOqvwvhgqDr0SdbawK0bwKS+KiuKoijHn2EbRa21FoDtbbsxZqkxptoYU51MhsU/UxRFUUaCobotHjbGlFhrG40xJQCaeqtorV0OYDkAnHrqqT1e/G1tJOjninCVzpuqULiqIUnDvYP/Qx+Fmt1+JttJGTpGqmFtUDatshwAUJDyBz6tler9ei81t7rWG6CmnUFxR+rra4Ky5gglOIhM8kPNkgQlIKhpJKPhho33+jZGyKWstMi7VMbTNMQ7st0bzNY2UzticZ5h2OoNUMeLjRyV9ZyYH69+6/b5AIBPf5EMhI1t/oM77x20bdePfbs/cjot33+xz3BxOc9SDHQG0u8zeYyXYhZimq95h7CEldLAtmQyz9ft8D5leTzt718v8gbbiZspqEx+3N/buk467i2P0P+FL30t2OZmAqYOe4uwU4780wVXYuiQCieRoKH95RdfGGw52kTt+cZ9PqZMXRs9/q3N/h78/CFypW1oJNXS5pqeLpPzKiqD9QsvJmNvyRle3xRjXUScffgKRYKLiYXcRvFTam2l57+zw9/vTp6heqCO9HR1u73L8NpfkjojhZGdKRqGc0N0qpaoEDnXrXsNAPDgj78blDVs51DEncKY7ILEuD53elVYSxvV23DY928DxxpyiT8AoJjdNqvOIfXfovnejPjBd00YRI9Gl6FK6KsBLOH1JQAe76OuoiiKMgoMxG3xYZABtMgYcxDAnQC+BODnxpgbAfwewPuG2oBUhow2kYT/whYVkuRV/5qIh8h2Rmc02bTRG4Um5tN6Yb7/PsVZCmrZ7r/OHc/Sl/uw+5rX+cNXbyaDXN184WLXTJbESL63LuZNJzG1/TBJne3im5jPWeBbWrxE2s7p1yrLvJtlsoNGF42HqQF76jfheHDThSK5Qh1JY2I+CuoKSTJfxDFf/tDgwxxeOJ2k5bVRL6FXsl2ttFCIeyVc6CTu+Bt+W5LvX74wRka4TRkRuY8jDmIvO4HKUHjsaopW/yz8n3I2aIlmdHTS+W/bTdJ7ix9ooU9/2gt6Fn3mmw9QEzv9yKytg+7jMTHrqbKCDKutHL1w0UffH2ybMZ9S59Ue8Cbv6q1HuNnehbByJl38uRfT9LiffPsHPdqz6L0LgvVzq2h0tKfOS5gN++oAAHX7yM/36GE/YC4pIkmztdmXvbiZDNf7anyija6xR8cel0tj5aO/CMp+9QsO89khjM/uhSADDAUjj5AYpzxJKiqsrfEgm4UfNf6hnkcqtXSdf/lzHwTp1iK6V/Mu9g/P9YsXAQAWzRlb6X0gXi7X97JpQS/liqIoyhigM0UVRVGyhDGP5XKsmX14o15NUfsqDf865GQ+Gq2i0Y1ghU2tiZOz79znjTatSRqWJQ/6zO2b9tCweSpHI/3e2T6x5fSDNASbu847xDcVU9sOCiPWPnbQPJyiNqZafUMaO6isrcUPX3N4dL1h4wZfxkkMisppSLhj33YcD2p2eKMXR3hFfoHf7vz8cznK6Ffv+HGw7blV3wQA3Pz33sCbriG1VGW5UIk4V2ZndZNBNVxChJS4WYGjsWhI9xQfMTEzss6pbURZEfuQC0NYDufHzOdDhQStHTCFk+hiFeR7I1lRnAyNHZ1eBsph02qMk2N2Cn/xGN/jeMbrhWZNIX/nXDHcT7MHQFMw1bFnyzes9z7Wzz1L9/RnP/qRqDGyiSdGC2cAlTlFW1jr9sMVNKt2x2afYxUuZ2ssZB5pREZ4cQFe3H4irg//RtPiGC6iTUQkJo3n0D2K5tD9jkT83JIOrveUmO381LO0nhD3du7sGQCAf1hMhuzL58h4OscHldAVRVGyhDGX0Df+lCViOYnPfVnD3NaPhZTxJ7ZWzDatfY2kt4Kk9Kgk89iFr9N/SxvL/aYjHDhml58tWYwKXnrW3UvGzeQcanAk6aXxOk7yUOm9+pBO0lf5hc1+pmPFWXTcRAkZY44eJwFLXr5W/nRPFCnlGrjpu7jrW1/027Y9S+2tyPFS6ra9dE07inzgDCdww6UzSwj/U+c+1i6k9ibubIc0WLnU7YmeDS9habxY+EOmWIxLigSDTRRfZkgBhbrxxGqSiKtmnhuUxdhtdtNL24KyqWz/mjvtLQCAsmuWBNtOKyN3zrKEfxicgHnLp28NylYudy54MoZLV1au+MIgezA+YCEYR47+OSj75re+AgCo38oj2i6BVvqQP6V/o5O+MyH57DL8ckn55y/Nue3SmW4jRQDRmJPM5TY6XizizxnhsJBpMUJd/yz1YePT5E49qWhisG3GLDKGnz/Hp+qZP/9tIR0bHCqhK4qiZAn6QlcURckSxlzlEgSQGkbA/HyOoJMrM9rzp6pluc/SPu9MsmhW7OHwr0e8GmQdyEFbRG5FPocvvT/iU85XTaUsDNVNNJzKiCHh6dyO3UL1k2I1Bfb4sp1JsuxGOAROkwhpO5JERcL5drZBTr5U6Fx4aHrhFGpH4hxvkNu6gXyUa3/jHdddoKIcmWLdWVmTrOxICX2Jq9YoDH1H2Yd4pp/9CA4qBWdATAsjVjPfvwYfshdFbCCVWptSCtb2kdO2AAB++AcMmWd+cj8v/VC9gCOBtRzz6qZneFnHz9p/XO2DJFew6kkGvr3ne+RHvXL5V4beuCwimaT7/PAjfuZn/Uaeo8i+3oiKZ03aPfuCA5IFapV0T1VKF1jVEsvx6sKYU9sExlYv+77BxxUzLhCPx7vuByCWR2pCp0ls7vDP9RP8u3p4lQ9m5zSUd35MpuoZHCqhK4qiZAljLqFHz6QvWrpBursN7hjlcTpGfsJ/wjtZfDs6138Vd2whKc/LqP6L7EyhXfKgV5A0vn/qO4Ky6TOvpT3X11HBS/6LXPQ29mmc5NNeNBwkiTXpw3ygcjLJbXku8MlhIX2OIB1CGK+8iM7VGPPDmB1baKSwYTVLna/7+o+BZrN+xXt9Yq5zuyop94VFfBJn2O0UZkl27ewSPvcV7muj6PMcnqo6lWfTyll/LgdZm5Dy3fGOelcy5FO/vvGvHwMA/PDmnjMuJbHiPjcz/tk5doz6VVY5MyirjNC9LW2kUUT7gSB2L47wiKGhybd72a1DijCNrunjejeejiz+HkRPovVEnn/W21qGH9dl0/M0Q3rreuGaOMnFoWGpWkrX0QEaRR0uc0a656au9WjfVLO/V6km5x/N3hqn+xGld030bUuxa2464w31b/BIM80jz4xoR4RHHvFc7w2SyYS4Yw4SldAVRVGyBH2hK4qiZAljrnJxhoB/+fqHgrIGDsj02Ipng7JUI1X85C2U7WfuBT5jkYsW2iLCtMbZf3ptzCcmfXATZeE5OygpD9auBm17QbTtkloyXNwcvygoS36eZlPeXEpDsJvPf8DvUERD8PqUt4qm3sHhSyu8/qPgLF6PUBtf/IQPtHTx2z+AkaJeTECtfsrpRLb0Wr/Ku8liF8/M3eibhjMraLgfr6sLyspKeMjo1CDtwqneGTdFEsgUq8U2rfLjzwIOhzt9FocRTgrFl9NenSPUMG00vN2xVQSU6iC1Ufl7yXR7Y5Xf9BjHoCoUapZ50vrdA3cub+BNs6qjvsaHbU6zb3xzPt3PBe1etVTWzmGeX/HBuTpeH6pqbbTULEDiFOqLnM2aYGNhRhirR0LlUvMK32+p53Sxjt38jr7ULANGqG1cF4QfeuC33mVuhLtX/Dy3+NxFyZRT7frrwXH5EIv5V2okMKiSGqZrT+hc8RzfNm9QzcVQUQldURQlSxhzCf36v/sEAGDZR+8MyvLY3y25zEsBP1pFKed/+QgZUOZd9pZg2zUX/z0AoE3ME0yxNLFrp4+DgQhJ4de4j+JE7x50c5KOcfPr3rAVsMuHGU3s4hCecCOEM329M2lWYFmJyERYwpLOZWKm40x3XvqCz5t1Tc9zjgBNh3qW5Qk74tnczMunUhuLRdKJHI5XHBEW6s4oXdPmNi8xljXwCCjCxqA2Oe2VJRgRIyP2NpIA503xskSSDUm1e0kqSgvhr4wvW7zDOwDe8RAZIb8oJoo6h8EP/47C/WaEUHvVJbTMCNti/C/og76MU75/DTzrtYEnIze3CpfNNB3jt0+LHKsnDCRO5pzin8ncPE6SwUlDokL6zGE3wFhEzAIehluoI+Ni/OQJiTTD13CgLophdLJ0H7gtim3uuGHZ0+Q5iznOU4KvUUSMEIPYMP73kuZzRcTs0Rgb9GWZw8eNETmPg4aqhK4oivJXz0ASXJwG4EFQImgLYLm19uvGmEIAPwMpousAvM9aO+hQGpVzKCLZMfEVzYuSKJUQ7lqfuvoOAMDUKkoJteyzPsVY7WESNf998ceDsrro7wEAK3/upWunrgockI7I2UxOWSs/5+4rLqVOJxY2d6sDYA+LxHukTxx/M9cJCX0nS7W3k7RaK2KjjCRVQvBP8KWsEIOHfBYO9v+W+rBNxHL5CHls4ppb/iEo27CKRjs1u317Z53hdOg8sikQfXe5BPNE3/ey3rTES3uJiyndXcU+vqbPe70zcuj4R1M+OuPvOv1kMcclZ5ME1RGlvjQKKb+RJeiMCPB4NgeRDM9uPrTgOv/4f/8lWC8u/QYAoKlha2/VjyN+hFFcQM9YosDfgwwLhxERoTDKumqvx/WSYzzGUntk+G51ktOmlAMAaut82zo6+JkJRE0Zh4V/m2GujDICY/eJRGKEGBqx0dkGpCSdT1JyHqelk4dMHnWjCF8/GnPrftiYDnaibSeJc0ZD4se0NLvXpwxsNTgGIqG/AeAWa20lgPMBfMoYUwngNgDrrLUVANbx/4qiKMoY0e8L3VrbaK3dyuttAF4FUApgEQCnUF4B4Orj1UhFURSlfwZlFDXGlAM4F8BmAJOstW5gewi9jV774eHfPQgA2LB3bVBWMfM0AMB3bl/jKzaQlWvOkssAALd+7qZg0wM/oGD/8z/5YFC2n2Oo1D/n47X0ZJVY52HfybN8UYLdC49ITZJzWwtzQeNQqad4N8dg5CotlF+jeBX1H78KALC7qEuM0BGjRsSUcZMwa171Za3ctnKO+TJJGA2fO0qGpTm1z4kj0vAwJcSA6u3k11hRSIaz/DZvRE3xELl2r1eRJNlYeeYZ3uUwD+S22MYZ7w+97I8fKaFG7m/w7oKfZJvyZ8XItLqRhsF376L/5RUtcTo2oXJpPYrjgD9oU8NxOUEoMTbYFRSQ4ThfJOaI8jA/LtwQ29lomJvrb7ib/ehULjJfhItdkh5hk1vVLPqtHTrsf6M1u+n56XBGyy5pQaNdFl2QRk4XyyXQggh9SUdI7tS4C/0sDZ+0T5vLD5zuOd00GhNG4gg1NBNi5IyyEfqNEFfJlqPi3bLf+RmL6dmDZMB3yBiTC+BRAP9kre0Sldxaa0H69bD9lhpjqo0x1ckwy7KiKIoyIgxIQjfGxEAv859aa13668PGmBJrbaMxpgRAU9i+1trlAJYDwKmnntrjpV+zigxgpVf76T7f+UhY0H+y5qX+yGLWUR9K8F8+RvXvuO3zQVn9um8PoGfiq1vJOa/L/dcxvZsMfdEzRMy8TW5QIsTfAJLKNh57KChxTpDvFxFkUEoSVGYa9WkqhC/hCJLc5dd38nqBkFLz+RKuYRvnB9/lJbZWvvabt/vZScUJEolzEt64U7OXpI4N++hj3XrEf7Sd2TjMUn6JsBXOmk3X1HlAxsV3v24bSVRynDWNJ0D9aocvezjkHI5OFu4TMpYGC0uz+9jvRMHFUgGAgkJ+FoXE6CYBOSm7sNBL6IGHnTDI5RfRfcwV7oKdbIyMsKFPBjkMhNou0vLwKSujQf273/33QVmSU0furnF3XJw00mPFJ6yQ9trAeOuSWQipPEh+IaR2ty7LRORF2q+n7JuW749MTwk+aHrGuS/Gem6U+50+E8OlXwndGGMA/BDAq9bae8Wm1QBcipYlAB4fdmsURVGUITMQCf1CAB8CsNMY48S1fwPwJQA/N8bcCOD3AN53fJqoKIqiDIR+X+jW2o0ATC+bFwy7Ba+TumHtw/2pSEgvsHXNfQCAL+ZIww8NNTdvFIbHk+YAAPLL/DCqdX/3OCZCleKsQGv8IOSXrAq5br8MXyqz1QPSQnM3+y/fEdL6PfCGwc/yTLBImo6VmxmSPXlItAj9R0t1120/+ZFXcc3ibsaEnWjBDFKiJJO+z87O8yuO/SInp7qBbrkocyFW2sXo825OHzmdVSnSTuSGfV3m7x5Bv8g7dhpru2Rq0/bjlMd1uBQUe0tvDs/alP7O+WzIzM3xN8apXNzoXRpFnQqlVcQpibHRMCosn84YmuIbGhUqhk7Ok9nePrI2MOdyPrXc512dez6Fqm7kpCitMmZMOkTnE/iOhykbuEyqT9wD2ymmGYcYPEPID8LkAAAPiUlEQVTLBkngh55xs0j9tiTHK4rFRVKNnOH7+etMUUVRlCxhzGO5yCQTjksW3gIAmPeuq4Kyx58mW+yOX9EMvOrHPiv2cBKJ/MKR9NG6v+dXfR0vFwg3M+yiOC94i3c5vM4Zl6Sn0/5HuxyrVRhGEvd8HwDwxq0fC8q++K1HAAA1N13vd7r16wCAsug7ZFNHnHIxmHCx8+tDTNdu8qicr7q1pesSAGpZTK4o8H1euIBGF7PqyY3zXiE9L3LHF08ZZ2aDjOXvbLePD0Dy7g8n307zoX7QydbZmLB5xcbQ4SohpLLSEho+5OTSqE3KhRk20qWEdJ3Ps25lNMQI75RiCbYj6R/YTLAUo6ooS+FCQneSvLPVJTu8BNvJ5z/aOrLDmoaD9PvrTPnj7n6VnCRa3VAyHWK8DImN0ieyvjOKSo87d9y+pHJpMB2w9O6SXvCsUNGMaEgUyc4wl8pBohK6oihKlqAvdEVRlCxhzFUuX7iX8iyu2egtdJ+69asAgCKR/PqqGyic5aVnUb22Q8+Lo9AwqrRyTlAyuZSGspu3bvbV2Ed6G+tQulp0WeFwSJjfDvXvH75YrD8pVC2Of/801Xgqf5ov/NDw/U0HwjQxk3IDJ6qoEGqYTh45NvM0sTnlftvmup7Hc6qqnUINU1RDqpawSQjuEPtFevSJPJG0SKQZHYk5lW72gFNmHBXWWaewKBdRSWN9xD9KnMwJHTJSXUcXKxJiSHTLeMw/L3mcqEH6eruZnHK47SbbRYKWe12Um1nYFvPG6lM4nKsMb9vOvtuRKIe5FTlZU6yySCR829LchY5OP8SX2eoBICNmNbpZmyOvGaTrUFvrs6g0NJARNMazN1NydmXQXiGHpkIMpdEg+ljv28ICdmUG6Wgv1TChaiCnC2P1VcrfszRf73Rs+IZQiUroiqIoWcKYS+g7dtDX+ZzpXl5+Jwvau0S4lCr2MDxtajkAoKaLhE5Wr4aaJ4KShprez/n/ePlvosx/m8PC5/bBKdP73FzHBrkrPu3P9g18CQBw04fO6f/4w0C65pWygFYhJFMOnRL0UkrlxWxUDEuSIaXxu/u4zjt639TFfN013QdwoZCk17Ak35/s5Jr5lpBtzoXxiBgVJPs4oDNUdkkxxskKpMSd6OYumBYSXoYlYyn5ZkJmE7p4KjH2D80IKa6j04VdlXvQ9pRw4TuJ3RBz4nSsnIR32ox2Ur2kaFt7O0n8XcLn8no6k+6yBPyIIixRw3CoPGsCAKC07B1B2aJraD2eQ5PK9+z2T9Hy7y8HADTt82klA0lbGiq7Gy3DjKJxMfp2Loxhts5MSFkYLllHWDYNt0kYmgOjrJTQY8MfA6mEriiKkiXoC11RFCVLGHOVy89WkPqh7OwbgrJvL7sWADA3ZAJlTW1f4XAHx4jEGjrm2/Pth0gNtHDhBUHZB264lev5YF633f45AMCNN6wEACSOkx96sxj9xTiRUINQY0kVRHfCVC0jibz2dbx0prxft4fX6wtnNnQPdL+pszp735TDszATuX447FQtUk3Rcz8RFCvMWBeyLS/hrNR0XLlXBxvTEsLnPNhXNCORRyqWaITUCB1CvdKZdqFv/cMQhMgVfXFqGIeMI+Xa0VefhoKbhTxFqAEn8nNaWEiT0+fM8Q4EBUWUd/hzy7z6svUgz8CWt8V1NVC9SLmVVS1R0ZfAaCmzI3VT5UjVSzrasyz4J+T5CGaMijKnfpH+8PHhh9FWCV1RFCVLGHMJ3VG/a12wvvyRlwAASxe/PSi7577VtNI0UAmdxPuEyHFZVkZzIk/jZbsIGtLJcSpOEfFlCwtpvajIS17Fk+i4xcV03JT40jfUkwFn5UPe9XHRFWThXfOLL/vjihSbx5N6IWUlWSAoFoKJ62lX2WxoOKe74cx1G4nJm4NOatsH6bCQqILueTjTYlajW5cuiq5ehzCOHW1t6bJNylgJDlMck7FsnegsXCR9eBJ3c8Ws0IxLvOD74nJbSsk/zoZVF0ZXJmrI4XNlek7qHhZuICFtmJ08OmsOsVMumE+/vVTqM0HZ5+6g0W6yWSaccf0PyUHqxGQ5BAmm08r4yt3vhziGC8fb34il++PTxajM6xnxi3l9+L8AldAVRVGyhDGX0Ff89LcAgBlVVUHZM+vWAwC+/q39QVlTPWUpuOmWuwEA113tU5jOu+is497O8UiyH5fDkWT4USg8OX5QhY7j1eABIGOddHbwBKConFhEkms83lN09cnle04cycnx0nX37VJP7WK5dAqdeCJBuvyUmNSSZMk8EDClDj04npjUku7pi+fcLN35MyHWi5F2Wyx1ET2l2rlbc+Um94xdteBvfWGSJiYuu+NOX6/ZBe9x11mIym7EJGOpuH6l+jCsSOndXb+Q64iQGC2hMWjcfX9Djik1louiKIrC6AtdURQlS+hX5WKMyQGwAeTvcxKAldbaO40xUwA8AmAigJcAfMha+5fBNuDDN8zvUTbjjPcM9jBKFjGWahbAx1dBRKpG3ODfy0ARNoi3tyW71QFSPOPz9WTPYbT0fIxx5vh0SPKG9jbat63d+3G643Z2sd854yYN47tGeo3yeXy7XYwaqXrxEWQ58YLIGxskv+js20g8WOr20etCqrHycml9IrsySgcCToWKYnH9Km8gdWtZ8Q+Cso9//BMAgLZGngrdRb3CHRUGYfBsXfSVxF4aQJ36JcxoLs/l2hl4NIqGH3MPuUjggT4CDA2QgUjonQAutdbOADATwEJjzPkA7gHwX9bavwU5F9w47NYoiqIoQ2YgKegsACcixPjPArgUgJsNtALAMgD3jXwTFWV02Vd3YKyb8FfBkitdFJ+eiWkQp23lld7hYe4F5wEA5s2fF5RVzX4bLWdNCMrOn30uAGDtapZ+pdXVJQuRkRWdkbpTJr1w21nmlcK4i9vSIY2iXD/UldH1ScSPyWVpPD3XlyWG7xc6IB26MSbKCaKbAKwFsA/An6y1LjDqQfgYS933XWqMqTbGVCf7GtIoiqIow2JAL3RrbdpaOxPAZADnAZjWzy5y3+XW2iprbZWcyKAoiqKMLIPyQ7fW/skYsx7AXAATjDEnsZQ+Gd0SsyuKovRNc++bOilGS902n6Cmbtv9AICHvz3Q47NFNTbLFzntjsjfGU1QbO503M8Sj6VI+IzGTqbqGaG2aXeaBpljlcvekEZwp35hVUqniAvtgjpP9OdMlIQqOQZFvxK6MebNxpgJvH4ygMsBvApgPYDruNoSAI8PuzWKoijKkBmIhF4CYIUxJgr6APzcWvvfxpgaAI8YY+4GsA3AD49jOxVFyTZOnk1LMVs2wb6JYTNsk+2UrDAjXDzjnMwjKgyrSU5VF2eX0PwC7/vYuGMrrWS84Tt9zIV19W6DKXYrTAURhg6KlkjJfJgc8atJt37tnaFVB8JAvFxeBnBuSPl+kD5dURRFOQHQmaKKoihZwpgH51IU5a+TyvlXAuiqXsnJITVJ3AU+yxHqGK6XL7zlErmJHvUchxrJT2PNk7/yhZlXQlpyoNty/KISuqIoSpZgaCLo6HDqqafapUuXjtr5FEVRsoG77rrrJWttVX/1VEJXFEXJEvSFriiKkiXoC11RFCVL0Be6oihKljCqRlFjzB8B/Bl9BnEYFxRhfPdhvLcfGP99GO/tB8Z/H8ZT+99qrX1zf5VG9YUOAMaY6oFYa09kxnsfxnv7gfHfh/HefmD892G8tz8MVbkoiqJkCfpCVxRFyRLG4oW+fAzOOdKM9z6M9/YD478P4739wPjvw3hvfw9GXYeuKIqiHB9U5aIoipIljOoL3Riz0BizxxjzmjHmttE891AwxpxmjFlvjKkxxrxijLmZywuNMWuNMbW8LOjvWGMJJ/neZoz5b/5/ijFmM9+Hnxlj3jTWbewLY8wEY8xKY8xuY8yrxpi54/Ae/DM/Q7uMMQ8bY3JO5PtgjLnfGNNkjNklykKvuSG+wf142Rgzq/cjjx699OEr/By9bIz5pcvGxts+w33YY4x559i0eniM2gudMx59G8AVACoBXG+MqRyt8w+RNwDcYq2tBHA+gE9xm28DsM5aWwFgHf9/InMzKG2g4x4A/2Wt/VsALQBuHJNWDZyvA3jKWjsNwAxQX8bNPTDGlAL4RwBV1tqzAUQBLMaJfR8eALCwW1lv1/wKABX8txTAfaPUxv54AD37sBbA2dbacwDsBfAZAODf9WIAZ/E+3+F31rhiNCX08wC8Zq3db639C4BHACwaxfMPGmtto7V2K6+3gV4kpaB2r+BqKwBcPTYt7B9jzGQA7wLwA/7fALgUwEqucqK3Px/APHCKQ2vtX6y1f8I4ugfMSQBONsacBCABoBEn8H2w1m4AcLRbcW/XfBGABy3xAiiBfMnotLR3wvpgrX2aE9sDwAugBPcA9eERa22ntfYAgNcwDjOyjeYLvRTAH8T/B7lsXGCMKQel4tsMYJK1tpE3HQIwqZfdTgS+BuBWABn+fyKAP4mH+kS/D1MA/BHAj1ht9ANjzN9gHN0Da20DgK8CqAe9yFsBvITxdR+A3q/5eP1tfxTAk7w+XvvQBTWKDgBjTC6ARwH8k7X2mNxmyU3ohHQVMsa8G0CTtfalsW7LMDgJwCwA91lrzwWFjuiiXjmR7wEAsK55EejjdCqAv0FPVcC44kS/5v1hjLkdpFL96Vi3ZSQZzRd6A4DTxP+TueyExhgTA73Mf2qtfYyLD7shJS+bxqp9/XAhgPcYY+pAKq5LQfroCTz0B078+3AQwEFr7Wb+fyXoBT9e7gEAXAbggLX2j9baFIDHQPdmPN0HoPdrPq5+28aYjwB4N4APWO+3Pa760Buj+ULfAqCCLftvAhkgVo/i+QcN65t/COBVa+29YtNqAEt4fQmAx0e7bQPBWvsZa+1ka2056Hr/1lr7AQDrAVzH1U7Y9gOAtfYQgD8YY87kogUAajBO7gFTD+B8Y0yCnynXh3FzH5jervlqAB9mb5fzAbQK1cwJhTFmIUgF+R5rbVJsWg1gsTEmboyZAjLwvjgWbRwW1tpR+wNwJciyvA/A7aN57iG29yLQsPJlANv570qQHnodgFoAvwFQONZtHUBfLgHw37x+OuhhfQ3ALwDEx7p9/bR9JoBqvg+rABSMt3sA4C4AuwHsAvBjAPET+T4AeBik70+BRkk39nbNARiQB9s+ADtB3jwnah9eA+nK3e/5u6L+7dyHPQCuGOv2D+VPZ4oqiqJkCWoUVRRFyRL0ha4oipIl6AtdURQlS9AXuqIoSpagL3RFUZQsQV/oiqIoWYK+0BVFUbIEfaEriqJkCf8L2p76GbMsk58AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "truck  frog truck  ship\n"
     ]
    }
   ],
   "source": [
    "# get some random training images\n",
    "dataiter = iter(trainloader)\n",
    "images, labels = dataiter.next()\n",
    "print(images.shape)\n",
    "\n",
    "# show images\n",
    "imshow(torchvision.utils.make_grid(images[:4]))\n",
    "# print labels\n",
    "print(' '.join('%5s' % classes[labels[j]] for j in range(4)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Define Lenet with/without dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lenet with Concrete dropout\n",
    "class Net_CDO(nn.Module):\n",
    "    def __init__(self, weight_reg_coef=5e-4, dropout_reg_coef=1e-2):\n",
    "        super(Net_CDO, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2)\n",
    "        self.cdropout1 = ConcreteDropout()\n",
    "        self.cdropout2 = ConcreteDropout()\n",
    "        self.cdropout3 = ConcreteDropout()\n",
    "        self.cdropout4 = ConcreteDropout()\n",
    "        self.weight_reg_coef = weight_reg_coef\n",
    "        self.dropout_reg_coef = dropout_reg_coef\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.leaky_relu(self.cdropout1(self.conv1(x))))\n",
    "        x = self.pool(self.leaky_relu(self.cdropout2(self.conv2(x))))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = self.leaky_relu(self.cdropout3(self.fc1(x)))\n",
    "        x = self.leaky_relu(self.cdropout4(self.fc2(x)))\n",
    "        x = F.softmax(self.fc3(x),dim=1)\n",
    "        return x\n",
    "    \n",
    "    def entropy(self,cdropout):\n",
    "        return -cdropout.p * torch.log(cdropout.p+1e-8) \\\n",
    "    -(1-cdropout.p) * torch.log(1-cdropout.p+1e-8)\n",
    "    \n",
    "    def calc_reg(self):\n",
    "        weight_reg = (self.fc1.weight.norm()**2+self.fc1.bias.norm()**2)/(1-self.cdropout3.p)+\\\n",
    "        (self.fc2.weight.norm()**2+self.fc2.bias.norm()**2)/(1-self.cdropout4.p)+\\\n",
    "        (self.fc3.weight.norm()**2+self.fc3.bias.norm()**2)+\\\n",
    "        (self.conv1.weight.norm()**2+self.conv1.bias.norm()**2)/(1-self.cdropout1.p)+\\\n",
    "        (self.conv2.weight.norm()**2+self.conv2.bias.norm()**2)/(1-self.cdropout2.p)\n",
    "        weight_reg *= self.weight_reg_coef\n",
    "        dropout_reg = -self.entropy(self.cdropout1)-self.entropy(self.cdropout2)-\\\n",
    "        self.entropy(self.cdropout3)-self.entropy(self.cdropout4)\n",
    "        dropout_reg *= self.dropout_reg_coef\n",
    "        return weight_reg + dropout_reg\n",
    "    \n",
    "# Lenet with Variational dropout\n",
    "class Net_VDO(nn.Module):\n",
    "    def __init__(self, reg_coef=5e-4):\n",
    "        super(Net_VDO, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2)\n",
    "        self.vdropout1 = VariationalDropout()\n",
    "        self.vdropout2 = VariationalDropout()\n",
    "        self.vdropout3 = VariationalDropout()\n",
    "        self.vdropout4 = VariationalDropout()\n",
    "        self.reg_coef = reg_coef\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.leaky_relu(self.vdropout1(self.conv1(x))))\n",
    "        x = self.pool(self.leaky_relu(self.vdropout2(self.conv2(x))))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = self.leaky_relu(self.vdropout3(self.fc1(x)))\n",
    "        x = self.leaky_relu(self.vdropout4(self.fc2(x)))\n",
    "        x = F.softmax(self.fc3(x),dim=1)\n",
    "        return x\n",
    "    \n",
    "    def kl_prior(self, vdropout):\n",
    "        c1 = 1.16145124\n",
    "        c2 = -1.50204118\n",
    "        c3 = 0.58629921\n",
    "        return -0.5*vdropout.log_alpha - c1*vdropout.alpha \\\n",
    "        - c2*vdropout.alpha**2 - c3*vdropout.alpha**3\n",
    "    \n",
    "    def calc_reg(self):\n",
    "        return self.reg_coef * (self.kl_prior(self.vdropout1)+self.kl_prior(self.vdropout2)+\\\n",
    "                               self.kl_prior(self.vdropout3)+self.kl_prior(self.vdropout4))\n",
    "        \n",
    "# Lenet with MCDO\n",
    "class Net_MCDO(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net_MCDO, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 192, 5, padding=2)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(192, 192, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(192 * 8 * 8, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 256)\n",
    "        self.fc3 = nn.Linear(256, 10)\n",
    "        self.leaky_relu = nn.LeakyReLU(negative_slope=0.2)\n",
    "        self.dropout = nn.Dropout(p=0.3)\n",
    "        \n",
    "        nn.init.xavier_uniform_(self.conv1.weight)\n",
    "        nn.init.constant_(self.conv1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.conv2.weight)\n",
    "        nn.init.constant_(self.conv2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc1.weight)\n",
    "        nn.init.constant_(self.fc1.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc2.weight)\n",
    "        nn.init.constant_(self.fc2.bias, 0.0)\n",
    "        nn.init.xavier_uniform_(self.fc3.weight)\n",
    "        nn.init.constant_(self.fc3.bias, 0.0)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(self.leaky_relu(self.dropout(self.conv1(x))))\n",
    "        x = self.pool(self.leaky_relu(self.dropout(self.conv2(x))))\n",
    "        x = x.view(-1, 192 * 8 * 8)\n",
    "        x = self.leaky_relu(self.dropout(self.fc1(x)))\n",
    "        x = self.leaky_relu(self.dropout(self.fc2(x)))\n",
    "        x = F.softmax(self.fc3(x),dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Define a Loss function and optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "CE = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, net, optimizer, log_freq=log_freq, is_calc_reg=False):\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs\n",
    "        inputs, labels = data\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = CE(outputs, labels)\n",
    "        if is_calc_reg:\n",
    "            loss += torch.sum(net.module.calc_reg())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if (i+1) % log_freq == 0:    # print every 2000 mini-batches\n",
    "            print('[Epoch : %d, Iter: %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / log_freq))\n",
    "    return running_loss / log_freq\n",
    "\n",
    "def test(net):\n",
    "    print('Start test')\n",
    "    class_correct = list(0. for i in range(10))\n",
    "    class_total = list(0. for i in range(10))\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            inputs, labels = data\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            output = 0\n",
    "            for i in range(50):\n",
    "                output += net(inputs)/10.\n",
    "            output = torch.log(output)\n",
    "            _, predicted = torch.max(output, 1)\n",
    "            c = (predicted == labels).squeeze()\n",
    "            for i in range(len(labels)):\n",
    "                label = labels[i]\n",
    "                class_correct[label] += c[i].item()\n",
    "                class_total[label] += 1\n",
    "\n",
    "\n",
    "    for i in range(10):\n",
    "        print('Accuracy of %5s : %.2f %%' % (\n",
    "            classes[i], 100 * class_correct[i] / class_total[i]))\n",
    "\n",
    "    test_score = np.mean([100 * class_correct[i] / class_total[i] for i in range(10)])\n",
    "    print(test_score)\n",
    "    return test_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "\n",
    "lenets = [Net_CDO, Net_VDO, Net_MCDO] \n",
    "epoch_num = 300\n",
    "test_freq = 10\n",
    "losses = list()\n",
    "net_scores = list()\n",
    "test_scores = list()\n",
    "cdropout_history = list()\n",
    "vdropout_history = list()\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "is_train = True\n",
    "\n",
    "# SAVE\n",
    "def save(name, net):\n",
    "    net_path = './model/'+name+'.pkl'\n",
    "    net = net.cpu()\n",
    "    torch.save(net.state_dict(), net_path)\n",
    "    # Place it to GPU back\n",
    "    net.to(device)\n",
    "    return net\n",
    "    \n",
    "def load(name, net):\n",
    "    net_path = './model/'+name+'.pkl'\n",
    "    # LOAD\n",
    "    net.load_state_dict(torch.load(net_path))\n",
    "    # Place it to GPU\n",
    "    net.to(device)\n",
    "    return net\n",
    "\n",
    "def main():\n",
    "    for lenet in lenets:\n",
    "        print(lenet.__name__)\n",
    "        net = lenet()\n",
    "        if torch.cuda.device_count() > 1:\n",
    "            print(\"Let's use\",torch.cuda.device_count(),\"GPUs!\")\n",
    "            net = nn.DataParallel(net)\n",
    "        net.to(device)\n",
    "        \n",
    "        if lenet.__name__ == 'Net_CDO':\n",
    "            optimizer = optim.Adam(net.parameters(), lr=5e-4, amsgrad=True)\n",
    "        else:\n",
    "            optimizer = optim.Adam(net.parameters(), lr=5e-4, weight_decay=5e-4, amsgrad=True)\n",
    "        scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.97)\n",
    "        \n",
    "        for i in tqdm_notebook(range(epoch_num)):\n",
    "            scheduler.step()\n",
    "#             net.train()\n",
    "            if lenet.__name__ == 'Net_CDO':\n",
    "                loss_avg = train(epoch=i, net=net, optimizer=optimizer, is_calc_reg=True)\n",
    "                cdropout_history.append((net.module.cdropout1.p.item(),\\\n",
    "                                         net.module.cdropout2.p.item(),\n",
    "                                        net.module.cdropout3.p.item(),\n",
    "                                        net.module.cdropout4.p.item()))\n",
    "            elif lenet.__name__ == 'Net_VDO':\n",
    "                loss_avg = train(epoch=i, net=net, optimizer=optimizer, is_calc_reg=True)\n",
    "                vdropout_history.append((net.module.vdropout1.alpha.item(),\\\n",
    "                                         net.module.vdropout2.alpha.item(),\n",
    "                                        net.module.vdropout3.alpha.item(),\n",
    "                                        net.module.vdropout4.alpha.item()))\n",
    "            else:\n",
    "                loss_avg = train(epoch=i, net=net, optimizer=optimizer)\n",
    "            losses.append(loss_avg)\n",
    "            if (i+1) % test_freq == 0:\n",
    "#                 net.eval()\n",
    "                net_score = test(net)\n",
    "                net_scores.append(net_score)\n",
    "        save(lenet.__name__, net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net_CDO\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9efc983c2dde47fb8c3ea12a24fb8cb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=300), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 2.955\n",
      "[Epoch : 2, Iter:    50] loss: 2.373\n",
      "[Epoch : 3, Iter:    50] loss: 2.219\n",
      "[Epoch : 4, Iter:    50] loss: 2.139\n",
      "[Epoch : 5, Iter:    50] loss: 2.084\n",
      "[Epoch : 6, Iter:    50] loss: 2.043\n",
      "[Epoch : 7, Iter:    50] loss: 2.017\n",
      "[Epoch : 8, Iter:    50] loss: 1.991\n",
      "[Epoch : 9, Iter:    50] loss: 1.973\n",
      "[Epoch : 10, Iter:    50] loss: 1.963\n",
      "Start test\n",
      "Accuracy of plane : 77.20 %\n",
      "Accuracy of   car : 86.10 %\n",
      "Accuracy of  bird : 36.20 %\n",
      "Accuracy of   cat : 29.00 %\n",
      "Accuracy of  deer : 46.70 %\n",
      "Accuracy of   dog : 69.40 %\n",
      "Accuracy of  frog : 82.70 %\n",
      "Accuracy of horse : 72.30 %\n",
      "Accuracy of  ship : 74.30 %\n",
      "Accuracy of truck : 72.90 %\n",
      "64.67999999999999\n",
      "[Epoch : 11, Iter:    50] loss: 1.951\n",
      "[Epoch : 12, Iter:    50] loss: 1.934\n",
      "[Epoch : 13, Iter:    50] loss: 1.923\n",
      "[Epoch : 14, Iter:    50] loss: 1.920\n",
      "[Epoch : 15, Iter:    50] loss: 1.906\n",
      "[Epoch : 16, Iter:    50] loss: 1.899\n",
      "[Epoch : 17, Iter:    50] loss: 1.892\n",
      "[Epoch : 18, Iter:    50] loss: 1.888\n",
      "[Epoch : 19, Iter:    50] loss: 1.886\n",
      "[Epoch : 20, Iter:    50] loss: 1.885\n",
      "Start test\n",
      "Accuracy of plane : 80.80 %\n",
      "Accuracy of   car : 85.30 %\n",
      "Accuracy of  bird : 53.40 %\n",
      "Accuracy of   cat : 48.70 %\n",
      "Accuracy of  deer : 57.80 %\n",
      "Accuracy of   dog : 62.30 %\n",
      "Accuracy of  frog : 88.30 %\n",
      "Accuracy of horse : 72.10 %\n",
      "Accuracy of  ship : 83.30 %\n",
      "Accuracy of truck : 80.90 %\n",
      "71.28999999999999\n",
      "[Epoch : 21, Iter:    50] loss: 1.872\n",
      "[Epoch : 22, Iter:    50] loss: 1.869\n",
      "[Epoch : 23, Iter:    50] loss: 1.860\n",
      "[Epoch : 24, Iter:    50] loss: 1.855\n",
      "[Epoch : 25, Iter:    50] loss: 1.853\n",
      "[Epoch : 26, Iter:    50] loss: 1.852\n",
      "[Epoch : 27, Iter:    50] loss: 1.844\n",
      "[Epoch : 28, Iter:    50] loss: 1.844\n",
      "[Epoch : 29, Iter:    50] loss: 1.837\n",
      "[Epoch : 30, Iter:    50] loss: 1.835\n",
      "Start test\n",
      "Accuracy of plane : 79.20 %\n",
      "Accuracy of   car : 84.80 %\n",
      "Accuracy of  bird : 67.00 %\n",
      "Accuracy of   cat : 50.90 %\n",
      "Accuracy of  deer : 63.00 %\n",
      "Accuracy of   dog : 72.60 %\n",
      "Accuracy of  frog : 83.30 %\n",
      "Accuracy of horse : 77.50 %\n",
      "Accuracy of  ship : 73.40 %\n",
      "Accuracy of truck : 89.40 %\n",
      "74.10999999999999\n",
      "[Epoch : 31, Iter:    50] loss: 1.831\n",
      "[Epoch : 32, Iter:    50] loss: 1.833\n",
      "[Epoch : 33, Iter:    50] loss: 1.826\n",
      "[Epoch : 34, Iter:    50] loss: 1.826\n",
      "[Epoch : 35, Iter:    50] loss: 1.822\n",
      "[Epoch : 36, Iter:    50] loss: 1.818\n",
      "[Epoch : 37, Iter:    50] loss: 1.820\n",
      "[Epoch : 38, Iter:    50] loss: 1.816\n",
      "[Epoch : 39, Iter:    50] loss: 1.812\n",
      "[Epoch : 40, Iter:    50] loss: 1.811\n",
      "Start test\n",
      "Accuracy of plane : 81.80 %\n",
      "Accuracy of   car : 91.00 %\n",
      "Accuracy of  bird : 56.30 %\n",
      "Accuracy of   cat : 59.40 %\n",
      "Accuracy of  deer : 76.20 %\n",
      "Accuracy of   dog : 65.20 %\n",
      "Accuracy of  frog : 90.20 %\n",
      "Accuracy of horse : 78.80 %\n",
      "Accuracy of  ship : 83.30 %\n",
      "Accuracy of truck : 84.50 %\n",
      "76.66999999999999\n",
      "[Epoch : 41, Iter:    50] loss: 1.809\n",
      "[Epoch : 42, Iter:    50] loss: 1.805\n",
      "[Epoch : 43, Iter:    50] loss: 1.809\n",
      "[Epoch : 44, Iter:    50] loss: 1.803\n",
      "[Epoch : 45, Iter:    50] loss: 1.800\n",
      "[Epoch : 46, Iter:    50] loss: 1.801\n",
      "[Epoch : 47, Iter:    50] loss: 1.796\n",
      "[Epoch : 48, Iter:    50] loss: 1.800\n",
      "[Epoch : 49, Iter:    50] loss: 1.793\n",
      "[Epoch : 50, Iter:    50] loss: 1.793\n",
      "Start test\n",
      "Accuracy of plane : 80.80 %\n",
      "Accuracy of   car : 91.30 %\n",
      "Accuracy of  bird : 66.20 %\n",
      "Accuracy of   cat : 44.50 %\n",
      "Accuracy of  deer : 62.90 %\n",
      "Accuracy of   dog : 77.50 %\n",
      "Accuracy of  frog : 90.60 %\n",
      "Accuracy of horse : 82.90 %\n",
      "Accuracy of  ship : 85.40 %\n",
      "Accuracy of truck : 88.10 %\n",
      "77.02000000000001\n",
      "[Epoch : 51, Iter:    50] loss: 1.789\n",
      "[Epoch : 52, Iter:    50] loss: 1.790\n",
      "[Epoch : 53, Iter:    50] loss: 1.791\n",
      "[Epoch : 54, Iter:    50] loss: 1.787\n",
      "[Epoch : 55, Iter:    50] loss: 1.791\n",
      "[Epoch : 56, Iter:    50] loss: 1.784\n",
      "[Epoch : 57, Iter:    50] loss: 1.784\n",
      "[Epoch : 58, Iter:    50] loss: 1.784\n",
      "[Epoch : 59, Iter:    50] loss: 1.781\n",
      "[Epoch : 60, Iter:    50] loss: 1.779\n",
      "Start test\n",
      "Accuracy of plane : 86.50 %\n",
      "Accuracy of   car : 89.70 %\n",
      "Accuracy of  bird : 61.90 %\n",
      "Accuracy of   cat : 65.00 %\n",
      "Accuracy of  deer : 68.60 %\n",
      "Accuracy of   dog : 72.60 %\n",
      "Accuracy of  frog : 82.70 %\n",
      "Accuracy of horse : 87.70 %\n",
      "Accuracy of  ship : 81.90 %\n",
      "Accuracy of truck : 87.70 %\n",
      "78.43\n",
      "[Epoch : 61, Iter:    50] loss: 1.781\n",
      "[Epoch : 62, Iter:    50] loss: 1.773\n",
      "[Epoch : 63, Iter:    50] loss: 1.774\n",
      "[Epoch : 64, Iter:    50] loss: 1.774\n",
      "[Epoch : 65, Iter:    50] loss: 1.771\n",
      "[Epoch : 66, Iter:    50] loss: 1.770\n",
      "[Epoch : 67, Iter:    50] loss: 1.769\n",
      "[Epoch : 68, Iter:    50] loss: 1.771\n",
      "[Epoch : 69, Iter:    50] loss: 1.766\n",
      "[Epoch : 70, Iter:    50] loss: 1.766\n",
      "Start test\n",
      "Accuracy of plane : 82.20 %\n",
      "Accuracy of   car : 88.80 %\n",
      "Accuracy of  bird : 69.30 %\n",
      "Accuracy of   cat : 57.60 %\n",
      "Accuracy of  deer : 79.90 %\n",
      "Accuracy of   dog : 70.50 %\n",
      "Accuracy of  frog : 85.90 %\n",
      "Accuracy of horse : 88.40 %\n",
      "Accuracy of  ship : 89.80 %\n",
      "Accuracy of truck : 86.80 %\n",
      "79.91999999999999\n",
      "[Epoch : 71, Iter:    50] loss: 1.764\n",
      "[Epoch : 72, Iter:    50] loss: 1.765\n",
      "[Epoch : 73, Iter:    50] loss: 1.765\n",
      "[Epoch : 74, Iter:    50] loss: 1.760\n",
      "[Epoch : 75, Iter:    50] loss: 1.759\n",
      "[Epoch : 76, Iter:    50] loss: 1.761\n",
      "[Epoch : 77, Iter:    50] loss: 1.758\n",
      "[Epoch : 78, Iter:    50] loss: 1.760\n",
      "[Epoch : 79, Iter:    50] loss: 1.760\n",
      "[Epoch : 80, Iter:    50] loss: 1.755\n",
      "Start test\n",
      "Accuracy of plane : 79.10 %\n",
      "Accuracy of   car : 88.10 %\n",
      "Accuracy of  bird : 72.90 %\n",
      "Accuracy of   cat : 53.00 %\n",
      "Accuracy of  deer : 78.00 %\n",
      "Accuracy of   dog : 69.20 %\n",
      "Accuracy of  frog : 88.90 %\n",
      "Accuracy of horse : 89.10 %\n",
      "Accuracy of  ship : 93.20 %\n",
      "Accuracy of truck : 86.50 %\n",
      "79.8\n",
      "[Epoch : 81, Iter:    50] loss: 1.757\n",
      "[Epoch : 82, Iter:    50] loss: 1.756\n",
      "[Epoch : 83, Iter:    50] loss: 1.755\n",
      "[Epoch : 84, Iter:    50] loss: 1.752\n",
      "[Epoch : 85, Iter:    50] loss: 1.754\n",
      "[Epoch : 86, Iter:    50] loss: 1.750\n",
      "[Epoch : 87, Iter:    50] loss: 1.750\n",
      "[Epoch : 88, Iter:    50] loss: 1.750\n",
      "[Epoch : 89, Iter:    50] loss: 1.750\n",
      "[Epoch : 90, Iter:    50] loss: 1.748\n",
      "Start test\n",
      "Accuracy of plane : 81.20 %\n",
      "Accuracy of   car : 89.90 %\n",
      "Accuracy of  bird : 66.40 %\n",
      "Accuracy of   cat : 66.30 %\n",
      "Accuracy of  deer : 84.00 %\n",
      "Accuracy of   dog : 73.00 %\n",
      "Accuracy of  frog : 86.30 %\n",
      "Accuracy of horse : 82.50 %\n",
      "Accuracy of  ship : 87.80 %\n",
      "Accuracy of truck : 90.10 %\n",
      "80.75\n",
      "[Epoch : 91, Iter:    50] loss: 1.747\n",
      "[Epoch : 92, Iter:    50] loss: 1.746\n",
      "[Epoch : 93, Iter:    50] loss: 1.747\n",
      "[Epoch : 94, Iter:    50] loss: 1.746\n",
      "[Epoch : 95, Iter:    50] loss: 1.743\n",
      "[Epoch : 96, Iter:    50] loss: 1.743\n",
      "[Epoch : 97, Iter:    50] loss: 1.741\n",
      "[Epoch : 98, Iter:    50] loss: 1.743\n",
      "[Epoch : 99, Iter:    50] loss: 1.740\n",
      "[Epoch : 100, Iter:    50] loss: 1.741\n",
      "Start test\n",
      "Accuracy of plane : 88.30 %\n",
      "Accuracy of   car : 91.00 %\n",
      "Accuracy of  bird : 70.40 %\n",
      "Accuracy of   cat : 56.30 %\n",
      "Accuracy of  deer : 82.00 %\n",
      "Accuracy of   dog : 75.90 %\n",
      "Accuracy of  frog : 83.80 %\n",
      "Accuracy of horse : 85.70 %\n",
      "Accuracy of  ship : 89.80 %\n",
      "Accuracy of truck : 88.00 %\n",
      "81.11999999999999\n",
      "[Epoch : 101, Iter:    50] loss: 1.738\n",
      "[Epoch : 102, Iter:    50] loss: 1.739\n",
      "[Epoch : 103, Iter:    50] loss: 1.737\n",
      "[Epoch : 104, Iter:    50] loss: 1.740\n",
      "[Epoch : 105, Iter:    50] loss: 1.737\n",
      "[Epoch : 106, Iter:    50] loss: 1.737\n",
      "[Epoch : 107, Iter:    50] loss: 1.736\n",
      "[Epoch : 108, Iter:    50] loss: 1.737\n",
      "[Epoch : 109, Iter:    50] loss: 1.736\n",
      "[Epoch : 110, Iter:    50] loss: 1.735\n",
      "Start test\n",
      "Accuracy of plane : 87.50 %\n",
      "Accuracy of   car : 90.20 %\n",
      "Accuracy of  bird : 68.50 %\n",
      "Accuracy of   cat : 70.00 %\n",
      "Accuracy of  deer : 74.00 %\n",
      "Accuracy of   dog : 69.80 %\n",
      "Accuracy of  frog : 89.10 %\n",
      "Accuracy of horse : 86.00 %\n",
      "Accuracy of  ship : 89.30 %\n",
      "Accuracy of truck : 89.80 %\n",
      "81.41999999999999\n",
      "[Epoch : 111, Iter:    50] loss: 1.733\n",
      "[Epoch : 112, Iter:    50] loss: 1.733\n",
      "[Epoch : 113, Iter:    50] loss: 1.733\n",
      "[Epoch : 114, Iter:    50] loss: 1.732\n",
      "[Epoch : 115, Iter:    50] loss: 1.732\n",
      "[Epoch : 116, Iter:    50] loss: 1.731\n",
      "[Epoch : 117, Iter:    50] loss: 1.731\n",
      "[Epoch : 118, Iter:    50] loss: 1.732\n",
      "[Epoch : 119, Iter:    50] loss: 1.728\n",
      "[Epoch : 120, Iter:    50] loss: 1.728\n",
      "Start test\n",
      "Accuracy of plane : 88.20 %\n",
      "Accuracy of   car : 91.40 %\n",
      "Accuracy of  bird : 64.70 %\n",
      "Accuracy of   cat : 66.00 %\n",
      "Accuracy of  deer : 85.80 %\n",
      "Accuracy of   dog : 68.30 %\n",
      "Accuracy of  frog : 90.40 %\n",
      "Accuracy of horse : 82.30 %\n",
      "Accuracy of  ship : 87.90 %\n",
      "Accuracy of truck : 89.70 %\n",
      "81.47\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 121, Iter:    50] loss: 1.729\n",
      "[Epoch : 122, Iter:    50] loss: 1.729\n",
      "[Epoch : 123, Iter:    50] loss: 1.728\n",
      "[Epoch : 124, Iter:    50] loss: 1.727\n",
      "[Epoch : 125, Iter:    50] loss: 1.726\n",
      "[Epoch : 126, Iter:    50] loss: 1.725\n",
      "[Epoch : 127, Iter:    50] loss: 1.724\n",
      "[Epoch : 128, Iter:    50] loss: 1.724\n",
      "[Epoch : 129, Iter:    50] loss: 1.725\n",
      "[Epoch : 130, Iter:    50] loss: 1.724\n",
      "Start test\n",
      "Accuracy of plane : 87.80 %\n",
      "Accuracy of   car : 92.10 %\n",
      "Accuracy of  bird : 70.30 %\n",
      "Accuracy of   cat : 69.20 %\n",
      "Accuracy of  deer : 85.40 %\n",
      "Accuracy of   dog : 70.70 %\n",
      "Accuracy of  frog : 88.30 %\n",
      "Accuracy of horse : 84.70 %\n",
      "Accuracy of  ship : 88.80 %\n",
      "Accuracy of truck : 88.90 %\n",
      "82.61999999999999\n",
      "[Epoch : 131, Iter:    50] loss: 1.723\n",
      "[Epoch : 132, Iter:    50] loss: 1.723\n",
      "[Epoch : 133, Iter:    50] loss: 1.723\n",
      "[Epoch : 134, Iter:    50] loss: 1.722\n",
      "[Epoch : 135, Iter:    50] loss: 1.725\n",
      "[Epoch : 136, Iter:    50] loss: 1.720\n",
      "[Epoch : 137, Iter:    50] loss: 1.721\n",
      "[Epoch : 138, Iter:    50] loss: 1.721\n",
      "[Epoch : 139, Iter:    50] loss: 1.719\n",
      "[Epoch : 140, Iter:    50] loss: 1.720\n",
      "Start test\n",
      "Accuracy of plane : 87.50 %\n",
      "Accuracy of   car : 90.40 %\n",
      "Accuracy of  bird : 66.80 %\n",
      "Accuracy of   cat : 65.60 %\n",
      "Accuracy of  deer : 82.90 %\n",
      "Accuracy of   dog : 75.30 %\n",
      "Accuracy of  frog : 90.10 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 86.20 %\n",
      "Accuracy of truck : 93.30 %\n",
      "82.53999999999999\n",
      "[Epoch : 141, Iter:    50] loss: 1.720\n",
      "[Epoch : 142, Iter:    50] loss: 1.718\n",
      "[Epoch : 143, Iter:    50] loss: 1.719\n",
      "[Epoch : 144, Iter:    50] loss: 1.717\n",
      "[Epoch : 145, Iter:    50] loss: 1.716\n",
      "[Epoch : 146, Iter:    50] loss: 1.718\n",
      "[Epoch : 147, Iter:    50] loss: 1.716\n",
      "[Epoch : 148, Iter:    50] loss: 1.717\n",
      "[Epoch : 149, Iter:    50] loss: 1.715\n",
      "[Epoch : 150, Iter:    50] loss: 1.716\n",
      "Start test\n",
      "Accuracy of plane : 86.40 %\n",
      "Accuracy of   car : 91.90 %\n",
      "Accuracy of  bird : 71.80 %\n",
      "Accuracy of   cat : 57.10 %\n",
      "Accuracy of  deer : 84.10 %\n",
      "Accuracy of   dog : 79.40 %\n",
      "Accuracy of  frog : 90.40 %\n",
      "Accuracy of horse : 87.20 %\n",
      "Accuracy of  ship : 91.70 %\n",
      "Accuracy of truck : 88.80 %\n",
      "82.88000000000001\n",
      "[Epoch : 151, Iter:    50] loss: 1.714\n",
      "[Epoch : 152, Iter:    50] loss: 1.716\n",
      "[Epoch : 153, Iter:    50] loss: 1.713\n",
      "[Epoch : 154, Iter:    50] loss: 1.715\n",
      "[Epoch : 155, Iter:    50] loss: 1.716\n",
      "[Epoch : 156, Iter:    50] loss: 1.713\n",
      "[Epoch : 157, Iter:    50] loss: 1.712\n",
      "[Epoch : 158, Iter:    50] loss: 1.712\n",
      "[Epoch : 159, Iter:    50] loss: 1.713\n",
      "[Epoch : 160, Iter:    50] loss: 1.712\n",
      "Start test\n",
      "Accuracy of plane : 89.00 %\n",
      "Accuracy of   car : 92.60 %\n",
      "Accuracy of  bird : 72.50 %\n",
      "Accuracy of   cat : 68.20 %\n",
      "Accuracy of  deer : 81.40 %\n",
      "Accuracy of   dog : 73.40 %\n",
      "Accuracy of  frog : 88.10 %\n",
      "Accuracy of horse : 85.30 %\n",
      "Accuracy of  ship : 90.30 %\n",
      "Accuracy of truck : 86.00 %\n",
      "82.67999999999999\n",
      "[Epoch : 161, Iter:    50] loss: 1.713\n",
      "[Epoch : 162, Iter:    50] loss: 1.712\n",
      "[Epoch : 163, Iter:    50] loss: 1.711\n",
      "[Epoch : 164, Iter:    50] loss: 1.710\n",
      "[Epoch : 165, Iter:    50] loss: 1.712\n",
      "[Epoch : 166, Iter:    50] loss: 1.710\n",
      "[Epoch : 167, Iter:    50] loss: 1.710\n",
      "[Epoch : 168, Iter:    50] loss: 1.708\n",
      "[Epoch : 169, Iter:    50] loss: 1.710\n",
      "[Epoch : 170, Iter:    50] loss: 1.709\n",
      "Start test\n",
      "Accuracy of plane : 84.10 %\n",
      "Accuracy of   car : 90.70 %\n",
      "Accuracy of  bird : 80.10 %\n",
      "Accuracy of   cat : 64.00 %\n",
      "Accuracy of  deer : 80.60 %\n",
      "Accuracy of   dog : 78.20 %\n",
      "Accuracy of  frog : 86.20 %\n",
      "Accuracy of horse : 88.10 %\n",
      "Accuracy of  ship : 89.30 %\n",
      "Accuracy of truck : 89.10 %\n",
      "83.03999999999999\n",
      "[Epoch : 171, Iter:    50] loss: 1.708\n",
      "[Epoch : 172, Iter:    50] loss: 1.709\n",
      "[Epoch : 173, Iter:    50] loss: 1.709\n",
      "[Epoch : 174, Iter:    50] loss: 1.709\n",
      "[Epoch : 175, Iter:    50] loss: 1.708\n",
      "[Epoch : 176, Iter:    50] loss: 1.708\n",
      "[Epoch : 177, Iter:    50] loss: 1.707\n",
      "[Epoch : 178, Iter:    50] loss: 1.707\n",
      "[Epoch : 179, Iter:    50] loss: 1.707\n",
      "[Epoch : 180, Iter:    50] loss: 1.708\n",
      "Start test\n",
      "Accuracy of plane : 85.00 %\n",
      "Accuracy of   car : 91.50 %\n",
      "Accuracy of  bird : 74.30 %\n",
      "Accuracy of   cat : 63.60 %\n",
      "Accuracy of  deer : 85.60 %\n",
      "Accuracy of   dog : 74.00 %\n",
      "Accuracy of  frog : 89.00 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 92.00 %\n",
      "Accuracy of truck : 89.40 %\n",
      "83.16999999999999\n",
      "[Epoch : 181, Iter:    50] loss: 1.706\n",
      "[Epoch : 182, Iter:    50] loss: 1.706\n",
      "[Epoch : 183, Iter:    50] loss: 1.707\n",
      "[Epoch : 184, Iter:    50] loss: 1.705\n",
      "[Epoch : 185, Iter:    50] loss: 1.706\n",
      "[Epoch : 186, Iter:    50] loss: 1.706\n",
      "[Epoch : 187, Iter:    50] loss: 1.705\n",
      "[Epoch : 188, Iter:    50] loss: 1.705\n",
      "[Epoch : 189, Iter:    50] loss: 1.706\n",
      "[Epoch : 190, Iter:    50] loss: 1.704\n",
      "Start test\n",
      "Accuracy of plane : 87.90 %\n",
      "Accuracy of   car : 93.30 %\n",
      "Accuracy of  bird : 72.30 %\n",
      "Accuracy of   cat : 67.50 %\n",
      "Accuracy of  deer : 81.20 %\n",
      "Accuracy of   dog : 77.90 %\n",
      "Accuracy of  frog : 88.50 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 90.30 %\n",
      "Accuracy of truck : 88.50 %\n",
      "83.47\n",
      "[Epoch : 191, Iter:    50] loss: 1.704\n",
      "[Epoch : 192, Iter:    50] loss: 1.704\n",
      "[Epoch : 193, Iter:    50] loss: 1.703\n",
      "[Epoch : 194, Iter:    50] loss: 1.703\n",
      "[Epoch : 195, Iter:    50] loss: 1.702\n",
      "[Epoch : 196, Iter:    50] loss: 1.703\n",
      "[Epoch : 197, Iter:    50] loss: 1.702\n",
      "[Epoch : 198, Iter:    50] loss: 1.703\n",
      "[Epoch : 199, Iter:    50] loss: 1.702\n",
      "[Epoch : 200, Iter:    50] loss: 1.702\n",
      "Start test\n",
      "Accuracy of plane : 83.70 %\n",
      "Accuracy of   car : 91.20 %\n",
      "Accuracy of  bird : 71.90 %\n",
      "Accuracy of   cat : 63.20 %\n",
      "Accuracy of  deer : 82.90 %\n",
      "Accuracy of   dog : 74.90 %\n",
      "Accuracy of  frog : 91.80 %\n",
      "Accuracy of horse : 89.30 %\n",
      "Accuracy of  ship : 93.70 %\n",
      "Accuracy of truck : 89.00 %\n",
      "83.16\n",
      "[Epoch : 201, Iter:    50] loss: 1.701\n",
      "[Epoch : 202, Iter:    50] loss: 1.702\n",
      "[Epoch : 203, Iter:    50] loss: 1.702\n",
      "[Epoch : 204, Iter:    50] loss: 1.701\n",
      "[Epoch : 205, Iter:    50] loss: 1.702\n",
      "[Epoch : 206, Iter:    50] loss: 1.701\n",
      "[Epoch : 207, Iter:    50] loss: 1.702\n",
      "[Epoch : 208, Iter:    50] loss: 1.702\n",
      "[Epoch : 209, Iter:    50] loss: 1.701\n",
      "[Epoch : 210, Iter:    50] loss: 1.700\n",
      "Start test\n",
      "Accuracy of plane : 87.10 %\n",
      "Accuracy of   car : 92.40 %\n",
      "Accuracy of  bird : 74.10 %\n",
      "Accuracy of   cat : 66.20 %\n",
      "Accuracy of  deer : 82.50 %\n",
      "Accuracy of   dog : 75.30 %\n",
      "Accuracy of  frog : 88.10 %\n",
      "Accuracy of horse : 90.30 %\n",
      "Accuracy of  ship : 91.40 %\n",
      "Accuracy of truck : 90.00 %\n",
      "83.74\n",
      "[Epoch : 211, Iter:    50] loss: 1.700\n",
      "[Epoch : 212, Iter:    50] loss: 1.700\n",
      "[Epoch : 213, Iter:    50] loss: 1.701\n",
      "[Epoch : 214, Iter:    50] loss: 1.699\n",
      "[Epoch : 215, Iter:    50] loss: 1.699\n",
      "[Epoch : 216, Iter:    50] loss: 1.700\n",
      "[Epoch : 217, Iter:    50] loss: 1.700\n",
      "[Epoch : 218, Iter:    50] loss: 1.700\n",
      "[Epoch : 219, Iter:    50] loss: 1.698\n",
      "[Epoch : 220, Iter:    50] loss: 1.700\n",
      "Start test\n",
      "Accuracy of plane : 88.30 %\n",
      "Accuracy of   car : 91.10 %\n",
      "Accuracy of  bird : 70.20 %\n",
      "Accuracy of   cat : 64.30 %\n",
      "Accuracy of  deer : 84.80 %\n",
      "Accuracy of   dog : 75.80 %\n",
      "Accuracy of  frog : 91.10 %\n",
      "Accuracy of horse : 88.40 %\n",
      "Accuracy of  ship : 90.70 %\n",
      "Accuracy of truck : 90.30 %\n",
      "83.5\n",
      "[Epoch : 221, Iter:    50] loss: 1.698\n",
      "[Epoch : 222, Iter:    50] loss: 1.699\n",
      "[Epoch : 223, Iter:    50] loss: 1.698\n",
      "[Epoch : 224, Iter:    50] loss: 1.698\n",
      "[Epoch : 225, Iter:    50] loss: 1.698\n",
      "[Epoch : 226, Iter:    50] loss: 1.698\n",
      "[Epoch : 227, Iter:    50] loss: 1.697\n",
      "[Epoch : 228, Iter:    50] loss: 1.697\n",
      "[Epoch : 229, Iter:    50] loss: 1.698\n",
      "[Epoch : 230, Iter:    50] loss: 1.698\n",
      "Start test\n",
      "Accuracy of plane : 86.10 %\n",
      "Accuracy of   car : 92.30 %\n",
      "Accuracy of  bird : 73.20 %\n",
      "Accuracy of   cat : 68.70 %\n",
      "Accuracy of  deer : 85.40 %\n",
      "Accuracy of   dog : 75.40 %\n",
      "Accuracy of  frog : 90.00 %\n",
      "Accuracy of horse : 86.60 %\n",
      "Accuracy of  ship : 90.30 %\n",
      "Accuracy of truck : 89.90 %\n",
      "83.78999999999999\n",
      "[Epoch : 231, Iter:    50] loss: 1.699\n",
      "[Epoch : 232, Iter:    50] loss: 1.698\n",
      "[Epoch : 233, Iter:    50] loss: 1.698\n",
      "[Epoch : 234, Iter:    50] loss: 1.697\n",
      "[Epoch : 235, Iter:    50] loss: 1.697\n",
      "[Epoch : 236, Iter:    50] loss: 1.697\n",
      "[Epoch : 237, Iter:    50] loss: 1.697\n",
      "[Epoch : 238, Iter:    50] loss: 1.696\n",
      "[Epoch : 239, Iter:    50] loss: 1.696\n",
      "[Epoch : 240, Iter:    50] loss: 1.697\n",
      "Start test\n",
      "Accuracy of plane : 87.30 %\n",
      "Accuracy of   car : 92.30 %\n",
      "Accuracy of  bird : 75.80 %\n",
      "Accuracy of   cat : 65.00 %\n",
      "Accuracy of  deer : 82.40 %\n",
      "Accuracy of   dog : 75.10 %\n",
      "Accuracy of  frog : 89.60 %\n",
      "Accuracy of horse : 87.40 %\n",
      "Accuracy of  ship : 92.50 %\n",
      "Accuracy of truck : 89.70 %\n",
      "83.71000000000001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 241, Iter:    50] loss: 1.696\n",
      "[Epoch : 242, Iter:    50] loss: 1.696\n",
      "[Epoch : 243, Iter:    50] loss: 1.695\n",
      "[Epoch : 244, Iter:    50] loss: 1.696\n",
      "[Epoch : 245, Iter:    50] loss: 1.697\n",
      "[Epoch : 246, Iter:    50] loss: 1.696\n",
      "[Epoch : 247, Iter:    50] loss: 1.695\n",
      "[Epoch : 248, Iter:    50] loss: 1.695\n",
      "[Epoch : 249, Iter:    50] loss: 1.696\n",
      "[Epoch : 250, Iter:    50] loss: 1.695\n",
      "Start test\n",
      "Accuracy of plane : 87.40 %\n",
      "Accuracy of   car : 90.90 %\n",
      "Accuracy of  bird : 74.90 %\n",
      "Accuracy of   cat : 65.30 %\n",
      "Accuracy of  deer : 82.00 %\n",
      "Accuracy of   dog : 78.20 %\n",
      "Accuracy of  frog : 90.30 %\n",
      "Accuracy of horse : 87.80 %\n",
      "Accuracy of  ship : 91.10 %\n",
      "Accuracy of truck : 89.20 %\n",
      "83.71000000000001\n",
      "[Epoch : 251, Iter:    50] loss: 1.695\n",
      "[Epoch : 252, Iter:    50] loss: 1.694\n",
      "[Epoch : 253, Iter:    50] loss: 1.694\n",
      "[Epoch : 254, Iter:    50] loss: 1.694\n",
      "[Epoch : 255, Iter:    50] loss: 1.694\n",
      "[Epoch : 256, Iter:    50] loss: 1.695\n",
      "[Epoch : 257, Iter:    50] loss: 1.694\n",
      "[Epoch : 258, Iter:    50] loss: 1.694\n",
      "[Epoch : 259, Iter:    50] loss: 1.694\n",
      "[Epoch : 260, Iter:    50] loss: 1.695\n",
      "Start test\n",
      "Accuracy of plane : 87.40 %\n",
      "Accuracy of   car : 92.30 %\n",
      "Accuracy of  bird : 74.80 %\n",
      "Accuracy of   cat : 67.50 %\n",
      "Accuracy of  deer : 83.70 %\n",
      "Accuracy of   dog : 76.60 %\n",
      "Accuracy of  frog : 88.60 %\n",
      "Accuracy of horse : 89.10 %\n",
      "Accuracy of  ship : 90.70 %\n",
      "Accuracy of truck : 88.80 %\n",
      "83.95\n",
      "[Epoch : 261, Iter:    50] loss: 1.693\n",
      "[Epoch : 262, Iter:    50] loss: 1.694\n",
      "[Epoch : 263, Iter:    50] loss: 1.694\n",
      "[Epoch : 264, Iter:    50] loss: 1.694\n",
      "[Epoch : 265, Iter:    50] loss: 1.695\n",
      "[Epoch : 266, Iter:    50] loss: 1.693\n",
      "[Epoch : 267, Iter:    50] loss: 1.694\n",
      "[Epoch : 268, Iter:    50] loss: 1.693\n",
      "[Epoch : 269, Iter:    50] loss: 1.695\n",
      "[Epoch : 270, Iter:    50] loss: 1.694\n",
      "Start test\n",
      "Accuracy of plane : 87.80 %\n",
      "Accuracy of   car : 92.80 %\n",
      "Accuracy of  bird : 75.40 %\n",
      "Accuracy of   cat : 68.90 %\n",
      "Accuracy of  deer : 84.40 %\n",
      "Accuracy of   dog : 74.60 %\n",
      "Accuracy of  frog : 87.90 %\n",
      "Accuracy of horse : 88.90 %\n",
      "Accuracy of  ship : 91.50 %\n",
      "Accuracy of truck : 88.30 %\n",
      "84.05\n",
      "[Epoch : 271, Iter:    50] loss: 1.695\n",
      "[Epoch : 272, Iter:    50] loss: 1.693\n",
      "[Epoch : 273, Iter:    50] loss: 1.693\n",
      "[Epoch : 274, Iter:    50] loss: 1.693\n",
      "[Epoch : 275, Iter:    50] loss: 1.694\n",
      "[Epoch : 276, Iter:    50] loss: 1.693\n",
      "[Epoch : 277, Iter:    50] loss: 1.691\n",
      "[Epoch : 278, Iter:    50] loss: 1.694\n",
      "[Epoch : 279, Iter:    50] loss: 1.692\n",
      "[Epoch : 280, Iter:    50] loss: 1.692\n",
      "Start test\n",
      "Accuracy of plane : 88.30 %\n",
      "Accuracy of   car : 92.10 %\n",
      "Accuracy of  bird : 74.90 %\n",
      "Accuracy of   cat : 68.80 %\n",
      "Accuracy of  deer : 84.40 %\n",
      "Accuracy of   dog : 74.00 %\n",
      "Accuracy of  frog : 89.80 %\n",
      "Accuracy of horse : 88.20 %\n",
      "Accuracy of  ship : 91.10 %\n",
      "Accuracy of truck : 89.70 %\n",
      "84.13000000000001\n",
      "[Epoch : 281, Iter:    50] loss: 1.692\n",
      "[Epoch : 282, Iter:    50] loss: 1.694\n",
      "[Epoch : 283, Iter:    50] loss: 1.694\n",
      "[Epoch : 284, Iter:    50] loss: 1.693\n",
      "[Epoch : 285, Iter:    50] loss: 1.693\n",
      "[Epoch : 286, Iter:    50] loss: 1.692\n",
      "[Epoch : 287, Iter:    50] loss: 1.694\n",
      "[Epoch : 288, Iter:    50] loss: 1.692\n",
      "[Epoch : 289, Iter:    50] loss: 1.693\n",
      "[Epoch : 290, Iter:    50] loss: 1.692\n",
      "Start test\n",
      "Accuracy of plane : 87.80 %\n",
      "Accuracy of   car : 92.30 %\n",
      "Accuracy of  bird : 75.10 %\n",
      "Accuracy of   cat : 65.80 %\n",
      "Accuracy of  deer : 84.30 %\n",
      "Accuracy of   dog : 77.50 %\n",
      "Accuracy of  frog : 90.30 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 91.30 %\n",
      "Accuracy of truck : 88.90 %\n",
      "84.05999999999999\n",
      "[Epoch : 291, Iter:    50] loss: 1.693\n",
      "[Epoch : 292, Iter:    50] loss: 1.693\n",
      "[Epoch : 293, Iter:    50] loss: 1.692\n",
      "[Epoch : 294, Iter:    50] loss: 1.691\n",
      "[Epoch : 295, Iter:    50] loss: 1.692\n",
      "[Epoch : 296, Iter:    50] loss: 1.693\n",
      "[Epoch : 297, Iter:    50] loss: 1.692\n",
      "[Epoch : 298, Iter:    50] loss: 1.692\n",
      "[Epoch : 299, Iter:    50] loss: 1.693\n",
      "[Epoch : 300, Iter:    50] loss: 1.691\n",
      "Start test\n",
      "Accuracy of plane : 87.30 %\n",
      "Accuracy of   car : 91.40 %\n",
      "Accuracy of  bird : 75.00 %\n",
      "Accuracy of   cat : 68.00 %\n",
      "Accuracy of  deer : 84.20 %\n",
      "Accuracy of   dog : 75.60 %\n",
      "Accuracy of  frog : 90.40 %\n",
      "Accuracy of horse : 88.00 %\n",
      "Accuracy of  ship : 91.80 %\n",
      "Accuracy of truck : 89.60 %\n",
      "84.13000000000001\n",
      "\n",
      "Net_VDO\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9258b46937c04ec5ae6448869a35f667",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=300), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 2.149\n",
      "[Epoch : 2, Iter:    50] loss: 2.044\n",
      "[Epoch : 3, Iter:    50] loss: 1.997\n",
      "[Epoch : 4, Iter:    50] loss: 1.961\n",
      "[Epoch : 5, Iter:    50] loss: 1.927\n",
      "[Epoch : 6, Iter:    50] loss: 1.906\n",
      "[Epoch : 7, Iter:    50] loss: 1.886\n",
      "[Epoch : 8, Iter:    50] loss: 1.870\n",
      "[Epoch : 9, Iter:    50] loss: 1.850\n",
      "[Epoch : 10, Iter:    50] loss: 1.841\n",
      "Start test\n",
      "Accuracy of plane : 80.70 %\n",
      "Accuracy of   car : 85.00 %\n",
      "Accuracy of  bird : 46.90 %\n",
      "Accuracy of   cat : 44.60 %\n",
      "Accuracy of  deer : 42.60 %\n",
      "Accuracy of   dog : 58.00 %\n",
      "Accuracy of  frog : 78.90 %\n",
      "Accuracy of horse : 77.10 %\n",
      "Accuracy of  ship : 62.30 %\n",
      "Accuracy of truck : 78.90 %\n",
      "65.49999999999999\n",
      "[Epoch : 11, Iter:    50] loss: 1.834\n",
      "[Epoch : 12, Iter:    50] loss: 1.825\n",
      "[Epoch : 13, Iter:    50] loss: 1.817\n",
      "[Epoch : 14, Iter:    50] loss: 1.807\n",
      "[Epoch : 15, Iter:    50] loss: 1.799\n",
      "[Epoch : 16, Iter:    50] loss: 1.789\n",
      "[Epoch : 17, Iter:    50] loss: 1.785\n",
      "[Epoch : 18, Iter:    50] loss: 1.780\n",
      "[Epoch : 19, Iter:    50] loss: 1.776\n",
      "[Epoch : 20, Iter:    50] loss: 1.769\n",
      "Start test\n",
      "Accuracy of plane : 85.40 %\n",
      "Accuracy of   car : 87.10 %\n",
      "Accuracy of  bird : 58.80 %\n",
      "Accuracy of   cat : 49.60 %\n",
      "Accuracy of  deer : 62.30 %\n",
      "Accuracy of   dog : 60.10 %\n",
      "Accuracy of  frog : 85.90 %\n",
      "Accuracy of horse : 80.50 %\n",
      "Accuracy of  ship : 76.20 %\n",
      "Accuracy of truck : 80.70 %\n",
      "72.66000000000001\n",
      "[Epoch : 21, Iter:    50] loss: 1.764\n",
      "[Epoch : 22, Iter:    50] loss: 1.756\n",
      "[Epoch : 23, Iter:    50] loss: 1.754\n",
      "[Epoch : 24, Iter:    50] loss: 1.754\n",
      "[Epoch : 25, Iter:    50] loss: 1.748\n",
      "[Epoch : 26, Iter:    50] loss: 1.743\n",
      "[Epoch : 27, Iter:    50] loss: 1.740\n",
      "[Epoch : 28, Iter:    50] loss: 1.740\n",
      "[Epoch : 29, Iter:    50] loss: 1.737\n",
      "[Epoch : 30, Iter:    50] loss: 1.727\n",
      "Start test\n",
      "Accuracy of plane : 83.70 %\n",
      "Accuracy of   car : 90.00 %\n",
      "Accuracy of  bird : 66.30 %\n",
      "Accuracy of   cat : 51.00 %\n",
      "Accuracy of  deer : 56.30 %\n",
      "Accuracy of   dog : 67.40 %\n",
      "Accuracy of  frog : 85.90 %\n",
      "Accuracy of horse : 85.70 %\n",
      "Accuracy of  ship : 84.90 %\n",
      "Accuracy of truck : 84.90 %\n",
      "75.60999999999999\n",
      "[Epoch : 31, Iter:    50] loss: 1.724\n",
      "[Epoch : 32, Iter:    50] loss: 1.725\n",
      "[Epoch : 33, Iter:    50] loss: 1.721\n",
      "[Epoch : 34, Iter:    50] loss: 1.720\n",
      "[Epoch : 35, Iter:    50] loss: 1.715\n",
      "[Epoch : 36, Iter:    50] loss: 1.717\n",
      "[Epoch : 37, Iter:    50] loss: 1.713\n",
      "[Epoch : 38, Iter:    50] loss: 1.711\n",
      "[Epoch : 39, Iter:    50] loss: 1.706\n",
      "[Epoch : 40, Iter:    50] loss: 1.705\n",
      "Start test\n",
      "Accuracy of plane : 84.90 %\n",
      "Accuracy of   car : 87.20 %\n",
      "Accuracy of  bird : 69.50 %\n",
      "Accuracy of   cat : 61.50 %\n",
      "Accuracy of  deer : 63.10 %\n",
      "Accuracy of   dog : 61.30 %\n",
      "Accuracy of  frog : 85.40 %\n",
      "Accuracy of horse : 80.40 %\n",
      "Accuracy of  ship : 88.80 %\n",
      "Accuracy of truck : 86.70 %\n",
      "76.88000000000001\n",
      "[Epoch : 41, Iter:    50] loss: 1.704\n",
      "[Epoch : 42, Iter:    50] loss: 1.699\n",
      "[Epoch : 43, Iter:    50] loss: 1.699\n",
      "[Epoch : 44, Iter:    50] loss: 1.700\n",
      "[Epoch : 45, Iter:    50] loss: 1.698\n",
      "[Epoch : 46, Iter:    50] loss: 1.691\n",
      "[Epoch : 47, Iter:    50] loss: 1.693\n",
      "[Epoch : 48, Iter:    50] loss: 1.694\n",
      "[Epoch : 49, Iter:    50] loss: 1.688\n",
      "[Epoch : 50, Iter:    50] loss: 1.687\n",
      "Start test\n",
      "Accuracy of plane : 82.10 %\n",
      "Accuracy of   car : 89.00 %\n",
      "Accuracy of  bird : 66.60 %\n",
      "Accuracy of   cat : 66.50 %\n",
      "Accuracy of  deer : 73.90 %\n",
      "Accuracy of   dog : 68.60 %\n",
      "Accuracy of  frog : 89.40 %\n",
      "Accuracy of horse : 82.50 %\n",
      "Accuracy of  ship : 88.90 %\n",
      "Accuracy of truck : 87.60 %\n",
      "79.50999999999999\n",
      "[Epoch : 51, Iter:    50] loss: 1.685\n",
      "[Epoch : 52, Iter:    50] loss: 1.685\n",
      "[Epoch : 53, Iter:    50] loss: 1.680\n",
      "[Epoch : 54, Iter:    50] loss: 1.680\n",
      "[Epoch : 55, Iter:    50] loss: 1.678\n",
      "[Epoch : 56, Iter:    50] loss: 1.674\n",
      "[Epoch : 57, Iter:    50] loss: 1.675\n",
      "[Epoch : 58, Iter:    50] loss: 1.675\n",
      "[Epoch : 59, Iter:    50] loss: 1.672\n",
      "[Epoch : 60, Iter:    50] loss: 1.672\n",
      "Start test\n",
      "Accuracy of plane : 83.20 %\n",
      "Accuracy of   car : 91.50 %\n",
      "Accuracy of  bird : 71.10 %\n",
      "Accuracy of   cat : 69.70 %\n",
      "Accuracy of  deer : 75.40 %\n",
      "Accuracy of   dog : 69.00 %\n",
      "Accuracy of  frog : 85.50 %\n",
      "Accuracy of horse : 81.20 %\n",
      "Accuracy of  ship : 90.20 %\n",
      "Accuracy of truck : 84.50 %\n",
      "80.13000000000001\n",
      "[Epoch : 61, Iter:    50] loss: 1.669\n",
      "[Epoch : 62, Iter:    50] loss: 1.669\n",
      "[Epoch : 63, Iter:    50] loss: 1.667\n",
      "[Epoch : 64, Iter:    50] loss: 1.665\n",
      "[Epoch : 65, Iter:    50] loss: 1.666\n",
      "[Epoch : 66, Iter:    50] loss: 1.664\n",
      "[Epoch : 67, Iter:    50] loss: 1.663\n",
      "[Epoch : 68, Iter:    50] loss: 1.661\n",
      "[Epoch : 69, Iter:    50] loss: 1.659\n",
      "[Epoch : 70, Iter:    50] loss: 1.657\n",
      "Start test\n",
      "Accuracy of plane : 83.10 %\n",
      "Accuracy of   car : 88.30 %\n",
      "Accuracy of  bird : 76.00 %\n",
      "Accuracy of   cat : 63.00 %\n",
      "Accuracy of  deer : 74.90 %\n",
      "Accuracy of   dog : 73.90 %\n",
      "Accuracy of  frog : 83.10 %\n",
      "Accuracy of horse : 84.40 %\n",
      "Accuracy of  ship : 91.50 %\n",
      "Accuracy of truck : 89.20 %\n",
      "80.74000000000001\n",
      "[Epoch : 71, Iter:    50] loss: 1.657\n",
      "[Epoch : 72, Iter:    50] loss: 1.655\n",
      "[Epoch : 73, Iter:    50] loss: 1.652\n",
      "[Epoch : 74, Iter:    50] loss: 1.654\n",
      "[Epoch : 75, Iter:    50] loss: 1.653\n",
      "[Epoch : 76, Iter:    50] loss: 1.650\n",
      "[Epoch : 77, Iter:    50] loss: 1.651\n",
      "[Epoch : 78, Iter:    50] loss: 1.650\n",
      "[Epoch : 79, Iter:    50] loss: 1.648\n",
      "[Epoch : 80, Iter:    50] loss: 1.649\n",
      "Start test\n",
      "Accuracy of plane : 86.20 %\n",
      "Accuracy of   car : 92.50 %\n",
      "Accuracy of  bird : 68.00 %\n",
      "Accuracy of   cat : 63.00 %\n",
      "Accuracy of  deer : 80.90 %\n",
      "Accuracy of   dog : 75.70 %\n",
      "Accuracy of  frog : 90.30 %\n",
      "Accuracy of horse : 84.30 %\n",
      "Accuracy of  ship : 87.50 %\n",
      "Accuracy of truck : 84.10 %\n",
      "81.25000000000001\n",
      "[Epoch : 81, Iter:    50] loss: 1.644\n",
      "[Epoch : 82, Iter:    50] loss: 1.645\n",
      "[Epoch : 83, Iter:    50] loss: 1.644\n",
      "[Epoch : 84, Iter:    50] loss: 1.642\n",
      "[Epoch : 85, Iter:    50] loss: 1.642\n",
      "[Epoch : 86, Iter:    50] loss: 1.642\n",
      "[Epoch : 87, Iter:    50] loss: 1.643\n",
      "[Epoch : 88, Iter:    50] loss: 1.639\n",
      "[Epoch : 89, Iter:    50] loss: 1.640\n",
      "[Epoch : 90, Iter:    50] loss: 1.639\n",
      "Start test\n",
      "Accuracy of plane : 88.10 %\n",
      "Accuracy of   car : 91.10 %\n",
      "Accuracy of  bird : 69.60 %\n",
      "Accuracy of   cat : 60.40 %\n",
      "Accuracy of  deer : 85.80 %\n",
      "Accuracy of   dog : 73.50 %\n",
      "Accuracy of  frog : 86.00 %\n",
      "Accuracy of horse : 88.60 %\n",
      "Accuracy of  ship : 90.50 %\n",
      "Accuracy of truck : 91.30 %\n",
      "82.48999999999998\n",
      "[Epoch : 91, Iter:    50] loss: 1.636\n",
      "[Epoch : 92, Iter:    50] loss: 1.635\n",
      "[Epoch : 93, Iter:    50] loss: 1.634\n",
      "[Epoch : 94, Iter:    50] loss: 1.634\n",
      "[Epoch : 95, Iter:    50] loss: 1.634\n",
      "[Epoch : 96, Iter:    50] loss: 1.631\n",
      "[Epoch : 97, Iter:    50] loss: 1.632\n",
      "[Epoch : 98, Iter:    50] loss: 1.633\n",
      "[Epoch : 99, Iter:    50] loss: 1.630\n",
      "[Epoch : 100, Iter:    50] loss: 1.628\n",
      "Start test\n",
      "Accuracy of plane : 86.80 %\n",
      "Accuracy of   car : 92.00 %\n",
      "Accuracy of  bird : 75.70 %\n",
      "Accuracy of   cat : 73.10 %\n",
      "Accuracy of  deer : 79.70 %\n",
      "Accuracy of   dog : 63.70 %\n",
      "Accuracy of  frog : 87.50 %\n",
      "Accuracy of horse : 83.30 %\n",
      "Accuracy of  ship : 91.60 %\n",
      "Accuracy of truck : 90.40 %\n",
      "82.38000000000001\n",
      "[Epoch : 101, Iter:    50] loss: 1.630\n",
      "[Epoch : 102, Iter:    50] loss: 1.628\n",
      "[Epoch : 103, Iter:    50] loss: 1.626\n",
      "[Epoch : 104, Iter:    50] loss: 1.624\n",
      "[Epoch : 105, Iter:    50] loss: 1.626\n",
      "[Epoch : 106, Iter:    50] loss: 1.623\n",
      "[Epoch : 107, Iter:    50] loss: 1.624\n",
      "[Epoch : 108, Iter:    50] loss: 1.621\n",
      "[Epoch : 109, Iter:    50] loss: 1.621\n",
      "[Epoch : 110, Iter:    50] loss: 1.622\n",
      "Start test\n",
      "Accuracy of plane : 86.30 %\n",
      "Accuracy of   car : 90.70 %\n",
      "Accuracy of  bird : 73.70 %\n",
      "Accuracy of   cat : 69.80 %\n",
      "Accuracy of  deer : 82.40 %\n",
      "Accuracy of   dog : 73.90 %\n",
      "Accuracy of  frog : 87.80 %\n",
      "Accuracy of horse : 87.90 %\n",
      "Accuracy of  ship : 90.80 %\n",
      "Accuracy of truck : 91.50 %\n",
      "83.47999999999999\n",
      "[Epoch : 111, Iter:    50] loss: 1.619\n",
      "[Epoch : 112, Iter:    50] loss: 1.618\n",
      "[Epoch : 113, Iter:    50] loss: 1.620\n",
      "[Epoch : 114, Iter:    50] loss: 1.616\n",
      "[Epoch : 115, Iter:    50] loss: 1.618\n",
      "[Epoch : 116, Iter:    50] loss: 1.618\n",
      "[Epoch : 117, Iter:    50] loss: 1.616\n",
      "[Epoch : 118, Iter:    50] loss: 1.614\n",
      "[Epoch : 119, Iter:    50] loss: 1.615\n",
      "[Epoch : 120, Iter:    50] loss: 1.615\n",
      "Start test\n",
      "Accuracy of plane : 85.90 %\n",
      "Accuracy of   car : 91.40 %\n",
      "Accuracy of  bird : 73.10 %\n",
      "Accuracy of   cat : 67.40 %\n",
      "Accuracy of  deer : 84.30 %\n",
      "Accuracy of   dog : 74.10 %\n",
      "Accuracy of  frog : 86.80 %\n",
      "Accuracy of horse : 91.40 %\n",
      "Accuracy of  ship : 93.50 %\n",
      "Accuracy of truck : 91.10 %\n",
      "83.9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 121, Iter:    50] loss: 1.613\n",
      "[Epoch : 122, Iter:    50] loss: 1.613\n",
      "[Epoch : 123, Iter:    50] loss: 1.614\n",
      "[Epoch : 124, Iter:    50] loss: 1.613\n",
      "[Epoch : 125, Iter:    50] loss: 1.612\n",
      "[Epoch : 126, Iter:    50] loss: 1.613\n",
      "[Epoch : 127, Iter:    50] loss: 1.610\n",
      "[Epoch : 128, Iter:    50] loss: 1.607\n",
      "[Epoch : 129, Iter:    50] loss: 1.609\n",
      "[Epoch : 130, Iter:    50] loss: 1.610\n",
      "Start test\n",
      "Accuracy of plane : 87.70 %\n",
      "Accuracy of   car : 92.90 %\n",
      "Accuracy of  bird : 73.40 %\n",
      "Accuracy of   cat : 73.80 %\n",
      "Accuracy of  deer : 87.90 %\n",
      "Accuracy of   dog : 68.50 %\n",
      "Accuracy of  frog : 88.60 %\n",
      "Accuracy of horse : 86.80 %\n",
      "Accuracy of  ship : 90.90 %\n",
      "Accuracy of truck : 89.60 %\n",
      "84.00999999999999\n",
      "[Epoch : 131, Iter:    50] loss: 1.608\n",
      "[Epoch : 132, Iter:    50] loss: 1.610\n",
      "[Epoch : 133, Iter:    50] loss: 1.607\n",
      "[Epoch : 134, Iter:    50] loss: 1.606\n",
      "[Epoch : 135, Iter:    50] loss: 1.607\n",
      "[Epoch : 136, Iter:    50] loss: 1.606\n",
      "[Epoch : 137, Iter:    50] loss: 1.607\n",
      "[Epoch : 138, Iter:    50] loss: 1.606\n",
      "[Epoch : 139, Iter:    50] loss: 1.605\n",
      "[Epoch : 140, Iter:    50] loss: 1.606\n",
      "Start test\n",
      "Accuracy of plane : 84.80 %\n",
      "Accuracy of   car : 93.10 %\n",
      "Accuracy of  bird : 74.40 %\n",
      "Accuracy of   cat : 71.40 %\n",
      "Accuracy of  deer : 83.90 %\n",
      "Accuracy of   dog : 70.50 %\n",
      "Accuracy of  frog : 91.90 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 92.30 %\n",
      "Accuracy of truck : 89.90 %\n",
      "83.94999999999999\n",
      "[Epoch : 141, Iter:    50] loss: 1.604\n",
      "[Epoch : 142, Iter:    50] loss: 1.603\n",
      "[Epoch : 143, Iter:    50] loss: 1.603\n",
      "[Epoch : 144, Iter:    50] loss: 1.601\n",
      "[Epoch : 145, Iter:    50] loss: 1.602\n",
      "[Epoch : 146, Iter:    50] loss: 1.602\n",
      "[Epoch : 147, Iter:    50] loss: 1.601\n",
      "[Epoch : 148, Iter:    50] loss: 1.600\n",
      "[Epoch : 149, Iter:    50] loss: 1.601\n",
      "[Epoch : 150, Iter:    50] loss: 1.598\n",
      "Start test\n",
      "Accuracy of plane : 86.70 %\n",
      "Accuracy of   car : 93.00 %\n",
      "Accuracy of  bird : 77.30 %\n",
      "Accuracy of   cat : 67.90 %\n",
      "Accuracy of  deer : 87.70 %\n",
      "Accuracy of   dog : 73.80 %\n",
      "Accuracy of  frog : 88.90 %\n",
      "Accuracy of horse : 87.40 %\n",
      "Accuracy of  ship : 92.10 %\n",
      "Accuracy of truck : 88.90 %\n",
      "84.37\n",
      "[Epoch : 151, Iter:    50] loss: 1.598\n",
      "[Epoch : 152, Iter:    50] loss: 1.599\n",
      "[Epoch : 153, Iter:    50] loss: 1.599\n",
      "[Epoch : 154, Iter:    50] loss: 1.599\n",
      "[Epoch : 155, Iter:    50] loss: 1.597\n",
      "[Epoch : 156, Iter:    50] loss: 1.599\n",
      "[Epoch : 157, Iter:    50] loss: 1.597\n",
      "[Epoch : 158, Iter:    50] loss: 1.595\n",
      "[Epoch : 159, Iter:    50] loss: 1.596\n",
      "[Epoch : 160, Iter:    50] loss: 1.595\n",
      "Start test\n",
      "Accuracy of plane : 89.30 %\n",
      "Accuracy of   car : 94.30 %\n",
      "Accuracy of  bird : 76.90 %\n",
      "Accuracy of   cat : 72.10 %\n",
      "Accuracy of  deer : 83.30 %\n",
      "Accuracy of   dog : 73.40 %\n",
      "Accuracy of  frog : 88.10 %\n",
      "Accuracy of horse : 88.00 %\n",
      "Accuracy of  ship : 91.70 %\n",
      "Accuracy of truck : 90.00 %\n",
      "84.71000000000001\n",
      "[Epoch : 161, Iter:    50] loss: 1.596\n",
      "[Epoch : 162, Iter:    50] loss: 1.594\n",
      "[Epoch : 163, Iter:    50] loss: 1.594\n",
      "[Epoch : 164, Iter:    50] loss: 1.595\n",
      "[Epoch : 165, Iter:    50] loss: 1.594\n",
      "[Epoch : 166, Iter:    50] loss: 1.594\n",
      "[Epoch : 167, Iter:    50] loss: 1.593\n",
      "[Epoch : 168, Iter:    50] loss: 1.593\n",
      "[Epoch : 169, Iter:    50] loss: 1.592\n",
      "[Epoch : 170, Iter:    50] loss: 1.593\n",
      "Start test\n",
      "Accuracy of plane : 89.20 %\n",
      "Accuracy of   car : 93.00 %\n",
      "Accuracy of  bird : 76.20 %\n",
      "Accuracy of   cat : 64.20 %\n",
      "Accuracy of  deer : 85.20 %\n",
      "Accuracy of   dog : 77.00 %\n",
      "Accuracy of  frog : 92.70 %\n",
      "Accuracy of horse : 88.60 %\n",
      "Accuracy of  ship : 91.50 %\n",
      "Accuracy of truck : 89.60 %\n",
      "84.72\n",
      "[Epoch : 171, Iter:    50] loss: 1.591\n",
      "[Epoch : 172, Iter:    50] loss: 1.592\n",
      "[Epoch : 173, Iter:    50] loss: 1.591\n",
      "[Epoch : 174, Iter:    50] loss: 1.592\n",
      "[Epoch : 175, Iter:    50] loss: 1.589\n",
      "[Epoch : 176, Iter:    50] loss: 1.588\n",
      "[Epoch : 177, Iter:    50] loss: 1.589\n",
      "[Epoch : 178, Iter:    50] loss: 1.590\n",
      "[Epoch : 179, Iter:    50] loss: 1.590\n",
      "[Epoch : 180, Iter:    50] loss: 1.589\n",
      "Start test\n",
      "Accuracy of plane : 87.00 %\n",
      "Accuracy of   car : 90.30 %\n",
      "Accuracy of  bird : 79.40 %\n",
      "Accuracy of   cat : 63.50 %\n",
      "Accuracy of  deer : 85.90 %\n",
      "Accuracy of   dog : 76.60 %\n",
      "Accuracy of  frog : 91.10 %\n",
      "Accuracy of horse : 86.40 %\n",
      "Accuracy of  ship : 94.00 %\n",
      "Accuracy of truck : 91.60 %\n",
      "84.58000000000001\n",
      "[Epoch : 181, Iter:    50] loss: 1.589\n",
      "[Epoch : 182, Iter:    50] loss: 1.588\n",
      "[Epoch : 183, Iter:    50] loss: 1.588\n",
      "[Epoch : 184, Iter:    50] loss: 1.588\n",
      "[Epoch : 185, Iter:    50] loss: 1.588\n",
      "[Epoch : 186, Iter:    50] loss: 1.586\n",
      "[Epoch : 187, Iter:    50] loss: 1.585\n",
      "[Epoch : 188, Iter:    50] loss: 1.586\n",
      "[Epoch : 189, Iter:    50] loss: 1.588\n",
      "[Epoch : 190, Iter:    50] loss: 1.586\n",
      "Start test\n",
      "Accuracy of plane : 89.00 %\n",
      "Accuracy of   car : 93.60 %\n",
      "Accuracy of  bird : 79.70 %\n",
      "Accuracy of   cat : 67.80 %\n",
      "Accuracy of  deer : 86.80 %\n",
      "Accuracy of   dog : 76.50 %\n",
      "Accuracy of  frog : 85.70 %\n",
      "Accuracy of horse : 90.40 %\n",
      "Accuracy of  ship : 90.10 %\n",
      "Accuracy of truck : 90.50 %\n",
      "85.01\n",
      "[Epoch : 191, Iter:    50] loss: 1.586\n",
      "[Epoch : 192, Iter:    50] loss: 1.588\n",
      "[Epoch : 193, Iter:    50] loss: 1.585\n",
      "[Epoch : 194, Iter:    50] loss: 1.585\n",
      "[Epoch : 195, Iter:    50] loss: 1.585\n",
      "[Epoch : 196, Iter:    50] loss: 1.584\n",
      "[Epoch : 197, Iter:    50] loss: 1.584\n",
      "[Epoch : 198, Iter:    50] loss: 1.584\n",
      "[Epoch : 199, Iter:    50] loss: 1.584\n",
      "[Epoch : 200, Iter:    50] loss: 1.584\n",
      "Start test\n",
      "Accuracy of plane : 87.90 %\n",
      "Accuracy of   car : 93.60 %\n",
      "Accuracy of  bird : 78.20 %\n",
      "Accuracy of   cat : 72.70 %\n",
      "Accuracy of  deer : 85.20 %\n",
      "Accuracy of   dog : 71.90 %\n",
      "Accuracy of  frog : 91.20 %\n",
      "Accuracy of horse : 88.40 %\n",
      "Accuracy of  ship : 91.80 %\n",
      "Accuracy of truck : 89.00 %\n",
      "84.99\n",
      "[Epoch : 201, Iter:    50] loss: 1.584\n",
      "[Epoch : 202, Iter:    50] loss: 1.584\n",
      "[Epoch : 203, Iter:    50] loss: 1.582\n",
      "[Epoch : 204, Iter:    50] loss: 1.583\n",
      "[Epoch : 205, Iter:    50] loss: 1.583\n",
      "[Epoch : 206, Iter:    50] loss: 1.582\n",
      "[Epoch : 207, Iter:    50] loss: 1.583\n",
      "[Epoch : 208, Iter:    50] loss: 1.582\n",
      "[Epoch : 209, Iter:    50] loss: 1.582\n",
      "[Epoch : 210, Iter:    50] loss: 1.582\n",
      "Start test\n",
      "Accuracy of plane : 86.60 %\n",
      "Accuracy of   car : 93.30 %\n",
      "Accuracy of  bird : 80.20 %\n",
      "Accuracy of   cat : 67.90 %\n",
      "Accuracy of  deer : 85.00 %\n",
      "Accuracy of   dog : 77.60 %\n",
      "Accuracy of  frog : 90.50 %\n",
      "Accuracy of horse : 88.00 %\n",
      "Accuracy of  ship : 92.20 %\n",
      "Accuracy of truck : 92.00 %\n",
      "85.33000000000001\n",
      "[Epoch : 211, Iter:    50] loss: 1.581\n",
      "[Epoch : 212, Iter:    50] loss: 1.582\n",
      "[Epoch : 213, Iter:    50] loss: 1.582\n",
      "[Epoch : 214, Iter:    50] loss: 1.580\n",
      "[Epoch : 215, Iter:    50] loss: 1.581\n",
      "[Epoch : 216, Iter:    50] loss: 1.579\n",
      "[Epoch : 217, Iter:    50] loss: 1.580\n",
      "[Epoch : 218, Iter:    50] loss: 1.580\n",
      "[Epoch : 219, Iter:    50] loss: 1.580\n",
      "[Epoch : 220, Iter:    50] loss: 1.580\n",
      "Start test\n",
      "Accuracy of plane : 88.60 %\n",
      "Accuracy of   car : 93.20 %\n",
      "Accuracy of  bird : 75.80 %\n",
      "Accuracy of   cat : 71.20 %\n",
      "Accuracy of  deer : 85.40 %\n",
      "Accuracy of   dog : 75.20 %\n",
      "Accuracy of  frog : 90.30 %\n",
      "Accuracy of horse : 90.00 %\n",
      "Accuracy of  ship : 89.40 %\n",
      "Accuracy of truck : 91.40 %\n",
      "85.05\n",
      "[Epoch : 221, Iter:    50] loss: 1.580\n",
      "[Epoch : 222, Iter:    50] loss: 1.579\n",
      "[Epoch : 223, Iter:    50] loss: 1.579\n",
      "[Epoch : 224, Iter:    50] loss: 1.580\n",
      "[Epoch : 225, Iter:    50] loss: 1.579\n",
      "[Epoch : 226, Iter:    50] loss: 1.579\n",
      "[Epoch : 227, Iter:    50] loss: 1.579\n",
      "[Epoch : 228, Iter:    50] loss: 1.579\n",
      "[Epoch : 229, Iter:    50] loss: 1.579\n",
      "[Epoch : 230, Iter:    50] loss: 1.578\n",
      "Start test\n",
      "Accuracy of plane : 90.40 %\n",
      "Accuracy of   car : 92.00 %\n",
      "Accuracy of  bird : 77.70 %\n",
      "Accuracy of   cat : 72.30 %\n",
      "Accuracy of  deer : 86.70 %\n",
      "Accuracy of   dog : 72.50 %\n",
      "Accuracy of  frog : 89.40 %\n",
      "Accuracy of horse : 88.40 %\n",
      "Accuracy of  ship : 91.00 %\n",
      "Accuracy of truck : 91.20 %\n",
      "85.16\n",
      "[Epoch : 231, Iter:    50] loss: 1.578\n",
      "[Epoch : 232, Iter:    50] loss: 1.578\n",
      "[Epoch : 233, Iter:    50] loss: 1.577\n",
      "[Epoch : 234, Iter:    50] loss: 1.578\n",
      "[Epoch : 235, Iter:    50] loss: 1.578\n",
      "[Epoch : 236, Iter:    50] loss: 1.579\n",
      "[Epoch : 237, Iter:    50] loss: 1.577\n",
      "[Epoch : 238, Iter:    50] loss: 1.577\n",
      "[Epoch : 239, Iter:    50] loss: 1.576\n",
      "[Epoch : 240, Iter:    50] loss: 1.577\n",
      "Start test\n",
      "Accuracy of plane : 87.40 %\n",
      "Accuracy of   car : 91.90 %\n",
      "Accuracy of  bird : 76.20 %\n",
      "Accuracy of   cat : 66.70 %\n",
      "Accuracy of  deer : 86.90 %\n",
      "Accuracy of   dog : 77.60 %\n",
      "Accuracy of  frog : 91.20 %\n",
      "Accuracy of horse : 90.50 %\n",
      "Accuracy of  ship : 92.50 %\n",
      "Accuracy of truck : 91.60 %\n",
      "85.25000000000001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 241, Iter:    50] loss: 1.577\n",
      "[Epoch : 242, Iter:    50] loss: 1.576\n",
      "[Epoch : 243, Iter:    50] loss: 1.576\n",
      "[Epoch : 244, Iter:    50] loss: 1.577\n",
      "[Epoch : 245, Iter:    50] loss: 1.576\n",
      "[Epoch : 246, Iter:    50] loss: 1.577\n",
      "[Epoch : 247, Iter:    50] loss: 1.576\n",
      "[Epoch : 248, Iter:    50] loss: 1.577\n",
      "[Epoch : 249, Iter:    50] loss: 1.576\n",
      "[Epoch : 250, Iter:    50] loss: 1.576\n",
      "Start test\n",
      "Accuracy of plane : 89.30 %\n",
      "Accuracy of   car : 92.60 %\n",
      "Accuracy of  bird : 75.60 %\n",
      "Accuracy of   cat : 67.30 %\n",
      "Accuracy of  deer : 86.00 %\n",
      "Accuracy of   dog : 78.30 %\n",
      "Accuracy of  frog : 91.10 %\n",
      "Accuracy of horse : 89.40 %\n",
      "Accuracy of  ship : 91.80 %\n",
      "Accuracy of truck : 90.90 %\n",
      "85.22999999999999\n",
      "[Epoch : 251, Iter:    50] loss: 1.576\n",
      "[Epoch : 252, Iter:    50] loss: 1.576\n",
      "[Epoch : 253, Iter:    50] loss: 1.576\n",
      "[Epoch : 254, Iter:    50] loss: 1.574\n",
      "[Epoch : 255, Iter:    50] loss: 1.575\n",
      "[Epoch : 256, Iter:    50] loss: 1.574\n",
      "[Epoch : 257, Iter:    50] loss: 1.576\n",
      "[Epoch : 258, Iter:    50] loss: 1.574\n",
      "[Epoch : 259, Iter:    50] loss: 1.575\n",
      "[Epoch : 260, Iter:    50] loss: 1.574\n",
      "Start test\n",
      "Accuracy of plane : 88.20 %\n",
      "Accuracy of   car : 93.10 %\n",
      "Accuracy of  bird : 76.10 %\n",
      "Accuracy of   cat : 70.60 %\n",
      "Accuracy of  deer : 85.70 %\n",
      "Accuracy of   dog : 75.60 %\n",
      "Accuracy of  frog : 91.20 %\n",
      "Accuracy of horse : 88.70 %\n",
      "Accuracy of  ship : 90.50 %\n",
      "Accuracy of truck : 92.10 %\n",
      "85.18\n",
      "[Epoch : 261, Iter:    50] loss: 1.574\n",
      "[Epoch : 262, Iter:    50] loss: 1.574\n",
      "[Epoch : 263, Iter:    50] loss: 1.574\n",
      "[Epoch : 264, Iter:    50] loss: 1.575\n",
      "[Epoch : 265, Iter:    50] loss: 1.573\n",
      "[Epoch : 266, Iter:    50] loss: 1.574\n",
      "[Epoch : 267, Iter:    50] loss: 1.573\n",
      "[Epoch : 268, Iter:    50] loss: 1.574\n",
      "[Epoch : 269, Iter:    50] loss: 1.573\n",
      "[Epoch : 270, Iter:    50] loss: 1.574\n",
      "Start test\n",
      "Accuracy of plane : 87.70 %\n",
      "Accuracy of   car : 92.20 %\n",
      "Accuracy of  bird : 78.00 %\n",
      "Accuracy of   cat : 70.30 %\n",
      "Accuracy of  deer : 87.30 %\n",
      "Accuracy of   dog : 76.00 %\n",
      "Accuracy of  frog : 89.80 %\n",
      "Accuracy of horse : 87.00 %\n",
      "Accuracy of  ship : 92.50 %\n",
      "Accuracy of truck : 91.90 %\n",
      "85.27000000000001\n",
      "[Epoch : 271, Iter:    50] loss: 1.573\n",
      "[Epoch : 272, Iter:    50] loss: 1.575\n",
      "[Epoch : 273, Iter:    50] loss: 1.573\n",
      "[Epoch : 274, Iter:    50] loss: 1.574\n",
      "[Epoch : 275, Iter:    50] loss: 1.573\n",
      "[Epoch : 276, Iter:    50] loss: 1.574\n",
      "[Epoch : 277, Iter:    50] loss: 1.573\n",
      "[Epoch : 278, Iter:    50] loss: 1.573\n",
      "[Epoch : 279, Iter:    50] loss: 1.572\n",
      "[Epoch : 280, Iter:    50] loss: 1.573\n",
      "Start test\n",
      "Accuracy of plane : 89.10 %\n",
      "Accuracy of   car : 92.60 %\n",
      "Accuracy of  bird : 77.60 %\n",
      "Accuracy of   cat : 67.80 %\n",
      "Accuracy of  deer : 85.10 %\n",
      "Accuracy of   dog : 76.70 %\n",
      "Accuracy of  frog : 91.20 %\n",
      "Accuracy of horse : 89.80 %\n",
      "Accuracy of  ship : 91.50 %\n",
      "Accuracy of truck : 91.40 %\n",
      "85.28\n",
      "[Epoch : 281, Iter:    50] loss: 1.573\n",
      "[Epoch : 282, Iter:    50] loss: 1.574\n",
      "[Epoch : 283, Iter:    50] loss: 1.571\n",
      "[Epoch : 284, Iter:    50] loss: 1.573\n",
      "[Epoch : 285, Iter:    50] loss: 1.573\n",
      "[Epoch : 286, Iter:    50] loss: 1.573\n",
      "[Epoch : 287, Iter:    50] loss: 1.572\n",
      "[Epoch : 288, Iter:    50] loss: 1.572\n",
      "[Epoch : 289, Iter:    50] loss: 1.573\n",
      "[Epoch : 290, Iter:    50] loss: 1.572\n",
      "Start test\n",
      "Accuracy of plane : 89.80 %\n",
      "Accuracy of   car : 93.10 %\n",
      "Accuracy of  bird : 79.40 %\n",
      "Accuracy of   cat : 66.10 %\n",
      "Accuracy of  deer : 85.90 %\n",
      "Accuracy of   dog : 77.90 %\n",
      "Accuracy of  frog : 90.50 %\n",
      "Accuracy of horse : 88.90 %\n",
      "Accuracy of  ship : 92.00 %\n",
      "Accuracy of truck : 92.10 %\n",
      "85.57000000000001\n",
      "[Epoch : 291, Iter:    50] loss: 1.573\n",
      "[Epoch : 292, Iter:    50] loss: 1.572\n",
      "[Epoch : 293, Iter:    50] loss: 1.573\n",
      "[Epoch : 294, Iter:    50] loss: 1.573\n",
      "[Epoch : 295, Iter:    50] loss: 1.572\n",
      "[Epoch : 296, Iter:    50] loss: 1.571\n",
      "[Epoch : 297, Iter:    50] loss: 1.572\n",
      "[Epoch : 298, Iter:    50] loss: 1.571\n",
      "[Epoch : 299, Iter:    50] loss: 1.572\n",
      "[Epoch : 300, Iter:    50] loss: 1.571\n",
      "Start test\n",
      "Accuracy of plane : 88.80 %\n",
      "Accuracy of   car : 92.60 %\n",
      "Accuracy of  bird : 77.10 %\n",
      "Accuracy of   cat : 70.20 %\n",
      "Accuracy of  deer : 86.20 %\n",
      "Accuracy of   dog : 76.00 %\n",
      "Accuracy of  frog : 91.00 %\n",
      "Accuracy of horse : 88.90 %\n",
      "Accuracy of  ship : 91.70 %\n",
      "Accuracy of truck : 91.70 %\n",
      "85.42\n",
      "\n",
      "Net_MCDO\n",
      "Let's use 4 GPUs!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "199401ce828442f5ac203df6a5acc0bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=300), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 1, Iter:    50] loss: 2.207\n",
      "[Epoch : 2, Iter:    50] loss: 2.065\n",
      "[Epoch : 3, Iter:    50] loss: 2.010\n",
      "[Epoch : 4, Iter:    50] loss: 1.975\n",
      "[Epoch : 5, Iter:    50] loss: 1.943\n",
      "[Epoch : 6, Iter:    50] loss: 1.921\n",
      "[Epoch : 7, Iter:    50] loss: 1.896\n",
      "[Epoch : 8, Iter:    50] loss: 1.883\n",
      "[Epoch : 9, Iter:    50] loss: 1.865\n",
      "[Epoch : 10, Iter:    50] loss: 1.862\n",
      "Start test\n",
      "Accuracy of plane : 72.50 %\n",
      "Accuracy of   car : 81.70 %\n",
      "Accuracy of  bird : 43.20 %\n",
      "Accuracy of   cat : 39.30 %\n",
      "Accuracy of  deer : 50.70 %\n",
      "Accuracy of   dog : 57.20 %\n",
      "Accuracy of  frog : 72.00 %\n",
      "Accuracy of horse : 73.50 %\n",
      "Accuracy of  ship : 73.10 %\n",
      "Accuracy of truck : 82.30 %\n",
      "64.55\n",
      "[Epoch : 11, Iter:    50] loss: 1.840\n",
      "[Epoch : 12, Iter:    50] loss: 1.836\n",
      "[Epoch : 13, Iter:    50] loss: 1.825\n",
      "[Epoch : 14, Iter:    50] loss: 1.812\n",
      "[Epoch : 15, Iter:    50] loss: 1.809\n",
      "[Epoch : 16, Iter:    50] loss: 1.804\n",
      "[Epoch : 17, Iter:    50] loss: 1.798\n",
      "[Epoch : 18, Iter:    50] loss: 1.790\n",
      "[Epoch : 19, Iter:    50] loss: 1.784\n",
      "[Epoch : 20, Iter:    50] loss: 1.782\n",
      "Start test\n",
      "Accuracy of plane : 79.30 %\n",
      "Accuracy of   car : 87.20 %\n",
      "Accuracy of  bird : 59.50 %\n",
      "Accuracy of   cat : 45.90 %\n",
      "Accuracy of  deer : 53.40 %\n",
      "Accuracy of   dog : 64.30 %\n",
      "Accuracy of  frog : 81.90 %\n",
      "Accuracy of horse : 82.60 %\n",
      "Accuracy of  ship : 83.80 %\n",
      "Accuracy of truck : 73.90 %\n",
      "71.17999999999998\n",
      "[Epoch : 21, Iter:    50] loss: 1.774\n",
      "[Epoch : 22, Iter:    50] loss: 1.771\n",
      "[Epoch : 23, Iter:    50] loss: 1.764\n",
      "[Epoch : 24, Iter:    50] loss: 1.755\n",
      "[Epoch : 25, Iter:    50] loss: 1.757\n",
      "[Epoch : 26, Iter:    50] loss: 1.751\n",
      "[Epoch : 27, Iter:    50] loss: 1.746\n",
      "[Epoch : 28, Iter:    50] loss: 1.747\n",
      "[Epoch : 29, Iter:    50] loss: 1.745\n",
      "[Epoch : 30, Iter:    50] loss: 1.746\n",
      "Start test\n",
      "Accuracy of plane : 79.50 %\n",
      "Accuracy of   car : 88.00 %\n",
      "Accuracy of  bird : 66.70 %\n",
      "Accuracy of   cat : 53.10 %\n",
      "Accuracy of  deer : 59.30 %\n",
      "Accuracy of   dog : 67.50 %\n",
      "Accuracy of  frog : 86.60 %\n",
      "Accuracy of horse : 78.00 %\n",
      "Accuracy of  ship : 87.90 %\n",
      "Accuracy of truck : 83.60 %\n",
      "75.02000000000001\n",
      "[Epoch : 31, Iter:    50] loss: 1.737\n",
      "[Epoch : 32, Iter:    50] loss: 1.732\n",
      "[Epoch : 33, Iter:    50] loss: 1.731\n",
      "[Epoch : 34, Iter:    50] loss: 1.729\n",
      "[Epoch : 35, Iter:    50] loss: 1.724\n",
      "[Epoch : 36, Iter:    50] loss: 1.726\n",
      "[Epoch : 37, Iter:    50] loss: 1.722\n",
      "[Epoch : 38, Iter:    50] loss: 1.721\n",
      "[Epoch : 39, Iter:    50] loss: 1.720\n",
      "[Epoch : 40, Iter:    50] loss: 1.717\n",
      "Start test\n",
      "Accuracy of plane : 82.10 %\n",
      "Accuracy of   car : 88.90 %\n",
      "Accuracy of  bird : 66.60 %\n",
      "Accuracy of   cat : 44.80 %\n",
      "Accuracy of  deer : 59.70 %\n",
      "Accuracy of   dog : 75.20 %\n",
      "Accuracy of  frog : 91.50 %\n",
      "Accuracy of horse : 77.60 %\n",
      "Accuracy of  ship : 90.20 %\n",
      "Accuracy of truck : 81.50 %\n",
      "75.81\n",
      "[Epoch : 41, Iter:    50] loss: 1.718\n",
      "[Epoch : 42, Iter:    50] loss: 1.714\n",
      "[Epoch : 43, Iter:    50] loss: 1.708\n",
      "[Epoch : 44, Iter:    50] loss: 1.707\n",
      "[Epoch : 45, Iter:    50] loss: 1.708\n",
      "[Epoch : 46, Iter:    50] loss: 1.705\n",
      "[Epoch : 47, Iter:    50] loss: 1.703\n",
      "[Epoch : 48, Iter:    50] loss: 1.700\n",
      "[Epoch : 49, Iter:    50] loss: 1.704\n",
      "[Epoch : 50, Iter:    50] loss: 1.699\n",
      "Start test\n",
      "Accuracy of plane : 81.70 %\n",
      "Accuracy of   car : 92.50 %\n",
      "Accuracy of  bird : 78.10 %\n",
      "Accuracy of   cat : 58.70 %\n",
      "Accuracy of  deer : 75.30 %\n",
      "Accuracy of   dog : 66.50 %\n",
      "Accuracy of  frog : 79.30 %\n",
      "Accuracy of horse : 78.50 %\n",
      "Accuracy of  ship : 88.20 %\n",
      "Accuracy of truck : 83.20 %\n",
      "78.20000000000002\n",
      "[Epoch : 51, Iter:    50] loss: 1.698\n",
      "[Epoch : 52, Iter:    50] loss: 1.696\n",
      "[Epoch : 53, Iter:    50] loss: 1.695\n",
      "[Epoch : 54, Iter:    50] loss: 1.693\n",
      "[Epoch : 55, Iter:    50] loss: 1.695\n",
      "[Epoch : 56, Iter:    50] loss: 1.689\n",
      "[Epoch : 57, Iter:    50] loss: 1.687\n",
      "[Epoch : 58, Iter:    50] loss: 1.691\n",
      "[Epoch : 59, Iter:    50] loss: 1.687\n",
      "[Epoch : 60, Iter:    50] loss: 1.685\n",
      "Start test\n",
      "Accuracy of plane : 89.00 %\n",
      "Accuracy of   car : 89.60 %\n",
      "Accuracy of  bird : 69.10 %\n",
      "Accuracy of   cat : 62.60 %\n",
      "Accuracy of  deer : 77.50 %\n",
      "Accuracy of   dog : 68.70 %\n",
      "Accuracy of  frog : 87.10 %\n",
      "Accuracy of horse : 84.90 %\n",
      "Accuracy of  ship : 87.40 %\n",
      "Accuracy of truck : 84.10 %\n",
      "80.0\n",
      "[Epoch : 61, Iter:    50] loss: 1.683\n",
      "[Epoch : 62, Iter:    50] loss: 1.686\n",
      "[Epoch : 63, Iter:    50] loss: 1.680\n",
      "[Epoch : 64, Iter:    50] loss: 1.681\n",
      "[Epoch : 65, Iter:    50] loss: 1.678\n",
      "[Epoch : 66, Iter:    50] loss: 1.679\n",
      "[Epoch : 67, Iter:    50] loss: 1.677\n",
      "[Epoch : 68, Iter:    50] loss: 1.676\n",
      "[Epoch : 69, Iter:    50] loss: 1.675\n",
      "[Epoch : 70, Iter:    50] loss: 1.673\n",
      "Start test\n",
      "Accuracy of plane : 83.30 %\n",
      "Accuracy of   car : 93.70 %\n",
      "Accuracy of  bird : 65.90 %\n",
      "Accuracy of   cat : 66.20 %\n",
      "Accuracy of  deer : 72.20 %\n",
      "Accuracy of   dog : 69.20 %\n",
      "Accuracy of  frog : 89.10 %\n",
      "Accuracy of horse : 87.30 %\n",
      "Accuracy of  ship : 90.70 %\n",
      "Accuracy of truck : 84.20 %\n",
      "80.18\n",
      "[Epoch : 71, Iter:    50] loss: 1.673\n",
      "[Epoch : 72, Iter:    50] loss: 1.672\n",
      "[Epoch : 73, Iter:    50] loss: 1.670\n",
      "[Epoch : 74, Iter:    50] loss: 1.670\n",
      "[Epoch : 75, Iter:    50] loss: 1.665\n",
      "[Epoch : 76, Iter:    50] loss: 1.665\n",
      "[Epoch : 77, Iter:    50] loss: 1.666\n",
      "[Epoch : 78, Iter:    50] loss: 1.664\n",
      "[Epoch : 79, Iter:    50] loss: 1.664\n",
      "[Epoch : 80, Iter:    50] loss: 1.665\n",
      "Start test\n",
      "Accuracy of plane : 86.90 %\n",
      "Accuracy of   car : 92.60 %\n",
      "Accuracy of  bird : 64.90 %\n",
      "Accuracy of   cat : 70.40 %\n",
      "Accuracy of  deer : 83.40 %\n",
      "Accuracy of   dog : 65.40 %\n",
      "Accuracy of  frog : 88.80 %\n",
      "Accuracy of horse : 82.40 %\n",
      "Accuracy of  ship : 91.20 %\n",
      "Accuracy of truck : 83.80 %\n",
      "80.97999999999999\n",
      "[Epoch : 81, Iter:    50] loss: 1.665\n",
      "[Epoch : 82, Iter:    50] loss: 1.662\n",
      "[Epoch : 83, Iter:    50] loss: 1.662\n",
      "[Epoch : 84, Iter:    50] loss: 1.659\n",
      "[Epoch : 85, Iter:    50] loss: 1.660\n",
      "[Epoch : 86, Iter:    50] loss: 1.659\n",
      "[Epoch : 87, Iter:    50] loss: 1.659\n",
      "[Epoch : 88, Iter:    50] loss: 1.657\n",
      "[Epoch : 89, Iter:    50] loss: 1.657\n",
      "[Epoch : 90, Iter:    50] loss: 1.656\n",
      "Start test\n",
      "Accuracy of plane : 81.70 %\n",
      "Accuracy of   car : 90.10 %\n",
      "Accuracy of  bird : 61.20 %\n",
      "Accuracy of   cat : 58.80 %\n",
      "Accuracy of  deer : 85.80 %\n",
      "Accuracy of   dog : 76.40 %\n",
      "Accuracy of  frog : 90.20 %\n",
      "Accuracy of horse : 87.80 %\n",
      "Accuracy of  ship : 90.90 %\n",
      "Accuracy of truck : 88.20 %\n",
      "81.11\n",
      "[Epoch : 91, Iter:    50] loss: 1.656\n",
      "[Epoch : 92, Iter:    50] loss: 1.654\n",
      "[Epoch : 93, Iter:    50] loss: 1.652\n",
      "[Epoch : 94, Iter:    50] loss: 1.653\n",
      "[Epoch : 95, Iter:    50] loss: 1.650\n",
      "[Epoch : 96, Iter:    50] loss: 1.651\n",
      "[Epoch : 97, Iter:    50] loss: 1.650\n",
      "[Epoch : 98, Iter:    50] loss: 1.651\n",
      "[Epoch : 99, Iter:    50] loss: 1.651\n",
      "[Epoch : 100, Iter:    50] loss: 1.652\n",
      "Start test\n",
      "Accuracy of plane : 89.50 %\n",
      "Accuracy of   car : 90.70 %\n",
      "Accuracy of  bird : 71.70 %\n",
      "Accuracy of   cat : 68.70 %\n",
      "Accuracy of  deer : 81.20 %\n",
      "Accuracy of   dog : 72.10 %\n",
      "Accuracy of  frog : 88.80 %\n",
      "Accuracy of horse : 84.00 %\n",
      "Accuracy of  ship : 91.40 %\n",
      "Accuracy of truck : 84.60 %\n",
      "82.27000000000001\n",
      "[Epoch : 101, Iter:    50] loss: 1.650\n",
      "[Epoch : 102, Iter:    50] loss: 1.648\n",
      "[Epoch : 103, Iter:    50] loss: 1.648\n",
      "[Epoch : 104, Iter:    50] loss: 1.648\n",
      "[Epoch : 105, Iter:    50] loss: 1.646\n",
      "[Epoch : 106, Iter:    50] loss: 1.646\n",
      "[Epoch : 107, Iter:    50] loss: 1.644\n",
      "[Epoch : 108, Iter:    50] loss: 1.641\n",
      "[Epoch : 109, Iter:    50] loss: 1.642\n",
      "[Epoch : 110, Iter:    50] loss: 1.643\n",
      "Start test\n",
      "Accuracy of plane : 85.30 %\n",
      "Accuracy of   car : 90.00 %\n",
      "Accuracy of  bird : 68.30 %\n",
      "Accuracy of   cat : 67.20 %\n",
      "Accuracy of  deer : 79.80 %\n",
      "Accuracy of   dog : 75.90 %\n",
      "Accuracy of  frog : 88.90 %\n",
      "Accuracy of horse : 87.00 %\n",
      "Accuracy of  ship : 91.40 %\n",
      "Accuracy of truck : 90.90 %\n",
      "82.47\n",
      "[Epoch : 111, Iter:    50] loss: 1.642\n",
      "[Epoch : 112, Iter:    50] loss: 1.643\n",
      "[Epoch : 113, Iter:    50] loss: 1.640\n",
      "[Epoch : 114, Iter:    50] loss: 1.641\n",
      "[Epoch : 115, Iter:    50] loss: 1.640\n",
      "[Epoch : 116, Iter:    50] loss: 1.641\n",
      "[Epoch : 117, Iter:    50] loss: 1.639\n",
      "[Epoch : 118, Iter:    50] loss: 1.637\n",
      "[Epoch : 119, Iter:    50] loss: 1.637\n",
      "[Epoch : 120, Iter:    50] loss: 1.637\n",
      "Start test\n",
      "Accuracy of plane : 86.30 %\n",
      "Accuracy of   car : 91.10 %\n",
      "Accuracy of  bird : 68.00 %\n",
      "Accuracy of   cat : 64.50 %\n",
      "Accuracy of  deer : 85.20 %\n",
      "Accuracy of   dog : 79.30 %\n",
      "Accuracy of  frog : 87.70 %\n",
      "Accuracy of horse : 87.10 %\n",
      "Accuracy of  ship : 89.80 %\n",
      "Accuracy of truck : 86.60 %\n",
      "82.56\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 121, Iter:    50] loss: 1.635\n",
      "[Epoch : 122, Iter:    50] loss: 1.637\n",
      "[Epoch : 123, Iter:    50] loss: 1.637\n",
      "[Epoch : 124, Iter:    50] loss: 1.636\n",
      "[Epoch : 125, Iter:    50] loss: 1.634\n",
      "[Epoch : 126, Iter:    50] loss: 1.634\n",
      "[Epoch : 127, Iter:    50] loss: 1.634\n",
      "[Epoch : 128, Iter:    50] loss: 1.634\n",
      "[Epoch : 129, Iter:    50] loss: 1.634\n",
      "[Epoch : 130, Iter:    50] loss: 1.633\n",
      "Start test\n",
      "Accuracy of plane : 85.20 %\n",
      "Accuracy of   car : 89.90 %\n",
      "Accuracy of  bird : 72.30 %\n",
      "Accuracy of   cat : 66.60 %\n",
      "Accuracy of  deer : 81.50 %\n",
      "Accuracy of   dog : 73.30 %\n",
      "Accuracy of  frog : 91.70 %\n",
      "Accuracy of horse : 88.60 %\n",
      "Accuracy of  ship : 93.10 %\n",
      "Accuracy of truck : 89.40 %\n",
      "83.16\n",
      "[Epoch : 131, Iter:    50] loss: 1.631\n",
      "[Epoch : 132, Iter:    50] loss: 1.631\n",
      "[Epoch : 133, Iter:    50] loss: 1.630\n",
      "[Epoch : 134, Iter:    50] loss: 1.630\n",
      "[Epoch : 135, Iter:    50] loss: 1.632\n",
      "[Epoch : 136, Iter:    50] loss: 1.629\n",
      "[Epoch : 137, Iter:    50] loss: 1.629\n",
      "[Epoch : 138, Iter:    50] loss: 1.629\n",
      "[Epoch : 139, Iter:    50] loss: 1.628\n",
      "[Epoch : 140, Iter:    50] loss: 1.629\n",
      "Start test\n",
      "Accuracy of plane : 86.70 %\n",
      "Accuracy of   car : 89.80 %\n",
      "Accuracy of  bird : 70.50 %\n",
      "Accuracy of   cat : 66.70 %\n",
      "Accuracy of  deer : 81.50 %\n",
      "Accuracy of   dog : 78.60 %\n",
      "Accuracy of  frog : 88.70 %\n",
      "Accuracy of horse : 88.10 %\n",
      "Accuracy of  ship : 93.30 %\n",
      "Accuracy of truck : 91.00 %\n",
      "83.48999999999998\n",
      "[Epoch : 141, Iter:    50] loss: 1.627\n",
      "[Epoch : 142, Iter:    50] loss: 1.627\n",
      "[Epoch : 143, Iter:    50] loss: 1.627\n",
      "[Epoch : 144, Iter:    50] loss: 1.628\n",
      "[Epoch : 145, Iter:    50] loss: 1.625\n",
      "[Epoch : 146, Iter:    50] loss: 1.625\n",
      "[Epoch : 147, Iter:    50] loss: 1.626\n",
      "[Epoch : 148, Iter:    50] loss: 1.626\n",
      "[Epoch : 149, Iter:    50] loss: 1.625\n",
      "[Epoch : 150, Iter:    50] loss: 1.626\n",
      "Start test\n",
      "Accuracy of plane : 87.00 %\n",
      "Accuracy of   car : 89.40 %\n",
      "Accuracy of  bird : 74.00 %\n",
      "Accuracy of   cat : 65.70 %\n",
      "Accuracy of  deer : 85.50 %\n",
      "Accuracy of   dog : 76.90 %\n",
      "Accuracy of  frog : 86.70 %\n",
      "Accuracy of horse : 90.10 %\n",
      "Accuracy of  ship : 91.70 %\n",
      "Accuracy of truck : 91.80 %\n",
      "83.88000000000001\n",
      "[Epoch : 151, Iter:    50] loss: 1.624\n",
      "[Epoch : 152, Iter:    50] loss: 1.624\n",
      "[Epoch : 153, Iter:    50] loss: 1.624\n",
      "[Epoch : 154, Iter:    50] loss: 1.625\n",
      "[Epoch : 155, Iter:    50] loss: 1.622\n",
      "[Epoch : 156, Iter:    50] loss: 1.624\n",
      "[Epoch : 157, Iter:    50] loss: 1.622\n",
      "[Epoch : 158, Iter:    50] loss: 1.621\n",
      "[Epoch : 159, Iter:    50] loss: 1.622\n",
      "[Epoch : 160, Iter:    50] loss: 1.622\n",
      "Start test\n",
      "Accuracy of plane : 86.10 %\n",
      "Accuracy of   car : 90.70 %\n",
      "Accuracy of  bird : 76.60 %\n",
      "Accuracy of   cat : 67.50 %\n",
      "Accuracy of  deer : 87.10 %\n",
      "Accuracy of   dog : 75.40 %\n",
      "Accuracy of  frog : 89.60 %\n",
      "Accuracy of horse : 85.80 %\n",
      "Accuracy of  ship : 92.10 %\n",
      "Accuracy of truck : 90.20 %\n",
      "84.11\n",
      "[Epoch : 161, Iter:    50] loss: 1.619\n",
      "[Epoch : 162, Iter:    50] loss: 1.622\n",
      "[Epoch : 163, Iter:    50] loss: 1.621\n",
      "[Epoch : 164, Iter:    50] loss: 1.620\n",
      "[Epoch : 165, Iter:    50] loss: 1.619\n",
      "[Epoch : 166, Iter:    50] loss: 1.619\n",
      "[Epoch : 167, Iter:    50] loss: 1.619\n",
      "[Epoch : 168, Iter:    50] loss: 1.619\n",
      "[Epoch : 169, Iter:    50] loss: 1.617\n",
      "[Epoch : 170, Iter:    50] loss: 1.619\n",
      "Start test\n",
      "Accuracy of plane : 87.60 %\n",
      "Accuracy of   car : 93.50 %\n",
      "Accuracy of  bird : 73.60 %\n",
      "Accuracy of   cat : 63.00 %\n",
      "Accuracy of  deer : 88.00 %\n",
      "Accuracy of   dog : 78.20 %\n",
      "Accuracy of  frog : 89.10 %\n",
      "Accuracy of horse : 88.80 %\n",
      "Accuracy of  ship : 90.60 %\n",
      "Accuracy of truck : 89.00 %\n",
      "84.14\n",
      "[Epoch : 171, Iter:    50] loss: 1.618\n",
      "[Epoch : 172, Iter:    50] loss: 1.618\n",
      "[Epoch : 173, Iter:    50] loss: 1.618\n",
      "[Epoch : 174, Iter:    50] loss: 1.618\n",
      "[Epoch : 175, Iter:    50] loss: 1.617\n",
      "[Epoch : 176, Iter:    50] loss: 1.616\n",
      "[Epoch : 177, Iter:    50] loss: 1.617\n",
      "[Epoch : 178, Iter:    50] loss: 1.616\n",
      "[Epoch : 179, Iter:    50] loss: 1.617\n",
      "[Epoch : 180, Iter:    50] loss: 1.615\n",
      "Start test\n",
      "Accuracy of plane : 89.30 %\n",
      "Accuracy of   car : 92.10 %\n",
      "Accuracy of  bird : 76.70 %\n",
      "Accuracy of   cat : 66.30 %\n",
      "Accuracy of  deer : 83.70 %\n",
      "Accuracy of   dog : 77.90 %\n",
      "Accuracy of  frog : 89.30 %\n",
      "Accuracy of horse : 88.50 %\n",
      "Accuracy of  ship : 90.70 %\n",
      "Accuracy of truck : 88.80 %\n",
      "84.33\n",
      "[Epoch : 181, Iter:    50] loss: 1.615\n",
      "[Epoch : 182, Iter:    50] loss: 1.615\n",
      "[Epoch : 183, Iter:    50] loss: 1.615\n",
      "[Epoch : 184, Iter:    50] loss: 1.615\n",
      "[Epoch : 185, Iter:    50] loss: 1.614\n",
      "[Epoch : 186, Iter:    50] loss: 1.613\n",
      "[Epoch : 187, Iter:    50] loss: 1.614\n",
      "[Epoch : 188, Iter:    50] loss: 1.614\n",
      "[Epoch : 189, Iter:    50] loss: 1.613\n",
      "[Epoch : 190, Iter:    50] loss: 1.613\n",
      "Start test\n",
      "Accuracy of plane : 86.20 %\n",
      "Accuracy of   car : 93.00 %\n",
      "Accuracy of  bird : 76.00 %\n",
      "Accuracy of   cat : 70.60 %\n",
      "Accuracy of  deer : 83.70 %\n",
      "Accuracy of   dog : 75.30 %\n",
      "Accuracy of  frog : 88.50 %\n",
      "Accuracy of horse : 88.40 %\n",
      "Accuracy of  ship : 92.90 %\n",
      "Accuracy of truck : 89.40 %\n",
      "84.39999999999999\n",
      "[Epoch : 191, Iter:    50] loss: 1.612\n",
      "[Epoch : 192, Iter:    50] loss: 1.613\n",
      "[Epoch : 193, Iter:    50] loss: 1.613\n",
      "[Epoch : 194, Iter:    50] loss: 1.613\n",
      "[Epoch : 195, Iter:    50] loss: 1.612\n",
      "[Epoch : 196, Iter:    50] loss: 1.612\n",
      "[Epoch : 197, Iter:    50] loss: 1.612\n",
      "[Epoch : 198, Iter:    50] loss: 1.614\n",
      "[Epoch : 199, Iter:    50] loss: 1.612\n",
      "[Epoch : 200, Iter:    50] loss: 1.611\n",
      "Start test\n",
      "Accuracy of plane : 88.70 %\n",
      "Accuracy of   car : 93.10 %\n",
      "Accuracy of  bird : 75.00 %\n",
      "Accuracy of   cat : 70.80 %\n",
      "Accuracy of  deer : 83.60 %\n",
      "Accuracy of   dog : 75.50 %\n",
      "Accuracy of  frog : 90.50 %\n",
      "Accuracy of horse : 87.60 %\n",
      "Accuracy of  ship : 91.30 %\n",
      "Accuracy of truck : 88.30 %\n",
      "84.43999999999998\n",
      "[Epoch : 201, Iter:    50] loss: 1.612\n",
      "[Epoch : 202, Iter:    50] loss: 1.611\n",
      "[Epoch : 203, Iter:    50] loss: 1.611\n",
      "[Epoch : 204, Iter:    50] loss: 1.610\n",
      "[Epoch : 205, Iter:    50] loss: 1.610\n",
      "[Epoch : 206, Iter:    50] loss: 1.612\n",
      "[Epoch : 207, Iter:    50] loss: 1.610\n",
      "[Epoch : 208, Iter:    50] loss: 1.610\n",
      "[Epoch : 209, Iter:    50] loss: 1.610\n",
      "[Epoch : 210, Iter:    50] loss: 1.611\n",
      "Start test\n",
      "Accuracy of plane : 87.70 %\n",
      "Accuracy of   car : 94.20 %\n",
      "Accuracy of  bird : 78.90 %\n",
      "Accuracy of   cat : 70.20 %\n",
      "Accuracy of  deer : 80.70 %\n",
      "Accuracy of   dog : 74.40 %\n",
      "Accuracy of  frog : 90.20 %\n",
      "Accuracy of horse : 87.70 %\n",
      "Accuracy of  ship : 91.60 %\n",
      "Accuracy of truck : 88.20 %\n",
      "84.38000000000001\n",
      "[Epoch : 211, Iter:    50] loss: 1.610\n",
      "[Epoch : 212, Iter:    50] loss: 1.609\n",
      "[Epoch : 213, Iter:    50] loss: 1.610\n",
      "[Epoch : 214, Iter:    50] loss: 1.610\n",
      "[Epoch : 215, Iter:    50] loss: 1.609\n",
      "[Epoch : 216, Iter:    50] loss: 1.609\n",
      "[Epoch : 217, Iter:    50] loss: 1.609\n",
      "[Epoch : 218, Iter:    50] loss: 1.609\n",
      "[Epoch : 219, Iter:    50] loss: 1.608\n",
      "[Epoch : 220, Iter:    50] loss: 1.609\n",
      "Start test\n",
      "Accuracy of plane : 85.20 %\n",
      "Accuracy of   car : 92.70 %\n",
      "Accuracy of  bird : 75.90 %\n",
      "Accuracy of   cat : 66.30 %\n",
      "Accuracy of  deer : 83.70 %\n",
      "Accuracy of   dog : 78.60 %\n",
      "Accuracy of  frog : 91.60 %\n",
      "Accuracy of horse : 88.20 %\n",
      "Accuracy of  ship : 92.90 %\n",
      "Accuracy of truck : 90.10 %\n",
      "84.52000000000001\n",
      "[Epoch : 221, Iter:    50] loss: 1.608\n",
      "[Epoch : 222, Iter:    50] loss: 1.609\n",
      "[Epoch : 223, Iter:    50] loss: 1.608\n",
      "[Epoch : 224, Iter:    50] loss: 1.607\n",
      "[Epoch : 225, Iter:    50] loss: 1.607\n",
      "[Epoch : 226, Iter:    50] loss: 1.608\n",
      "[Epoch : 227, Iter:    50] loss: 1.608\n",
      "[Epoch : 228, Iter:    50] loss: 1.607\n",
      "[Epoch : 229, Iter:    50] loss: 1.608\n",
      "[Epoch : 230, Iter:    50] loss: 1.607\n",
      "Start test\n",
      "Accuracy of plane : 88.20 %\n",
      "Accuracy of   car : 91.90 %\n",
      "Accuracy of  bird : 76.00 %\n",
      "Accuracy of   cat : 68.10 %\n",
      "Accuracy of  deer : 86.60 %\n",
      "Accuracy of   dog : 76.80 %\n",
      "Accuracy of  frog : 87.00 %\n",
      "Accuracy of horse : 89.20 %\n",
      "Accuracy of  ship : 91.10 %\n",
      "Accuracy of truck : 90.30 %\n",
      "84.52\n",
      "[Epoch : 231, Iter:    50] loss: 1.607\n",
      "[Epoch : 232, Iter:    50] loss: 1.605\n",
      "[Epoch : 233, Iter:    50] loss: 1.607\n",
      "[Epoch : 234, Iter:    50] loss: 1.606\n",
      "[Epoch : 235, Iter:    50] loss: 1.607\n",
      "[Epoch : 236, Iter:    50] loss: 1.607\n",
      "[Epoch : 237, Iter:    50] loss: 1.606\n",
      "[Epoch : 238, Iter:    50] loss: 1.606\n",
      "[Epoch : 239, Iter:    50] loss: 1.607\n",
      "[Epoch : 240, Iter:    50] loss: 1.607\n",
      "Start test\n",
      "Accuracy of plane : 87.90 %\n",
      "Accuracy of   car : 92.50 %\n",
      "Accuracy of  bird : 76.70 %\n",
      "Accuracy of   cat : 69.40 %\n",
      "Accuracy of  deer : 86.10 %\n",
      "Accuracy of   dog : 73.80 %\n",
      "Accuracy of  frog : 90.60 %\n",
      "Accuracy of horse : 88.60 %\n",
      "Accuracy of  ship : 92.50 %\n",
      "Accuracy of truck : 91.00 %\n",
      "84.91\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Epoch : 241, Iter:    50] loss: 1.606\n",
      "[Epoch : 242, Iter:    50] loss: 1.606\n",
      "[Epoch : 243, Iter:    50] loss: 1.606\n",
      "[Epoch : 244, Iter:    50] loss: 1.604\n",
      "[Epoch : 245, Iter:    50] loss: 1.603\n",
      "[Epoch : 246, Iter:    50] loss: 1.605\n",
      "[Epoch : 247, Iter:    50] loss: 1.605\n",
      "[Epoch : 248, Iter:    50] loss: 1.606\n",
      "[Epoch : 249, Iter:    50] loss: 1.605\n",
      "[Epoch : 250, Iter:    50] loss: 1.604\n",
      "Start test\n",
      "Accuracy of plane : 86.00 %\n",
      "Accuracy of   car : 92.80 %\n",
      "Accuracy of  bird : 75.90 %\n",
      "Accuracy of   cat : 68.50 %\n",
      "Accuracy of  deer : 85.80 %\n",
      "Accuracy of   dog : 75.20 %\n",
      "Accuracy of  frog : 90.90 %\n",
      "Accuracy of horse : 88.70 %\n",
      "Accuracy of  ship : 92.60 %\n",
      "Accuracy of truck : 90.20 %\n",
      "84.66000000000001\n",
      "[Epoch : 251, Iter:    50] loss: 1.605\n",
      "[Epoch : 252, Iter:    50] loss: 1.603\n",
      "[Epoch : 253, Iter:    50] loss: 1.604\n",
      "[Epoch : 254, Iter:    50] loss: 1.605\n",
      "[Epoch : 255, Iter:    50] loss: 1.604\n",
      "[Epoch : 256, Iter:    50] loss: 1.605\n",
      "[Epoch : 257, Iter:    50] loss: 1.603\n",
      "[Epoch : 258, Iter:    50] loss: 1.604\n",
      "[Epoch : 259, Iter:    50] loss: 1.604\n",
      "[Epoch : 260, Iter:    50] loss: 1.605\n",
      "Start test\n",
      "Accuracy of plane : 88.50 %\n",
      "Accuracy of   car : 92.70 %\n",
      "Accuracy of  bird : 76.40 %\n",
      "Accuracy of   cat : 67.00 %\n",
      "Accuracy of  deer : 84.80 %\n",
      "Accuracy of   dog : 77.10 %\n",
      "Accuracy of  frog : 91.00 %\n",
      "Accuracy of horse : 88.70 %\n",
      "Accuracy of  ship : 92.20 %\n",
      "Accuracy of truck : 90.10 %\n",
      "84.85000000000001\n",
      "[Epoch : 261, Iter:    50] loss: 1.603\n",
      "[Epoch : 262, Iter:    50] loss: 1.603\n",
      "[Epoch : 263, Iter:    50] loss: 1.603\n",
      "[Epoch : 264, Iter:    50] loss: 1.604\n",
      "[Epoch : 265, Iter:    50] loss: 1.604\n",
      "[Epoch : 266, Iter:    50] loss: 1.603\n",
      "[Epoch : 267, Iter:    50] loss: 1.605\n",
      "[Epoch : 268, Iter:    50] loss: 1.605\n",
      "[Epoch : 269, Iter:    50] loss: 1.602\n",
      "[Epoch : 270, Iter:    50] loss: 1.601\n",
      "Start test\n",
      "Accuracy of plane : 86.80 %\n",
      "Accuracy of   car : 91.60 %\n",
      "Accuracy of  bird : 74.40 %\n",
      "Accuracy of   cat : 67.70 %\n",
      "Accuracy of  deer : 84.90 %\n",
      "Accuracy of   dog : 78.40 %\n",
      "Accuracy of  frog : 91.50 %\n",
      "Accuracy of horse : 88.00 %\n",
      "Accuracy of  ship : 92.30 %\n",
      "Accuracy of truck : 92.00 %\n",
      "84.75999999999999\n",
      "[Epoch : 271, Iter:    50] loss: 1.604\n",
      "[Epoch : 272, Iter:    50] loss: 1.603\n",
      "[Epoch : 273, Iter:    50] loss: 1.602\n",
      "[Epoch : 274, Iter:    50] loss: 1.602\n",
      "[Epoch : 275, Iter:    50] loss: 1.603\n",
      "[Epoch : 276, Iter:    50] loss: 1.602\n",
      "[Epoch : 277, Iter:    50] loss: 1.602\n",
      "[Epoch : 278, Iter:    50] loss: 1.603\n",
      "[Epoch : 279, Iter:    50] loss: 1.602\n",
      "[Epoch : 280, Iter:    50] loss: 1.603\n",
      "Start test\n",
      "Accuracy of plane : 88.10 %\n",
      "Accuracy of   car : 92.10 %\n",
      "Accuracy of  bird : 77.50 %\n",
      "Accuracy of   cat : 69.80 %\n",
      "Accuracy of  deer : 84.50 %\n",
      "Accuracy of   dog : 76.60 %\n",
      "Accuracy of  frog : 90.50 %\n",
      "Accuracy of horse : 87.60 %\n",
      "Accuracy of  ship : 91.40 %\n",
      "Accuracy of truck : 90.10 %\n",
      "84.82000000000001\n",
      "[Epoch : 281, Iter:    50] loss: 1.603\n",
      "[Epoch : 282, Iter:    50] loss: 1.601\n",
      "[Epoch : 283, Iter:    50] loss: 1.603\n",
      "[Epoch : 284, Iter:    50] loss: 1.602\n",
      "[Epoch : 285, Iter:    50] loss: 1.602\n",
      "[Epoch : 286, Iter:    50] loss: 1.602\n",
      "[Epoch : 287, Iter:    50] loss: 1.602\n",
      "[Epoch : 288, Iter:    50] loss: 1.601\n",
      "[Epoch : 289, Iter:    50] loss: 1.601\n",
      "[Epoch : 290, Iter:    50] loss: 1.601\n",
      "Start test\n",
      "Accuracy of plane : 87.10 %\n",
      "Accuracy of   car : 92.40 %\n",
      "Accuracy of  bird : 77.60 %\n",
      "Accuracy of   cat : 67.20 %\n",
      "Accuracy of  deer : 84.30 %\n",
      "Accuracy of   dog : 77.70 %\n",
      "Accuracy of  frog : 91.00 %\n",
      "Accuracy of horse : 89.00 %\n",
      "Accuracy of  ship : 91.40 %\n",
      "Accuracy of truck : 90.20 %\n",
      "84.78999999999999\n",
      "[Epoch : 291, Iter:    50] loss: 1.601\n",
      "[Epoch : 292, Iter:    50] loss: 1.602\n",
      "[Epoch : 293, Iter:    50] loss: 1.602\n",
      "[Epoch : 294, Iter:    50] loss: 1.601\n",
      "[Epoch : 295, Iter:    50] loss: 1.601\n",
      "[Epoch : 296, Iter:    50] loss: 1.600\n",
      "[Epoch : 297, Iter:    50] loss: 1.601\n",
      "[Epoch : 298, Iter:    50] loss: 1.602\n",
      "[Epoch : 299, Iter:    50] loss: 1.601\n",
      "[Epoch : 300, Iter:    50] loss: 1.601\n",
      "Start test\n",
      "Accuracy of plane : 87.50 %\n",
      "Accuracy of   car : 92.70 %\n",
      "Accuracy of  bird : 75.40 %\n",
      "Accuracy of   cat : 68.10 %\n",
      "Accuracy of  deer : 85.00 %\n",
      "Accuracy of   dog : 78.70 %\n",
      "Accuracy of  frog : 90.90 %\n",
      "Accuracy of horse : 88.60 %\n",
      "Accuracy of  ship : 92.80 %\n",
      "Accuracy of truck : 90.10 %\n",
      "84.97999999999999\n",
      "\n"
     ]
    }
   ],
   "source": [
    "if is_train:\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEUCAYAAABkhkJAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8VPW9+P/XObPPZN9DEhIIBAKCQNxxF1EURetStbe1tV4ttVavS2uXr1uv3h/WXlvb2l6t2rp00SoqS6VuuFSpymJYwg4hhOyZJJPZz/L7Y8JAZMkEsvN+Ph7zmDMnc855fzIw73zOeZ/PRzFN00QIIYQYZOpgByCEEEKAJCQhhBBDhCQkIYQQQ4IkJCGEEEOCJCQhhBBDgiQkIYQQQ4IkJCGEEEOCJCQhhBBDgiQkIYQQQ4IkJCGEEEOCJCQhhBBDgiQkIYQQQ4IkJCGEEEOCdbAD6Aterx/DOLJByzMzk2hp6ezjiIaOkd4+GPltlPYNfyO9jV9un6oqpKd7er2fEZGQDMM84oS0d/uRbKS3D0Z+G6V9w99Ib2NftE9O2QkhhBgSJCEJIYQYEiQhCSGEGBIkIQkhhBgSJCEJIYQYEiQhCSHEMcA0NExzaFf6jYiybyGEEN2ZYT96/Wa0uk3odZswmqvBakdNykJJzkRNzkZNzkJJzoovY3ejKMqgxSwJSQghRgAj2IHelXz0+k0YLbsBEyxWLDml2I+fg6lFMX1NGJ3NROs2QzTYfSc2F2pKFmpSFrYps7GOKh/QNkhCEkIMONPQ0Ru3oe+qpK6jBs2ThyWvDEveeFRXyuDEZBqAMig9BNPQMcN+zHAnRMNgGmCasVNsXctgxp5NY9/Pw370+i2xHlDbntjOLHYseeOwn3AZlvyJWLLHoFjtBx7TNCESwPA1YfiaMX3N8WXD14TR0QgjNSG99957/OpXv8Ls+iV/73vfY/bs2Zx77rnY7XYcDgcAd911F2ecccZAhSWEGCBGoA29Zi1aTSXa7vUQCYCiYs8uJFq9gejaZQAoqXlYu5KTJX8CSnJ2r5OEqWuYwXbMkA8z1Lnfw4cZ3u/1fsto4djGqgUUS+xZVVHUrmVFBdUSe61YwGpHsTliz1YHWB0otu7LWB0oVgcdNSrh5uYvHdeHGepKQmH/kf9ibU4seWXYy2ZizZ+AmlWCYun5q11RFHB4sDg8WLJKjvz4fWhAEpJpmvzgBz/gxRdfpKysjI0bN3Lttdcya9YsAB5//HHKysoGIhQhxACJ9YK2o9dUotVUxq5hAIo7DduYCixFU7EWTCKnMI/G+laM5mq0us3o9ZuJ7lxJdNMH8fdb8sbHelC540FVMQPtmMEOjEB7LPHsfQ62YwTaD/8Fb3ejOJNRnB4UVypqeiGKMwnF5oz3PkxDB8MAQwcztmzut4yhYWoR0CKY/jZMLYypRTC1cKyHY2jdDhnau2B1xI7lSEJxJqEmZ8eXY4/kWJJT1K6Hsu8ZJZYgUfatt9pR0/JjSXIEGLAekqqq+Hw+AHw+Hzk5OaiqFPkJMZSZhoFetxFt6ycYfi8oB/YY4r0IJbaMasX0e9Fq18cSg6JiyR2H/cQrsRZNQc0cfUCPR7HYsOSOw5I7DrgI0zQwvHXo9ZvjD237ZwcP0mpHcaehuFJQU/Ox5E9EcaeiuFJRXMmxL/m9X/oOz4B8eZuGHktWXQkqMyedVj8HPXU2lPijAer9jTQGmihNG0OOO2tAj6+YA1QH+Mknn3D77bfjdrvx+/08+eSTTJs2jXPPPZekpCRM06SiooI77riDlJTBOYcshIiJNFbjW/cBnes+QPe1ojjc2DMLunoPe3sQGmZXL8LUY70Hs2tZdbhwjTke97gZuMYcj8XZ+5Gfv0zraCZUuxkUBasnHUtSGhZPKqrddcT7NEyDzc07WLF7FTXteyhOK2RcRgnjMkvIdmcc9fWkqB6lobMZE5Nku4ckRxLWQe7NmKZJa7CN2o56dnfUUdtRH3+0h33x9101+WKuOm7ugMY2IAlJ0zRuvPFGbr31VioqKli5ciV33nknS5YsoaOjg/z8fCKRCA899BB+v59HH320V/tvaek84pFms7OTaWry9fzGYWqktw9GfhsHqn2G34u2bQXRLR9jtNSAYsFSdBy28TOxFk/rt7/uB/rzM0yDbW07WN20ljWN62iPdGBVreS5c6gPNKJ1nW5LsnkoSSmiOKWI4pTRFKcUkmQ7eGINaSEaAk3U+xup8zdQH2ikwd9Ic6gVwzS6vddpcZJkc+OxefB0Pe//2qpaCethwnqYkNb1rIcJa7Hn/ZejRhSLYsGqWrF2PVtUC1bFilW17PuZakFRVFqDXhoCjYT0cDwel9VFnjuHPE/Xo2s505l4Qv7yZ6iqCpmZSb39aAbmlF1VVRWNjY1UVFQAUFFRgcvlYtu2bUydOhUAu93Oddddx/z58wciJCGGLNM0Mdr2dF17WUt1RwOGru+rtjJNTMxur+PLioriSUdNykBJykTteijx54xuicWMhtB2rCS65WP0PRvANFGzx+A47WtYS08etIq3vqYbOlv3JqGmtfgindhUK5MzJzI9ewqTs8pxWZ3ohk6tv47qjhp2dtRQ3VHD+pZNsd83kOXKpDi5kIKkfDoiPur9jdQHGmkLt8ePZVEsZLuzGJWUz4zc48l1Z5Oe6qGupYXOqB9/NIA/GqAz6qcz6qch0Ig/GuiWJPayKhYcVgdOiwOHxYHT6sBtdZHhSMNhcWC32NBNHc3Q0Qyta1mLvTY1wnoYf9SPZuropk66I42T8yviSSfXnUuKPWlQ7z3a34AkpLy8POrr69m+fTtjx45l27ZttLS0kJubi8/nIzk5GdM0Wbp0KeXlA1tmKMRQYEZD6LVVsQq0mkrMzhYA1PQC3GOmEI4YwJcucn/pgreiqrHqMr8Xw9+KsXsdWqAd6H72QHGloCRlojg86PWbQYugJGdhnzYX2/jTUNPyB7z9iQppITRDT+i9Jia7fXtY3VTJF03r6Yz6sas2JmeVx5JQ5kScVke3bSyqhdHJhYxOLuSMglPjx9zlq40nqe3t1axs/AKHxU6uO4ey9NJ9PQx3DlmuTCxfOi2XnZ1Mk+vwvUDN0PBHA2iGFk9CVvXYujNnQFqbnZ3N/fffz2233RbPxA8//DCRSISbb74ZXdcxDIPS0lLuu+++gQhJiEFlmiZGex36rlgvSK/bFKvMsjmxjirHMv2SWAFAUuZRndKKJ6jOFszOlu7PgTZs407FOv40LHnjUZSei4xM00QzdaJ6lKgRJWposWd9v2UjimGaOLv+onfs9+yw2FEPcRzDNGgPd9AaasMb8tIabtu3HGrDG24jqIUOuu3hOCx2pmRNYnr2FCZlTsBu6d2pR6fVSVl6KWXppfF1QS2E0+Lo056FVbWS6hgZPdIjNWBFDf1JriEd2khvHwyfNpqmidG4jejWFWi71mD6mgFQ00fFSqCLpmLJKzvgHpK+bp9pmrRHOmgMNNMZ9RPSQgS7HvuWg7Fnfd/6sB5BM7T46asjZbfYY8nK4sBhdeCyO2j2e2kLtx9wvcVjdZPuTCPdmUaGM410Rxo2iy3hY2U40ijPKOvVNv1huPwbPVLD6hqSEMcy3bsHbesnRLeuwPQ1gcWGpWAy1uMvwlo0NTaGWD8wTIPmYCsNgcbYtY6u6x0NgcZD9jScFicua+zhtDpJsSeT687GaXXisNixqTZsqg27asWq2rBZ9i3bVRtW1Yq968s/rEdiF+cPcjF+/wv2itWkNHUMmV2JJ92Z3pV8UnFanf3yuxFDkyQkIfqB0dmKtu3fRLd+gtGyCxQFS8FkbBXzsJZUoCRYquyLdOJr9dLU3oG+38VqzdDRDY2oqXdbv3+1V2OwOV4xBpBiTybPncOJudPJ7brekWxPiichp9VxyNNp/Wmk9x76gmmahCI6gZCGPxTtetaIRHUcdgsuhxW3w4rLEVt2OaxYLT1/loZpEgrrBMMawbBGoOs5GNGYUJROerKjx330JUlIQuzHNE0I+zGC7RAJdg0B4wS7E8XmRDnMqR8z7Ce643O0LZ/ErglhomaPxXHqdVhLT0J1px32uG3hdmp8tbFH5x5qfLXdqrcSoaCQ4Uwn35NDeWYZee7crovt2bht7l7tS/QP0zQJhjV8gSgdgQgd/ii+YASfP0JHIIo/GKVzv6QT6FrWe3lZwm5V48nJ7bTitFuIaEY8+QTDGqGwfsgTsBedUsyVZ5ce4qf9QxKSOGaYoU50b+1+Q810YAbaMYL7ls1ge2y4mENRLWDrSk42VyxRdVVq7S1MUFLzsFdchm3cyaipeQfsYu+ptBpfLbu7Ek+Nr5bOaGy4GwWFXHc249PGUpg8inF5RQR80a77SmL3lOy978TSdY+JVYndf2JXbQdUeImBF9UMdjd1srOug+oGH76gRktbEF8wSoc/csjk4nJYSXJZ8ThteJxWMlOceJxWPC4bbmdsvdsRe+1xWnHYLLGe035JZv/l2Gu9K/lo2KwqOWmurt6UtVvC6r7OQm76wP8BIwlJjGimFkbbuZro1hXou9d2TzaKGiuBdqWiuGNjmqnufa8VuwszGoZoCLPrQTSEGdlvORrCjAZBj2KbfB62cafEBrf8UvWVbuhsadvO6sZYCbIv2gnE7lkZ5cllatYkCpMLKEouoCApH8d+lWBySuvIBMOx01sepw2n3dJv99poelfyqfexs87HzvoOapv88aST5LJRkJ1ERoqT0XnJpLjtJLttsWePjWSXnRSPnSSXDZv12B5OTRKSGHFMQ0evXU90yydoO1eBFkbxpGM77nyshcftG+fMmZRQqfOR0gyNTd5trGms5Ivm9fijAewWO1Myy5mQMY7RyYXke3KPuXtN9jJNk/U7Wln77laSHBZK8lMo7vrCPhKdwShbatrY1PXY1eBjbw2xRVVwOawH9jac+3ojLkfin0MsCfnZWdfB7qZOND12II/TSkleMheePJri3GRK8pPJTHGSk5Mif1Qk4Nj8nyBGnH0l1Z+gbfsUM+QDuxvbuJOxjjsVS94ElAEYzDdqaGxs3czqxrVUNm8gqAVxWhyx+2ByplKeURavQjtWmabJF1tbWPTxTnbUdeB2WgmE9hVfZKY4KcmLfZmX5MWSVJLrwN9Zuz/C5po2Nu3ysrmmjd1NsVOeVotK6agULjmthIwUZ7wQYO/1GH9Iwx+M0tgajBUIhDWO5OYXl8NCSV4Ks04o6oo3hexU55AZ9WA4koQkhjWjvYHW9YvxV74fL6m2Fk/DOu4UrEVTD1uE0FdCWoiN3q2saVzL2uYNhPQwLquLqVmTmJ4zhYkZZdiGeS8oEtVRFOWoTikZpsmqTU0s/ngnuxo7yUp1cv2FE5h3Thl76trZ1eCLnfaq72BnvY+Vm5vi22alOinJT2F0ThItHSE27WqjvjUAgMNmYVxBCieW5zKhKI0x+Sm9inNvpVkoovX85i6KopCaZEeV5NOnhvf/EnFMi+74nNA7vwfTwDKqHNuMS7GOOSHhkuojZZgGuzv3UNWymarWzWxr34lhGnisbmbkTGVazlQmpJcO61Nx4ajOttp2Nu7ysrG6jR11HVhUhfGFqUwsTmdSSQbFucmoas9fyIZh8tnGRhZ/vJPaZj+56S6+fXE5J0/KxWpRsVlV3E4rE4vTmVicHt/OH4pSXb83SfnYWdfB5xsbcTksjC9M44zj8ykrSqM4NzmhEudDURUldlHfOXw/r5FCPgExLEU3fkDow2dRc0opvPoHeMP9O89MR8QXT0AbW7fEixIKkvI5r+hMJmWWUZo6ZthWuEU1na21HWza5WVjtZftdR1ouomqKJTkJzP7pCKiUYOqai+vvL+dV97fjtthZcLoNCaVZFBenE5+prvb6SrdMFixvoEln1RT3xpgVJaHmy6dxEkTcxNKZB6njUklGUwqyYivC4Q0nHZLQtuL4UcSkhh2Il/8g/C//4al8Dhc59+KNSUT+viC8d4pCja0bqaqZRM1nXuA2JQEEzPGMyljAhMzykh1JPfpcQdCVNPx+sI0t4fYujvWC9pa24GmGygKFOcmM+uEIiaOTmd8YeoBF/vb/RGqqlup2umlqtrL6i2xIZBSk+xM6urlGIbJ0hXVNLWFKMpJ4ruXHceMCdlHfYpLejEjm3y6YtgwTZPIZ68QWbMY69iTcJ5z0wHjvh0twzRY3VjJ0h1vUx9oRFVUxqYWc8nYC5mUUUZh8qhBGc0gUeGITqsvhNcXxusL09r17O0IxV93BqPx9yvA6Nxkzp1RwMTidMoK03r80k/12DllUh6nTIrdY9XYFqRqZytV1V7W72jlk/UNAJTkJXPNFeOZNi5LLvSLhEhCEsOCaRiE//Uc0arl2MrPxjHzG31aNWeYBmua1rF0x1vU+RvI9+TyzUnXclzXPDlDlWma7G7ys2ZLE2u2NrOj7sCeYpLLRnqyg/RkB2NHpXQtO0lPcVCSl4zHeXSFHzlpLnKmFXDWtAJM06S2yU8oolNakCKJSPSKJCQx5Jm6Rui9J9G2f4p92lzsJ17RZ190hmlQ2bSeJTveYo+/njx3DjdMvo7pOVP7vCdkmiYN3iDrd7SiqgoFWR5GZXkOWtJ8OJpusGlXG2u2NLNmaxMtHbGJ3caOSuHSmSXkprtjSSfFQXqSA7tt4K5rKYpCYU7vR3kWAiQhiSHOjIYJvvVr9N3rcJzyVexT5/TNfk2TyuYNLN3xFrs795Drzuabk66lIvf4Pk1E4ajOpl1e1m5rZe32Fhrbgge8J9VjZ1SWh4IsDwXZHgqykhiV5el26qwzGGXtthZWb21m3fYWQhEdu1VlUkkGl8wcw/GlmaQmDexAmEL0NUlIYsgyQ50Elv0So3EbzjNvwDbxzKPfp2myrqWKpTveYpevlmxXJtdPuoYTcqcdkIgMwzyiaq6G1gCV21tYu72FTbvaiGoGdqvKxOJ0Zp9UxHFjM7EoCrXNfvY0+6lt7mRPs58PKvcQie6bDyg92cGoLA8oClU7WjFMk9QkOyeV5zJtfBblxek4BrD3I0R/k4QkhiQj0EZwyaMY7fU4Z92CbcwJCW9rmiZhPYw/GsAfDdAZ9cefP6tfTbWvhixnBl8vv5oTc6cfUKptmiZ/X76NNz/dhc2qHjDEzMGGnbFaVDbVtLF2275eUG6Gm7OnFTClNIMJRWnYrN2Pk5nqZGpp5r42myYt7aF9iaop9myxqlx0ajHTx2dRnJcsN2OKEUsSkuh30S0fE61ajuLw7BtHrutZjb9OiY+abXQ0Eljyc8xgB645d2AtmNRtf51RP3WdDdT5G6gPNBDaHKS1s70rAcWSj2YefMTuTGc6X5t4FSfnzTjoPUOGafLntzbz7qpaTpiYQ2aKA39w3xw0TW0hAuEo/qBGONr9GHt7QeefWMSU0kxy0np3g66qKGSnuchOczFt3L5J+2RwVXGskIQk+lV0y8eE3nsKNTUXMxLAbNiKGeqEg83CYnOiuFMxQ50oKDDndqpdLupqV1DnjyWgPf56fJHO+CZOi4NMTzpOxUmWK5OSlCI8Ng8emzv+nLTfs9vmOuQ1IsMweW7ZRj74oo4LTiri6nPGHbZ4QtON+Dhp4ahOQZbngF6QECJxkpBEv4nu+JzQ8j9gGTUR14X/hWKNjaZgGjpmyBebf6hrDiKja46ixkAzb9DCHoeF9qpn4/uyW+zku3OZnDGR/KRc8j15jPLkkuZI7ZORlHXD4JklG/lkfT1zTyvm8jPG9ljJZ7WopHhiUwcIIY6eJCTRL7RdXxB653eoOWNxXXBbPBkBKKoFxZ0GX5pBtd7fyO9X/x+aoXJc5kTyPbnxR7ozrd9uSNV0g6cWbeCzjY1cfsYYLpk5pl+OI4Q4PElIos9pe6oIvvUb1IxC3Bf+V2wK8B7s6azn8dVPggK3z/gOo5IOnGm1P0Q1g9+/vo7VW5q5+pxxXHjy6AE5rhDiQJKQRJ/S67cQfPOXqCk5uC66C8Xh6XGb3b49/HrNU1gUC7dNv4lcT84ARBqbUuG3C9exdnsLXzu/jPMqCgfkuEKIg5OEJPqM3ryTwD/+F8Wdhuviu1GdPQ88uqtjN79e8xQOi4PvT7+JHHdWj9v0hXBE5/FXKtlY7eX6Cydw1rSCATmuEOLQJCGJPqG37ia45FEUhxv33B+gfun60MHsaN/Fb7/4Ay6ri9um30yWK6PHbfpCMKzxq5e/YEttOzdcXM7MKfkDclwhxOFJQhJHzWivJ7jkEbBYcc/9IWpSZo/bbGvbyRNfPE2SzcNtM24mw5ne4zZ9IRCK8thLX7CjzsfNl07mpPLcATmuEKJnkpDEUTF8zQQWPwKmiWvu3agpPV//2eLdzhOVz5BmT+G2GTeT5kjt3xgNk3BUxxeM8rvX1rG7sZP5lx1HxYTsfj2uEKJ3JCGJI2b4vQQWL8CMhnDP/SGW9J6vw2xs3cLvK/9IhjOd26bfRKoj5YiO3dYZZuMuL1tq2gnrBh2dYSIRnXDUIBzViWg64a7Xmr5vfDirReXWK6YwtXRgrlUJIRInCUkcESPYQXDJI5ghH+6L78aSVdzjNhtaNvHk2j+R7cri1un/SYo98dlWO/wRNu7ysnFXG5t2ealrCQDgtFvITndjVcFutZCaZMdus+CwqThsFhw2S9fr2LrSglRG5w6/WV6FOBZIQhK9YoR8RDe+T3T9O5ghP645d2DJKe1xu3XNVTy19jnyPLncOu0/SbIfvhzcF4iwaVdbPAntafYD4LBbKCtM4/Sp+Uwcnc7o3CTyclNlrDchRgBJSCIhessuouveJrr1E9CjWAom4ai4HEve+Ph7TNPErwVoC7XTFm7HG+56DrXxecMaCpLy+N60/8Rjcx+wf8M0qa73sWpzE19sbWF3U2y8OrtNZXxhGqdOzmVicTrFuclYLUN3CnEhxJGThCQOyTR0tJ2riK5/G71uE1js2MpmYpt8PpvMTra0bcW7fiVt4TbaupJP1NC67UNBIdWRwpSscr428Srctn0jYO+d+XTVlibWbGnG6wujKgrjC1O5/MyxlI9OpyRfEpAQxwpJSOIAZqiTyN7Tcv5WlOQsHKd8FduEMwmoCn/d/DqfNazGolhIc6SQ5khldHIhU7Mnk+5II82RSpojlXRnKsm2pG7TPATDGmu3t7B6SzOV25oJhnXsNpUpYzKZdmYWx4/L6vWU3kKIkUESkojTW3cTXfdPolu6TsuNKsc282tYR09HUVXWNVfx541/xxf1c9GY87mg+Bysas//hIJhjX9XNbB6czNV1a1oukmSy0bFhBxmjM9mUkk6dpn5VIhjniQkgd5aS2TlQrQdn8dOy40/Ddtxs7BkFAEQ1IL8fdMiVtR9zihPHt85/luMTk5s3LfaZj+/eaWSBm+QnDQX51UUMn18NuMKUo9oenAhxMglCekYZrTVE171GtrWf4PNgX3GPOzHnY/iTIq/p6plMy9sfJmOiI8Li8/lwjGzsCXQKwJYtbmJpxZvwGGzcPe105k4Oq3HOYaEEMcuSUjHIKOjifCqN9C2/AssVuzHz8F+/EXdElFIC/Hq1iX8a8+/yXPncFPFNyhOKUps/6bJ6x/uYNHHOxk7KoVbLp9CerKjv5ojhBghJCEdQ4zOViKrFxHd+AGoCrbJs7BPuxjV3X3ons3erTxf9TLeUBuzRp/F3DGzsVkSKzQIhKI8uWgDldtaOH1qPl+fPQGbVarkhBA9G7CE9N577/GrX/0K0zQxTZPvfe97zJ49mx07dnDPPffQ1tZGWloaCxYsoKSkZKDCOiYYgTYia5YQrXoPTBNb+VnYp1+C6uk+oGlYj/D6tqW8v/tjclxZ3FExn7GpJQkfZ+/1oub2EP8xu4xzphfIKTohRMIGJCGZpskPfvADXnzxRcrKyti4cSPXXnsts2bN4r777uO6665j3rx5vP7669x7770899xzAxHWiGeGOml59zX8ny0FXcNWNhP7jEtRkrLwRTtp8G6nMdhEQ6CJxkATuzp20xHp5Jyi07l07IXYLfaeD9Lly9eLyop6nn5CCCH2N2A9JFVV8fliw7v4fD5ycnLwer1s2LCBZ599FoC5c+fys5/9jNbWVjIyBmZunJHIjASJrF1GpPJNGtFoKimndVQpTUaIhk1/pTHQRFALxd9vVa3kuLIYm1rCWYUzGZ8+NuFj7X+9aEx+CrdcfhwZKT1PWS6EEF82IAlJURR++ctf8t3vfhe3243f7+fJJ5+krq6O3NxcLJbYPSgWi4WcnBzq6uokIR0BU4sQ3fAOkdVLMMOd/HtMGQstbUAD7GkgzZFKrjubE3Onk+POJtedTY47mwxnGqqisn5nKx2NOtuC7aQnOUjx2A87SsKB14vKsFnlfiIhxJEZkISkaRr/93//xxNPPEFFRQUrV67k9ttv55FHHumT/WdmJvX8psPIzh7eoz+buoZvzTt4P/o7emcrrrHHEznpIpasfo7js8u5burl5Cfn4LQeutLtzU928tu/f3HA+tQkOxkpzn2PVCeZKU6SXHZeeLOKhtYA3/nKVC46rWRQrxcN98+wJ9K+4W+kt7Ev2jcgCamqqorGxkYqKioAqKiowOVy4XA4aGhoQNd1LBYLuq7T2NhIfn7vppRuaenEMMwjii07O3nYjhRtGgbathWEP1+I6WtCzR2H6+z/RM2fwJOrn0TFwndO/DqG34rPG8FH5KD72Vrbzu9freS4sRlccWYpbZ3hrkck9uwL09wWZOvuNjr8EcyuX3WK2xa/XtTc3DmALe9uOH+GiZD2DX8jvY1fbp+qKkfUURiQhJSXl0d9fT3bt29n7NixbNu2jZaWFoqLiykvL2fx4sXMmzePxYsXU15eLqfremCaJlr1KiKfvYrhrUXNLMJ54X9hKZqKoih8sPtjtrRt57qJV5DpTqfJf+j/CG2dYX67cC0ZKQ5uvnQyHqeNYg79l45uGHT4o7R1hslJd+FxyrhzQoi+MSAJKTs7m/vvv5/bbrstflrn4YcfJi0tjfvvv5977rmHJ554gpSUFBYsWDAQIQ1bessuQh88i9G0AyU1D+d538U69gQUJXatpyXYysJtS5mYPp7T8k867L403eCJheuTUjZVAAAgAElEQVQIhjXuvPqEhJKLRVVJT3bIja5CiD43YFV2l156KZdeeukB60tLS3n55ZcHKoxhzTRNQsufxgx4cZ55A9aymSj7jaRtmiZ/3vgKCnDdxCt7vKbzl7e3sLW2ne/Mm0xhztFdhxNCiKMlt9API/qeKoyWauwnXoFt4pndkhHAx3WfstG7hctKLybTlX6IvcR88MUe3ltdy5yTR3NSeW5/hi2EEAlJKCH99a9/paOjo79jET2IVL6J4krBNu7UA37mDbXx6pYljE8by+kFJx92P9v3dPDCPzcxuSSdK87qefpxIYQYCAklpLfeeouzzjqL73//+7z99ttomtbzRqJP6d5a9JpKbJPPQ7F2H0HBNE3+sulVDFPnaxOvQlUO/bG2+yP8duFa0pIc3DzvOJkCQggxZCSUkJ5++mmWLVvGtGnT+M1vfsPpp5/Ogw8+SGVlZX/HJ7pEK5fF5iqadO4BP/u0fhXrWzZyaekcst2Zh9yHphv87rV1+INRvveVKTIzqxBiSEn4GlJOTg433HADr732Gk8//TSff/45X/3qV7ngggt45plnCIVCPe9EHBEj0EZ0y8fYJpyO6uxekt0e7uDlLW90Dftz2mH389K7W9lc08Y350xkdO7IvklPCDH89KrKbuXKlbzxxhssW7aM0tJSHnzwQUaNGsXzzz/P8uXLZVDUfhJd/w4YOvYps7utN02Tv25aiGZE+Y/yw5+q+9faOt5euZvZJxZxyuS8/g5ZCCF6LaGE9Nhjj7F48WJUVWXevHm8/PLLFBXtm6ztxBNP5OSTD38hXRwZMxomsuFdrCXTUVO7J5KVDWuobF7PZaUXkevOPuQ+dtZ38NyyTZQXp3PVOVLEIIQYmhJKSM3NzSxYsIATTjjhoD+32+385S9/6dPAREx084cQ9mObOqfbel+kk5e2vE5xShHnjT7zkNu3d4b57atrSXHbuHneZCyqVPoLIYamhBLSHXfcgcPR/c78zs5OIpFIfJifiRMn9n10xzjTMIis/SdqzlgsueO6/exvm18jrIX5evnVhzxVpxsGjzz/OR2BKD/+jwpS3InPbySEEAMtoT+Xv/Od77B79+5u62pqapg/f36/BCVitOpVmB2N2KfO6TbqwurGtaxurGTOmPPJ9xz6pta/vr2Vyq3NfOOCCRTnSRGDEGJoSyghbd++/YAeUHl5OVu3bu2XoERMpPJNlORsrCUV8XWdET9/27SQouQCzh991iG3fWflbt5ZtZvLzipl5pTejZ4uhBCDIaGElJ6eTk1NTbd1NTU1pKam9ktQAvSGrRgNW7FPmY2y33Wf17f9A78W4D8mXoVFPfhkeOt2tPCXt7cwbVwW35w7eaBCFkKIo5JQQrrsssu47bbb+Pjjj6mpqeFf//oXt99+O5dffnl/x3fMilS+CXY3tglnxNc1B1tYUf85ZxScSmHyqINuV9vs53evrWNUloebLp2ERUZiEEIMEwkVNcyfPx9FUfjpT39KQ0MDubm5XHnlldx00039Hd8xyehoRNuxEvu0i1Fszvj6ZTvfRUVhdvHZB93OF4jw+N+/wGa1cNuVU3HaB2wwdyGEOGoJfWNZLBZuueUWbrnllv6ORwCRymWgqtiOmxVf1xJsZUX9Ss4oOIU0x4GnSqOawW9fXYvXF+GHX5tOZqrzgPcIIcRQlvCf0LquU1NTg9frxTT3TRc+Y8aMfgnsWGWGOolu/hDruFNR3Wnx9cuqY72j80effeA2pslzb25k8+7Y3Ealo+TanhBi+EkoIX3xxRfcdttttLW1EYlEcDgcRCIR0tPT+eijj/o7xmNKZMO7oEWwT70wvq4l6OWTus85fdTJpDvTDtjmH//exb/W1TPv9DEyt5EQYthKqKjhoYce4tprr2X16tV4PB5WrVrFjTfeKNeQ+pipR4mufxtL4XFYMgrj6/9Z/S4KCrOLzzlgm5Wbmvj78m2cVJ7DpTNLBjBaIYToWwklpG3btnHjjTfGb85UFIVbbrmFp59+ul+DO9ZoWz7BDHZg32+YoNZQrHd06qgTD+gdVdf7eGrxesaOSuGGi8p7nLJcCCGGsoQSUlJSEsFgEIDMzEy2b99OZ2cnnZ2d/RrcscQ0TSJr30TNKMJSMCm+/p/VywG44Eu9I68vzK/+/gVJLhu3fmUKdtvB70kSQojhIqGEdM455/DOO+8AcPnll3P99ddz5ZVXct555/VrcMcSvWYthncP9qkXxns63lAbH+/5lFPzTyDDmR5/bziq8/grlQQjOrddeTypSY5D7VYIIYaNhIoa7r///vjyzTffzJQpU/D7/ZxzzoHXNMSRiax9E8WTjrV03zQe/6x+D4DZxftmiTVMkz8s3sCueh+3XjGVopykAY9VCCH6Q48JSdd1LrnkEl577TXs9tho0aeddviZSUXv6M3V6LUbsJ90NYol9pHs7R2dkl9Bpiud1o4QX2xr4fONjVRVe/nqueOYNj5rkCMXQoi+02NCslgsRKNRIpFIPCGJvhX54h9gc2Iv3zdY6j+rl2OYJkb9OO7916fsbopdr8tKdXLFWWOZfWLRoXYnhBDDUkKn7G644Qbuuusu5s+fT15e91lLc3PlvpejEVn3Ftq2FdinXUzAtLNuQwMrt+9ivXMFWvMolld7GV+YytXnjGNqaSb5mW6pphNCjEgJJaQHHngAgOXLl3dbrygKVVVVfR7UsSK6+SPCH7+IL3Myz28ew5b3PsIwTdxjN4HL5OrJF3DqZWNxO22DHaoQQvS7hBJSZWVlf8dxzInuXEno/Wcwcifw4MbppKdrzDllNKUlDv64420qcis4r3zCYIcphBADJqGEJNeO+pZWu4HQ279DzS5hqesSdKWJu6+ZRkaKk79veQPdNLiwWErqhRDHloQS0re+9a1DXrd45pln+jSgkU5v3EZw2a9QU/Mwzv4+7z29hpMn5ZKR4qQ93MFHtSs4KXcG2e7MwQ5VCCEGVEIJ6YILLuj2urm5mVdffZV58+b1S1Ajld5aQ+Af/4viTsV18V0sXu0lEjW48OTRALy963100+CCknN72JMQQow8CSWka6655oB1c+bM6XbDrDg8o6OR4JJHUSw23BfdTdSWzNufr2VqaSaF2Ul0RHx8WLuCE3Onk+OW+4uEEMeehIYOOpiioiI2bNjQl7GMWIbfS2DJz8HQcV18N2pKNh9V1tEZjHLRKcUAvF39PpqhcaH0joQQx6iEekiLFi3q9joUCvHmm29y3HHH9UtQI4kZ6iS49OeYIR/ui3+AJb0A3TBY9ukuSgtSGF+Yii/SyQe1n3BC7nRy3NmDHbIQQgyKhBLSCy+80O21y+WivLycG2+8sV+CGinMSJDAP36B0dGIa86dWHLGAvD5xiaa20Ncc954FEXhrerlaIbGHOkdCSGOYQklpL/97W/9HceIY2oRgv98HKO5GtfsW7GOKo+tN03+saKa/Ew3E8Z4eKHqZT6p+4yT8yrI9eQMctRCCDF4ErqGtGTJEjZt2tRt3aZNm1i6dGm/BDXcmYZB6J3foe/ZiPPsG7EWT4//bP3OVnY1djJ1ms7Dnz7GirrPmV18DtdO+MogRiyEEIMvoYT0i1/8gpyc7n+9Z2dn8+ijj/ZLUMOdtmsNWvVqHKdeg21895HRF/97K57x6/nA9zpOq5O7T/ge80rnYLPI8EBCiGNbQqfsOjo6SE1N7bYuNTWV9vb2fglquItWLUfxpGObPKvb+ne3rGJX2mIUe4TZxedwUcksSURCCNEloR5SaWkp7777brd17733HmPGjOmXoIYzw9eEXrMW24QzUdTYtOKBaJDnq17ilZq/ohg2vj91vvSKhBDiSxLqId1xxx185zvf4dxzz2X06NHs2rWL5cuX88QTTyR0kN27d3PLLbfEX/t8Pjo7O/n0008599xzsdvtOByxabjvuusuzjjjjCNoytAQ3fgBKGCbeCYA65qr+PPGV+iI+ND2jOW8gnOZkFUyuEEKIcQQlFBCOvnkk3nttdd44403qKuro6SkhIULFzJ69OiEDlJYWMjrr78ef/3QQw+h63r89eOPP05ZWVkvQx96TEMjuvEDLEVTCTncvLLhJVbUf06+J5fCwNmsqYsy+7LiwQ5TCCGGpIQSkmEYjB49mltvvTW+zjRNDMNAVXs32EMkEmHRokU8/fTTvYt0GNCq12AG24mWncbPP32M9kgHFxSfy8ycM/nR7z/ltOPySUtyDHaYQggxJCWUkK6//npuv/12Kioq4utWrVrF448/zp/+9KdeHfDdd98lNzeXyZMnx9fdddddmKZJRUUFd9xxBykpKb3aZ2ZmUq/e/2XZ2clHtf1edW9/iCUli+2ZDrw72/jpWd9nal45zy3dgG4YXDennOzso4v1SPRV+4aykd5Gad/wN9Lb2BftSyghbdy4kRkzZnRbN3369COaLfaVV17hiiuuiL9+8cUXyc/PJxKJ8NBDD/Hggw/2upy8paUTwzB7HQvEfolNTb4j2nZ/Rkcjwe1fYK+4nI93ribDmU6eWsCu3V6WfLSDGeOzsWP2ybF6o6/aN5SN9DZK+4a/kd7GL7dPVZUj6igkdL7N4/Hg9Xq7rfN6vfFChEQ1NDTw2Wefcckll8TX5efnA7FJAK+77jpWrVrVq30OFdGN74OiYow/hU3eLUzNmoSiKHzwxR4CYY05p8i1IyGEOJyEEtKsWbO4++67qa6uxjAMdu7cyT333MPs2bN7dbCFCxdy1llnkZ6eDkAgEMDni2VV0zRZunQp5eXlvWzC4DN1jeimD7GOPp5N4SaihsbUrMlousE/P6th4ug0xo7q3WlIIYQ41iR0yu7OO+/kZz/7GRdffDG6rmOxWLjsssu46667enWwhQsX8pOf/CT+uqWlhVtvvRVd1zEMg9LSUu67777etWAI0KpXYQY7sJWfQ2XTelxWF+PSxrBifQNeX5jrL5w42CEKIcSQl1BCcrlcPPzwwzz44IM0NTWRnZ2N1ZrQpt0sW7as2+uioiJee+21Xu9nqIlWLUdJyoRR5az75FWOy5yIoqj849+7KMxOYsrYjMEOUQghhrxe1WwbhoHFYqGlpYWGhgYaGhr6K65hw2hvQK/dgG3iWezw1eCPBpiaPZnKrS3safYz55TRKIoy2GEKIcSQl1A3Z8eOHdxzzz1UVlYe8LMjqbQbSSJVy0GxYJt4JpW1H2JVLEzKKON//7mezBQnJ06UKSWEECIRCfWQ7r//fiZNmsQHH3xAUlISH374IV/5ylf4n//5n/6Ob0gz9Sja5o+wFk9DcaVS2bSesoxx1DaG2bq7ndknFWG1HPEs8UIIcUxJ6NuyqqqKH/3oR2RnZ2OaJllZWfz4xz/mN7/5TX/HN6RpO1ZihnzYJp1Dnb+B5lArU7Mm8+7K3TjtFk6fkj/YIQohxLCRUEKy2WwYhgFAWloa9fX1KIpCS0tLvwY31EWrlqMkZ2MpmERl83oAxrjH8dnGRmZOycfl6H3hhxBCHKsSSkgVFRXxCrlZs2Zx8803881vfpOTTjqpX4Mbyoy2OvS6jdjKz0JRVCqbNlCcUsSaqk403eTcGQWDHaIQQgwrCf0J/9hjj8WX7777bl599VX8fj9XXnllvwU21MWLGcrOoC3cTrWvhrljLuDtpbVMKkknP9Mz2CEKIcSwklBCslgs3ZavuuqqfgtoODC1CNHNH2EdMwPVncra2k8AsPnz8frq+Y/zh/9UGkIIMdCkBOwIaDs+h7AfW/k5AFQ2bSDLlcnKyiCZKQ6OH5c1yBEKIcTwIwnpCESrlqOk5GIZNZGgFmKzdyulnjI27Wrn7OkFqKrcCCuEEL0lCamXdG8tev1m7F3FDBtaNqGZOr76DKwWlTOOHzXYIQohxLCUUEJasGDBQdf3dt6ikSBatRxUK9ay0wGobF6Px+pm3VqTk8tzSHHbBzdAIYQYphJKSH/7298Ouv7ll1/u02CGulgxw7+wjqlAdaWgGzrrWzaRqRQTjhqcW1E42CEKIcSwddgqu0WLFgGg6zqLFy/GNPfNylpTU0Nqamr/RjfEaNs/g0gAW/nZAGxp205QC9Jcl8qY/BTG5MucR0IIcaQOm5BeeOEFAKLRKM8//3x8vaIoZGZm8tBDD/VvdENMpOo91NQ8LPmx+Y0qmzdgUay01CZx+UVyI6wQQhyNwyakvafqFixYwA9/+MMBCWio0lt3YzRsxXHKNSiKgmmascn4wrkoTpeM6i2EEEcpoWtI3/3udwmFQkBsqvHFixezdOnSfg1sqNGqVwNgLZsJwO7OOrzhNry16Zx5/ChsVsvhNhdCCNGDhBLSjTfeyNatWwH45S9/ya9//Wt+85vf8Mgjj/RrcEOJ4a1FScpEdSYDxAdT1duyOXuanK4TQoijlVBC2r59O5MmTQLg9ddf5+mnn+aFF17gjTfe6NfghhLDuwc1fV/iqWxaD/50po8pJDPVOYiRCSHEyJDQWHaqqqJpGtXV1bhcLgoLCzFNE7/f39/xDQmmYWC01WEriCXl1pCX3Z17iLZM4LyzpXckhBB9IaGENHPmTO6++25aW1uZM2cOEOs15eQcGxfyTV8T6FEsXT2kyqYNAGQwmonF6YMZmhBCjBgJJaSHHnqIl19+GavVGp9yorGxkfnz5/drcEOF7q0FQE2PDQv079ovMIIezp9SjqLIuHVCCNEXEkpILpeLb3zjGwC0tbVht9s59dRT+zWwocTw7gFATRtFIBpkl78apWMsp83KG+TIhBBi5EioqKGzs5Mf//jHTJs2jbPPPhuA9957j9/+9rf9GduQYXhrUTwZKHYXn+9ZB4rJ1MxJMkW5EEL0oYQS0gMPPIBpmixatAibzQbA1KlTj5kqO8O7BzUjdv3o/R2rMCN2Lp0xfZCjEkKIkSWhP/E/+ugj3n//fex2e/yaSWZmJs3Nzf0a3FAQq7Dbg23UREJahPpoNclaMaOykgY7NCGEGFES6iF5PB46Ojq6rauvrycra+TPjGp2NoMeRU0fxbL1a8CicWrR8YMdlhBCjDgJJaTLL7+c22+/ndWrV2OaJhs2bODHP/4xV111VX/HN+iMrgo7S3oBn9SsAcPChZPkdJ0QQvS1hE7ZzZ8/H5vNxl133UUgEOB73/se11xzDd/+9rf7O75Bt7fku05PpcNaQ561GKfNMchRCSHEyHPYhLR48WLmzp2LqqrcdNNN3HTTTQMV15BhePegeDJYvn07ij3MmSXSOxJCiP5w2FN2995770DFMWQZ3lrU9FFs79gOwAmjjhvkiIQQYmQ6bELaf4bYY5FpGhjeOtT0AlqjTdh0D0l2z2CHJYQQI9JhT9kZhsGKFSsOm5hG8ogNpq8F9AhmSh7h1i3kWEd+VaEQQgyWwyakSCTCT37yk0MmJEVReOedd/olsKFgb4VdAykoTj+jPHK6Tggh+sthE5LL5RrRCacnetcYduv8GopiMj6zcJAjEkKIkSuh+5COVYa3FsWdxtaO2IgUE3OKBzkiIYQYuaSo4TCMttgssQ3BBjAs5HrkGpIQQvSXwyak1atXD1QcQ06swi5W8t1hNOMmDYtqGeywhBBixBqQ+RN2797NLbfcEn/t8/no7Ozk008/ZceOHdxzzz20tbWRlpbGggULKCkpGYiwDsvsbAEtQtidi+moIstROtghCSHEiDYgCamwsJDXX389/vqhhx5C13UA7rvvPq677jrmzZvH66+/zr333stzzz03EGEd1t4Kux2aE8UeoTh11CBHJIQQI9uAFzVEIhEWLVrEFVdcQUtLCxs2bGDu3LkAzJ07lw0bNtDa2jrQYR1g7yyx64JhQAoahBCivw14Qnr33XfJzc1l8uTJ1NXVkZubi8USuzZjsVjIycmhrq5uoMM6gO7dg+JOo7qzBYDSDCn5FkKI/jTgc3C/8sorXHHFFX26z8zMo5ssLzs7+YB1tb56bDlFtIabsdpcjC3IP6pjDKaDtW+kGeltlPYNfyO9jX3RvgFNSA0NDXz22Wc88sgjAOTn59PQ0ICu61gsFnRdp7Gxkfz83n35t7R0YhhHVqKenZ1MU5Ov2zrTNAk31WApO4NAeCeZauYB7xkuDta+kWakt1HaN/yN9DZ+uX2qqhxRR2FAT9ktXLiQs846i/T0dCA2DXp5eTmLFy8GYtNdlJeXk5GRMZBhHSBWYRemw5qB4uok3507qPEIIcSxYEB7SAsXLuQnP/lJt3X3338/99xzD0888QQpKSksWLBgIEM6qL0FDRt1K4pqyPUjIYYgXdfwepvQtMhgh9KjxkYVwzAGO4x+YbXaSUtz9s2++mQvCVq2bNkB60pLS3n55ZcHMowe7S353hgMgwoTc6XCToihxuttwul04/HkoSjKYIdzWFariqaNvIRkmiZ+fwc1NTUkJ2cf9f5kLLuD0L17UFyp1IRawVQoSM4b7JCEEF+iaRE8npQhn4xGMkVR8HhSCAZDfbI/SUgHER8ySG/GaaZiUwe8GFEIkQBJRoOvLz8DSUhfYpomRtse9OQ8NHs7mbaj74YKIYTomSSkLzH9rRANUa+mojpCFKXIkEFCCDEQ5FzUl+wtaKiKxl6XZY8exGiEEMPFlVdegsvl4k9/+iuqqsbXPfLIY5SVlR1yu5de+jPnn38h6emHv91F0zT++Mc/8Pbb/8ThsKOqKjNmnMj8+bdSWbmGu+++jdGji4lGNQBOPvkUrr/+RlJSUgAwDIMXXvgjS5cuxmJRURSFq6++jksvvbyPfgNHTxLSl+wt+d4cDoMDyrKKBjkiIcRwEQwGWbZsKXPmzE14m5de+gsnnHBSjwnp4YcfIBwO8cwzz+N2e9A0jSVL3iASiZW9l5SM5emnnwcgEPDz618/xu23z+epp57DYrHw3HPP8Pnnn/Lkk38kJSWFhoZ67rzz+zgcDi644KIjb3QfkoT0JYa3FsWVQn3Ei2qzkeZIHeyQhBA9+NfaOj6q7J8xME+fms/MKYmNHnPDDTfxzDNPMWvWBdhstvj65uYmHn10AQ0N9YTDYWbNuoBvfOMG/vSnp2lubuKnP/0hdruD++77b8aMGXvAfmtqdvHBB+/x6qtLcbs9AFitVubN+8pB43C7Pdx55z1cffU8/v3vT6ioOJHnn3+WZ555Md5jys3N45ZbbuOxxx6RhDRU6d49KGmjCJheUpVMqeIRQiRs4sRyJkyYyMKFf+fqq6+Nr3/ggXu5/vpvM23aDKLRKLfdNp/y8klcf/23WbToNf77vxcwduy4Q+538+ZNFBaOjieTRFitVsrKJrBjxzby8vKw2WwUF5d0e8/kyVPYs6eWQCCA2+3udXv7miSk/ZimieHdQ6joBDC3kuuaMtghCSESMHNK4r2Y/nbTTfO59dbvMHfuPAB03WDVqpV4vd74ewIBPzt37uTEE0/p11hM0+x67vGd/RpHoiQh7cf0eyEapFr1oKAzNr1gsEMSQgwzo0eXcOqpM/nb314EQFVVFAX+8IfnsFqP7Cu3rGwCu3fvoqOjI+FekqZpbNmymcsuu4LCwiKi0SjV1Tu79ZLWr19Lfn5B/DTgYJOy7/3srbDbFI39tVCeKxV2Qojeu+GGm3j11ZcJBAIoCkybNp0XXvhj/OcNDfW0tDQD4PF46OzsPOz+iopGM3Pmmfz85w8TCPgB0HWdRYteIxAIHPD+QCDAY489QlpaGieddCoOh4Ovfe16fv7zh+no6IjH8MQTv+KGG/6zj1p99KSHtJ+9FXbbIiGwIfcgCSGOSE5OLhdccBF//esLADzwwEP87/8+yje+8VUgVnTwox/dS2ZmFldeeQ0PP/wgTqfzkEUNAD/96QM888yT3HDD17HZrJimySmnzMRutwOwc+d2vvnN69A0DTA56aRT+OUvfxefAPWb37wRVVW56abr4+uuuuqaXlUE9jfFNHs+uzjU9dV8SKEPnkHbuZofp5Zi8fh4bNb/68swB8VIn4cFRn4bpX0HV19fTV7e8Bj4eKQOrrpXY2MNOTn7bpEZFvMhDXW6dw9maj5Razvp1qzBDkcIIY4pcsquS6zCrpb2vOkotq0UJA2Nih0hxLHj2Wef4v333ztg/WOP/abHG2dHAklIXcxAG0SCbFYcKAqUZUlBgxBiYH3rW//Jt741dIoMBpqcsuuyt8JuS9d53rJsGTJICCEGkiSkLnsTUrUeQjEsZLszBzkiIYQ4tkhC6mJ494AjiTazA4+SgarIr0YIIQaSfOt2Mbx70FPywNlBtjNnsMMRQohjjiQkYhV2ureWelsaii1KSaoMGSSE6J0rr7yEr3/9agzD6LZu+/ath93upZf+jNfbetj33Hnn93nttb93W2eaJlddNY/Vq1eydOkiLrzwbL71rev42teu5Prrr+WZZ54kHA7F3x+JRPj1rx/j6qvncd11V/Ctb13HBx8s731D+5EkJMAMtkMkwGYldsezDBkkhDgSe+dD6o2XXvpLjwnp4osvZenSxd3WrV69ElVVmDZtBgAnnHASzz77Z1588e/88pe/ZdOmKu6990fx9//iF/8fTU2NPP/8S/z5z6/w//7fgzz22COsWbOqV/H2Jyn7BozWWEHDtq6/bErSCgczHCFEL0U3/4vopg/6Zd+2CWdiK5uZ0Hv7az6kM844i1/84n/YuXMHJSVjAFiy5A0uuuiSg06Rk56ewU9+8gCXXz6H7du34Xa7effdt3jllcU4HA4Axo4dxze+cQPPPvsUv/rV747kV9PnpIcEGG2xMexq9RBWw43HNvjzggghhp/950Pa3wMP3MuVV17DU089x9NPv8CKFR/z2WcruP76b5OVlc1///cC/vjHPx9yHDubzcb5589h6dI3gNj0FR9++P5hx6FLSUmhsHA0O3ZsZ9u2rRQUFJGS0n3C0cmTj2Pr1s1H2eq+Iz0kunpIDg9+awfZFin3FmK4sZXNTLgX09/6az6kiy++lNezmDgAABHASURBVP+/vXsPi7pMGzj+nRkYTokgBg7qxoqCiAdQysNmCdp7WbKVqay5sq3a0RfzsG666lpqJqCXuhKvVFy9Vhdt1mvhiRIPV7nXVqyVa3kOIh0BMRaUAJlxZp73D5bZTEHlNDPM/fmL+c3Mb+57Hua653n48dwLF87hqadS2L9/L4MGDSE4OOQGz2rsh+QaW5ZKQaJhhlTvF4zGu5ZQvwGODkcI4cLaox8SQL9+EQQF3c7nn39KXt4OpkyZ1uzjq6urOXfOSJ8+4Xh7e1NSYqS6+tJVs6Rjx44SHt6vxTG1Nbdfsmu8wq5Y3xWNVtE3SP5+JIRonbbuh9RowoQHef31VzEazzJ69L1NPq6qqoo1a1YSF3cXv/xlHwyGUOLjx7FuXSomkwmA774r5M03X3eqrYrcfoZkrb0IploK//1W9A92je3shRDOqz36IQHcd994MjP/woMPTrzqogmAL774BzNmTMNkMuHpqeeee8Ywffpj9vv/8IdFvPJKJtOnJ+Hp6YFe78XcuX8gNnZYO7wDLeP2/ZBuq/2espwXyAiOpaRLGZviV6PT6to4Qsfp7L10oPPnKPldn/RDch7SD6mNmH8wAlCuqcdHde1UxUgIIVyJ2y/ZmSuMKL0vV7xqMHiFOTocIYQbk35Ibu7KD0aqfG9Ho7/ML/xDHR2OEMKNST8kN6aUwlxh5Dt9w1qnXNAghBCO494F6XI1tss1FGoaJorh3eSSbyGEcBS3LkiNWwYZlQWdzQt/fRcHRySEEO7LvQvSvzdVrdTV00UbdN1NCoUQQnQM9y5IF0ux6X2weddh8O3h6HCEEMKtufVVdhpPb0qC+qLRldNH/n4khGiFyZN/jY+PD2+88Q5ardZ+LD19AxEREU0+79133+a++8bf8LLuyZN/zZUrZt5/Pw+druH/JfPydvLSSyuYP/+PTJrUsAvEiRPHeOWVTEpLS/D374qvry8zZz5JTMxQUlKepLy8HD8/Py5frsNgCCUpaRqjRt1tf53vvivk5Zf/QkmJEZtNERkZyZw5CwgJaf8v7W49Q/IansSngQ3NraKCpSmfEKJ12qtBX6OgoNv5xz8+s9/Oy9tJRER/++2iokL++Md5PPpoMu++u53s7DdZtGgZFy/+Z6fxefMWsmXL22zdmsvvfjeT1NRVfPLJAaBhQ9Z58/6bxMSH2Lo1l/fe286gQUNYsCAFi8VyS3m1RIfNkEwmEy+99BKfffYZXl5exMTEsGrVKhISEtDr9famUQsXLmT06NEdFRbG6hLwgp5dZMlOCFdVUPYln5UdapdzjzTcyXDDze331l4N+ho98EAieXm7GDnybkpKzlFfX094eF/7/Tk5b5CY+BDDh4+0H+vZsxc9e15/BWjo0DhmznyCt97awr33JrBt21ZiY4eSkDDO/pjf/Oa37NuXz759exg/fsJNvQ8t1WEFae3atXh5ebFnzx40Gg0VFRX2+zZt2tTslLY9VV75AS+9P3qd3iGvL4ToPH7aoC8p6VH78RUrlvPYY7OIiRnKlStXmDv3GaKiBvDYY7PYuTOXF19Mo0+fvs2cuUFsbBwffPB/VFdX89FHuxk//gFOnTppv//06ZOMGTP2lmIeMGAgGRkbACgq+pbBg2OueUx09CAKC7+9pfO2RIcUpNraWnJzc/nkk0/sV7J17969I166WdV1ZiyelwjxNDg6FCFEKww3DLvpWUx7a68GfQAaDSQk3Mf+/Q0zlqys168qSC3x0+21Hb3XdocUJKPRSEBAAC+//DIFBQX4+fkxd+5c4uLigIZlOqUUw4YNY8GCBfj7+9/S+VuyqyzA9yfPofWuo1/wL7j99s77P0idObdGnT1Hye9aFy5o8fBwrj+D63Ra+vTpw6hRd/Pee28D4OmpQ6OBLVvewsPDs8nn3UwuOp2WxMRfM2vW74iJGUpQUDc0Gg1arQYPDy2Rkf05deoYCQkJ132+RqNBp9Nc9VqnTx8nPLwvHh5aIiIiOX786DWxnDhxlEmTkpqNsS1+RzukIFmtVoxGIwMGDGDRokUcOXKEp59+mr1795KTk4PBYMBsNrN69WpWrlzJunXrbun8LW0/8fm3Db3kf3FbaKfd3r+zty6Azp+j5Hd9NpvN6Vo6WK0NMc2Y8QSzZiVjtVqx2RQxMbFs2fK//P73jwMNDfo8PDwICuqOn58fly5V31QuVquNkJBQnnhiNgMGRGOx2FBKYbMpLBYbU6cmM3/+bGJi4rjzzuEAlJaW2JfylFJYrcr+WkeOHCY7+xXmz38Oi8XGxIlT+O1vp5Cfn2//O9LWrTn8+OOPJCT8V7Mx/nQMW9p+okMKksFgwMPDg8TERACGDBlCYGAgxcXFDBo0CAC9Xs+0adN45plnOiIkAM5cLAE99Ove+8YPFkKIm9ReDfoaPfTQI9c93q9fBKmpG3jttf9h7dqX8Pb2JiAgkFmznrY/ZuPGdbz22mbq6y8TEmLgueeW8atfNVxI5u/flQ0bMsnM3EhWVgZKNZxz/fqXW9V+/WZ1WIO+mTNnMnPmTO6++26Ki4uZOnUqO3bswNfXly5duqCUYuPGjRQWFpKZmXlL527pDGnDpzl8bzrBhjEr0Wqca+rfVjr7t2vo/DlKftcnDfqcR1s16Ouwq+xWrFjBkiVLSEtLw8PDg/T0dMxmM0899dS/p7U2wsPDef755zsqJO4I7kqQLbrTFiMhhHAlHVaQevfuzVtvvXXN8dzc3I4K4RqP9E2ke/fbqKiocVgMQgjRSBr0uTnZUFUI4SykQZ8QQrgoR//fjGjbMZCCJIRwSR4eemprq6UoOZBSitraanx8vNvkfG6/ZCeEcE2BgbdTVfUDNTUXHR3KDWm1Wmy2znmVnYeHnr59f8nFi/WtP1cbxCOEEB1Op/Oge3fX2Pars1+637CRbOsLkizZCSGEcApSkIQQQjiFTrFkp9W27tLt1j7f2XX2/KDz5yj5ub7OnuNP82tprh22dZAQQgjRHFmyE0II4RSkIAkhhHAKUpCEEEI4BSlIQgghnIIUJCGEEE5BCpIQQginIAVJCCGEU5CCJIQQwilIQRJCCOEUOsXWQS1RXFzM4sWLuXjxIgEBAaSlpREWFubosFolISEBvV6Pl5cXAAsXLmT06NH885//ZPny5ZhMJnr27MnatWsJCgpycLQ3lpaWxp49eygpKWHnzp1EREQAzY+dq41rUzk2NZaAS41nVVUVzz33HGfPnkWv13PHHXewcuVKunXr1mwerpJjc/lFRkYSERGBVtvwvT89PZ3IyEgADhw4QHp6OlarlejoaNasWYOPj48jU2nW7NmzOXfuHFqtFl9fX/785z8TFRXV9p9F5aaSk5NVbm6uUkqp3NxclZyc7OCIWi8+Pl6dOnXqqmNWq1WNGzdOHTp0SCmlVGZmplq8eLEjwrtlhw4dUqWlpdfk1dzYudq4NpXj9cZSKdcbz6qqKvX555/bb6empqo//elPzebhSjk2lZ9SSkVERKiampprnlNTU6NGjRqliouLlVJKLVmyRGVkZHRIvC1VXV1t/3nv3r3q4YcfVkq1/WfRLZfs/vWvf3H8+HESExMBSExM5Pjx41RWVjo4srZ39OhRvLy8iIuLA2Dq1Kl89NFHDo7q5sTFxWEwXN3vprmxc8VxvV6OzXG18QwICGD48OH22zExMZSWljabhyvl2FR+zTl48CADBw60zxamTp3Khx9+2J5htlqXLl3sP9fU1KDRaNrls+iWS3ZlZWWEhISg0+kA0Ol0BAcHU1ZWRrdu3RwcXessXLgQpRTDhg1jwYIFlJWVERoaar+/W7du2Gw2+zTa1TQ3dkqpTjWuPx9Lf39/lx5Pm83GX//6VxISEprNw1Vz/Gl+jZKTk7Fardxzzz3MmTMHvV5/TX6hoaGUlZU5IuRbsnTpUv7+97+jlCI7O7tdPotuOUPqrHJyctixYwfbtm1DKcXKlSsdHZJooc44lqtWrcLX15fp06c7OpR28fP8Pv74Y95//31ycnIoLCwkMzPTwRG2zurVq/n444+ZP38+6enp7fIablmQDAYD5eXlWK1WAKxWKxcuXLilpRNn1Bi/Xq9n2rRpfPXVVxgMhquWECorK9FqtU79TbM5zY1dZxrX641l43FXHM+0tDTOnDnDxo0b0Wq1zebhijn+PD/4zxjedtttTJkypckxLC0tdanf0YcffpiCggJ69OjR5p9FtyxIQUFBREVFsWvXLgB27dpFVFSUSy7rNKqrq+PHH38EQClFXl4eUVFRDBw4kPr6er744gsA3nnnHcaPH+/IUFulubHrLOPa1FgCLjme69ev5+jRo2RmZqLX64Hm83C1HK+X36VLl6ivrwfAYrGwZ88e+xiOHj2ab775hu+//x5oyO/+++93SOw3o7a29qolxQMHDtC1a9d2+Sy6bYO+oqIiFi9eTHV1Nf7+/qSlpdGnTx9Hh9ViRqOROXPmYLVasdlshIeHs2zZMoKDg/nqq694/vnnr7qEtnv37o4O+YZefPFF8vPzqaioIDAwkICAAHbv3t3s2LnauF4vx6ysrCbHEnCp8fz2229JTEwkLCwMb29vAHr16kVmZmazebhKjk3l9/jjj7N8+XI0Gg0Wi4XY2FiWLFmCn58fAPv27WPt2rXYbDaioqJITU3F19fXkak0qaKigtmzZ3P58mW0Wi1du3Zl0aJFREdHt/ln0W0LkhBCCOfilkt2QgghnI8UJCGEEE5BCpIQQginIAVJCCGEU5CCJIQQwilIQRLCRZw7d47IyEgsFoujQxGiXUhBEkII4RSkIAkhhHAKUpCEaIXy8nLmzJnDiBEjSEhI4M033wQgIyODZ599lnnz5hEbG8vEiRM5efKk/XlFRUUkJycTFxfHhAkT2L9/v/2++vp6UlNTiY+PZ9iwYTz66KP2bWgAdu7cyZgxYxg+fDibN2+2H//666955JFHGDp0KKNGjWLNmjUd8A4I0YZa2bdJCLdltVrVxIkTVUZGhjKZTOrs2bMqISFBHTx4UG3atEkNGDBAffjhh8psNqvs7GwVHx+vzGazMpvNaty4cWrz5s3KZDKpTz/9VMXExKiioiKllFIvvPCCmj59ujp//ryyWCzqyy+/VCaTSRmNRhUREaGWLl2qLl++rE6cOKGio6NVYWGhUkqppKQk9cEHHyilGprAHT582GHvjRAtITMkIVrom2++obKykpSUFPR6Pb179yYpKYm8vDwAoqOjGT9+PJ6ensyYMQOz2cyRI0c4cuQIdXV1PPnkk+j1ekaOHEl8fDy7d+/GZrOxbds2li5dau8nM3ToUPumnQApKSl4e3vTv39/+vfvb595eXh4cPbsWSorK/Hz8yMmJsYh74sQLeWWDfqEaAslJSVcuHDB3tkUGrbZj4uLIzQ0lB49etiPa7VaQkJCuHDhAgA9evSwtymAhiZt5eXlVFVVYTKZ6N27d5Ov+9NNRn18fKirqwMa+tVs2rSJ+++/n169epGSkkJ8fHyb5StEe5OCJEQLGQwGevXqRX5+/jX3ZWRkcP78efttm81GeXm5fcfu8+fPY7PZ7EWprKyMsLAwAgMD8fLywmg00r9//1uKJywsjPXr12Oz2cjPz+fZZ5+loKDAaXeRFuLnZMlOiBYaPHgwfn5+vPrqq9TX12O1Wjl9+jRff/01AMeOHSM/Px+LxcIbb7yBXq9nyJAhDB48GG9vb7Kzs7ly5QoFBQUcOHCABx54AK1Wy6RJk1izZo29wdnhw4cxm803jGf79u32Znb+/v4AV83ChHB28tsqRAvpdDqysrI4efIkY8eOZcSIESxbtoyamhoAxo4dS15eHnfeeSfbt28nIyMDT09P9Ho9WVlZHDx4kBEjRrBixQrS09MJDw8HYNGiRURERDB58mTuuusu1q1bh81mu2E8f/vb35gwYQKxsbGsXr2aDRs22Hv0COEKpB+SEO0gIyODM2fOsG7dOkeHIoTLkBmSEEIIpyAFSQghhFOQJTshhBBOQWZIQgghnIIUJCGEEE5BCpIQQginIAVJCCGEU5CCJIQQwilIQRJCCOEU/h9FgQHjrxI2GAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set()\n",
    "epochs = [10*i for i in range((30))]\n",
    "plt.plot(epochs, net_scores[:30],label='Net_CDO')\n",
    "plt.plot(epochs, net_scores[30:60],label='Net_VDO')\n",
    "plt.plot(epochs, net_scores[60:],label='Net_MCDO')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('Test accuracy')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEUCAYAAABkhkJAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xd8W9X9//HXvRresi1ZtuUdZ5rEziCDTEYIARowq9BSxrfQfFvgW2ZbAuVLgOCWpN+mg5b+2lJCU2gphZYUk4SSAWQHJ4HsQeLEsS3LQ/KQZVvj3t8fcpyYLMWxZUk+z8fDD2tcSedYid665557PpKqqiqCIAiC0M/k/m6AIAiCIIAIJEEQBCFEiEASBEEQQoIIJEEQBCEkiEASBEEQQoIIJEEQBCEkiEASBEEQQoIIJEEQBCEkiEASBEEQQoIIJEEQBCEkiEASBEEQQoIIJEEQBCEkiEASBEEQQoK2vxsQTA5HK4rSs8XNTaZ4Ghqcvdyi0BCpfYvUfoHoW7iK1L59tV+yLJGcHHfBzzOgAklR1B4H0onHR6pI7Vuk9gtE38JVpPatN/olhuwEQRCEkCACSRAEQQgJA2rI7kx8Pi8ORx1er/uc29XWyiiKEqRWBVd/9E2WNcTExBMfn4gkSUF9bUEQQtOADySHo47o6Fji4tLP+cGo1cp4vZEZSMHum6qq+HxeWloacTjqMBpTg/bagiCErqAN2ZWXl3PHHXcwe/Zs7rjjDo4ePXraNuvXr+eWW25h1KhRLFy4sNt9P/rRjyguLu76GTFiBKtXr77odnm9buLiDOJbehBJkoRWqyMpyYTb3d7fzREEIUQEbQ9p/vz53HnnnRQXF7Ns2TKeffZZli5d2m2b7OxsSkpKWLlyJW539yG0RYsWdV3ev38/9957L9OnT++Vtokw6h+SJAOROeNIEMKdqqpB/2wMyh5SQ0MDe/fuZc6cOQDMmTOHvXv3Yrfbu22Xm5tLQUEBWu25c/Kdd97hhhtuQK/X91mbBUEQBpLmVjdb99lYunI/T/1hMw//ah0dbl9Q2xCUPSSr1UpaWhoajQYAjUZDamoqVqsVo9F4Qc/ldrt5//33ef311/ugpeHhtttuYNGiX5CfP6S/m8Lbb/+VWbOuJTnZ/z4eOnSAxYsXcvDgASZPnsqLLy46zzMIgtAfnG0eDlQ42H+skf0VDqrqWwGI1msYlp3EtROz0euCOxE77CY1rFq1ioyMDAoKCi74sSZT/Gm31dbKaLWB/dED3S4YNJrT2+31es+7d3k2Pe3b22//jUmTLsNsTgEgJSWFRx55gkOHDrB165bzPq8sy5jNCT167UD05XP3N9G38NRfffN4FXYcrGXnoXp2fVlPubUJVYUovYZL8ozMnJhD0ZAUhmQlodFc+OdBb/QrKIFksViw2Wz4fD40Gg0+n4/a2losFssFP9e7777Lrbfe2qN2NDQ4TzubWFGUgGaYBWsm2u7dO/ntb3+Fy+UC4KGHHiEqKoqf//wlAMaMGdc5S83f7ttuu4GZM69h+/bPyM8fwlNPPcsbb7zOhx8uB6CgYCSPPvpDYmNj+dOffs/Ro+U0NTVSX1/HoEH5PPXUfJKSDDQ3O/nlL3/Gvn17ALj22q/xrW/dC5y+R3bi+rp1n1BfX8dTT/0QvT6K+fNfZNCgfJKTTRw5cgRVVc/7N1MUhbq6lj75W5rNCX323P1N9C089UffmlvdfPx5FWt3VNHkdKPVyAzJNFA8bRAjcpLJzzCgPSWA7PbWC36Nr/ZLlqUz7gCcT1ACyWQyUVBQQGlpKcXFxZSWllJQUHDBw3U1NTVs27aNxYsX90k7N+yysn6n9Yz3SRKoF3H8fVqRhamF5w7g5uYmnn76h5SULKKwcDQ+n4+mpkbuu+8unn12AePGjWf16o/45z//0e1xra2t/PGP/gkimzZt4MMPl/P//t9rxMbG8eKL83n99Vd58MGHAdi5cwdLlvwVo9HET37yPK+//iqPPvo4r7/+KoqisHTp33G5Wvnud+8jP38IkydPPWt77733ft5//z1efHFhSAwfCoJwUoWthY/KjrNlby1en8KoQUbuvTaLkXnJ6LSa/m7eGQVtDOq5557jjTfeYPbs2bzxxhs8//zzAMydO5ddu3YBUFZWxowZM1iyZAlvvfUWM2bMYN26dV3P8a9//Ysrr7ySxMTEYDU7qHbv3kVe3iAKC0cD/mNtdrud6Ohoxo0bD8DMmbOIj+/+zePaa7/WdbmsbCszZ15DXFw8kiRx4423UFa2tev+KVOmYzSaAJgzp5jt2z/retwNN9yMJEnExcVz9dXXdHucIAihT1FUth2o5aU3t/Pcks/4bH8t04ssvPidSTx+xxjGDEkJ2TCCIB5DGjx4MP/4xz9Ou/2Pf/xj1+Xx48fz6aefnvU5HnjggT5p2wlTC8++FxNaJ8Z2n4oZGxvTp6+m0Wi6DXV+dUq+IAj9q7Xdw7ovrKzeVklDczsmQzS3XzmE6aMtxEXr+rt5AQudo/QCo0YVcvRoObt37wTA5/NhNBrp6Ojgiy92ALB27SqczrOPQY8fP5E1az7C5WpFVVVKS99jwoRJXfdv3Lgeh8MBwPLl7zNu3ISux33wwTJUVcXlamX16v90PS4zM5v9+/3HlsrKtmK3N3Q9X1xcHE5n5C2nLwihTlFVDlU2svTDAzzx2w28vfZLTInRPHTzKF763mVcOyknrMIIwnCWXSQzGBIpKVnEyy//gvb2NiRJ5qGHHuG550r4+c9fQpIkRo8eS1pa+lmfY/LkqRw+fIjvfvfbAIwYcQn33nt/1/2jR4/lueeepq6ulry8fP7nfx4D4L/+6zv84heLuOeeOwCYPft6LrtsCgBz536PkpLneOedt7n00vHdXv+2277BT37yAtHR0cyf/yLR0dE8+OB3aG9vx+3u4Oabr+f++/+bOXNu6vW/lyAMNIqicvB4I2UHatl2sK5rksKkS1KZNT6bnLTwnp0oqerFHKoPL2eaZVdTc4z09NzzPja0hux65k9/+j1tbW38z/882u32/uxboH//nhCztcKT6Ft3Xp/C/goH2w7Usf1gHS0uDzqtTFG+iUuHmxk9JIWYqP7dtwirWXaCIAhC4Dxehb1H7Ww7UMeOQ3W0tnuJ0mkYPcTEpcNTKcw3Eq2PvI/vyOuRcFb33//d/m6CIAhnoagqh443smlPDZ/tr6Otw0tMlIYxQ1IYPzyVkYOM6HWhO0OuN4hAEgRB6Ec1dhcbd9ewaXcNDc3tROk0XDrczMSCVApyjehCaIWYviYCSRAEIchaXG627qtl054ajlQ3I0lwSZ6RW2bkM26YmSh9ZO8JnY0IJEEQhCDweBU27Kzmw43l7DzcgE9RyTLHc/uVQ5h0SRrJCVH93cR+JwJJEAShj9ib29ldbmf3kQb2HHXQ1uElMU7P1eOzmDwyPeynafc2EUiCIAi9xOP1ceB4I7uP2Nldbqe6s6RDckIUlw43M2tSHhnJUWjkgXNc6EKIQApDoVwP6d///hfvvvv3rmqTd955D7NnX9/PrRSEvqGqKjV2F7uO2Nld3sCBikY8XgWtRmZ4diLTCi0U5hvJSIlDkqSIPseqN4hAiiAXUw+pp95++2+MHz+xK5CysrJ5+eXfYzAkUltr49vfvpOiojFYLBlBbZcg9KW2Di/rdlpZve04dY3tAKQbY7l8dAaj8k0Mz0kiKsKnaPcFEUin8BzcgOfAmRd3lSSJi1nUQjd8BrphZy/lcEKg9ZBO6K16SC6Xq8f1kJ555smuekgnViUHSE1Nw2RK6ax9JQJJCH+Olg5WbTvOxzuqaevwMiwrkesm5TJqkJGUpL5d5HggEIEUQiKtHtL27WU4nU5GjBjRi38lQQi+yjonH26tYPMeG4qqcunwVGZPzGZwRmSWwukvIpBOoRs29ax7McFY7y3Qekg/+1lJt8edrR4SwI033sKvfvV/Xfd/tR7SL3/5s67HPfLID06rh3SuQDqX8vIjvPjifObPLyEqKrpHzyEI/UlVVfYfc7BiawW7j9jR62SuGJPJrInZpIq9oT4hAikshXY9pOPHK/jhDx/hhz98mtGjx/Rp2wSht3l9CmUHavlwy3GO2VowxOq4eUY+V47NJD4mvMo5hBsx9zCEREI9pKqqSh5//Ps8+ugPerx3JQj9wdHSwb83lDPv95v4w7/34vb6+K/rRvCzB6dww5Q8EUZBIPaQQkgk1EP6059+T3NzI6+++nteffX3ADzwwPeZNGly7/6xBKEXKKrKnnI7H++o4osvG1BUlZF5ydw1azhFQ0zIknT+JxF6jaiHJOohiXpIYUj07eI0OTtYv8vKJ59XU9/UTkKsjmmFFmaMySAtObbPXjdS3zdRD0kQBOECKKrKvmMOPtlRxY5D9fgUlRE5Sdx2xWDGDjUPqFW1Q1XQAqm8vJx58+bR2NhIUlISCxcuJC8vr9s269evZ/HixRw8eJC7776bJ598stv9y5cv53e/+13XKgBLliwhJSUlWF0Ie6IekjAQNTk72Li7hk8+r6a2sY24aC1Xj8/i8jGZpBv7bm9IuHBBC6T58+dz5513UlxczLJly3j22WdZunRpt22ys7MpKSlh5cqVp83k2rVrF7/5zW/485//jNlspqWlBb1eH6zmC4IQRnyKwq7DdtbtrO46NjQsK5Hi6YMYP9yMTitWUQhFQQmkhoYG9u7dy5IlSwCYM2cOCxYswG63YzQau7bLzfUfS1i1atVpgfT6669z3333YTabAUhIEKvkCoLQnc3uYt1OKxt2W2lyujHE6rhmYjbTiyxYTHH93TzhPIISSFarlbS0NDQa/7cSjUZDamoqVqu1WyCdy+HDh8nKyuJb3/oWLpeLWbNm8cADDyCJWTCCMKB1eHyU7a9l3U4rB483IklQlG9i+jUZFA02odWIY0PhImwmNfh8Pg4cOMCSJUtwu9185zvfISMjg5tuuing5zjTrI/aWhltgAczA90uHPVX32RZxmzuu73dvnzu/jaQ+6aqKoeON/LR1go+3VGJq92LJSWOe64v4Krx2ZgSQ3clhUh933qjX0EJJIvFgs1mw+fzodFo8Pl8nQtuWgJ+joyMDK699lr0ej16vZ6ZM2eyc+fOCwqkM037VhQloCnPoTTtu7fLT1xM375afmLz5o387ncvI0kSXq+X6dMv57//+8Gz7skqitJn02AjdYotDNy+tbZ72LS7hk+/sFJZ50Svlbl0eCozRlsYlp2EJEkobm/I/m0i9X0Lq2nfJpOJgoICSktLKS4uprS0lIKCgoCH68B/3OmTTz6huLgYr9fL5s2bmT17dh+2OvyEQvmJoqIxvPbaG2g0GrxeLw88cB8jR45i2rTLg9ouIXKoqsqBikY+3VlN2f46vD6F3PQE7p49nEkFacRGh81Aj3AeQXsnn3vuOebNm8crr7yCwWBg4cKFAMydO5eHH36YwsJCysrKePzxx3E6naiqygcffEBJSQnTp0/na1/7Grt37+b6669HlmWmTZvGbbfdFqzmB024l58YNCi/q21udwcejxdJityhTqHvNDo72LDLyrqdVmodbcREaZkx2sL0ogxy0yNz2GugC1ogDR48mH/84x+n3f7HP/6x6/L48eP59NMz1yOSZZmnnnqKp556qs/auMW6jU3Wz854nyTBxaxpMdkygUmWS8+5TaSUn9i/fy8//ekLHD9+nJtvvpUpU6b19M8mDDA+RWHr3hpKPz18crp2dhI3Ts3j0uGpouhdhBNfXUNIoOUn4uO7j82erfyEJEnceOMtlJVt7br/q+Untm//rOtxN9xw82nlJ3pixIhL+POf3+Kf//yAAwf2dy0MKwhnU1nr5O21X/KDVzay4E9bOFzVxOyJ2ZTMncS8b41jyiiLCKMBQAy+nmKS5dKz7sWE0qSGUC8/cUJSUhKTJk1h7dpVjBkzrq+aJ4SpplY3W/bUsHF3DRW1TjSyRGG+iWunDiI/NU5M1x6AxDseQiKh/ERFxTEUxR/cbW1tbNmysddmAwrhz+3xsXWfjV/+4wue+M0G3lrzJbIscefVQ/n5/0zl4duKmFqUIcJogBJ7SCEkEspPbNq0nuXLSzv3qnzMmHElN9wQ+NR8IfKoqsqhyiY27rby2f5a2jp8JCdEce2kHCaPSiczRaygIPiJ8hOi/IQoPxGGwqFviqKyZZ+N0o1HsTa4iNJpuHS4mSmj0hmRk4wsn/nctHDoW09Fat/C6jwkQRAGDv9MuVre33iUGruLLHMc93+tgPHDU4nSi4kJwtmJQBpARPkJoS/5FIUte228v/EYNruLLHM8D908irHDzKLyqhAQEUiCIFwUn6KweY9/aM7maCM7NZ6Hbi5k7LAUEUTCBRGBJAhCj5wIovc3HqXW0UZOajz/c0shY4aKIBJ6RgSSIAgXxNXuYeu+WlZuqaC20R9E3+8MIlEORrgYIpAEQTgvRVHZc9TOhl1Wdhyqx+NVyE1L4Pu3FjJmiAgioXeIQBIE4ayq6lvZuMvKpj01NDrdxEVrmV5kYWqhhbz0BBFEQq8SgRSGerse0sX4aj2kExwOB/fccwejR4/hxRcX9VPrhJ5wtnnYus/Ghl1Wyq0tyJJE0WATU0alM3pICroILlQp9C8RSBEkFOohnfDzn7/E5MlTcblag9oeoef2HXOwdnsln39Zj9enkp0azzeuGsKkkekkxun7u3nCACAC6RTNGzfQtP7M5S8kSeJiFrVInDYDw5Szl3I4IRLqIf3nPyswGo2MGHEJGzeu6/HfTAiOw9VN/POTI+w75iAhVseVY7OYWphOTpqoOSQElwikEBIJ9ZDq6+t46603+c1v/sDHH6/uo7+U0Bsq65z869Mj7DhUT0Ksjm/OHMoVYzPQacVqCkL/EIF0CsOUqWfdiwnGem+B1kP62c9Kuj3ubPWQAG688RZ+9av/67r/q/WQfvnLn3U97pFHfnBaPaRzBdKZLFz4Ig8++DCxsbEX2HshWGodLpatL2fzHhvRURpunpHPrPFZROvFx4HQv8S/wLAUuvWQdu/eRXn5AgDa2lx0dHTwgx88zP/936/7tI3C+TlaOnh/41HWfVGNRpa49rIcrpuUS3yMrr+bJgiACKSQcmo9pFGjik6rhzR69NiA6iH97ne/5vbbv0lMTOxZ6yElJyefsR5SUdFo2tpcrF79Hx56yL8q+Il6SEOGDD1vPaQVK9Z0XV6+/H02blwnZtn1M2ebh+Wbj7F6WyWKojJjTAY3TMkjKT6qv5smCN2IQAohkVAPadCg/F7/uwgXTlVVjtla2LLXxqdfVNPe4WPyqHRunDaI1KS+3aMWhJ4KWj2k8vJy5s2bR2NjI0lJSSxcuJC8vLxu26xfv57Fixdz8OBB7r77bp588smu+15++WX++te/kpqaCsC4ceOYP3/+BbVB1EMS9ZAixdn6Vl3fypa9Nrbus2FztKGRJcYMTeGmaYPINF94fZr+MBDft3AXdvWQ5s+fz5133klxcTHLli3j2WefZenSpd22yc7OpqSkhJUrV57xOMVNN93ULaQEQYC6xja27rOxZW8tlXVOJGBEbjLXTsrh0uGp4hiREDaCEkgNDQ3s3buXJUuWADBnzhwWLFiA3W7HaDx5QmVurv+b8qpVq8554FzoGVEPKXLYm9v5qOw4W/faOFzdDMDgTAPfvHooE0akiuNDQlgKSiBZrVbS0tLQaPznN2g0GlJTU7Fard0C6Xw++OAD1q9fj9ls5vvf/z5jx47tlfapqirW5OoHQRotjhhtHV62Hahj054aDlQ4UFTITo3ntisGM3FEKini2JAQ5sJmUsM3vvENvve976HT6diwYQMPPvggy5cvJzk5OeDnONOYZkODDklS0GrPP6yhjeA1vPqjbx0d7URFRWE2992KAH353MHg9SlsP1DLx9sq2bLbiturYDHFcfvVw5kxNpPsCF1NIdzft3OJ1L71Rr+CEkgWiwWbzYbP50Oj0eDz+aitrcVisQT8HGazuevy1KlTsVgsHDp0iIkTJwb8HGea1KDXx+Jw2ElKMiFJZ/9QjoRJDWcT7L6pqorH46axsY6EhOQ+O8gbrgeQVVXlSHUzm/bUsHVfLc42D/ExOqYVWZg8Mp38DAOpqQbq6lrCsn/nE67vWyAitW9hNanBZDJRUFBAaWkpxcXFlJaWUlBQcEHDdTabjbS0NAD27dtHVVUVgwYNuui2xccn4nDUYbNVAmcfQpJlGUWJzEDqj75pNFoSEpKJiYkL6uuGMpvdxaY9NWzeY6O2sQ2dVmbs0BQuG5nOqEFGtJrI3UMXBAjikN1zzz3HvHnzeOWVVzAYDCxcuBCAuXPn8vDDD1NYWEhZWRmPP/44TqcTVVX54IMPKCkpYfr06SxevJg9e/YgyzI6nY5FixZ122vqKUmSMBpTz7tdpH6zgcjuW6hTVJXdRxpYuaWC/RWNXTPk5kzJ49LhZmKiwmZUXRAuWsDnIR0+fJiVK1dSX1/P/PnzOXz4MB6PhxEjRvR1G3vNmYbsAhXJH9qR2rdQ7pfHq7B5Tw0ffnac6vpWkhOiuGpcJlNGWUhOOP8MuVDu28USfQs/vTVkF9AYwIoVK7jrrruw2WwsW7YMAJfLxUsvvXTBLygIA5mzzcMHm47yo99tZMmK/Whkibk3XMLC703ma5PzAgojQYhUAY0H/PrXv2bJkiWMGDGCFStWADBixAj279/fp40ThEhR19jGR58dZ91OKx0eH6PyjcydmENBbrI45UAQOgUUSHa7neHDhwN0/eeRJEn8RxKE8zhS3czKrRVsO1CLLElcdkkasyfmkJUaHsv4CEIwBRRII0eOZNmyZdx0001dt33wwQcUFRX1WcMEIZx9WdXEe+uOsPeog5goLddOyuHqS7PFkJwgnENAgfTjH/+Y+++/n3feeQeXy8X9999PeXk5r732Wl+3TxDCypHqZt5bf4TdR+wYYnXcfuUQLh+TIWbLCUIAAvpfMnjwYFasWMHatWu54oorsFgsXHHFFcTFiXNIBAHgWE0L7607wheHG4iP0fH1Kwdz1dgsovSiHLggBCqgQHrxxRd55plnuP7667vdXlJSwo9//OM+aZgghIPjtU6WrS9n+8E64qK13Hp5PleNyxJ7RILQAwFN+/7nP/95xtv//e9/92pjBCFcVNU5eeW93cx/bSv7jjm4adogFn5vCl+bnCfCSBB66Jz/c9555x0AfD5f1+UTjh8/TlJSUt+1TBBCUGWdkw82HWPrXhtReg03TMnjmonZxEWLmkOCcLHOGUgnToL1eDxdl8E/5TslJaVr+R9BiGSqqrLnqJ3/bD3O7nI7UToN112Wy7WTckTxO0HoRecMpL/85S8A/OIXv+Cxxx4LSoMEIVR4vApb9tr4z2cVVNa1khin55YZ+VwxNlMEkSD0gYAGu08NI1VVuxVWk2WxArEQWZxtHtbuqGLNtkqaWt1kmeO4/2sFTCxIQxfBNbEEob8FFEg2m40XXniBsrIympubu923b9++PmmYIASbze7iP58dZ8MufyG8UflGvjMxh0vE8j6CEBQBBdL8+fOJjo7m9ddf56677uLNN9/k5Zdf5vLLL+/r9glCn1JVlQMVjXxUdpzPD9Wj0UhMHpnONROyyTSL5X0EIZgCCqQdO3awdu1aYmNjkSSJESNGUFJSwje+8Q1uv/32vm6jIPQ6t8fH5r02VpVVUlnnJD5Gx5wpeVx1aRaJcfr+bp4gDEgBBZIsy2i1/k0NBgN2u534+HhsNlufNk4Qepu9uZ21O6r45PNqnG0esszxfPu6EUy6JA29TqyqIAj9KaBAGj16NJ988gmzZs1i2rRpPProo0RHRzNq1Ki+bp8gXDRVVTlc1cyqbccp21+HisqYISnMGp/N8JwkcXxIEEJEQIG0aNEiFEUB4Omnn+a1116jtbWVe++9t08bJwgXw+P1sXG3lVVllRytaSEmSsusCVlcNS4Lc1JMfzdPEISvOG8g+Xw+SkpKWLBgAQDR0dE8+OCDfd4wQegpV7t/2vbq7VU0tnSQbozlrmuGMWVUOtF6sayPIISq8/7v1Gg0bNiwQQxrCCGv0dnBR58dZ+2OKtrdPsYOM3PldRlcMsiILP79CkLIC+gsv3vvvZeXX34Zj8fT4xcqLy/njjvuYPbs2dxxxx0cPXr0tG3Wr1/PLbfcwqhRo866LNGRI0cYPXq0WLZI6FLrcLF05X5+9LtNrNxaQdFgE/P/awIvfHcKo/JNIowEIUwENH7xxhtvUF9fz5IlSzAajd32lj7++OOAXmj+/PnceeedFBcXs2zZMp599lmWLl3abZvs7GxKSkpYuXIlbrf7tOfw+XzMnz+fq6++OqDXFCJbha2F5ZuP8dn+WjSyxNRCC9dOyiEtOba/myYIQg8EFEg/+9nPLupFGhoa2Lt3L0uWLAFgzpw5LFiwALvdjtFo7NouNzcXgFWrVp0xkP7whz9wxRVX4HK5cLlcF9UmITypqsrB440s31zBriMNROs1XDsxh1kTskmKF+XBBSGcBRRIEydOvKgXsVqtpKWlodH4z/PQaDSkpqZitVq7BdK57N+/n/Xr17N06VJeeeWVHrXDZLq4M+/N5oSLenwoC/W+ebwKm3dZ+fe6w+w/5iAxXs/d1xVw/dRB51zoNNT7dTFE38JTpPatN/oVFlOOPB4P//u//8tPf/rTrlDriYYGJ4qinn/DMzCbE6ira+nxa4eyUO6bvbmdTz6v5tMvqmlqdWNOiuaua4YxrdCCXqehzdlOm7P9jI8N5X5dLNG38BSpfftqv2RZ6tEOQFACyWKxYLPZ8Pl8aDQafD4ftbW1WCyWgB5fV1dHRUUF//3f/w1Ac3MzqqridDq7pqMLkUNVVfYdc7BmexWfH6pHVVWKBpu4clwWo/LFjDlBiFRBCSSTyURBQQGlpaUUFxdTWlpKQUFBwMN1GRkZbNmypev6yy+/jMvl4sknn+yrJgv9wNXuYcOuGtbuqKLG7iI+RsfsSdlcMSZTnMgqCANAQNO+//SnP53x9hOTFALx3HPP8cYbbzB79mzeeOMNnn/+eQDmzp3Lrl27ACgrK2PGjBksWbKEt956ixnFvSfvAAAgAElEQVQzZrBu3bqAX0MITxW2Fl5fsZ/Hf7uBv60+RFyMlrlzLuHnD03h61cMEWEkCAOEpJ5abe8sxo0bx/bt20+7feLEiWzdurVPGtYXxDGkM+uvvtmb23l77Zds3VeLXitz2cg0rhybRW567xz0Fe9ZeBJ9Cz9BOYa0adMmABRFYfPmzd0qxVZWVhIXF3fBLygIHq+PlVsq+GDTMVTgxql5zJqQTVy0KAsuCAPZOQPpxz/+MQAdHR08/fTTXbdLkoTZbOaZZ57p29YJEUVVVT4/VM/fVh+ivqmdS4ebueOqIaQkiiE5QRDOE0hr1qwB4Ec/+hGLFi0KSoOEyFRd38rfVh9iT7mdzJQ4fvCNMVySF9ikFkEQBoaAy08IQk+42r38e0M5q7dVotdp+ObVQ7lybCZaTUDzaQRBGEACCqTLL7/8rKt9B7qWnTCwKKrKhl1W3v34MC0uD9NHZ3DL5fkYYkV5cEEQzqxHa9nV1dWxdOlSrr/++j5plBDeKmwt/HnlAcqtzQzONPDo7aPJSzf0d7MEQQhxPV7LbuLEiXznO98RVWOFLh6vj/c3HmXF5griorV8Z04Bk0emi1pagiAEpMcrNej1eiorK3uzLUIYO1TZyOsr9mNtcDF1VDp3zBx6zkVPBUEQviqgQPrVr37V7Xp7ezuffPIJM2bM6JNGCeGjrcPLPz85wprtlRgN0Tx++2hG5Zv6u1mCIIShgAKppqam2/WYmBi+/e1vU1xc3CeNEsLDriMNLF25H3tzBzMvzeKWy/OJ1ofFAvKCIISggD49fvrTn/Z1O4Qw4mzz8LdVh9i0pwaLKZan7rqUIVmJ/d0sQRDCXMBfZ7ds2cJ7771HbW0tqampFBcXc9lll/Vl24QQo6oqn+2v5c2PDuJq93LDlDzmTMlDpxXnFAmCcPEC+iT5xz/+waOPPorZbGbWrFmkpqbyxBNP8Pbbb/d1+4QQ0dDUzsvv7uL/LduDyRDNs/81gZtn5IswEgSh1wS0h/Tqq6+yZMkSRowY0XXbddddx8MPP8ztt9/eZ40T+p/Xp7CqrJJl68tRVZWvXzmYayZko5FFEAmC0LsCCqTGxkYGDx7c7bb8/Hyampr6pFFCaDhc1cSfVx6gss5J0WATd80aRoqoTSQIQh8J6GvuuHHjeOmll2hrawPA5XKxaNEixo4d26eNE/pHa7uHpSv385O/bKO13cNDN4/ikduKRBgJgtCnAtpDev7553nssccYP348iYmJNDU1MXbsWH7+85/3dfuEIFJVlc17bLy15hDONg+zJmRTPG0QMVFiKrcgCH0voE+a1NRU3nzzTWpqarpm2aWnp/d124Qgsja08pcPD7C/opFBFgOP3z6m1yq3CoIgBCLgr77Nzc1s3bq1K5CuuOIKDAaxYGa483h9vLlyP++sOYhOq+Hu2cO5fHQGsizWnxMEIbgCCqRNmzbx/e9/n0GDBpGRkYHVauWFF17g5ZdfZvLkyX3dRqEPeH0Km/fYeH9jOXWN7Vw2Mo07rhxCYnxUfzdNEIQBKqBAWrBgAS+88EK3chMrVqzg+eefZ+XKlQG9UHl5OfPmzaOxsZGkpCQWLlxIXl5et23Wr1/P4sWLOXjwIHfffTdPPvlk133vvvsur7/+OrIsoygKX//617nnnnsCem3hJI9XYcNuK8s3HaO+qZ2ctHgWfHcymcliwoIgCP0roECqra1l9uzZ3W6bNWsW//u//xvwC82fP58777yT4uJili1bxrPPPsvSpUu7bZOdnU1JSQkrV67E7XZ3u2/27NnccsstSJKE0+nkhhtuYOLEid3OjRLOzuP18ekXVpZvPoajpYNBFgN3zhrG6MEmUlMN1NW19HcTBUHoB6riQ22pR2myojTW+H+arKAoxMx5EkkTvElNAb1ScXExb775Zrc9kr/97W/cdNNNAb1IQ0MDe/fuZcmSJQDMmTOHBQsWYLfbMRqNXdvl5uYCsGrVqtMCKT4+vutye3s7Ho9H1NkJQIfHxyc7qlixtYImp5shWYl8+/oRjMwzir+fIAwgarsTpakGpdHa+dsfPEpTLSjekxtGxSEnWdCkDYEgf0QEFEh79+7lrbfe4tVXXyUtLQ2bzYbdbqeoqIhvfetbXdu9+eabZ3y81WolLS0NjUYDgEajITU1FavV2i2Qzmf16tUsXryYiooKnnjiCYYPHx7wYwFMpvjzb3QOZnP4zDpztXtYvvEo733yJU1ON0VDUrhj1jAKB6ecMYjCqW8XIlL7BaJv4aov+6Z43XjtNXjs1bgbqvHYO38aqlHaThkFkbXoktOINmeiGzERnTEDvSkTnSkDTWzPJqv1Rr8CCqTbb789JJYImjlzJjNnzqS6upqHHnqIGTNmkJ+fH/DjGxqcKIrao9c2mxPCYlirrcPLqrLj/Oez47S2exk1yMiDN+UxNCsJgPp652mPCZe+XahI7ReIvoWrQPumdrSidrhQvR3g7UD1nPytejvg1N9uF0pzLUpTDWpLA3DyM06KTUJOTEeTdym6xHTkxHTkpHSkBDOS7N9BUICOzh9agdYL/9t/tV+yLPVoByCgQLr55psv+IlPZbFYsNls+Hw+NBoNPp+P2tpaLBZLj54vIyODwsJCPv744wsKpEjW1uFl1bZK/rO1gtZ2L6MHm7hh6iDyM8TUfEEIZUpbM0rdUXz1R1HqyvHVH0NttQf2YEkGfQxyghlN6hDkYdO6Qkc2pCHpw2uyUsBHq959912WLVuGzWYjLS2N4uJibr311oAeazKZKCgooLS0lOLiYkpLSykoKLig4brDhw93radnt9vZsmUL11xzTcCPj1RtHV5Wb6vkw84gGjMkhRun5ZGXLoJIEEKNr7UJ7/Hd+OqOotQfxVd3tFv4SInpaCzD0JhykKITQBeFpI0CbRSSLqrruqT1X0bWRtSx4IAC6Xe/+x3vvfce9913HxkZGVRXV/Pqq69SW1vLAw88ENALPffcc8ybN49XXnkFg8HAwoULAZg7dy4PP/wwhYWFlJWV8fjjj+N0OlFVlQ8++ICSkhKmT5/O3//+dzZs2IBWq0VVVe666y6mTZvW856HubYOL2u2V7Jyiz+IigabKJ42iEEWEUSC0B9URUFta0J1NqC0OlBb7ShOO2qrA6XVjupsoKXV0bW9lJiOJn0YGnMucsogNCm5YbdH09skVVXPe1Dlqquu4i9/+QuZmZldt1VVVXHXXXexdu3aPm1gb4qEY0jtbi9rtlexcksFzjZPrwRRqPStt0Vqv0D0rT+pqoLisOKz7sdnO4TSUo/qtKO6GkFVum+s0SHFG5HjjEhxRgw5Q2iLSe8Mn9j+6UAfCOoxpLa2ttOG15KSkmhvb7/gFxR6psPtY832SlZ0BlFhvokbp+UxOEOUDheEvqQqCoq9Al/1AXw1B/BZD6J2+CcHSbFJyEkW5IwC5HgjUlxy529/CBEV121ILcmcgCeEw7a/BRRI06dP5wc/+AFPPPEEGRkZVFVV8ctf/nJAD5kFS4fbx9odVazYcowWl4dR+UaKpw5icKYIIkHoC6rPi1J/FK91Pz7rQXw1h8DjL70jJZjR5I5FaxmGxjLcP1stgo7h9LeAAunZZ5/lhRde4MYbb8Tn86HVarnuuut45pln+rp9A1aHx8fa7VWs3HKMZpeHkYOMFE8bxBARRIJwUVSfB9VpR3E2+FcocDagOOu7LqtOB6g+AOSkDHRDLkNjGY4mfRhyfOATsYQLd95AUhSFXbt28eKLL/LSSy/hcDhITk5GFiWs+0SHx8fHO6pYsbkziPKSKZ6Wz5AsEUSCcCGU9haU+gqUhgp8DRUoLXWoLfWoriZOPVcHpM6hNhOatCHIg1OQzXn+AIoRk4SC6byBJMsyDz74IDt27AD8U7iF3ufuDKLlWypobnVzSV4yN04dxLDspP5umiCENFVVUFvq8dUfOxk+Dce7T6eOS/afn5M1CjnehJyQgpSQghxvQoozBnW9NuHsAnoXJkyYwOeff86YMWP6uj0Djtvj4+PPq1mx+RhNrW4KcpN58KZRIogE4QxURfGvv1Z3FF9deVcA4emcYCXJ/nXYLMPRmHKQTTnIKTnI0ZG7FFEkCSiQMjIymDt3LjNnziQ9Pb3bQbxHHnmkzxoXyTo8Pj79vJrlW47R5HQzIieJ7xWPZHhOcn83TRBCgqoqqE02fHXlJ08krT8G3g7/BtooNKYcdEOnIqfk+AMoORNJq+/fhgs9FlAgdXR0cPXVVwNgs9n6tEGRztXuZe2OSv7z2XFaXB6GZyfxvRtFEAmC6nbhrdpLwxcVuCoO+MPnxJ6PRo+ckoNuxAw0KXnI5jzkRAuSOJYdUQIKpJ/+9Kd93Y6I1+Jy81FZJau3VdLW4WVUvpE5k/PE0JwwYKmqiuKoxnf8C7wVO/3Tq1UfHRodkikb3dCpaMyd4ZOU0bUYqBC5AgqkL7/8krKyMpqamkhMTGT8+PEMGTKkr9sWERwtHXy4tYKPP6/C7VG4dJiZr03JFWvNCQOS6unAV70P7/GdeCu+QHU2ACAbs9GPvg5NThHpl4ym3t7Wzy0V+sM5A0lVVZ5++mnee+890tPTSU1NxWazUVtbS3FxMT/5yU/ESWFnUdfYxorNx1i/y4qiwKRL0rh+ci6ZKXH93TRBCJoTkxB8VXvxVnyBz7offF7QRqHNGolm7A1os4u6nd8jZrwNXOd85//+97+zdetW/v73v1NUVNR1+86dO3niiSd46623+OY3v9nnjQwn1oZWSjceY8teG7IM0wotXHtZLqlJA3vRRCHyqaqK2lzrn4RwSimFE8eB5MR0dJfMRJszGk36UCSNrp9bLISacwbSsmXLeOaZZ7qFEUBRURFPP/00v//970UgdXK1e1i2/iirt1Wi1UpcPT6L2RNzSE6I6u+mCUKvU1UV1dngn3rdWUbBV1cObpd/A40W2ZSLbthUNOZB/pNMDan922gh5J0zkA4fPsyECRPOeN+ECRP40Y9+1CeNCieKqrJhp5V3PzlMi8vD5WMyuGlGPoZYMfVUiAyq143iqEaxH/efdGqv9J942rnAKJIG2ZSFbvBEZPMg/yw4YyaSLIbehAtzzn8xPp+P+PgzLyEeHx+PoihnvG+gOFLdzJsfHaDc2sKQzEQeu30YueniBDwhPKmq6q/hYz+Or+E4SsNxf/g0WeFElRqtHtmYhXbQpcimHDTmQcjGLHHuj9ArzhlIXq+XzZs3c7aSST6fr08aFeqaWt28+/Fh1u+ykhivZ+6cS7hsZJqY4CGENFVVTxaQa6nvrOPT0Pnbfx2vu2t7KSEFjTEbbf54ZGM2GlM2UkKqOPdH6DPnDCSTycTTTz991vsvpAR5JPD6FNZsq2TZhnLcHoVrJ+Vww5Q8YqLE0IQQGk6EjuKoRmms9v9uru1ayRqfp/sDouKQ41OQEy1osgqRDan+PR9jZkQVkBPCwzk/SdesWROsdoS8zw/W8so7X2BtcDEq38g3Zw7FYhJTuIX+oaoKSnNtZ+hYURqr8XWGEO5TzuHRxSAnpSMbs9DkjvGHz6kLiw7wktlCaBFf7QPwr0+P8P7Go5iTovn+rYWMGZIihueEPuXf02nuHErrPqymOhtwttShnjq8FmPorN0z2V/BNDkTOcmCFJsk/q0KYUMEUgAspljuu2Ekk4anoNOK5UuE3qGqKqqrEcVR1Tl5oMYfOGcbXtPHIiekIBtSiRs6lvaoFOTkTDRJFqToM08+EoRwErRAKi8vZ968eTQ2NpKUlMTChQvJy8vrts369etZvHgxBw8e5O677+bJJ5/suu+3v/0ty5cvR5ZldDodjz32GNOnTw9K2y8bmY7ZnEBdXUtQXk+IPEpbc2fwVKE4Kv1ruNkrT563A0hR8UgG8ynDa511e+JTkBNM3Y7pmMS/RyECBS2Q5s+fz5133klxcTHLli3j2WefZenSpd22yc7OpqSkhJUrV+J2u7vdV1RUxH333UdMTAz79+/nrrvuYv369URHRwerC4JwXqetVlB/DMVRhdrWfHKjqDg0yZnoBk9CNmb6h9eSM0V1UmHAC0ogNTQ0sHfvXpYsWQLAnDlzWLBgAXa7vdtMvdzcXABWrVp1WiCdujc0fPhwVFWlsbGR9PT0IPRAEE534rwdX125f5mcM61WYMxBmzPGHzid4SOO6wjCmQUlkKxWK2lpaWg0/uMvGo2G1NRUrFZrj6aOv/fee+Tk5FxwGJlMFzfObjZH7kmvkdq33uqXqvjwOGx46irosB2lw3oYd81hfK1N/g1kDfrUXGJHTiXKMoQoy2D05uw+XSg0Ut8zEH0LR73Rr7Cb1LB161Z+9atf8dprr13wYxsanCjKmU/yPZ9IPoYUqX3rSb9UVfGfLOqowmfvnGzgqPJPp/Z5/RtJEnJSJnJmIVHmPDTm/G6rFXR0/tCHJRQi9T0D0bdw9NV+ybLUox2AoASSxWLBZrPh8/nQaDT4fD5qa2uxWCwX9Dw7duzghz/8Ia+88gr5+fl91FphIOg6gfRE4Nir8HVONuiqUgpIcUZkYya6zEvQJGciG7P8xeJ0YtFcQehtQQkkk8lEQUEBpaWlFBcXU1paSkFBwQUN1+3cuZPHHnuMX//614wcObIPWytEGrXdic9xyt6Oo8o/w62jtWsbKToB2ZiFbtg0/yy35Ezk5AykKHHysyAEi6SebaG6Xnb48GHmzZtHc3MzBoOBhQsXkp+fz9y5c3n44YcpLCykrKyMxx9/HKfTiaqqJCQkUFJSwvTp07n11lupqqoiLS2t6zkXLVrE8OHDA26DGLI7s0jqm+p24bN9ia/mEJrGCtptx1BdjSc30MegSc7qnGCQdXKWW5jNcIuk9+yrRN/CT28N2QUtkEKBCKQzC9e+qaqK2lKPz3YIX80hfLZDKPYqQAVJRp+ai5KY2TnU5g8gKS45Ima4het7FgjRt/ATVseQBOFiqYoPtb3FXxSucw/IZ/vy5N6PLhpN2hD0gyagSR+KJjWf1AxzRP7nF4RIJQIpAJUt1RxsbSRJSsUcY4qIb9ihQFUV1LYWVFejf922tmYUV5O/RELndfXE9XYncHLvVoo3ockYgSZtKJr0of69H1EWQRDCmgikAGz+4Dccc7dyJFNPjCaKHEN2508WOQlZmKIjYxiot6iqgtra6F+nzeX/rbqaTrne1BVCqGco8qjRI8UakGISkQ1mpLQhSLGJSDEGpLgk/zTruOTgd0wQhD4lAikAU5otFG3dhjtB5sgIPXty2ljTeARf5zf2OG0s2QmZ5BiyyE3IIj0uDWN0MnqNrp9b3ndUbwdKcx1qc52/DEJLbef1Wn+hN8X7lUdISDEJ/mCJTUI2ZiN3Xvbflogc4w8ddNEi4AVhABKBFADLdx6CqZ9T/pe/MuKzeorKtcSkNWI3SlQZkrCaYqlsb2RV42GUU77xG/QJmKKTMUYnY4oxYopOxhRtxBjjv00nh+6fX/V2oDgbUJ32k5VFnQ1dAaS2NXV/gC7aX9zNmIUmdyyywYwcl9wZOEn+MArh/gqC0P/EJ0QAJFnGfMXlMLyIpvWf0vD+v2na4yN+UBbjB0WjObAfVAVfsoXa3NHYEwzYVQ92Xxt2j5NjLZXsqNvVLawAEvUGEqMMGPTxJOgTSNDHY+j6ffK2OG1sr+4xqN4OVFeT/3iNq5HGI62011T5w8fZgOpsQG3/ymQASUKKTUY2mNHmFCElmP0BZEhFMpj9K1WLvRpBEC6CCKQLIGm1JF1xFYbJU2n8eA325aW0lbcSN2YMiaNz0Dv2Y/n8I05bf0IfgxqdQEtMHI7oaBx6HQ6thB2Vlo42HG3NVCgdOBU3CqdPS5eRSdDFkKCNJV4bQ4ImhgRtNPGaEz9RJMh64mU9CVIUOsl/MmjXBIHO3yeO35y6EgFAO4A2yl/iIN6ExpyHFG/yVxQ9UQIhNglJFrWgBEHoOyKQekCOisI4+zoSp1+O46MPcfznQ1q/+ALD5KkkX3sHGq3bP0W53emfRdZ5Obm9hSRnC3ntLahtLacdZ1GANlnCqZVp0cg4T/xoZVo0rbR2Xq/p/O2Vz7xHEuVTSPQqJHt9JCsSRikKozYWoykdY1YBCbEmNCeG0mITMefm0NCiij0cQRD6lQiki6CJjSWl+GaSrpqJY8VyGtesonnLJgyXTSHx8iuJLphw1g95VVXB0+4vQ614wedF9XmJV7yYfR5UxedfzFPxoPp8/uqhkgSSDLIMSLhRaVHcOJUOWnxunEo7Tl8HLb42HL4O7B4nxzsacXnbgFb/T4cVnUeHqSMZY1syRlcyg6UaEkjCEpdGot4ggkkQhH4hAqkXaBMMmG//BklXX4N9eSnNG9fTvGEdUTm5JF5+BYZJlyFHx3R7jCRJoI9B0sec5VnPTwcEstJam7cNe3sjDW12Gtod2Dt/GtodlDdXsL5qc9e20ZpoLHGppMelkR6XiiUujfTYNJKjE5ElcZ6PIAh9RywdFKALWfLD53LRsmUTjZ98jLvyOFJUNIbLLvPvNeXk9uj1+4qqqkQZYHfFEWpabVhba/2/XTZa3M6u7fQaPemxqaTFmkmNTSH1xO+YFKK1oVm1N1KXaQHRt3AVqX0TSweFME1sLElXziTxiqtoP3KYpk/W0rxxA02ffExU3iCSLr+ChImXIUf1fwkDSZJIjE5gWPJghiUP7naf09NKTWdA1bTWYm21cbjpKGW2z1FPmXyRqE8gNdaMOSalW1gl6g3EaMU5RYIgBEbsIQXoYr/Z+Fpbad60kaZP1+KurkaOiSHhsskYJk8jetCgfv3QvtC+uX0e6tsaqHXVUeuqx9bm/13rqsPpae22rVbWkqCLP20q+6nT2w36hM4TifX92q9wIvoWniK1b2IPKcxo4uJIvnoWSTOvpu3QQZo++ZjmdZ/StHYN2mQj8WPHEj9uPDFDhyFpQnt6tV6jIyM+nYz400vIuzwuatvqqXXV0+xuocXtpMXtpNndQlNHM8dbqmnxOE87Jwv852WZY02kxJgwd/6cuByriw1G1wRB6EcikIJMkiRihw0ndthwfN/8Fq07v6BlexlN6z6lcc1q5Ph44kePJX7sOGJHjkTW9e5eQ1+L1cWSp8shz5Bz1m0UVcHlbesMqxaaOlpoaLdT52qgrq2BfQ0H2Ozu/i0yThvrD6dYE2mxZtJiU0mPSyU1JgVdBC/RJAgDiQikfqSJj8cwZSqGKVNROjpo3b0T5/btOLeX0bxhHVJUNHGFRcSPG0dc4Wg0MT2fkRdKZEkmXhdHvC4OS1zaGbfp8Lmpb2ugvs0fUnVtDdS7GihvOsY22xddx7AkJEzRyaTFpfonXcSZSY9NIy3OjJmEYHZLEISLJAIpRMhRUSRcOoGESyeger249u/DuX0bzh3bcZZtRdJqib1kJAkTJhI3eiya2MgeworS6MmMt5AZf9q6F7h9bmpd9dS4arG11mJz1VHjquWA40u8p5xsnKCPwxRtIiXGSMopw38pMUZxvpUghCARSCFI0mqJG1VI3KhCUu+6h/bDX9KyfRvOss9o3fmFP5xGFZIwfoI/nCJkzylQeo2erIQMshIyut2uqAr29kZqWm3YXHU0KY1UOmpO26sC0Mk6TDFGzCfCKtpEUpSBxKhEkqIMGPQJaMRSSYIQVCKQQpwky8QMHUbM0GGYv34H7eVHaPlsK85tn9H6+Q5/OBUWkTB+IvGjR592Au5AIkty596QkVEUdJv541W82Nsd1LfZu4YBT1w+YP8St+Lp9lwSEvH6OJKiEknUGzrDyuC/3hlayVGJxGhjxJ6WIPQSEUhhRJJlYgYPIWbwEMy3f4P2w4dp2baVlrLPaN2xHUmnI64znGILiwbcntO5aGVt5/lR5tPuU1UVp6eVxo5mmjqaaOpoprGjyX/d3Yyjo5GjzRWnTWkH0Ms6kqITSYpKIjkqkaTOn+Tok5fjdXEitAQhAEELpPLycubNm0djYyNJSUksXLiQvLy8btusX7+exYsXc/DgQe6++26efPLJgO4biPx7TkOJGToU8+3f9A/rfbaVlm1lOLdvA42GmKHD/JMiikajS7eID8WzkCSJBL3/XKnsrwwDnsqjeGnuaOkMq5M/jo4mGtubOOg4TJO7+bQp7TpZhzE6ieSoJIzRyf7L0UkYo/3Xk6IS0YpaUYIQvBNj77nnHm699VaKi4tZtmwZ7777LkuXLu22zbFjx3C5XKxcuRK3290tdM51X6D688TYYFEVhbYvD9G68wtad+3EXVUJgC7FTFxREXGFo4kZPgJZf3I6ebj07UL1R78UVaHZ3Rla7f6wsrc7cLQ3Yu9oxN7u6LYkE/iHBw36eJKjk/17WZ17V8lRiSRHJ3UNG556TCtS3zMQfQtHYXVibENDA3v37mXJkiUAzJkzhwULFmC32zEajV3b5eb613lbtWoVbre723Oc6z7hJEmWu85zMt92O56GBlp3+cOpaf06GtesRtLriR1RQFxhEXGFRWAW06N7iyzJXUN1GM68jcfnwdHR1C2kHO2NONobqW61scd+ALev+7/xE6GVFO0fGkxPTEHriyJBH3fK6hfxxOviidb2/5JUgtATQQkkq9VKWloams4VCDQaDampqVit1m6B1Nd6ktinMofjB7c5AUbkwdeLUdxumnbvwVG2Hce2bdS++QUAxw0GYrOziMnKIjY7k9jsbGKystCbjGE/zBeq71kGZ/93r6oqLk8bDS4H9rZG6l0O7G0O/29XI3Vt9Rw49iVtXym0eEKURo8hOoGkqAQM0QmYY03kJGWSl5RFdmIGUdrQP9k6VN+33hCpfeuNfg2ogeuBMGR3XtlDSMgeQvxNX8djs9G6ZxdyvY2m8mM4121AcZ08cC9HR6O3ZKC3WDp/Z6Azm9EaTWExYSLc37MYDGRqDWQack7b2zKbE6iuseP0tJ5cosnTSsspyzW1uJ3UNNez2+vNXXwAABdySURBVHaAjs49LgmJ1NgUMuItZHWe55UZbyE5KilkvnyE+/t2LpHat7AasrNYLNhsNnw+HxqNBp/PR21tLRbL6Sc9CsEhSRL69HT06eld/5hUVcXX3IzbWt3101FdTeuePTRv3NDt8XJsLFqjCZ3J1PVbZ0pBazSiM6WgMRiQZFE/qS/pNDqSNf4JEufiPz/LQaXTSlXnz/HmSnbU7uzaJlYbQ2a8BUtcOqmxKZ1lRswYo5NEHSwhaIISSCaTiYKCAkpLSykuLqa0tJSCgoKgDtcJ5ydJEtrERLSJicSOKOh2n8/VittqxdvQgKehAY+9AW9DPZ6GBtoOHkBpa+v+XFot2mQjWpMJndHoDy2jqdv1UCi/MRD4z8/yr1Qxxjyq6/Z2bzvVrTVUtliparVS1WJla8122n0nhwK1shZzjMk/ZT7mZFClxqaI6exCrwvaLLvDhw8zb948mpubMRgMLFy4kPz8fObOncvDDz9MYWEhZWVlPP744zidTlRVJSEhgZKSEqZPn37O+wIlhuzOrDf65nO58NpPBJU/tLx2u/+6vQGvwwFf+acmx8f7Q8poRJucjDbpxE9S13U5pucnnor37MKdOCfL5qo7WV6k83JdWwM+1de1rV7WkaCPJ/5EaRHdicsJJOjiTl7WxxOniw14T0u8b+Gnt4bsRD2kAEXqPyQITt9UrxdvU2NnUHWGVedlj92O1+HodvzqBCkq6mRIJSWjNRqJysomOjcPXWrqOYcFxXvWu3yKD3t7I7VtddhcdTS2N9HicXY7ZnW20iKyJJOoN3Q7Ydg/xT2JpBMrYHRObRfvW/gJq2NIgiBptehMKehMKWfdRunowNvYiLepEa/DgbfR0fm7EW+jg7bDh/CWOcDn/5Yux8QQlZ1DdG4eUXl5ROfkoktLF8eu/n97dx4bxXn+Afw75x5e23v4JHZwQmK8MT/C4TYk+gXFDmpIYzWkLSiN4j8qtfkjhaitokJDShIaBCaIVrFQUIVUNUrUSFUOinGIS2gKaRR+EC6DQ1KoD8D3rr322t5jZt7fH7M73mXt5fKxa56PNJq9DO/Ly/IwM888zxQReAG5Vr0FSLmrbNzPaEzDqBIYS66IJF1EbyjuD/pw2d+Bpr6vER6nXFOWbENeZg7soj2mGK6+Zck2OkU4y1FAIimDN5kg5+dDzh+/JQWgH2kFO64g2NaKQFsbgm2tGPjnp2CKXuWbM5lhvvNOmOaWgJWXYlS0QsjSr4vxGXTNY6rxHI8MyYoMyYqCCVqLAPqpwVFlVK9yEbmJOBqwBlUfLvpacbz7VFxBXJmXjOCUY3Ei1+JCtikLNskGm5yBTCmDagumOQpIJK1wogjznXNhvnMusiOXD5miINTZiUBbKwJtrQi2tcL3r39i4GBj/A8LAsSsbAiRxA0xO9sIVkK2XU+2cDj1DEH6R21KcRwHq2SFVbImtBiJnv5RNAWeQH9MMVx96xntw9febxCOaTUSFdtryybbItey9OeZMde5oo8topnWOoVQQCJpjxNFmIqLYSouRvb/6lGKqSpsyjB6Wzug+nxQBn1QfL6xx14PAi3/hTo0lJBsEZch6HBCdEWzBMeyBXmzeSamelsReTHSHTixIG60RNNgcAhD4WH4Q374w8P6FjlV6A8Noz1wGf7wMEaV8W8iFjghEpwy4gJVlikT2XIWso19FswirflUo4BEZiVOEGAtuANW0wT1eyKYqkL1D+nXqbxehPu9UDweKP160sXIN1+PmyEo2DIh5uRAysmFlJur73NyIOXmQXI6wYn01ZpKcSWaroOiKfCHhzEUigYsvxG4YhMyekZ6MRjyJ1zfAvQ+XHY5KyZY6ZvDZIfL4oDT7ECmRNe5bgV9a8htjRMEiNl2iNl2YG7JuJ9hqqonWni8Y/df9fUh3NeLYHsb/Ce/MhIt9F+Ug+hwQsrNhVxQCKvbDWvZfRBst1a6itw8kRevO4AxxhBQg/AFB/UtpO8HQ0OR1iSDaB+6DF/fYEIfrWhld6fZAZfZMba3OOEyO5DD6O9AMhSQCLkGThAgRU7VWXBvwvtM06D09yPc14twb6++jzwe+r8v4fvXPwGOg7nkLljLy2G9bwEsd8+jo6gUxXEcLKIZFtGMgoy8CT+nB64AvAG9QK5ntF/fB/T9paErCT20TKIJeWb9RuPo6ci8jFzkWXKpKC4oIBFyyziej5ROcgHz49Ohmaoi0PJfjDSfw/C5s/A27Ie3fh84kxnWsjJYyxcg475yPV2dTvWkFT1wWXCHzZKQmBEVVEORYOXVW48wH1o9HWgdbMeJnjNxWYR2U3ZcoNIrZOTAZXbGtR6ZzSggETKFOEGA5Z57YbnnXrh+sArqyDBGzp/HyLmzepA6fQq9AESnC+a77gInSeAEEZzAA4IAzthEgOeN58EME4YG/GChEFg4BC0UjuxDkdfC+uNwWL8HLCdH31w5EHPGrnnF9sUik88kyCjMyEdhJAU+9gbSkBpG7+hYJYzuyHa8+2RcEgbP8XCZHci15iDPkmPs86w5cJods6rWIAUkQqaRYM1A5pKlyFyyFAAQ6unBSLMenIJXLgOqChaz4arHUX2RPSdJ4CQZnCyBl2RwsgxOksDLMgSbDbwkQwsFEbxyGcOnTxn3axnjycqKBCc9QImuHIgORySj0AneYqUjtykiC5JRbT0WYwxDYT96R/QU996RPmN/YaAlrleWwAnIsTjhjDR3dJjtRqNHp9kOu8kOWZCme2o3jQISITNIzsuDnFcF+yNV1/wsY8wIUDm5mfAMBG6oKgXTNKiDg2PXuPrGkjMCLRcx9NWx+OQM6KWbJId+f5YYCVJjAUuv8E5FcicXx3HIkjORJWdinr0k7j3GGAZDQ+gd9aBnpA+9kUDlDQ6gw98JXyixLJFNyoDTbIfDpFeGj024cFocyBBT5z8dFJAISRMcxwGiCE4UIZhM4Pgb65zM8XykJqAdlnvGSc5QVSg+H5R+LxSvV099j+yVfi+GzzVB9fkSU+Azs4yjK+PUYPT0oItOC04mjuOMdPN77HclvB/WFPiCPngjHYj7gwP64+AAekb78E3/BQTUYNzPyIIcnxEY2YpshchPktQxFSggEUIARLMJnZCcTmDe+J/Ri+T6xiq5e/qMo63gpTYMnzqReFowOxtSbh7kvHzIBQWQ8vIh5xdAysujo6tJJvGiUV5pPNGSTdFMwOjeG8kQbPG1YUTRW8nwHI8dyzfDJEzffygoIBFCrpteJFfPKJwwBd7ng9LXh7Cn1zglGO7pwXDzWQx+8Xnc50WHE1KkfqEepPJhvrMAgcEgOFEAJ4p6QocQeSzqCR6cGEnySJFTTekitmRTceYd435mVAnAG+gHgGkNRgAFJELIJOJ4HpLDAcnhgOXexIClBUYR6ulBuLsboe4uhLq7EO7uxtCxY0b7kY7r/s048GYzhAwbeJsNgs0GISOyj3kefU+0OyBkZlIQuwaLaJ4wjX2qUUAihEwb3mwxiuNeTfX7EeruQqYMDHiGwBQFTFXAFBVMVQAl+liNvB6GNhqA6h+C6vdD9fsR7uqCOuxP6GAcxUlSTPdiZ0wXYxfESLNIXkqfrLTZhgISISQlCDYbLLZ74MjNhHKLTeyYokAdHtYD1bAf6tCQ3l/L6zEaQw43NUH1DSSOIzs7clryqkQNStKYchSQCCGzDieKRouRZLRwWG8C6fUg7OmLdDLug+LxINDaivCJrxJS4fV7t2IClNkcub4lRa57SeAk0bjWxUlS5H0R/iE7QsNhcJIMXh67b4yaSuooIBFCblu8JEXuBRs/vZlpml4JPlpQ16NvSp8Hgba2cQNWMpcmeJ0TxUhwksHLkZudJUm/yTmyNwJb9HVx7DUgcp9azHb1czC9UBHHcXpCCM8DPAdw0ceRPcfptwg4nbAtqZjWYDltAamlpQUbNmzAwMAA7HY7amtrUVJSEveZzz//HDt37sS3336LmpoarF+/3nhPVVW8/vrrOHLkCDiOw3PPPYfVq1dP1/AJIbchjueNVHjLvaUJ7xs3KytKzBbW9+HINbDw2GuZFhEDHh9YKFrmKabkU/iq8k+KopeACofBRkaM59FNC4fBlHBCpXkA8RmI0YASLTHENEDT9LFrWsJ9ZVG8xYKM/7kf3DSm5k9bQHrllVfwzDPP4Mknn8TevXuxadMmvP3223GfKS4uxpYtW3DgwAGEQvE3/e3btw/t7e1obGzEwMAAVq1ahQcffBBFRUXTNQVCCIkTe7Py9XDlZkK7xetjV2Oaph/V3GT2YDQwsWhwijzmJBG8NL3Xy6blWMzj8aC5uRnV1dUAgOrqajQ3N8Pr9cZ9bu7cuXC73RDHWdyGhgasXr0aPM/D6XRixYoVOHDgwHQMnxBCUhZ3i/djcRwHThDAR2og8mYzBKt12oMRME0BqbOzE/n5+RAEvYS6IAjIy8tDZ2fnDf0ac+bMMZ4XFhaiq6tr0sdKCCFkZtxWSQ0u1611a8zNzZykkaSe2Tq32TovgOaWrmbr3CZjXtMSkAoLC9Hd3Q1VVSEIAlRVRU9PDwoLr/9u4MLCQnR0dGDhwoUAEo+YrofH44emjX8B71pi+5jMNrN1brN1XgDNLV3N1rldPS+e527qAGBaTtm5XC643W7U19cDAOrr6+F2u+F0Oq/711i5ciX+9re/QdM0eL1eHDx4EI899thUDZkQQsg0m7YE81dffRXvvPMOHnvsMbzzzjt47bXXAAA///nP0dTUBAA4fvw4li9fjj//+c947733sHz5chw5cgQA8OSTT6KoqAjf+973sGbNGvziF79AcXHxdA2fEELIFOMYmyAJfRaiU3bjm61zm63zAmhu6Wq2zm2yTtndVkkNPH9rVX5v9edT2Wyd22ydF0BzS1ezdW6x87rZOd5WR0iEEEJSF1X0I4QQkhIoIBFCCEkJFJAIIYSkBApIhBBCUgIFJEIIISmBAhIhhJCUQAGJEEJISqCARAghJCVQQCKEEJISbqvSQTejpaUFGzZswMDAAOx2O2pra1FSUjLTw7ppVVVVkGUZJpMJAPDiiy/i4YcfxqlTp7Bp0yYEg0HccccdeOONN+ByuWZ4tBOrra3FJ598gitXrmDfvn0oLS0FkHy90mUtJ5rbRGsHIC3Wr7+/H7/5zW/Q3t4OWZYxd+5cbN68GU6nM+n4031u8+fPR2lpKXhe////9u3bMX/+fADAoUOHsH37dqiqivLycmzduhUWi2UmpzKu559/HpcvXwbP87Barfjd734Ht9s9+d83RpKqqalhH330EWOMsY8++ojV1NTM8IhuTWVlJfvmm2/iXlNVla1YsYIdO3aMMcbYrl272IYNG2ZieNft2LFjrKOjI2E+ydYrXdZyormNt3aMpc/69ff3sy+//NJ4vm3bNvbb3/426fjTfW6MMVZaWsr8fn/Cz/j9fvbQQw+xlpYWxhhjL730Equrq5uW8d6owcFB4/E//vEPtmrVKsbY5H/f6JRdEh6PB83NzaiurgYAVFdXo7m5GV6vd4ZHNrnOnj0Lk8mEiooKAMDTTz+NAwcOzPCokquoqEho8JhsvdJpLcebWzLpsn52ux0PPPCA8XzRokXo6OhIOv50n1syhw8fxoIFC4yjhqeffhoff/zxVA7zpmVmjnWD9fv94DhuSr5vdMouic7OTuTn50MQBACAIAjIy8tDZ2fnDTUXTDUvvvgiGGNYunQpfv3rXyd033U6ndA0zTjUThfJ1osxNivW8uq1y8rKSsv10zQNf/3rX1FVVZV0/Ok+t6iamhqoqorly5dj3bp1kGU5YW5z5sxBZ2fnTAz5umzcuBH//ve/wRjDnj17puT7RkdIt5l3330Xf//73/H++++DMYbNmzfP9JDIdZpNa/f73/8eVqsVzz777EwPZdJdPbfPPvsMH3zwAd59911cuHABu3btmuER3pwtW7bgs88+w69+9Sts3759Sn4PCkhJFBYWoru7G6qqAgBUVUVPT88NnU5JNdGxy7KMZ555BidOnEBhYWHc6QWv1wue51P2f6ATSbZes2Etx1u76OvptH61tbVoa2vDH//4R/A8n3T86T43YGzdbDYbVq9ePeG6dXR0pMXfx1WrVuHo0aMoKCiY9O8bBaQkXC4X3G436uvrAQD19fVwu91pdYon1sjICIaG9K6OjDE0NDTA7XZjwYIFCAQCOH78OADgvffew8qVK2dyqDcl2Xql+1pOtHYA0mr9du7cibNnz2LXrl2QZRlA8vGn+9x8Ph8CgQAAQFEUfPLJJ8a6Pfzww2hqakJraysAfW6PP/74jIw9meHh4bhTiYcOHUJ2dvaUfN+oQd81XLx4ERs2bMDg4CCysrJQW1uLu+++e6aHdVMuXbqEdevWQVVVaJqGefPm4eWXX0ZeXh5OnDiBV155JS61NicnZ6aHPKHXX38djY2N6Ovrg8PhgN1ux/79+5OuV7qs5Xhz271794RrByAt1u8///kPqqurUVJSArPZDAAoKirCrl27ko4/nef2s5/9DJs2bQLHcVAUBYsXL8ZLL72EjIwMAMDBgwfxxhtvQNM0uN1ubNu2DVardSankqCvrw/PP/88RkdHwfM8srOzsX79epSXl0/6940CEiGEkJRAp+wIIYSkBApIhBBCUgIFJEIIISmBAhIhhJCUQAGJEEJISqCAREgau3z5MubPnw9FUWZ6KITcMgpIhBBCUgIFJEIIISmBAhIhk6y7uxvr1q3DsmXLUFVVhbfffhsAUFdXhxdeeAG//OUvsXjxYjz11FM4f/688XMXL15ETU0NKioq8MQTT+DTTz813gsEAti2bRsqKyuxdOlS/OQnPzFK0gDAvn378Mgjj+CBBx7AW2+9Zbx+5swZ/PCHP8SSJUvw0EMPYevWrdPwJ0DITbrFvk2EkBiqqrKnnnqK1dXVsWAwyNrb21lVVRU7fPgwe/PNN9l9993HPv74YxYKhdiePXtYZWUlC4VCLBQKsRUrVrC33nqLBYNB9sUXX7BFixaxixcvMsYYe/XVV9mzzz7Lurq6mKIo7KuvvmLBYJBdunSJlZaWso0bN7LR0VH29ddfs/LycnbhwgXGGGNr1qxhH374IWNMbwh38uTJGfuzIeRa6AiJkEnU1NQEr9eLtWvXQpZlFBcXY82aNWhoaAAAlJeXY+XKlZAkCT/96U8RCoVw+vRpnD59GiMjI3juuecgyzIefPBBVFZWYv/+/dA0De+//z42btxo9JhZsmSJUcATANauXQuz2YyysjKUlZUZR16iKKK9vR1erxcZGRlYtGjRjPy5EHI9qEEfIZPoypUr6OnpMTqcAnrp/YqKCsyZMwcFBQXG6zzPIz8/Hz09PQCAgoICo2UBoDds6+7uRn9/P4LBIIqLiyf8fWOLjVosFoyMjADQe9i8+eabePzxx1FUVIS1a9eisrJy0uZLyGSigETIJCosLERRUREaGxsT3qurq0NXV5fxXNM0dHd3GxW7u7q6oGmaEZQ6OztRUlICh8MBk8mES5cuoays7IbGU1JSgp07d0LTNDQ2NuKFF17A0aNHU66iNCEAJTUQMqkWLlyIjIwM/OlPf0IgEICqqvj2229x5swZAMC5c+fQ2NgIRVHwl7/8BbIs4/7778fChQthNpuxZ88ehMNhHD16FIcOHcL3v/998DyPH/3oR9i6davR9OzkyZMIhULXHM/evXuNpnZZWVkAEHcURkgqob+ZhEwiQRCwe/dunD9/Ho8++iiWLVuGl19+GX6/HwDw6KOPoqGhAd/5znewd+9e1NXVQZIkyLKM3bt34/Dhw1i2bBlee+01bN++HfPmzQMArF+/HqWlpfjxj3+M7373u9ixYwc0TbvmeI4cOYInnngCixcvxpYtW/CHP/zB6NdDSKqhfkiETJO6ujq0tbVhx44dMz0UQlISHSERQghJCRSQCCGEpAQ6ZUcIISQl0BESIYSQlEABiRBCSEqggEQIISQlUEAihBCSEiggEUIISQkUkAghhKSE/weB0XiVzSX5RQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cdropout1_history = [cdropout_history[i][0] for i in range(30)]\n",
    "cdropout2_history = [cdropout_history[i][1] for i in range(30)]\n",
    "cdropout3_history = [cdropout_history[i][2] for i in range(30)]\n",
    "cdropout4_history = [cdropout_history[i][3] for i in range(30)]\n",
    "\n",
    "plt.plot(epochs, cdropout1_history,label='cdropout1')\n",
    "plt.plot(epochs, cdropout2_history,label='cdropout2')\n",
    "plt.plot(epochs, cdropout3_history,label='cdropout3')\n",
    "plt.plot(epochs, cdropout4_history,label='cdropout4')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('Dropout rate')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAAEUCAYAAABkhkJAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl8lNW9+PHP7Nn3bbKwJSwBEnYEARFZhUAii1rUuqG31kJta5WrrXBbu6D3elttbW/9VaxrVRQwIewim7IkQCAkJCEEErLvySyZ7Xl+fwSikS1AMpkZzvv1yivJ5HlmzplJ5ptznnO+X4UsyzKCIAiC0MuUvd0AQRAEQQARkARBEAQXIQKSIAiC4BJEQBIEQRBcgghIgiAIgksQAUkQBEFwCSIgCYIgCC5BBCRBEATBJYiAJAiCILgEEZAEQRAElyACkiAIguASREASBEEQXIIISIIgCIJLUPd2A1xFY6MRSbqxxOehoX7U1xu6uUW9yxP7BKJf7sQT+wSe2a/v90mpVBAc7Hvd9yMC0gWSJN9wQLp4vqfxxD6B6Jc78cQ+gWf2qzv6JKbsBEEQBJcgApIgCILgEsSU3VU4HHYaG2ux261XPa6mRokkSU5qlXP0dp/Uai3BweGoVOJXVBBuFeKv/SoaG2vx8vLB1zcKhUJxxePUaiV2u2cFpN7skyzLGI0tNDbWEham75U2CILgfGLK7irsdiu+vgFXDUZC91MoFPj6BlxzZCoIgmcRAekaRDDqHeJ5F4RbjwhIgiAIQge7Q2LboVJ++vpejhbVOvWxxTUkD/PPf/4fZrOZn/zkmd5uCkeOZGG32xk/fkLHbf/1X7/iyJEs6uvr2LZtDz4+Pr3YQkEQvqugtJH3txVSXmckOT6UgbFBTn18EZBuEZIkoVAonDoVdvRoNmazuVNASklJZcWKnzN//iyntUMQhKtrbLXwya7THMyrJizQi+WLkhiZEOb0qXMRkNzEO+/8P1pamlmx4hcANDc3sXTpIj7+eCN//ONvOHOmmJCQUCIjIwkODgXaR0slJWcwGg1UV1fx97+vpby8jD/96b9pazPj5eXNM888S2LiMCorK1i27CHmzEkhK+sgAD//+fOMGDEKgM2bM/joo/dQKBRER8fy3HMvEBwccsmI7OL3d9+dwsaNnyNJEllZh5g+fRYPPfQIY8aM64VnTxCEy7E7JHZknWfj/hIcDpkFk/oxd0JftBpVr7RHBKQu2n+ikn3HKy/7M4UC5JvImjE5Wc+kpKsvb54zJ4X/+I+H+fGPf4parWb79i1MmnQHa9e+hY+PLx9++BlNTU089tgD3HXXzI7z8vJyefvtDwgKCsJms/Hii8/xwgurGDt2PIcPH+TFF5/j4483ANDc3ExCwkCWL/8ZOTlHWL36RT7+eAPnz5fy97//hX/+833CwsJ4662/8b//+yq/+c0frtje+PgEUlMXusz0oSAIneWfa+SD7YVU1BkZER/KD2YMJCK4d6fQxaIGNxEVFUW/fvEcOLAfgMzMDObOnc/Ro1mkpKQCEBQUxNSpd3U6b+LESQQFtc8Dl5aeQ6PRMHbseADGjbsNjUZDaek5ADQaDbNnzwVgzJix6HQ6SkvPceRIFhMnTiIsLAyA1NSFZGUd6vlOC4LQ7RpbLfx9Yy6vfnQUq83BikXJ/HTJiF4PRiBGSF02KenKoxhnbSKdOzeFzZsz0OtjMBoNHdNpV+Pt3bO/ZCqVCln+tu9Wq9g7JAiuyGZ3sD3rPOn7zyLJMqmT+3P3bX16bXrucsQIyY1MnXoXOTlH+fe/3+fuu1NQKBSMHj2OzMx0oP260p49u654fp8+fbHZbBw5kgVAdvZh7HY7ffr0BcBms7F9+xYAjh07gsVioW/ffowePZZvvtlPfX0dAOnpGxg3rn2UFRsbR0HBKSRJwmQy8vXXezsez9fXF6PRs9LsC4K7kWWZQ/nVvPCPg6z7qpjEvsH8dtltpE7u71LBCMQIya14eXkxefJUMjPT+eSTLwB45JFl/OEP/8XSpYsICQll5Mgrj5o0Gg2/+90rnRY1vPzyGjQaDQCBgYEUFRXy4YfvArB69e/QaDQMGJDAj370E372s6cvLGqI4Ze/fAFoD5I7d27ngQcWExkZxeDBiR2Pd8cd03jhhV/yyCNLOxY1vPDCL8nPPwnA0qWLGDAgntde+0uPPF+CcKsrLm/m3zuLKK5oIS7Cj0fnjmRov5DebtYVKWT5Zi7He476esMl9Tyqqs4RFdX3mud6Qi67i6vsNm3aCbhGn7r6/F+P8HB/amtbu/U+XYEn9ssT+wTO6Vddk5l1u4s5lF9DoK+WhXcMYFKSHqWyZ5Zxf79PSqWC0FC/674fMUISBEHwEKY2O5sOnGX74fMoFbBgUj/m3NYHL617vNW7RyuFHqfXR3eMjgRBcC8OSWJPTiUb9p6h1WTj9uFRLLxjACEBXr3dtOsiApIgCIIbO3m2gY92FFFRZ2RQXBDPLEmgvz6gt5t1Q0RAEgRBcEONrRY+/rKIQ/k1hAd58fQ9SYwe5Px0P91JBCRBEAQ34pAkvswuZ/3eM9gd7fuJ5k7og0btWku4b4QISIIgCG6iuLyZ97YWUFpjYHj/EB6YNYhIF8iw0F1EQBIEQXBxBrONdV8VsyengmB/HT9OG86YweFuPT13OSIgeRhXrodUWnqOV1/9PfX1dahUKhITh/GLXzyPTudeK4EEwVkkWWb/8Uo+/aoYU5udOeP7MH9SP7x1nvnW7Zm9Ei7hCvWQNBoNy5f/jEGDhiBJEqtXv8hHH73PI48sc1qbBMFdlNUYeG9bAafPNzMwNpCHZg0mNuL6N5u6ExGQushWuB9bwZ7L/kyhUHAzCS80g+9AM2jSVY/xlHpIen00AEqlksTEYZw7V3LDz5sgeCKzxc7GfSXsyDqPj5eax+YmcntSFEoPm567HBGQ3ISn1UOyWNrYtOkLfvSjp7v/yRIENyTLMtkFtXy0s4imVgt3jIxm0dR4/Lw1vd00p3FaQCopKWHlypU0NTURFBTEmjVr6NevX6djHA4HL7/8Mnv37kWhUPDkk0+yZMkSAJ577jkKCgo6ji0oKOCvf/0r06dP54033uDDDz8kIiICgNGjR7Nq1apubb9m0KQrjmKckfftu/WQ2hOsZrBixc95/fX/4ZlnfgncXD0kHx+fK9ZDOnYs+5J6SI88svSG+2K321m16gXGjBnL5MlTb/h+BMFT1DSaeH97IblnGugT4ceP04YTHxPY281yOqcFpFWrVrF06VJSU1PZuHEjL730Eu+++26nY9LT0yktLWXbtm00NTWRlpbGxIkTiY2N5ZVXXuk47tSpUzz88MNMmTKl47a0tDSef/55Z3WnV3hCPSSHw8FvfvNr/P0DOgKpINyqbHaJzQfPsembc6iUCn4wfSB3jYlBpbw1KwM5pdf19fXk5eWRkpICQEpKCnl5eTQ0NHQ6LjMzkyVLlqBUKgkJCWHGjBls2bLlkvtbt24d8+fPR6vVOqP5LsPd6yFJksTvf78apVLJypW/9rglq4JwPfLONvDS24fYsLeEkQlh/O6JCcwcF3fLBiNw0gipsrKSyMhIVKr2ncQqlYqIiAgqKysJCQnpdFx0dHTH93q9nqqqqk73ZbVaSU9P55133ul0+6ZNm9i3bx/h4eEsX76cUaOuPXpwN+5eDyk+PoGtWzczYEA8jz/+EABJSSP4xS88e2QrCN/V0NLG/31xkoN51UQEefPz+0YwvH9obzfLJTilHlJubi7PP/88mzZt6rht7ty5vPrqqwwbNqzjtvnz5/O73/2O5ORkAN566y2qq6v51a9+1XFMZmYmb731FuvXr++4rba2lqCgIDQaDfv37+fZZ58lMzOT4ODgm2r3yZN5REd3bz0eV1VRUcGjjz7I1q1f9nZTOlRUnGPYsKG93QxB6BYOh0Tm12d5f0s+VpvEkukDWXzXQJer2tqbnDJC0uv1VFdX43A4UKlUOBwOampq0Ov1lxxXUVHREZC+P2IC+Oyzz1i0aFGn28LDwzu+njRpEnq9nqKiIsaPH9/lNl6uQJ8kSV1arOAKxexulsMhAXJHP1yhT5IkdXshM1H0zX14Up9Olzfz/oWUPyMHhXPfnfFEhvjQ3GTq7aZ1i+4q0OeUycrQ0FASExPJyMgAICMjg8TExE7TdQBz5szh008/RZIkGhoa2LFjB7Nnz+74eVVVFdnZ2cyfP7/TedXV1R1f5+fnU15eTv/+/XuwR55H1EMShO7XYrLydmY+v38vm1azjafShvObJycSGeI5+ee6k9NW2a1evZqVK1fy5ptvEhAQwJo1awB44oknWLFiBUlJSaSmppKTk8OsWbMAePrpp4mLi+u4j/Xr1zNt2jQCAzsvh3zttdc4efIkSqUSjUbDK6+80mnUJAiC4EySJLM7p4LPdxfTZnVw923tKX+8tGqxmOcqnHINyR1cbsququocUVHXvobkCtNb3c0V+tTV5/96eNI00Hd5Yr/ctU8llS28t7WAs1WtDOkTxAOzBhMT5tvxc3ft19V015SdyNQgCILQDQxmG5/tLmbPsQoC/LQ8uWAotyVGihHRdRABSRAE4SZIssy+45Wsu5CRe+a4OFIn9/fYjNw9STxjHsaVy09IksRTTz1OW1sbAKGhYfzyl//ZkXBVENxNaXUr720toLiihUGxgTx4C2Tk7kkiIN0iXKH8hFKp5H/+5w38/Nr/YD/55CPeeON/+f3vX3VamwShO5gtdjbsLWFHdhl+3hoen5fI7cOjxPTcTRIByU14SvmJi8EIwGQyolSKP2DBfVzMyP3hjkKaDVamjoxm0Z3x+HrdOhm5e5IISF10sDKbbyoPX/ZnCgXczFrFifpx3KYfc9VjPKn8xLPPrqCwsIDAwED+93//euNPnCA40fczcj+9MIn46FsvI3dPunWz+LmZ75afAMjMzGDu3PkcPZpFSkoqcHPlJ4Arlp84ciTrkvITWVmHbrgv//3fr7Nhw2ZmzJjNO+/884bvRxCcwWaXSN9fwq//eYjT55v5wfSB/PqRsSIY9QAxQuqi2/RjrjiKcdaeHU8oP3GRUqkkJSWV++9fyLPPruyp5gnCTck/28B72wqpajAxbkgE908fSLC/rreb5bHECMmNuHv5icbGRpqamjq+37VrB/HxCd3x1AhCt2o2WvnHFyd59d/HkCSZn987gqfShotg1MPECMmNuHv5idtvn8zvfrcah8OOLMvo9dH8+te/6amnSxCumyzLfHOyio92FGGxOVgwqR9zJ/QVGbmdRKQOuuBWTx10cZXdxQSrrtAnkTqo6zyxX87uU31zG//aeorcMw0kxATy6Nwh6EN9r33idboVXiuROkgQBOEGSLLMriPlrNtdDDI8MHMQ00bHoBR7ipxOBCQBEOUnhFtTZb2Rdzafouh8M8P6h/Dw7MGEBXn3drNuWSIgCYJwy7E7JLYeKmXjvrPoNEqRacFFiIAkCMIt5VxVK2s351NabWDM4HAenDmIQD+xes4ViIAkCMItwWJzkPH1WTYfKMXPR8OP04YzdkhEbzdL+A4RkARB8GgX8899/GUR9S0WJg2P4r7pA/HzFvnnXI0ISIIgeKzyWgMf7igi/1wjseF+PL90KIP7BPd2s4QrEAHJw7hyPaTv+v3v/4vMzHS2bduDj0/PpjcSbj2mNhsb9pXwZXY53joVD8wcxJ2jolEpRXIaVyYC0i3CFeohXbRv3x6xmknoEZIss/94JZ/tLqbVZOOOkdEsvGMA/j7a3m6a0AUiIHVRy9f7ad6357I/UygU3EzCi8DJdxBw+6SrHuMp9ZCam5tYu/Yt/vznv7Fp0xc3/JwJwvedqWjhg+2FlFS2kBATyM/uHUTfKP/ebpZwHURAchOeUg/ptdfW8Pjj/9GpUJ8g3IwWo5V1u4vZd7ySQF8ty1ISmThM7ClyRyIgdVHA7ZOuOIpxRt6379ZDak+wmsGKFT/n9df/h2ee+SVwc/WQfHx8rlgP6dix7EvqIT3yyNLr7sPOndtRqzXcfvvkG34eBOEim11iR1YZGd+cxWqTmDO+D/Mn9cNbJ97W3JV45dyIu9dDOno0myNHsli8eH7HbQ89dC///d+v07//gB5tp+A5Li7j/mTXaeqa20iOD+W+uxJ6JBGq4FxiyYkbcfd6SM8+u5L16zNZty6ddeva2/zee5+IYCR02dmqFtZ8cIQ3N+Si06r4xX0jeWbJCBGMPIQYIbkRd6+H9NBDj/TQMyN4usZWC5/vLmZ/bhX+Php+OHswU0boxTJuDyPqIV0g6iGJekjuzBP7FR7uz/mKJrYeLCXz4DkkSWbm2DjmTeyHj5f7/i/tqa+VW9VDKikpYeXKlTQ1NREUFMSaNWvo169fp2McDgcvv/wye/fuRaFQ8OSTT7JkyRIAnnvuOQoKCjqOLSgo4K9//SvTp0+/6nmCILgfSZbZlV3G2vSTNLZaGDs4nMXTEogQpSE8mtMC0qpVq1i6dCmpqals3LiRl156iXfffbfTMenp6ZSWlrJt2zaamppIS0tj4sSJxMbG8sorr3Qcd+rUKR5++GGmTJlyzfOErhH1kARXUVZj4N2tpygub6FvlD//sWAYg+KCertZtwzZbsVWuA9bwV5045egjhnqtMd2ygRsfX09eXl5pKSkAJCSkkJeXh4NDQ2djsvMzGTJkiUolUpCQkKYMWMGW7ZsueT+1q1bx/z589Fqtdd13o0QM5q9Qzzvt542q52Pvyziv9YeprrBzE/vG8WvHx4rgpGTyDYL1uNbMf77OSz73gWFEmVgpFPb4JQRUmVlJZGRkahUKqB9qXBERASVlZWEhIR0Oi46Orrje71eT1VVVaf7slqtpKen884771zXeddyufnO+noNCoWEWn3trMBqteddXO3NPtntNrRaDeHh3b/Tvifu0xW4a79kWeabE5W8teEEdc1tzJ7Ql4fnDfXodD+u9FpJFhMt2VtoOpiOZGrBq+8wgic/g1ff4de1ubg7+uR2VwZ37NhBdHQ0iYmJ1z74OlxuUYNW60NjYwNBQaEoFFd+c3aFBQDdrTf7JMsSTU31aDQ+3X7x1xMvKIP79qu2ycwH2ws5XlxPbLgfLzw0hoSYQNqMFvx9tG7Zp2txlddKbjNgzd2ONXc7WE2o4pLwnjEfddQgDIChznDN+7jIrRY16PV6qqurcTgcqFQqHA4HNTU16PX6S46rqKggOTkZuHTkA/DZZ5+xaNGi6z7vRvj5BdLYWEt19XngylNISqUSSfKsgNS7fVKg1Xrh5xfYS48v9DS7Q2LLwVIyvj6LQqng/rsSmD42VizjdgLJ3ILtxFasJ3eCrQ1131FoRy9AFd6/t5vmnIAUGhpKYmIiGRkZpKamkpGRQWJiYqfpOoA5c+bw6aefMmvWLJqamtixYwcffPBBx8+rqqrIzs7mtddeu67zbpRCoSAk5NoVJV3lP57u5Il9ElzDqXONvLetgMp6E2MGh/OD6QMJCfDq7WZ5PLnNgOVYBraTX4LDhnrAOLSj5qMKjevtpnVw2pTd6tWrWblyJW+++SYBAQGsWbMGgCeeeIIVK1aQlJREamoqOTk5zJo1C4Cnn36auLhvn6z169czbdo0AgM7/+d8rfMEQeh9LUYrn+w6zde5VYQFevHMkmSS48N6u1keT7ZbsZ3cgeVoBljNqAdORDdqPsog/bVPdjKxMfaCy11D6ipPHE14Yp9A9Ks3SJLM7mPlfLb7DBabg7sn9GHexH7oNKqrnufKfboZzuqXLEvYi77BkvU5sqEeVVwyutuWoArp/n/W3eoakiAIt6azVS28t7WAkspWEvsG8+CsQSLvnBPYz+diOfgxUn0ZyrB+eE193Kn7iW5UlwNScXExW7Zsoa6ujlWrVlFcXIzNZmPIkCE92T5BENyQqc3G53vOsOtIOQG+Wp5cMJTbEiNFjaIe5qg7h+XgJzjKT6LwD8frrh+hjh9/1VXCrqRLrdy8eTMPPvgg1dXVbNy4EQCTycQf//jHHm2cIAjuRZZlvjlZxQtvHWTX0XKmj4nld09MYMJQUTCvJ0mtdZh3/QPT56tx1J1FN/EH+N77ezQJE9wmGEEXR0ivv/46a9euZciQIWzevBmAIUOGcOrUqR5tnCAI7qOizsj72wo4VdpEf30AP1syQpQQ72GSqQnrsU3Y8ncBCrQj7kY7ch4KnXtOi3YpIDU0NDB48GCAjv9yFAqF+I9HEAQsVgfpX59l66FSvLQqfjhnMHeMiEYp3h96jGRqwpqzGVvelyA5UA+chG5sGkq/0N5u2k3pUkAaNmwYGzduJC0treO2TZs2dWxEFQTh1iPJMofyqvlsdzH1LRYmJ+lZPC2eAA9O+dPbJFMz1pxMbHm7QLKjHnh7+xJuJ+ec6yldCkgvvvgijz/+OOvWrcNkMvH4449TUlLC22+/3dPtEwTBBRWUNvLxl6c5W9VK30h/npgvMnL3JMnc0h6ITn4Jkg11wu3oRi/wmEB0UZcCUnx8PJs3b2bXrl3ceeed6PV67rzzTnx93XOeUhCEG1PVYOLTXac5WlRHsL+OZSmJTBgWJabnekh7INqMLW9ne3aF+AnoRqeiDIrq7ab1iC4v+/b29mbu3Lk92RZBEFxUq8nKF/vO8tWxctRqJQvvGMCscXFor7G5VbgxcpsBa05me745uxV1woT2EZELZlfoTl0KSEuXLr3iAobuyBknCIJrstkd7Mg6T8Y3Z7FYJe4YGU3q5P4E+orrRD1BtlvaM3Af2wTWNtTxt6EdswBV0M0ni3YHXQpI3y8HXltby2effcb8+fN7pFGCIPQuSZY5lF/NZ1+dob6ljRHxoSyelkBMmJim7wmy5MBWuA9r1npkUxOqPiPQjV/cI2l+XFmXAtI999xzyW2zZ8/mP//zP/nJT37S7Y0SBKH3lFa38t7WAoorWugT4cejc0cytF/ItU8Urpssy9jPHsF6eB1SUyXKiHi8pj+FWj+4t5vWK244l11kZCQFBQXd2RZBEHqR2WJn474SdmSdx9dbzWNzE7k9SSxY6Cn2ygIsBz9BqilGGaTHa9Zy1H1H39L7O7sUkNatW9fp+7a2NrZt28bIkSN7pFGCIDiPLMscKazjwx2FNLZamDoymkVT4/Hz1vR20zySteYcpq3/wlGag8I3GN0dj6IZNBmFUiwQ6VJAupi/7iIfHx9GjRrFI4880hNtEgTBSeoulBDPuVBC/Km04STEiEq9PUEy1GM5/DmtRV+D1hvt+HvRDp+BQu1aC0RkScJSeg7zmWL8R49FHeS8/WVdCkjvvfdeT7dDEAQnsjskth4qJX3/WRQKBffdlcAMUUK8R8hWE9ajGVhztwEQOGEBjsGzXCrfnL25GVNeLsbcE5hOnsRhaAWFAm1klGsEpLKysi7dgajMKgjupbCsiXe3FlBRZ2T0oHCWzhAlxHuC7LBjy/sS65EvkC3G9jQ/4xYS2r9frxcelO12zMWnMZ1sD0KW0nMAqPwD8ElKwnfYcHyGDkcdEODUdl0xIM2cOROFQsHVCsoqFAry8/N7pGGCIHSvVpOVT3cVs+9EJaEBXqxYlMzIgaKEeHeTZRn7mcNYDn2K3FqLKmYYutvuRRXWt1fbZauvx5h7HGPuCcz5eUhtbaBS4R2fQNjCxfgMT0IXG4eiF0fJVwxIorSEIHgGhyTx1dEK1u/5toT4gtv7o9OKi+jdzV5ZgOXAx0i1Z1CGxOJ19y9QxQ7vlZVzst2OuaiwPQidOI61ogIAdWgo/rdNwHd4Et5DhqLy9nZ6265ElDAXBA9WWNbE+9sKOV9rILFvMEtnDCQm3K+3m+VxHE0VWA9+iv3cURS+we0lwwdOcvpow9ZQj/HECYwncjDl5yNb2kdBPoOGEDj5DnyTktFE6V12aXmXApLdbufDDz/k8OHDNDY2dprGE6mDBMH1NLZa+HTXaQ7kVRMSoOPHacMZMzjcZd+I3JVkasaavQHbqd2g1qIdtxht0kwUap1THl92ODAXn8aYc+zCKKgcAHVIKAETJuKblIzPkESUXu5xjbBLAekPf/gDBw4c4N577+VPf/oTzzzzDB999BHz5s3r6fYJgnAd7A6J7YfL+GL/WRySzPzb+zF3Yl90Iglqt5JtFqzHt2A9vhnsNjRDp6EdnYrSu+cXAUhtZoy5uRhyjmI8noNkNLZfCxo4iLBJk/FNSkarj3bLfz66FJC2bdvGxx9/THR0NG+88QYPP/wwkydPZtWqVSxfvryn2ygIQhfknqnngx1FVDeYGJkQxv0zBhIR5DrXBzyBLEnYCvd25JxT9x+LbvxilIE9Ww7C1tCAMecYhpyjmE/lI9vtKH198U0egd+IUfgMG+5S14JuVJcCUltbG3p9e9pzLy8vzGYz8fHx5OXl9WjjBEG4ttomM/+XnsfBk1VEBnvzzJIRJMe7dylrVyPLMo6y4+2pfhrLUUYm4D3jaVRRA3vm8SQJS1kpxuM5GI4e6ViWrYmIJOiuGfiOHIV3fAIKlWeNfLtcoO/EiRMkJyczfPhw3njjDfz8/IiM9KxqhYLgTi5ubv1i/1lUSgWL74xn5tg4NGqxufVmme1ttFhbUSmUKJoqcWSno6gqRO0Xjs/0H6MdMK5bp8SkNjNtJSWYTxdhPl1E25liJLMZFAq8BsQTtmgJfiNHufSChO7QpYD0wgsvoLoQiVeuXMnq1asxGo389re/7dHGCYJweWcqWnhn8ynO1xoYMzic5feNQrLae7tZTiHJEq1WAw1tjdS3NdLQ1r7Qykfjg6/GBx+194XPPvhqvNGpdJe8idslOw1tTdS3NVBvbqDO3EB927efjTZT5wf1AQaEt399bh3K0s9RKVSolSpUChVeai+CdAEE6QIJ1AYQpAsgUBdIkC7wwtcBqJXtb7eyLGOrr8N8+nR78Ck+jaWsFGS5PTtCdAz+4yfgnTAQn2HO35zamxTy1Xa+dqOSkhJWrlxJU1MTQUFBrFmzhn79+nU6xuFw8PLLL7N3714UCgVPPvlkp1pMmZmZ/O1vf0OWZRQKBWvXriUsLIw33niDDz/8kIiICABGjx7NqlWrrqt99fUGJOnGnorwcP8OFK2dAAAgAElEQVRe33nd3TyxT+D+/Wqz2lm/p4Qd2WUE+el4cOYgRg0Kd/t+fZcsyzRbW3DoLBRXnaehrYmGtgbqzY00WBppaGvCLnU9+CoVSnzV7cFKp9bRYmmlydKMzLd/7yqFilCvYEK9QwjV+BPYUIlfeSGSQgGxwyBmKJJSiV1yYJftOCQHtguf7bIDs81Mk6WFZkszTdaWS9qns0oMrlYQX2EjqqYNbWsbAAqdDu8B8XglDMQ7PgGvAfGofHy654l0ou///imVCkJDr397QZdGSAsWLGDBggXMmzev41rS9Vq1ahVLly4lNTWVjRs38tJLL/Huu+92OiY9PZ3S0lK2bdtGU1MTaWlpTJw4kdjYWE6cOMFf/vIX/vWvfxEeHk5rayta7bdJCdPS0nj++edvqG2C4A6OF9fz3tYC6lvamDY6hsVT4/HWuedWQptkp8HcQK25nrq2BurM9Rc+2kcpNsnW6Xh/jR8h3sHE+EWTHDaMUK9gQr7zoVQoMdlNmGxmjDYTJrsJo8184bMJk82E0W7GbDMTERROmHcwod6hhHkFE+YdSqAuAIXdivXENqyHNoO9DfXASejG3oPS7/qux8myjNFuovH8GQzHjuE4mY/mXCUKWcbiraE0SkPZYD+qwrR4x/VhcNhgEkMG0T+wLyqle76e3aVLvV++fDkZGRn89a9/ZdiwYaSkpDBnzhyCuph0r76+nry8PNauXQtASkoKv/3tb2loaCAk5NvCX5mZmSxZsgSlUklISAgzZsxgy5YtLFu2jHfeeYfHHnuM8PD2YbO/v//19lUQ3FKL0cq/dxZxIK8afagP//ngaAbGOi/h5c1ySA7OtpRxqqGQ081nqTXVXTJC0So1hHmHEu4dRmLIIMK8Q4mPikFt8SLEKxit6toZsbWq9imy6yU77NhOfon16BfI5hbU/cagHbcQVXDM9d2PJNFWfBrDsaMYc45hrapEBXjHxOJ39zx8R47Cq19/7gz3J+tMPqcaCslvKGJH6W62nduFVqlhYHA8Q0IGkhgyiCifCI++XnQ5XQpIM2fOZObMmRgMBrZv305GRgZ//OMfmTBhAn//+9+veX5lZSWRkZEd16FUKhURERFUVlZ2CkiVlZVER39bO16v11NVVQVAcXExsbGxPPDAA5hMJmbOnMlTTz3V8YJt2rSJffv2ER4ezvLlyxk1alTXnwVBcEGyLPN1bhX/3llEm9VB6uT+zJ3Q1+UXLciyTK25vuMNt7CxmDZHGwoUxPlHkxA0gHDvEMK8Qzs+ArR+l7z59vQ0pCxJ2IsPYMn6HLm1DpV+CLpZK1BFJnT5PuzNTZhOncKUewLDiRwkg6E9M8LgIQROuwu/ESPRhIV3OkepVNI/sA/9A/twd/8ZmO1tFDUWk99QxKnGQk4WtadtC9T6E+4TRrAuqP1alFcgwR3XpYLw1/qiVLj278L1uq7xoZ+fHykpKfj7+2Oz2dizZ09PtesSDoeDgoIC1q5di9VqZdmyZURHR5OWlsb999/Pj370IzQaDfv37+fHP/4xmZmZBAcHd/n+b2S+87vCwz1vxOaJfQL36FdVvZG/fprDsaJaEvuF8JMlI+gTdfWL273ZL4PVSG51Acer8smpzqfWWN/eJt9QJvcdS3JUIsMjBuN3nSUXeqJPsixjOp1N41cfYK0pRRvZn5B5P8J7wMhrjkhszc00556k+UQuzSdyMZ+/kBnB34/QsaMJGT+OoFEjUV/jOlDnfvnTRx/OdCYAUGOs53hVPvm1RdSZGjhnKOVobfMl16VUShUhXoGE+gQT4h1EiE8wod5BhPgEEerdfluQdyBqJxX+647XqksBSZZlDhw4QHp6Ojt27CA6OpqUlBTWrFnTpQfR6/VUV1fjcDhQqVQ4HA5qamouuR6l1+upqKggOTkZ6Dxiio6OZs6cOWi1WrRaLdOnT+f48eOkpaV1TOMBTJo0Cb1eT1FREePHj+9S+0Asavg+T+wTuH6/zBY7mQfOsfVQGWqVgodmDWLqqBiUCsVV2+2sfsmyTJOlmQpjNZXGKioN1ZQbKznfWoGMjJfKi0HB8dwVM4UhIYMI9w7teJM3t0iY6Xobe6JP9soCrIfW4aguQhEQidf0p1APGIdRocRYZ7jkeIfBgKngFOaCfEynTnWk5lHovNozI0yYhM+QRHR9+qJQKpGBRqMDjDf+WinQMiJgBCMCRnTcJskSRpuJRksTTW3NNFmaabS0f25qa6ao7ixNluOXXHtToMBf60eQrn10FagLJMQriBCvYMK8QwjxCsZP43vTU4NOXdQwZcoUfHx8mDt3Lh999BHx8fHX9SChoaEkJiaSkZFBamoqGRkZJCYmdpquA5gzZw6ffvops2bNoqmpiR07dnTkyktJSWH37t2kpqZit9s5cOAAs2fPBqC6urpjT1R+fj7l5eX079//utooCL1JkmT2n6jk8z1naDZamTgsksV3JhDs75ycaN8nyzIt1lYqjFVUGqupNFwIQMYa2hxtHccFaP3R+0Zyd7/pDAkZRL+AOFQuWIrbUXcOy+HPcJQdR+EThG7yw2iGTEHxvUUEHbnhjudgOnkCy/nzIMsotFq8EwYSMGEi3oOH4NW3Hwq18xYgKBVK/LV++Gv96OMfe9ljZFnGZDe3B6nvBKv2r1uoNddT2HQGs93c6TytStu+wvDCKsMQr2DCvNqnVGP8nLvvqUvP6JtvvtkxarlRq1evZuXKlbz55psEBAR0jK6eeOIJVqxYQVJSEqmpqeTk5DBr1iwAnn766Y4CgPPmzSM3N5e5c+eiVCqZPHkyixcvBuC1117j5MmTKJVKNBoNr7zySqdRkyC4slPnGvn3ziJKawzExwSwfFEyA6Kdt/fE6rBSaaym3FDZ8VFhqMJo/3Yvjp/GF71vJOOjRqP3jWz/8IvET+M6VU8vR2qqwpL1OfYzh0Dni3b8kgtlw78N9I7WVoy57Rmyjbm5SCZjR52g0AVp+AxJxKv/AKcGoBuhUCjwvbAXK8bvyquhzfa29j1c5gbq2xqpb2ugwdxIXVsDp5vOdvqH4/HhDzI64ube+6+H0/YhuToxZdeZJ/YJXKtfNY0mPtlVzJHCWkIDdCy+M4HxiTe2sqor/ZJlmfq2BsoNVVRcDD7GSmpN9R0r3rQqLTG+UUT76Yn2jSLaLxK9bxT+WueXrLiZ10oyNGA9sgFbwT5QqdEmzUabPAeFzhdZlrGeL8NwPAfj8RzazhSDLKPyD8A3KQnf5JH4DB3WY/uBXOl38HJMNhP1bY20WFsZGDSgSyscnTplJwhC9zG12cn4+iw7sstQKZXcc8cAZo+LQ9uNGbllWabO3EBpaxmlreWUtpZT1noes/3ChkwUhHmHEOOnZ2zkKGL89MT46gn1DnbrlVuSuQXrsU3Y8naCDJph01HGT8Xe2ErT/m+wlJZiOpmLvbEBAF3ffoSkLMA3aQRe/fr1arVUV+Gj8cFH0zubc0VAEgQncUgSe3Iq2bD3DAaTjUlJehZOHUCQ381dJ5JlmSpDLceqCyhrLedc63nKWss7rhWoFSqi/aIYHTGCOP8YYv2i0ftG4uWkmj3OIFvNmA9uxJT9JXaDFUmrx+7wxvbJARyG7R3HKX188BmciG9qGr7Dk1F3cS+l4BwiIAmCE5ytauGdzFOU1hgYFBfED+4dSN+oG18m65AcFDYVk12dw/G6kx2511QKFTF+UYyOSKavfyxxATFE+0Z15FFzZ7IsIxkMWGtrsdXWtH/UVGM5V4itpg7J9u2Uu9KrEW20D76jRqGLjkEbHYMuJgZVYNAtt9nUnXT5t9RqtVJSUnJJxdiJEyf2SMMEwRNYrA7W7z3D9qwyAny1PJU2nLE3WLlVkiVKmkvJqj7G0ZrjtNoMeKl0JIUNZVRcIsGEofeLQuMBwcdSUU5l1jkaS0qx1da2f9TVtmfA/g6lVoFKK+OlD8J7+Di8BiWhjY5BHRwiAo8b6tJvblZWFs888wxWqxWDwYCfnx9Go5GoqCh27tzZ020UBLeUe6aed7cWUNfcxp0jo1l8Zzw+Xprrug9ZlikzlJNVfYwj1cdptDShUaoZHprI2MiRDA0dglalcfkL5V1hra6i9fAhWg8fwlp+HgCFWo0mLBxNeDjeAwehDgtDaa1DrjiM0tGMJmYg2rELUUcP6eXWC92hyyXMly1bxiOPPMK4ceM4dOgQf/nLX/D2gAqFgtDdWkxW/r3j29xzKx8YzaC467tWUWWsJqs6h+yaY9SY6lAqlAwNGcSC+Dkkhw3FS+3VQ613Llt9XUcQspw7C4D3wEFELH2QuGmTaJF17RtOL6b5yd6I3FKNMnwAunFPoIoZJkZCHqRLAens2bP88Ic/7HTbk08+yfTp03n88cd7pGGC4G6+n3tuwaR+zJvYr8u552pMtRypOU52dQ4VxioUKBgYHM+MuKmMiBju8nt+usre1ERr1mFaDx+krfg0ALp+/Qm/9378xo5DE9KeXdsr3J+WmmZsZw5jzV6P1FiBMiQOr1k/RdX32ml+BPfTpYDk7++PwWAgICCA8PBwTp8+TVBQECaT6donC8ItoKbRxLtbC8g720hCTCAPzxlMTPi192HUmRs4UpPDkeocygwVAAwI7MeSgamMikgiUOcZxdlkScJw7ChNO7djLiwAWUYXF0fYwsX4jRuPNjyi8/GyjKkoG9POD5Dqz6EM0uM1/ceoB4xF4cbL0oWr63K27927dzN//nwWLVrED3/4Q9RqdUfqHkG4VdkdEtsPl7FxXwlKpYIHZw3izgu5566ksa2pfSRUk8O5ljIA+gbEsTAhhdERyQR7ec5SZNlup/XwQRoyN2GtrEATFk7o/FT8x41Hq4++9HhZxlGRj+XwZxhqilH4h+N15xOoEyaKPUK3gBvK1JCVlYXRaGTKlCkoPeSXRGRq6MwT+wTd26/jxfV8/GURlfUmRg0M44GZgwgJuPy1HUmWOFR1hP0VBznTfA6AOP8YxkSMYHREMqHeIZc9r6tc7fWSrFZa9u+lYUsm9vp6tDGxhMxNwX/sOBSqy28AtlcVYT38GY7KUyh8Qwi9YwltMeMuyTfn7lztteoOvZqpYezYsTdymiB4hPI6Ix/vLCK3pIGIYG+WL0piZELYZa9pyLJMTm0u6We2UmWqIco3kvkDZjM6IpkIH8/Lt+gwm2n+aheN27fgaGnBa0A8EUsfwjd5xBWv+ThqS7BkfY6j7AQK7wB0tz+AZshUAvShWDzsjVu4ui4FpLKyMv70pz+Rn59/yXWjr776qifaJQgup9VkZeO+Er46WoFOq+L+uxK4a0wsatXlZwlONRTxxZktnGspI9IngmXDH2Jk+HCPvBjvaG2lced2mr7cgWQy4TN0GCFzU/AePOTKgaihDGvWBuxnsy8kPr0X7bDpKDSek0FCuD5dCkjPPvsscXFxPP/882Kpt3DLsTskvsw+zxf7z9JmdTB1VDRpk/vj73P5pJPnWsrYWLyZgsbTBOuCeGDIEm6LGu2SZRluhixJmE8XYcg6RPO+vchWK36jxhAyLwWvflcu/yI1VWHJ3oC9+CBovNCOSUObNBuFVry33Oq6FJCKior46KOPPOZ6kSB0hSzL5Jxuv05U3WhmeP8Q7ps+kJiwyy+/rjJWk35mK8dqc/HT+LJo4HymRE9Ao7q+zbCuTLJaMeWdxHD0CMacYzgMrSjUavzGjSfk7nnoomOufG5rLZbsL7AX7QOVBu3IuWiT70bh5fxM4oJr6lJAGjduHHl5eQwfPryn2yMILuF8jYGPdhaRf64RfagPzywZQXJ86GWPbWhrZFPJdg5WZqNTaZnbfybT46Z4zOZVh8GAIecYhmNHMJ3MRbZaUXp745s8Ar+Ro/EZnoTqKjMnkrER69F0bKd2Awo0w2agHZmC0ifQeZ0Q3MIVA9Kf//znjq9jYmJYtmwZM2fOJCwsrNNxP/3pT3uudYLgZG1WOxv2lrA9qwwfnZoHZg5i6sjoS64TybJMScs59pUfJLv6GCgUTIubzKy+03qldlB3s9XVYjh6BMPRI5iLCkGWUQcHEzBpCn6jRuMzaPA1C9ZJ5hasOZnYTu4ESUIzZAraUfNR+l0+sAvCFX+jqqqqOn0/bdo07Hb7JbcLgqfIOV3H+9sKqG+xcOfIaBZOjcfPu/N0m8lm7li+XWGswkul4/bo8czqO83t9w9Za2owZB+mNTsLy9kSgPbl2vNS8Bs5Gl3ffl1akCFbjFiPb8Gaux3sFtQJt6Mbk4oyIOKa5wq3tisGpD/84Q/ObIcg9Jpmg4UPdxRx+FQN0WG+rHxgWKfcc7Isc7altH00VJODTbLRxz+WpUMWMSZipFvXFbJWVdKadRhDdhaWslIAvPoPIGzxvfiNGoM2MrLL9yXb2rDmbseasxmsJtQDxqEdcw+q4Es3wArC5XR5H9LZs2fZvHkzNTU1REREcPfdd9OvX78ebJog9CxJltmTU8Gnu4qx2SXumdKfuyf07ZieM9vNHK46yr6Kg5QbKtGqtIyPGs3kmNvo4x/by62/cZaKcgzZWbRmHe7Iqu0Vn0D4vT/Ab8wYNKFh17iHzmS7FVveLqzHMpDbWlH1GYFu7EJUYX17ovmCB+tSQEpPT+ell15i6tSpREdHU1hYyD/+8Q9+85vfMH/+/J5uoyB0u/I6I//acorT55sZ0ieIH84ZQlRIe9nmckMlX5XtJ6v6KFbJRpxfNPcPXsi4yJFuu1DBUlFB6+GDGLIOY62sAIUC74SBhN//AH6jx6AJuf5MEbJkx1awD+uRjcjGRlQxQ9sDUWRCD/RAuBV0KSD96U9/4h//+Afjxo3ruC0rK4vnnntOBCTBrVhtDj7fc4bNB87hpVXx2NxEJiVFISNzrDaXr8r2UdR0Bo1Sw7jIkUyOmUAf/1i33Mxqra3BcPgQLYcOYj1f1h6EBg0m4q7p+I0ac8Plu2VZwl58CEv2euTmapQR8XhNexJ1dGI390C41XQpIBmNRkaOHNnpthEjRohs34JbKSht5L3/d5CKOiMTh0Vx3/QE1BoHO8v2sOf819S3NRKsCyItfi63R4/HV+PT202+braGBgwXSzuUnAEuTMfd/wD+Y8fdcBCCC4lPy05gObwOqb4UZXCsKAUhdKsuBaRHH32U1157jWeeeQadTkdbWxuvv/46jz76aE+3TxBums3u4LPdZ9h2uAx9qC+/uG8koRF2Mss2cbAyC6tkIz6wP/ckpJAcNtTtMirYW1qoPLyfyi93ty/RBnR9+hK2+F78x42/7mtCl32MqiKshz7FUVXYnoF72pOo4yeIDNxCt+pStu+pU6dSV1eHQqEgICCAlpYWZFkmPLxzckh3zmsnsn135il9Kq1u5a30PMrrjEwbHc2kKVoyC74kv6EQtULF2MhR3Bk3iTj/K2cYcFXmM2do2rGV1qzDIElo9dH4j78N/3G3oY2K6pbHcNSXYTm8DkdpDgrvQLSjF6AZMhWFquczcHvK7+D3eWK/nJrt+9VXX73uOxaE3iRJMlsOlbJ+zxl8fdQsmKfjhGkTB76pJkDrT0r/WUyOmeB2m1hlScJwNJvG7dtoO12E0tub4Okz6ZsyG6NPcLdNnUktNViyPsd++iBovdGOX4x22EyR+FToUV0KSOPHj+/pdghCt6ltMvP/MvIoOt/E4GFW7OGn2F5bSZRPBD+57REGeg9C7WY1dhxmMy379tC0cwe2ulo0YeGE37+UgElTUHl74xvuj6kb/uuWjI1Yj2zEdmovKFXt+eZGzEWh84zy6YJr61LqoKsRqYMEVyHLMvtOVPLhjkKU/nXE3l5Kqb2KMDmUh4fez9jIkURGBLrVdImtvo6mHdtp3rcHyWzGe+Agwpbch9+o0d16/UZuM2A5ltGe5keW0CTeiXb0fJQ+7p19QnAvXU4ddLNKSkpYuXIlTU1NBAUFsWbNmks21jocDl5++WX27t2LQqHgySefZMmSJR0/z8zM5G9/+xuyLKNQKFi7di1hYWHXPE/wfC0mK//afIqcykICEkuw6GqRVEEsTVjEhKixbrdQoa3kDA1bN2PIzgKFAv+x4wmeOQuv/gO69XFkqxlr7jasOVvA1oZ64O3oxqShDPC84oGC67up1EGSJHX5gVatWsXSpUtJTU1l48aNvPTSS7z77rudjklPT6e0tJRt27bR1NREWloaEydOJDY2lhMnTvCXv/yFf/3rX4SHh9Pa2opWq73meYLnO3a6jrd3fY0t7BS6oXXotP4s6JfKpOjb0LjZ1Jy9pYW6dZ/Q8vU+lD4+BM++m6C7pqMJ6d6EpLLdii1/F9aj7dkV1P3GoB27EFWI+y3uEDzHDf21FhQUsGHDBtLT09m3b981j6+vrycvL4+1a9cCkJKSwm9/+1saGhoI+c4O8czMTJYsWYJSqSQkJIQZM2awZcsWli1bxjvvvMNjjz3WsbLP39+/S+cJnsvUZmftroOcMH2DKr4GX5UPc/rP446YiWhVly+e56pkSaJ5z1fUfb4OyWIh+O55hM6bj9KrezNDyJIDW+E+rNkbkY0NqGKGoRu3CFVE9468BOFGdDkgNTQ0kJ6ezoYNGzh16hRjxozhxRdf7NK5lZWVREZGolK1T5uoVCoiIiKorKzsFJAqKyuJjv42EaNer++YOiwuLiY2NpYHHngAk8nEzJkzeeqpp1AoFFc9r6tuZInid4WH+1/7IDfjyn3adOQ47x/ZiMO/Ap1Oyz3DUkgZPB1vzbXfwF2tX61Fpznz939gOF1MYNJwBvzHE/jEXf/o/mr9kmUJY97XNO75GFtDBbrogYSkrcC7X9LNNL3Hudpr1V08sV/d0aerBiSbzcaXX37J+vXr2bdvH3369GHevHlUVFTw5z//mdBQ59U1cTgcFBQUsHbtWqxWK8uWLSM6Opq0tLRuuX+xD6kzV+1TcX05b2dvpFF1FoWvmolhU1iYOAMfjTeGJhsGbFc935X65TAaqVv/Gc27d6EKCCDqif/Af/wEjAoFxuts45X6Jcsy9rPZWLM2IDWeRxkcg9esFaj7jsKgUGBwkeficlzptepOntgvp+xDmjRpEgqFgoULF7J8+XKGDRsGwEcffXRdD6LX66mursbhcKBSqXA4HNTU1KDX6y85rqKiguTkZKDziCk6Opo5c+ag1WrRarVMnz6d48ePk5aWdtXzBM9Qbarl/ZwMzpjykRUqBqhG8cSE+QS6YflrWZZp+Xo/des+xmEwEDR9BqEL7kHl032pimRZxlF6DEvWBqT6cygCo/C660eoB4wX2RUEl3XVgDR48GCys7PJycmhb9++xMbGEhh4/WWHQ0NDSUxMJCMjg9TUVDIyMkhMTOw0XQcwZ84cPv30U2bNmkVTUxM7duzggw8+ANqvO+3evZvU1FTsdjsHDhxg9uzZ1zxPcG915no2Fm3jSO0xZEmJr2Ewj4+fx5CY7slE4GyW82XUfPAe5qJCvOITiPnZs3j16b4yDbIs4zifiyVrPVLtmfY0P3c+gTphAgo3W2ko3HquGpDee+89ysvL2bBhA2+//TYvv/wykydPxmQyYbfbr+uBVq9ezcqVK3nzzTcJCAhgzZo1ADzxxBOsWLGCpKQkUlNTycnJYdasWQA8/fTTxMXFATBv3jxyc3OZO3cuSqWSyZMns3jxYoCrnie4p3pzI1vO7uCbyixkSYGjth+z+k4ldfoQVG72H77scGA6lU/rwW9oOfANSh8fIh9+lIBJU7p1tGKvyMeatb4935xfKLo7HkUzaBIKN1tpKNy6upTL7qKsrCw2btzI5s2bUalULFq0iOeee64n2+c04hpSZ73VJ6vDyuazO9lZugdJkrFVxxEjJ/PEnNHoQ28+W4Cz+iVLEuaiQloPHcSQnYXD0IrS2xv/CRMJS12Iyq/7phrtVUXIORtpO5eLwicI7aj5aIbcgUKlufbJLswT/67AM/vl1Fx2F40dO5axY8fyq1/9iu3bt7Nhw4brfkBBuJLcunw+KdxAfVsjNMTiKB/I4tuHM31MLEql65c3kGWZtpIztB46SGvWIRxNTSi0WvxGjsJ/3Hh8hieh1HTfcnRH3Vkshz/HUXYclW8guok/QJM4DYXavZa8C8JFNzSW1+l0pKSkkJKS0t3tEW5BjW1NrCtK51jtCbSOQCyF40kIGsBjPxxCRLBr1ySSZRnr+TJaDh2k9fBB7HV1KNRqfJKSCRh3G74jRqLUdW9CUkdjBdasz7GXZIHOF+34xURPvYf65quvMBQEVycml4Ve45Ac7Cn/hvQzW7BLEsrqRIzlfVg8ZSCzxsW57KhIlmUs587Smp2FITsLW001KJX4DB1G6Pw0/EaN7tYVcxdJLbVYjmzAXvQ1qHVoRy9AmzQbhc4XpdYLrrHkXRBcnQhIQq8411LGRwWfU9ZaTqAUQ/WJeGICInjih0OJi3C9pdyyJNFWXEzrkSwMR7Kw19e3B6EhiQTPnoP/6LGo/Htms6NkbMR6NB3bqd2gUKAZPgvtyHkovQN65PEEobeIgCQ4ldlu5ovirewt/wYflS9eFeOpPh/MnNv6kjZlABq166ygkx0OzIUFF4LQERzNTe3TcUOHEbogDb8Ro7p1ccIlj99mwHJsE7aTO0CS0AyZgnbUApR+Idc+WRDckAhIglPIssyRmhzWFaXTajUQoxjG6UNRhPr58dzSRAb3Ce7tJgJgb27GVJCPKe8kxmPHcBhaUWi1+A5Pwm/MWHyTR6Ly9u7RNkjmFmwnd2I9sRVsFtQJE9CNvQdlQESPPq4g9DYRkIQeV2Oq5eOCDZxqLCLKW4+ychxFZVomJ+n5wYyBeOt679fQYTBgKizAfCoP06l8rBUVACi9vfFNGoHfmDH4Dk/u9oUJlyM1VWE9sRVb4T5w2EQGbuGWIwKS0GNsDhtbz+1i+7ldqJVqknR3cGSfL15aNT9ZOITRg5xfc8duMmM4noP5VFehyOcAACAASURBVD6mU/lYykpBllFotXgPHETAxMn4DBmCrk9fFKqez2wgyzKO6iJsOZuxnzsGShWaQbejSZqDKlikvxJuLSIgCT3iZH0BnxRuoM5cz9DA4dTm9edQuY0R8SE8MjeRQF/n7JWRZRlrZQXGnGMYj+dQWHwaJAmFWo1XfAKhC9LwGZKIV/8BKNTO+3OQJak96enxzUg1Z9qXb49KQTNsuqjSKtyy/n97dx5fZXUuevy3x8wzmRMIU0IghIR5EBREZIgSBygi9LRX5FoU7rH1HFLtEWvtR6G9tJ/yoXrPsceWFqvltIpACAERg6BMQoCEmUCmnQQyz3vv9133j62pyAwZduLz/Ss7ed+915OVvZ+s9a73WZKQRLuqaa3lf05/xOFLxwjzCmUoMzm4Q+HrBYsfGsyYweEYDB27nFs5nTSfOU1D7mEac4/guHQJAI/efYh5NB36DMCz/wCM1s6/gVQ5WnCc2o39WDaq/hIG/zA8JizEEn8PBkvHTwsK4c4kIYl2oekau4r3sKUgG13pjAyYSN7+AC7WOpk0LIrH7+uPr1fHlbLRGhpoPJZLQ+4RmvKOozc3Y7BY8E4cTND0mfgkp2AJCuqysi2qpcG1VXjex9DaiDF8AB5j52HukyrVt4X4iiQkcdfO117gvVMfUNJgIz4gHlU8hN1fNBPVy4OMJ5OJj+2YKSjldFL3+R7q9u6h+ewZUApTQAC+I0fhOywV78TBnbIY4YZtbGnAfjQLe94OcLS4FiokT8cUMbBL2yWEO5KEJO5Yg72Rjecy2Ws7QKBHAGO9Z/FFjsLhbOWRSf2YMaY3ZlP7//evt7ZSu/tTqrdl4ayuwhoVTfCsh/AdloJHnzi3GHHoLfU4jma5RkSOVsz9RmEd/jCm4NvfCVaI7wpJSOK26Urnc9sBNp7dSrPWwuhe4yjMjeCTkmYS+wTx/ekJhHdADTqtqZGaT3ZSsz0braEer/gEwv/lh3gPSerw61K3Sm+uw567FUf+TnDaMfcfjTX1YVm6LcQtkIQkbktRfQnvn/qAgrpC+vn3JaRuBJ9lNeLl4eTptMGMHdL+ixac9XXUbM+m5pOP0Zub8U5KJmRWGl4D49v1de6G3lSL/ehXiUhzYO4/FmvqQ7J0W4jbIAlJ3JJmZzObzmeTU7wXX4sP9wbPZP9eM3l1DdwzNJK5Uwa0+6IFR1Ul1dlZ1OZ8inI48B0+guBZD7XrDqt3S2+sxn40C0f+J6A7MA8Yh0fqQxgDI7u6aUJ0O5KQxA0ppThQfph/nN1Mg72RUaGjqD4TR9aeOqJ7Wch4MqldFy18XcS0ds9u6j7fA0rhP3YcwTNmYY10n9GGXleB/Uimq6qC0jEP/CoRBXTPrdWFcAeSkMR1lTWW896pDzhTc54+frEMM87g0+xGoIE5k/vzwMjYdlm08PXuqg2HDlD/5SHXxnZmMwGT7iX4wRlYenV+RYfr0SoLsR/ZgvP8fldVhYSJWIfNkDpzQrQDSUjiKq2anfW5H7Dp1A48TR5MCZvB4X1ebL9UT+rAXjwxdSC9Au6uwGhbJe1Dru0ctLo6DBYLPknJ+I7snCKmt8NZdhr74c1oRUfB4ok1eQaWodOkqoIQ7UgSkrjCuZoLrDvxPpebKxkROhxHYQJb9lYT4q+x9LGhpA6889GKcjppOnWShkMHXNs5fF1JO3kYfiNG4TM0GaOnZztGc3eUUmhFx7Af2YxWdhqDpx/WkY9iHXI/Bg+frm6eED2OJCQBuAqhbi7I5uPCHII9g5gVMZ+t2XW02GuYMbY3D4/vi4f19ouNKl2n+ewZ6vd9Tv3BA+iNjRg8PPEdluLaziFpaJffvPptStdxFhzAfmQLemUhBp9gPMY/iWXQJAxm92qrED2JJCTBxboi1p34G2WN5YwIGUF5Xhz/U1hFfEwACx9MIDr09jahU0rRWlRI/b4vqN+/D2d1FQarFd+U4fiNGo13UhJGS+fXkbsZpTlwnNmL/Ugmqq4cY0AEnvc+hXnAOAwmeasI0dHkXfYd5tSdZF3YybaLO/Gz+DLG4yH2bNcwm1pZOjeFlL5Bt3VPkf1ShSsJ7fsCu60UTCZ8hiTR6/G5+Kakut1I6GvK0YLjxC7sR7NQTTUYe8XhMfVZzHEj3KLqgxDfFZKQvqNKGmz8Of99ihpKSQpMpuJ4X3aVtpI6sBcLpiUQ36/XLRUh1errqdvvSkIt588B4DUwnrAF38dvxChMfn4dHcodcxU83e6qM9faiCkqEet9izBFD3Gbyg9CfJdIQvqO0XSNjwtz2FyQjZfZixTzdPbvMODtqfPM7CGMGhR2Sx/GzpoaqrdtpebTT1B2Ox6xsfR6bC5+o8dgCQnphEjuXNvNrCd2gbMVc59UrCmzMIUP6OqmCfGdJgnpO6S86RJ/zn+fgrpCBvoN4vLxAXxe7mTs4DCemDoQP++bX9dxVFdTnZVJbc4ulKbhP3YcQQ/OwCPa/YuG6rVlXNr/FxqPfgJKYe4/xpWIpOCpEG6h0xJSQUEBGRkZ1NTUEBgYyMqVK4mLi7viGE3TeO2119i9ezcGg4HFixczZ84cANasWcO7775LWJjrBsThw4ezYsUKADIyMti7dy9BQUEATJ8+nR/96EedFZrb05VOTvHnfHguE4vRTCJTOLzTQqCviWWPDyZlQK+bPoejspKqrC3U7c5BKYX/uPEEz0jDGh7eCRHcHa3iPPbcTJwFhzCYzFgG3Ys1eQZGf/e54VYI0YkJacWKFcyfP5/Zs2ezceNGXn75ZdatW3fFMZs2baKwsJDs7GxqampIT09n3LhxxMS4/oNNT09n+fLl13z+xYsXs2DBgg6Po7upbqnhLyc2cLL6DH28+3H5eDxfXoZJw6KYO3kA3p43/hNwXL5EVeYWavfsBiBgwkSCZ8zCEureH+ZKKbTiY9iPZKLZToLVC2vKLCLuTae6WSYGhHBHnfLOrKysJD8/n3feeQeAtLQ0fvGLX1BVVUVwcHDbcZmZmcyZMwej0UhwcDBTp04lKyuLRYsWdUYze5yDZYd57/SHaLqTBMNEjuzypleAJ/82bxCJccE3PLfZVkbZX96j7vO9GAwGVxmf6bPc/vqQ0p04z+3HnrsVvaoIg08QHmO/h2XQfRisXph9/aC583eMFULcXKckJJvNRnh4OCaT68ZKk8lEWFgYNpvtioRks9mIivpnAc3IyEjKysraHm/ZsoXPPvuM0NBQli5dSmpqatvP3nnnHd5//31iY2P5yU9+Qv/+/TshMvfU6Gji/VMfcKgil1ifWBznh3Lkos6kYZE8cX/8DW9wdVRepvKjjdR9vgeDyUTgfVMImj4Ty1fToe5KOVpxnPwU+7FtqIZKjEFRcg+REN1Mt3mnzps3j2eeeQaLxcKePXtYsmQJmZmZBAUF8fzzzxMaGorRaOTDDz9k0aJF7Nixoy0B3oqQkNu7+fPbQkPdY3nzEVs+bx5cR11LPZMiprD3Y28cDsVP5g/nvhGx1z3PXl1N8YZ/ULYtG4DIWTOIefQRrMHunYicDdXUHcqi7lAWenMDnrGJBMxcjPeA4RgM176HyF36qr31xLh6YkzQM+Nqj5g6JSFFRkZSXl6OpmmYTCY0TaOiooLIyMirjistLSU5ORm4csQU+o1rFhMmTCAyMpIzZ84wevRowr9xYT09PZ3XX3+dsrIyoqNvfZfOysoGdF3dUXyhoX63dM9OR7Jrdj44m0lOyV7CvcMYbrifbR81ER3qwZL0JCJDfK7ZRq2xkaqsTGo+3o5yOgm4ZyLBaQ8TlRDnOr6L47oWpWtoxcdwnMzBeTEXlIY5bjieyTMwRQykCWi63HjNc92hrzpCT4yrJ8YEPTOub8dkNBru6J/8TklIISEhJCYmsnnzZmbPns3mzZtJTEy8YroOXKvjNmzYwLRp06ipqWHHjh2sX78egPLy8rbEc+LECUpKSujbt+9VP9u9ezdGo/GKJNXTXawr4o/5f6Wi6TLjw8dx8Ug0OUUNTEyOZP4D8XhYrh4p6i0tVO/IpnrbVvTmZvxGjyVkdjrWcPfdz0evu4TjVA6O05+hGqsxePphGToN66B7MQa6b7uFELem06bsXnnlFTIyMvj973+Pv78/K1euBODpp59m2bJlDB06lNmzZ5Obm8u0adMAePbZZ4mNdU0zrV69mry8PIxGIxaLhVWrVrWNmpYvX05lZSUGgwFfX1/efPNNzOZuMxt5x3Sls/XCx2Rd+JgAqz8PRcxj645GWh1NLEpLZHzS1buW6g47tZ/uomrLJrT6enxSUuk1+1E8Yq8/ndeVlObAeeFLHCdz0EryAAOm2KFYxj+JuXeKXB8SogcxKKXubJ6qh+luU3a1rfX8Me9dTtecY2RYCt6VqWR/biMq1IcfzU4iqteV2yMop5O6vXuo3LwRZ1UVXoMS6fXIY3j1v3Z1gq6eVtCqSnCc/BTnmb2o1gYMviFYEiZhSbgHo++dr/Tr6rg6Sk+MqyfGBD0zrm41ZSfa1+nqc7yT9y7NzhYe7fsIB/d6cKrIds0pOt1up+6zHKqytuKsqsSzbz8ifrgI78TBXRjBtSl7M45z+3CcykGvOA9GE+a44VgG3YsparAUOhWih5OE1I3oSif74i42n99GmHcvJgc8ysaPqrA7W3lqViIThv5zik5rbqZ21ydUb89Cq6vDs/8AwhYsxGfoMLcqHKqUQis/41qgcH4/OO0Yg6LwGDsP88DxGL38u7qJQohOIgmpm2iwN/Kn/PfIrzrFsJBkWguG8N7JcuIi/FiUNrhtik5raKB6RzY1O3egNzXhPSSJ4JlpeMUnuFUi0ptqcJzei/NUDnptGVg8sQwYiyVhEsaw/m7VViFE55CE1A2cr73AH46vp8HewLiAB9i3y4NWew2P3duP6WN6YzIacdZUU70ti5qcXajWVnxTRxA8cxaefft1dfPbKF1DKzrqGg0V5oLSMYUPxDNlFuZ+ozBY3Gf7ciFE55OE5MaUUnxclMPGc1sJtAbQu346O/dpxEV48dSsRKJDfXFcusTlrC3U7fkMpev4jR5D8Iw0PG7jHqyOptdfxnHyUxyndqOaajB4+WMZOg3LoEmYAqNu/gRCiO8ESUhuqsnRxJ9PbODo5Tz6eA6k6FB/ylv0tlGR0WCgZucOLm14H5TCf8I9BE2fiTU0rKubDnw1Gio8iv3EJ2hFxwBcy7UnLMTcZxgGo/zpCSGuJJ8KbuhiXRF/OP4XqltriWgZxcn9wcRF+PHUPNeoyFlXR+kf/0Dj0Vy8k5IJ//4PsATfuFhqZ9Ebqv45GmqswuAVgDU1DcugSRj93LtCuBCia0lCciO60tl+cZdrN1ejD5wZT1GtH4/d27ftWlHj8WOU/fd/oTc1EfrEkwROmdrlCwCUrqMVH8WevwutKBeUwhSThGX8fMx9UmQ0JIS4JfJJ4SYqm6tZd+I9ztYU4O/oQ/nRAcSFBvPUD1yjIt1hp+Jv/0PNjmysUdHE/Pjf8Ijp2uoKWk0pzrP7XKV8GioxePljHTYTy6B7Mfq7x9ShEKL7kITkBg6WH+G9U//AoWkYi1OoLI/gsXv+uYKutaQE23+9hb24iMApU+n1+FyM1ptvN94R9PpLOM7tx3luH3plIWDAFJ2IZez3MPcZLqV8hBB3TD49ulCzs4W/nf6Q/WVf4q2F0nA8kd5B4Tz1g0RiQn1RSrUtXDB6ehG17Hl8k4d1ejv1xmqc5w/gOLcPveIcAMaw/niMm4+53yiMPu69RYUQonuQhNRFztde4I9571HVUo2xIoGawjjSJ/Rjxtg+mE1GnHV1lH9j4ULED5/CHBDQae3Tmuqwn/gU57l9aKUnAYUxpDfW0XOw9BuN0V8WKAgh2pckpE6m6VpbhW6L7kPLydHEeMfyv/4lkd7hfihdp/7gfire/UunL1zQW+pxXvgS5/kD1JeeAF3DGBCBdfjDmAeMkXuGhBAdShJSJ7rUVMmf8v9KQV0hxpoYGs4PIm3MANLGx2FSOnWf76Eqcwt2WynW6JhOWbigN9XivHAIZ8FB10hI6Rj8wwgY8xCOyFSMIb27fBWfEOK7QRJSJzlUnsv6Extwagr7uWFEmgbw1ILBxIZ4ULd7F1VZmTgvX8YaE0vk4h/hO3JUh1W31hurcRYcwllwAM12GlAYAiKwpszC3HckxpDehIT597gS+UII9yYJqYPpSmfjua3sKPwUQ1MwLWeSmTliEGkjImj47FMKsrPQamvx7NefsCcW4JPcMdW49boKnBcOu0ZC5WcAMAZFu6bj+o3CGBQtIyEhRJeShNSBGh1N/Pfx9ZysPoNW0ZvA2lR+/HA/AvL2Ufji/0VvbMQ7cQjBTz+DV8Kgdk0IStfRKs6iXTyC8+IR9JpSAIwhsVhHPoq570hMQXJNSAjhPiQhdZCSBhv/7+ifqGqpwX4+iSSPgcwJuEDj6j9T2dqCT0qqa1uIfv3b7TWVvQln0XGcFw/jLDoKrY1gMGGKSsAj8T7MfVLkhlUhhNuShNQBDpYfYf2JDehOM80nRvNwgDdJuX+jobbGVY17Zhoe0THt8lp6bTnOwiM4C3PRSk+B0jB4+GLuPQxznxTMMUkYrN7t8lpCCNGRJCG1I03X+Oh8FjsKP8XcEkJL3mCWWEvxy/kcU2QU0Uv/D5594u7qNZS9CWfJCbTi4ziLj6PqLwFgDIrCmvwgpj4pmMIGyHbfQohuRxJSO2lwNPLO8Xc5WX0GY1UcQWfCeKLmMwyV5QTe/wC9HptzR+V+lK6hXyrAWZyHs/gYesV5UDpYPDFHJWJKfhBzbLJMxQkhuj1JSO3gQnUxqw68SXVLLVpBEhMK6xlpy8Tk70/E8y/gMyTptp5Pr7+Ms/i4axRUkg/2JsCAMTQOa8osTDFJmML7SxVtIUSPIp9od+lg+RHWn9yAQbNg+XIw80vzCamz4TdqNGFPfh+Tr+9Nn0M5WtFsJ1wLEoqPo2rLADD4BGPpOxJTTBLm6MEYPG/+XEII0V1JQroLDt3JX05swOoIIirHi2mXdmO1mAh7+n/jN3rsdZdxK6XQq4rRio+5RkK206A7wWTFFJWAefBkTDFDMQZGyr1BQojvDElId8FsMBFx4R6SDu8hvvE4XgmDiHjqaSzBIVcdqzfXoZXk4yw+hlach2qqAcAYFIMlaSrmmCRMEfEYzF2zrYQQQnQ1SUh3wWl3MPPQVjwdTYTOfYLAqQ+0rW5Tjla0slM4S/LRSvLQK4tcJ3n4YI4egjl2KKaYJNm6QQghviIJ6S6YrRZi0x8ietxIGj390S+db0tAWvlZ0DUwmjFFDMQ66jHM0YMx9uorS7KFEOIaOi0hFRQUkJGRQU1NDYGBgaxcuZK4uLgrjtE0jddee43du3djMBhYvHgxc+bMAWDNmjW8++67hIW5ljcPHz6cFStWANDc3MxPf/pT8vLyMJlMLF++nMmTJ3d4TAaDAd/ePtTve5emC8fB0QwYMPbqjXXog5iiB2OKGIjB7NHhbRFCiO6u0xLSihUrmD9/PrNnz2bjxo28/PLLrFu37opjNm3aRGFhIdnZ2dTU1JCens64ceOIiXFVNUhPT2f58uVXPfcf/vAHfH192b59OxcuXODJJ58kOzsbHx+fDo1JaU5a9q7H7O2HZcAYVwKKSsTo6dehryuEED1Rp8wdVVZWkp+fT1paGgBpaWnk5+dTVVV1xXGZmZnMmTMHo9FIcHAwU6dOJSsr66bPv3XrVr73ve8BEBcXR1JSEjk5Oe0fyLcYTGZ8v7+G3kvW4jnxB66dVCUZCSHEHemUEZLNZiM8PByTyQSAyWQiLCwMm81GcHDwFcdFRf2zAnVkZCRlZWVtj7ds2cJnn31GaGgoS5cuJTU1FYDS0lKio6Ove96tCAm5u3t8QkN7XiLqiTGBxNWd9MSYoGfG1R4xdZtFDfPmzeOZZ57BYrGwZ88elixZQmZmJkFB7bNKrbKyAV1Xd3RuaKhfj9vMrifGBBJXd9ITY4KeGde3YzIaDXf0T36nTNlFRkZSXl6OpmmAa/FCRUUFkZGRVx1XWlra9thmsxEREQFAaGgoFosFgAkTJhAZGcmZM66N5qKioigpKbnmeUIIIbqHTklIISEhJCYmsnnzZgA2b95MYmLiFdN1ANOnT2fDhg3ouk5VVRU7duzgwQcfBKC8vLztuBMnTlBSUkLfvn3bznv//fcBuHDhAseOHWPixImdEZoQQoh20mlTdq+88goZGRn8/ve/x9/fn5UrVwLw9NNPs2zZMoYOHcrs2bPJzc1l2rRpADz77LPExsYCsHr1avLy8jAajVgsFlatWkVoaCgATz31FBkZGTzwwAMYjUZeffVVfG+hhpwQQgj3YVBK3dmFkx5GriFdqSfGBBJXd9ITY4KeGVe3uoYkhBBC3Ey3WWXX0YzGu6uqfbfnu6OeGBNIXN1JT4wJemZc34zpTuOTKTshhBBuQabshBBCuAVJSEIIIdyCJCQhhBBuQRKSEEIItyAJSQghhFuQhCSEEMItSEISQgjhFiQhCSGEcAuSkIQQQrgFKR10FwoKCsjIyKCmpobAwEBWrlxJXFxcVzfrtk2ZMgWr1YqHhwcAL7zwAhMnTuTIkSO8/PLLtLa2Eh0dza9+9StCQkK6uLXXt3LlSrZt20ZJSQmbNm0iPj4euHE/uXsfXi+m6/UZ0C36rbq6mn//93+nsLAQq9VKnz59ePXVVwkODr5h+905thvFlJCQQHx8PEajawywatUqEhISANi5cyerVq1C0zSGDBnC66+/jpeXV1eGcpUlS5ZQXFyM0WjE29ub//iP/yAxMbH931tK3LGFCxeqDz/8UCml1IcffqgWLlzYxS26M5MnT1anTp264nuapqmpU6eqAwcOKKWUWrt2rcrIyOiK5t2yAwcOqNLS0qviuVE/uXsfXi+ma/WZUt2n36qrq9UXX3zR9viNN95QP/3pT2/YfneP7XoxKaVUfHy8amhouOqchoYGNX78eFVQUKCUUurFF19Ua9as6ZT23o66urq2r7dv367S09OVUu3/3pIpuztUWVlJfn4+aWlpAKSlpZGfn09VVVUXt6x9HD9+HA8PD0aOHAm4tpDPysrq4lbd2MiRI6/ahfhG/dQd+vBaMd1Id+m3wMBAxowZ0/Y4JSWF0tLSG7bf3WO7Xkw3kpOTQ1JSUtvIYd68eWzdurUjm3lH/Pz82r5uaGjAYDB0yHtLpuzukM1mIzw8HJPJBIDJZCIsLAybzXbVTrjdwQsvvIBSihEjRvDjH/8Ym81GVFRU28+Dg4PRdb1t+N1d3KiflFLdug+/3Wf+/v7dst90Xeevf/0rU6ZMuWH7u1Ns34zpawsXLkTTNCZNmsTSpUuxWq1XxRQVFYXNZuuKJt/USy+9xJ49e1BK8fbbb3fIe0tGSIL169fz0Ucf8fe//x2lFK+++mpXN0ncRE/qs1/84hd4e3uzYMGCrm5Ku/l2TLt27eIf//gH69ev5+zZs6xdu7aLW3j7fvnLX7Jr1y6ef/55Vq1a1SGvIQnpDkVGRlJeXo6maQBomkZFRcVtTa+4i6/bbLVamT9/Pl9++SWRkZFXTDdUVVVhNBrd7j/Rm7lRP3XnPrxWn339/e7UbytXruTixYv89re/xWg03rD93SW2b8cE/+wvX19f5syZc93+Ki0tdfu/v/T0dPbt20dERES7v7ckId2hkJAQEhMT2bx5MwCbN28mMTGxW0z1fFNTUxP19a6th5VSZGZmkpiYSFJSEi0tLRw8eBCA9957j+nTp3dlU+/Ijfqpu/bh9foM6Fb9tnr1ao4fP87atWuxWq3AjdvfHWK7Vky1tbW0tLQA4HQ62bZtW1t/TZw4kWPHjnHhwgXAFdOMGTO6pO3X09jYeMU04s6dOwkICOiQ95Zs0HcXzp07R0ZGBnV1dfj7+7Ny5Ur69evX1c26LUVFRSxduhRN09B1nf79+/Ozn/2MsLAwvvzyS1asWHHFEttevXp1dZOv67XXXiM7O5vLly8TFBREYGAgW7ZsuWE/uXsfXiumt95667p9BnSLfjtz5gxpaWnExcXh6ekJQExMDGvXrr1h+905tuvFtGjRIl5++WUMBgNOp5PU1FRefPFFfHx8ANixYwe/+tWv0HWdxMRE3njjDby9vbsylCtcvnyZJUuW0NzcjNFoJCAggOXLlzNkyJB2f29JQhJCCOEWZMpOCCGEW5CEJIQQwi1IQhJCCOEWJCEJIYRwC5KQhBBCuAVJSEL0EMXFxSQkJOB0Oru6KULcEUlIQggh3IIkJCGEEG5BEpIQHai8vJylS5cyduxYpkyZwrp16wBYs2YNy5Yt41//9V9JTU3lkUce4eTJk23nnTt3joULFzJy5EhmzZrFxx9/3PazlpYW3njjDSZPnsyIESN44okn2krTAGzatIn77ruPMWPG8Oabb7Z9/+jRozz66KMMHz6c8ePH8/rrr3fCb0CI23CX+zYJIa5D0zT1yCOPqDVr1qjW1lZVWFiopkyZonJyctTvfvc7NXjwYLV161Zlt9vV22+/rSZPnqzsdruy2+1q6tSp6s0331Stra1q7969KiUlRZ07d04ppdQrr7yiFixYoMrKypTT6VSHDh1Sra2tqqioSMXHx6uXXnpJNTc3qxMnTqghQ4aos2fPKqWUmjt3rvrggw+UUq6N4Q4fPtxlvxshrkVGSEJ0kGPHjlFVVcVzzz2H1WolNjaWuXPnkpmZCcCQIUOYPn06FouFH/7wh9jtdnJzc8nNzaWpqYnFixdjtVoZN24ckydPZsuWLei6zt///ndeeumltv1mhg8f3lbIE+C5557D09OTQYMGMWjQoLaRl9lsprCwkKqqKnx8fEhJSemS34sQ1yMb9AnRQUpKSqioqGjb4RRcZfhHjhxJVFQUERERbd83Go2Eh4dTUVEBQERERNvWBeDauK28vJzq6mpaW1uJ22iLKQAAAdRJREFUjY297ut+s9iol5cXTU1NgGs/m9/97nfMmDGDmJgYnnvuOSZPntxu8QpxtyQhCdFBIiMjiYmJITs7+6qfrVmzhrKysrbHuq5TXl7eVrG7rKwMXdfbkpLNZiMuLo6goCA8PDwoKipi0KBBt9WeuLg4Vq9eja7rZGdns2zZMvbt2+dWlaXFd5tM2QnRQZKTk/Hx8eE///M/aWlpQdM0Tp8+zdGjRwHIy8sjOzsbp9PJn/70J6xWK8OGDSM5ORlPT0/efvttHA4H+/btY+fOncycOROj0chjjz3G66+/3rYB2uHDh7Hb7Tdtz8aNG9s2tfP39we4YhQmRFeTv0YhOojJZOKtt97i5MmT3H///YwdO5af/exnNDQ0AHD//feTmZnJqFGj2LhxI2vWrMFisWC1WnnrrbfIyclh7Nix/PznP2fVqlX0798fgOXLlxMfH8/jjz/O6NGj+fWvf42u6zdtz+7du5k1axapqan88pe/5De/+U3bvj1CuAPZD0mILrBmzRouXrzIr3/9665uihBuQ0ZIQggh3IIkJCGEEG5BpuyEEEK4BRkhCSGEcAuSkIQQQrgFSUhCCCHcgiQkIYQQbkESkhBCCLcgCUkIIYRb+P9ELPpZCEkXiQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vdropout1_history = [vdropout_history[i][0] for i in range(30)]\n",
    "vdropout2_history = [vdropout_history[i][1] for i in range(30)]\n",
    "vdropout3_history = [vdropout_history[i][2] for i in range(30)]\n",
    "vdropout4_history = [vdropout_history[i][3] for i in range(30)]\n",
    "\n",
    "plt.plot(epochs, vdropout1_history,label='vdropout1')\n",
    "plt.plot(epochs, vdropout2_history,label='vdropout2')\n",
    "plt.plot(epochs, vdropout3_history,label='vdropout3')\n",
    "plt.plot(epochs, vdropout4_history,label='vdropout4')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('Alpha value')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}