pipeline_text_val.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__Python VERSION: 3.7.5 (default, Oct 25 2019, 15:51:11) \n",
      "[GCC 7.3.0]\n",
      "__pyTorch VERSION: 1.2.0\n",
      "__CUDA VERSION\n",
      "/bin/sh: 1: nvcc: not found\n",
      "__CUDNN VERSION: 7301\n",
      "__Number CUDA Devices: 2\n",
      "__Devices\n",
      "Active CUDA Device: GPU 0\n",
      "Available devices  2\n",
      "Current cuda device  0\n",
      "3.7.5 (default, Oct 25 2019, 15:51:11) \n",
      "[GCC 7.3.0]\n",
      "96.1\n",
      "svmem(total=67477729280, available=22301544448, percent=66.9, used=40141840384, free=6282264576, active=47955124224, inactive=6594109440, buffers=120909824, cached=20932714496, shared=4288757760, slab=5068201984)\n",
      "memory GB: 0.250274658203125\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import sys\n",
    "import torch\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_auc_score, log_loss, roc_auc_score, roc_curve, auc\n",
    "from sklearn.model_selection import StratifiedKFold, ShuffleSplit, cross_val_score, train_test_split\n",
    "\n",
    "print('__Python VERSION:', sys.version)\n",
    "print('__pyTorch VERSION:', torch.__version__)\n",
    "print('__CUDA VERSION')\n",
    "from subprocess import call\n",
    "# call([\"nvcc\", \"--version\"]) does not work\n",
    "! nvcc --version\n",
    "print('__CUDNN VERSION:', torch.backends.cudnn.version())\n",
    "print('__Number CUDA Devices:', torch.cuda.device_count())\n",
    "print('__Devices')\n",
    "# call([\"nvidia-smi\", \"--format=csv\", \"--query-gpu=index,name,driver_version,memory.total,memory.used,memory.free\"])\n",
    "print('Active CUDA Device: GPU', torch.cuda.current_device())\n",
    "\n",
    "print ('Available devices ', torch.cuda.device_count())\n",
    "print ('Current cuda device ', torch.cuda.current_device())\n",
    "\n",
    "import numpy\n",
    "import numpy as np\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "Tensor = FloatTensor\n",
    "\n",
    "import pandas\n",
    "import pandas as pd\n",
    "\n",
    "import logging\n",
    "handler=logging.basicConfig(level=logging.INFO)\n",
    "lgr = logging.getLogger(__name__)\n",
    "%matplotlib inline\n",
    "\n",
    "# !pip install psutil\n",
    "import psutil\n",
    "import os\n",
    "def cpuStats():\n",
    "        print(sys.version)\n",
    "        print(psutil.cpu_percent())\n",
    "        print(psutil.virtual_memory())  # physical memory usage\n",
    "        pid = os.getpid()\n",
    "        py = psutil.Process(pid)\n",
    "        memoryUse = py.memory_info()[0] / 2. ** 30  # memory use in GB...I think\n",
    "        print('memory GB:', memoryUse)\n",
    "\n",
    "cpuStats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "# use_cuda = False\n",
    "\n",
    "FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "Tensor = FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/projects/audio-pipelines/data\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyps = pd.read_csv(\"sets/full_set/hyps\", names=['audio_id'])\n",
    "# hyps[['audio_id','hyps']] = hyps[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "# targets = pd.read_csv(\"sets/full_set/targets\", names=['audio_id'])\n",
    "# targets[['audio_id','target']] = targets[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "# text = pd.read_csv(\"sets/full_set/text\", names=['audio_id'])\n",
    "# text[['audio_id','text']] = text[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "# speakers = pd.read_csv(\"sets/full_set/utt2spk\", names=['audio_id'])\n",
    "# speakers[['audio_id','speaker']] = speakers[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "# images = pd.read_csv(\"sets/full_set/wav.scp\", names=['audio_id'])\n",
    "# images[['audio_id','audio_path']] = images[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "# dataset = pd.merge(hyps, text, how=\"left\")\n",
    "# dataset = pd.merge(dataset, speakers, how=\"left\")\n",
    "# dataset = pd.merge(dataset, images, how=\"left\")\n",
    "# dataset = pd.merge(dataset, targets, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset['image_path'] = dataset['audio_path'].apply(lambda x : x[:-4] + \".png\")\n",
    "# def remove_absolute(string, prefix='/home/raznem/projects/audio-pipelines/data/'):\n",
    "#     if string.startswith(prefix):\n",
    "#         string = string[len(prefix):]\n",
    "#     return string\n",
    "    \n",
    "# dataset['image_path'] = dataset['image_path'].apply(remove_absolute)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.to_csv('data/dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## from allennlp.commands.elmo import ElmoEmbedder\n",
    "from torchvision import transforms\n",
    "import numpy as np\n",
    "\n",
    "try:\n",
    "    from PIL import Image\n",
    "except ImportError:\n",
    "    import Image\n",
    "    \n",
    "toImg = transforms.ToPILImage()\n",
    "toTensor = transforms.ToTensor()\n",
    "    \n",
    "class WavImagesLoader(Dataset):\n",
    "    def __init__(self, csv_path, transform=None):\n",
    "        self.dataset = pd.read_csv(csv_path)\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        image_path = self.dataset['image_path'][key]\n",
    "        image = Image.open(image_path)\n",
    "        image = image.convert('RGB')\n",
    "        image = toTensor(image)\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        \n",
    "        return image, target\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "    \n",
    "    @staticmethod        \n",
    "    def imshow(img):\n",
    "        img = img / 2 + 0.5     # unnormalize\n",
    "        npimg = img.numpy()\n",
    "        plt.imshow(np.transpose(npimg, (1, 2, 0)))    \n",
    "\n",
    "#     @staticmethod    \n",
    "#     def flaotTensorToImage(img, mean=0, std=1):\n",
    "#         \"\"\"convert a tensor to an image\"\"\"\n",
    "#         img = np.transpose(img.numpy(), (1, 2, 0))\n",
    "#         img = (img*std+ mean)*255\n",
    "#         img = img.astype(np.uint8)    \n",
    "#         return img    \n",
    "    \n",
    "    \n",
    "class ElmoWavImagesLoader(WavImagesLoader):\n",
    "    def __init__(self, csv_path, text_vecs_path, transform=None):\n",
    "        super().__init__(csv_path, transform=None)\n",
    "        self.text_vecs_path = text_vecs_path\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        image_path = self.dataset['image_path'][key]\n",
    "        image = Image.open(image_path)\n",
    "        image = image.convert('RGB')\n",
    "        image = toTensor(image)\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        \n",
    "        text_vec = np.zeros((322, 1024))\n",
    "        tmp = np.load(self.text_vecs_path + self.dataset[\"audio_id\"][key] + \".npy\")\n",
    "        tmp = np.sum(tmp, axis=0)\n",
    "        text_len = tmp.shape[0]\n",
    "        text_vec[:tmp.shape[0],:tmp.shape[1]] = tmp\n",
    "        \n",
    "        return image, text_vec, text_len, target\n",
    "    \n",
    "    \n",
    "class ElmoWavVecLoader(ElmoWavImagesLoader):\n",
    "    def __init__(self, csv_path, text_vecs_path, transform=None):\n",
    "        super().__init__(csv_path, text_vecs_path, transform=None)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        image_vec = self.dataset['image_vec'][key]\n",
    "        text = self.dataset['hyps'][key]\n",
    "        text = text.split(' ')\n",
    "        \n",
    "        text_vec = np.zeros(322, 1024)\n",
    "        tmp = np.load(text_vecs_path + self.dataset[\"audio_id\"][key] + \".npy\")\n",
    "        text_len = tmp.shape[0]\n",
    "        text_vec[:tmp.shape[0],:tmp.shape[1]] = tmp\n",
    "\n",
    "        return image_vec, text_vec, text_len, target\n",
    "    \n",
    "    \n",
    "class ElmoLoader(WavImagesLoader):\n",
    "    def __init__(self, csv_path, text_vec_list, transform=None):\n",
    "        super().__init__(csv_path, transform=None)\n",
    "        self.text_vec_list = text_vec_list\n",
    "            \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        text_vec = np.zeros((322, 1024))\n",
    "        tmp = self.text_vec_list[key]\n",
    "        text_len = tmp.shape[0]\n",
    "        text_vec[:tmp.shape[0],:tmp.shape[1]] = tmp\n",
    "\n",
    "        return text_vec, text_len, target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "with open(\"text_vec_list.p\", \"rb\") as f:\n",
    "    text_vec_list = pkl.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv net to get vector from Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout = torch.nn.Dropout(p=0.30)\n",
    "class ConvRes(nn.Module):\n",
    "    def __init__(self, insize, outsize):\n",
    "        super(ConvRes, self).__init__()\n",
    "        drate = .3\n",
    "        self.math = nn.Sequential(\n",
    "            nn.BatchNorm2d(insize),\n",
    "            # nn.Dropout(drate),\n",
    "            torch.nn.Conv2d(insize, outsize, kernel_size=2, padding=2),\n",
    "            nn.PReLU(),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.math(x)\n",
    "\n",
    "\n",
    "class ConvCNN(nn.Module):\n",
    "    def __init__(self, insize, outsize, kernel_size=7, padding=2, pool=2, avg=True):\n",
    "        super(ConvCNN, self).__init__()\n",
    "        self.avg = avg\n",
    "        self.math = torch.nn.Sequential(\n",
    "            torch.nn.Conv2d(insize, outsize, kernel_size=kernel_size, padding=padding),\n",
    "            torch.nn.BatchNorm2d(outsize),\n",
    "            torch.nn.LeakyReLU(),\n",
    "            torch.nn.MaxPool2d(pool, pool),\n",
    "        )\n",
    "        self.avgpool = torch.nn.AvgPool2d(pool, pool)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.math(x)\n",
    "        if self.avg is True:\n",
    "            x = self.avgpool(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, output_dim):\n",
    "        super(Net, self).__init__()\n",
    "\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d(1)\n",
    "\n",
    "        self.cnn1 = ConvCNN(3, 32, kernel_size=7, pool=4, avg=False)\n",
    "        self.cnn2 = ConvCNN(32, 32, kernel_size=5, pool=2, avg=True)\n",
    "        self.cnn3 = ConvCNN(32, 32, kernel_size=5, pool=2, avg=True)\n",
    "\n",
    "        self.res1 = ConvRes(32, 64)\n",
    "\n",
    "        self.features = nn.Sequential(\n",
    "            self.cnn1, dropout,\n",
    "            self.cnn2,\n",
    "            self.cnn3,\n",
    "            self.res1,\n",
    "        )\n",
    "\n",
    "        self.classifier = torch.nn.Sequential(\n",
    "            nn.Linear(5184, output_dim),\n",
    "        )\n",
    "        self.sig = nn.Sigmoid()\n",
    "  \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "#         x = self.sig(x)\n",
    "        return x\n",
    "\n",
    "    \n",
    "# if use_cuda:\n",
    "#     lgr.info (\"Using the GPU\")\n",
    "#     model = Net(output_dim).cuda() # On GPU\n",
    "# else:\n",
    "#     lgr.info (\"Using the CPU\")\n",
    "#     model = Net(output_dim) # On CPU\n",
    "\n",
    "# lgr.info('Model {}'.format(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get maximum length of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max_text_length = dataset_dropna['hyps'].apply(lambda x: len(x.split(' '))).max()\n",
    "# max_text_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_data = ElmoLoader('data/dataset_dropna.csv', text_vec_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN to encode Elmo sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextModel(Net):\n",
    "    def __init__(self, rnn_dim, layers_rnn=1, text_emb_dim=1024):\n",
    "        super(Net, self).__init__()\n",
    "        self.layers_rnn = layers_rnn\n",
    "        self.text_emb_dim = text_emb_dim\n",
    "        self.rnn_dim = rnn_dim\n",
    "        self.text_lstm = nn.LSTM(text_emb_dim, self.rnn_dim, num_layers=self.layers_rnn, batch_first=True)\n",
    "        self.fc_last = nn.Linear(rnn_dim, 1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, text, text_len):        \n",
    "#         cnn_out = self.cnn_model(img)\n",
    "        seq_lengths = text_len\n",
    "        pack = torch.nn.utils.rnn.pack_padded_sequence(text, seq_lengths, batch_first=True)\n",
    "        _, (rnn_out, _) = self.text_lstm(pack)\n",
    "#         print(rnn_out.shape)\n",
    "        rnn_out = rnn_out.reshape(-1, self.rnn_dim)\n",
    "#         x = torch.cat([cnn_out, rnn_out], dim=1)\n",
    "#         print(rnn_out.shape)\n",
    "        y_pred = self.fc_last(rnn_out)\n",
    "        y_pred = torch.squeeze(y_pred)\n",
    "        y_pred = self.sig(y_pred)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextModelDout(Net):\n",
    "    def __init__(self, rnn_dim, layers_rnn=1, text_emb_dim=1024, dout_rate=0.3):\n",
    "        super(Net, self).__init__()\n",
    "        self.layers_rnn = layers_rnn\n",
    "        self.text_emb_dim = text_emb_dim\n",
    "        self.rnn_dim = rnn_dim\n",
    "        self.text_lstm = nn.LSTM(text_emb_dim, self.rnn_dim, num_layers=self.layers_rnn, dropout=dout_rate, batch_first=True)\n",
    "        self.fc_last = nn.Linear(rnn_dim, 1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        \n",
    "    def forward(self, text, text_len):        \n",
    "#         cnn_out = self.cnn_model(img)\n",
    "        seq_lengths = text_len\n",
    "        pack = torch.nn.utils.rnn.pack_padded_sequence(text, seq_lengths, batch_first=True)\n",
    "        _, (rnn_out, _) = self.text_lstm(pack)\n",
    "#         print(rnn_out.shape)\n",
    "        rnn_out = torch.mean(rnn_out, 0)\n",
    "        rnn_out = rnn_out.reshape(-1, self.rnn_dim)\n",
    "#         x = torch.cat([cnn_out, rnn_out], dim=1)\n",
    "#         print(rnn_out.shape)\n",
    "        y_pred = self.fc_last(rnn_out)\n",
    "        y_pred = torch.squeeze(y_pred)\n",
    "        y_pred = self.sig(y_pred)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextModelDout_v2(Net):\n",
    "    def __init__(self, rnn_dim, layers_rnn=1, text_emb_dim=1024, dout_rate=0.3):\n",
    "        super(Net, self).__init__()\n",
    "        self.layers_rnn = layers_rnn\n",
    "        self.text_emb_dim = text_emb_dim\n",
    "        self.rnn_dim = rnn_dim\n",
    "        self.text_lstm = nn.LSTM(text_emb_dim, self.rnn_dim, num_layers=self.layers_rnn, dropout=dout_rate, batch_first=True)\n",
    "        self.fc_last = nn.Linear(rnn_dim, 1)\n",
    "        self.sig = nn.Sigmoid()\n",
    "        self.dout = nn.modules.Dropout(dout_rate)\n",
    "        \n",
    "    def forward(self, text, text_len):        \n",
    "#         cnn_out = self.cnn_model(img)\n",
    "        seq_lengths = text_len\n",
    "        text = self.dout(text)\n",
    "        pack = torch.nn.utils.rnn.pack_padded_sequence(text, seq_lengths, batch_first=True)\n",
    "        _, (rnn_out, _) = self.text_lstm(pack)\n",
    "#         print(rnn_out.shape)\n",
    "        rnn_out = torch.mean(rnn_out, 0)\n",
    "        rnn_out = rnn_out.reshape(-1, self.rnn_dim)\n",
    "#         x = torch.cat([cnn_out, rnn_out], dim=1)\n",
    "#         print(rnn_out.shape)\n",
    "        y_pred = self.fc_last(rnn_out)\n",
    "        y_pred = torch.squeeze(y_pred)\n",
    "        y_pred = self.sig(y_pred)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset_val = pd.read_csv('data/dataset_val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_data = WavImagesLoader(csv_path)\n",
    "# dataloader = DataLoader(image_data, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rval_acc(result, target):\n",
    "    result = (result > 0.7).type(torch.int)\n",
    "    target = target.type(torch.int)\n",
    "    acc = target == result\n",
    "    acc = acc.type(torch.float32).sum()/len(acc)\n",
    "    return acc\n",
    "\n",
    "def validate(model, loader_val, dtype):\n",
    "    prediction_list = []\n",
    "    target_list = []\n",
    "    with torch.no_grad():\n",
    "        for i, (text, text_len, y) in enumerate(loader_val):\n",
    "            text, text_len = text.to(dtype=dtype), text_len.to(dtype=dtype)\n",
    "            text_len, perm_idx = text_len.sort(0, descending=True)\n",
    "            text = text[perm_idx]\n",
    "            y = y[perm_idx]\n",
    "            pred = model(text, text_len).to(\"cpu\")\n",
    "            prediction_list.append(pred)\n",
    "            target_list.append(y)\n",
    "    result = torch.squeeze(torch.cat(prediction_list))\n",
    "    target = torch.squeeze(torch.cat(target_list))\n",
    "    acc = rval_acc(result, target)\n",
    "    print(f'Acc {acc}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "def textspec_trainer(loader_train, loader_val, model, optimizer, exp_name='', val_every=None, \n",
    "                     save_every=None, print_every=None, epochs=1, use_gpu=True, \n",
    "                     dtype=torch.float32):\n",
    "    if use_gpu and torch.cuda.is_available():\n",
    "        device = torch.device('cuda:0')\n",
    "    else:\n",
    "        device = torch.device('cpu')\n",
    "        \n",
    "    model.train()\n",
    "    model.to(device=device)\n",
    "    model.eval()\n",
    "    model.to(device='cpu')\n",
    "    validate(model, loader_val, dtype)\n",
    "    model.to(device=device)\n",
    "    model.train()\n",
    "    for e in range(epochs):\n",
    "        print('Epoch %d' %e)\n",
    "        acc = 0\n",
    "        files_no = 0\n",
    "        for t, (text, text_len, y) in enumerate(loader_train):\n",
    "#             img = img.to(device=device, dtype=dtype)\n",
    "            text = text.to(device=device, dtype=dtype)\n",
    "            y = y.to(device=device, dtype=dtype)\n",
    "            text_len, perm_idx = text_len.sort(0, descending=True)\n",
    "            text = text[perm_idx]\n",
    "#             img = img[perm_idx]\n",
    "            y = y[perm_idx]\n",
    "            \n",
    "            y_pred = model(text, text_len)\n",
    "            \n",
    "            loss = loss_fn(y_pred, y)\n",
    "            \n",
    "            acc += (torch.round(y_pred.cpu()) == y.cpu()).sum().type(torch.float32)\n",
    "            files_no += len(text)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "            if print_every is not None and t % print_every == 0:\n",
    "                print('Iteration %d, loss = %.4f; epoch %d' % (t, loss.item(), e))\n",
    "                print(f'Acc {acc / files_no}')\n",
    "                \n",
    "        if val_every is not None and e % val_every == 0:\n",
    "            model.eval()\n",
    "            model.to(device='cpu')\n",
    "            validate(model, loader_val, dtype)\n",
    "            model.to(device=device)\n",
    "            model.train()\n",
    "        if save_every is not None and e % save_every == 0:\n",
    "            torch.save(model.state_dict(), f'models/{exp_name}_e%d_cnn_rnn.pt' % e)\n",
    "            gc.collect()\n",
    "    torch.save(model.state_dict(), f'models/{exp_name}_cnn_rnn.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle as pkl\n",
    "with open(\"text_vec_list.p\", \"rb\") as f:\n",
    "    text_vec_list = pkl.load(f)\n",
    "\n",
    "with open(\"text_vec_list_val.p\", \"rb\") as f:\n",
    "    text_vec_list_val = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc 0.5184999704360962\n",
      "Epoch 0\n",
      "Iteration 0, loss = 0.7068; epoch 0\n",
      "Acc 0.4765625\n",
      "Iteration 100, loss = 0.6463; epoch 0\n",
      "Acc 0.6116955280303955\n",
      "Iteration 200, loss = 0.5916; epoch 0\n",
      "Acc 0.6344449520111084\n",
      "Acc 0.5220000147819519\n",
      "Epoch 1\n",
      "Iteration 0, loss = 0.6024; epoch 1\n",
      "Acc 0.703125\n",
      "Iteration 100, loss = 0.6421; epoch 1\n",
      "Acc 0.6911355257034302\n",
      "Iteration 200, loss = 0.6163; epoch 1\n",
      "Acc 0.6937189102172852\n",
      "Epoch 2\n",
      "Iteration 0, loss = 0.5390; epoch 2\n",
      "Acc 0.7734375\n",
      "Iteration 100, loss = 0.5479; epoch 2\n",
      "Acc 0.7267171740531921\n",
      "Iteration 200, loss = 0.5345; epoch 2\n",
      "Acc 0.7248134613037109\n",
      "Epoch 3\n",
      "Iteration 0, loss = 0.4569; epoch 3\n",
      "Acc 0.7734375\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-71a9ff42a045>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m textspec_trainer(loader_train, loader_val, model, optimizer, exp_name='text_val', \n\u001b[1;32m     18\u001b[0m                  \u001b[0mval_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprint_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNUM_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m       use_gpu=USE_GPU, dtype=TE_DTYPE)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-21-0a4e4e969eb4>\u001b[0m in \u001b[0;36mtextspec_trainer\u001b[0;34m(loader_train, loader_val, model, optimizer, exp_name, val_every, save_every, print_every, epochs, use_gpu, dtype)\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m \u001b[0;31m#             img = img.to(device=device, dtype=dtype)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m             \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0mtext_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperm_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtext_len\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdescending\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 1e-4\n",
    "USE_GPU = True\n",
    "TE_DTYPE = torch.float32\n",
    "CNN_DIM = 128\n",
    "RNN_DIM = 128\n",
    "NUM_EPOCHS = 100\n",
    "\n",
    "loss_fn = torch.nn.BCELoss()\n",
    "model = TextModel(RNN_DIM)\n",
    "image_data = ElmoLoader('data/dataset_dropna.csv', text_vec_list)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
    "loader_train = DataLoader(image_data, batch_size=BATCH_SIZE, shuffle=True, num_workers=6)\n",
    "val_data = ElmoLoader('data/dataset_val.csv', text_vec_list_val)\n",
    "loader_val = DataLoader(val_data, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
    "\n",
    "textspec_trainer(loader_train, loader_val, model, optimizer, exp_name='text_val', \n",
    "                 val_every=5, save_every=5, print_every=100, epochs=NUM_EPOCHS,\n",
    "      use_gpu=USE_GPU, dtype=TE_DTYPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc 0.5184999704360962\n",
      "Epoch 0\n",
      "Iteration 0, loss = 0.6957; epoch 0\n",
      "Acc 0.4140625\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "DataLoader worker (pid 55404) is killed by signal: Killed. ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-408eaaa437a3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m textspec_trainer(loader_train, loader_val, model, optimizer, exp_name='text_dout_val_simpler', \n\u001b[1;32m     18\u001b[0m                  \u001b[0mval_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msave_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprint_every\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNUM_EPOCHS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m       use_gpu=USE_GPU, dtype=TE_DTYPE)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-21-0a4e4e969eb4>\u001b[0m in \u001b[0;36mtextspec_trainer\u001b[0;34m(loader_train, loader_val, model, optimizer, exp_name, val_every, save_every, print_every, epochs, use_gpu, dtype)\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mperm_idx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m             \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-16-b7ea0ee27c85>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, text, text_len)\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;31m#         x = torch.cat([cnn_out, rnn_out], dim=1)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m#         print(rnn_out.shape)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfc_last\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrnn_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py\u001b[0m in \u001b[0;36mlinear\u001b[0;34m(input, weight, bias)\u001b[0m\n\u001b[1;32m   1367\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mbias\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1368\u001b[0m         \u001b[0;31m# fused op is marginally faster\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1369\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddmm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1370\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1371\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/signal_handling.py\u001b[0m in \u001b[0;36mhandler\u001b[0;34m(signum, frame)\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;31m# This following call uses `waitid` with WNOHANG from C side. Therefore,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m         \u001b[0;31m# Python can still get and update the process status successfully.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m         \u001b[0m_error_if_any_worker_fails\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     67\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mprevious_handler\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m             \u001b[0mprevious_handler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msignum\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mframe\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: DataLoader worker (pid 55404) is killed by signal: Killed. "
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 1e-4\n",
    "USE_GPU = True\n",
    "TE_DTYPE = torch.float32\n",
    "CNN_DIM = 32\n",
    "RNN_DIM = 32\n",
    "NUM_EPOCHS = 100\n",
    "\n",
    "loss_fn = torch.nn.BCELoss()\n",
    "model = TextModelDout(RNN_DIM, layers_rnn=2, dout_rate=0.5)\n",
    "image_data = ElmoLoader('data/dataset_dropna.csv', text_vec_list)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)\n",
    "loader_train = DataLoader(image_data, batch_size=BATCH_SIZE, shuffle=True, num_workers=6)\n",
    "val_data = ElmoLoader('data/dataset_val.csv', text_vec_list_val)\n",
    "loader_val = DataLoader(val_data, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n",
    "\n",
    "textspec_trainer(loader_train, loader_val, model, optimizer, exp_name='text_dout_val_simpler', \n",
    "                 val_every=5, save_every=10, print_every=100, epochs=NUM_EPOCHS,\n",
    "      use_gpu=USE_GPU, dtype=TE_DTYPE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}