NU_Rad_Model_3D.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NU_Rad_Model_3D.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyN73L3jmjEKMFwIFtCwQ2S7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/eyang9001/NU_Radiology/blob/master/NU_Rad_Model_3D.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cFkQyKdNgYo",
        "colab_type": "code",
        "outputId": "4d492ce5-d12a-4dc6-dff1-160bb3ce81ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Ynj65yfNkOS",
        "colab_type": "code",
        "outputId": "9a34cda4-3207-4b8f-d1c3-6e65b88e09ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install --upgrade wandb\n",
        "!wandb login 3aa7250d739679a5cc8a09410800a87bc578ba26"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wandb\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1d/17/7f632c1c700758822f9e41aa7b025ea15f017f2a43611efa7b64341303ea/wandb-0.8.34-py2.py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 2.7MB/s \n",
            "\u001b[?25hCollecting GitPython>=1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/19/1a/0df85d2bddbca33665d2148173d3281b290ac054b5f50163ea735740ac7b/GitPython-3.1.1-py3-none-any.whl (450kB)\n",
            "\u001b[K     |████████████████████████████████| 460kB 16.6MB/s \n",
            "\u001b[?25hCollecting shortuuid>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: Click>=7.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (7.1.1)\n",
            "Collecting subprocess32>=3.5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 9.1MB/s \n",
            "\u001b[?25hCollecting sentry-sdk>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/20/7e/19545324e83db4522b885808cd913c3b93ecc0c88b03e037b78c6a417fa8/sentry_sdk-0.14.3-py2.py3-none-any.whl (103kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 18.7MB/s \n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (1.12.0)\n",
            "Collecting watchdog>=0.8.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/73/c3/ed6d992006837e011baca89476a4bbffb0a91602432f73bd4473816c76e2/watchdog-0.10.2.tar.gz (95kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: PyYAML>=3.10 in /usr/local/lib/python3.6/dist-packages (from wandb) (3.13)\n",
            "Collecting configparser>=3.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/4b/6b/01baa293090240cf0562cc5eccb69c6f5006282127f2b846fad011305c79/configparser-5.0.0-py3-none-any.whl\n",
            "Collecting gql==0.2.0\n",
            "  Downloading https://files.pythonhosted.org/packages/c4/6f/cf9a3056045518f06184e804bae89390eb706168349daa9dff8ac609962a/gql-0.2.0.tar.gz\n",
            "Requirement already satisfied, skipping upgrade: requests>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: psutil>=5.0.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (5.4.8)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: nvidia-ml-py3>=7.352.0 in /usr/local/lib/python3.6/dist-packages (from wandb) (7.352.0)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/52/ca35448b56c53a079d3ffe18b1978c6e424f6d4df02404877094c89f5bfb/gitdb-4.0.4-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 8.0MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: urllib3>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from sentry-sdk>=0.4.0->wandb) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi in /usr/local/lib/python3.6/dist-packages (from sentry-sdk>=0.4.0->wandb) (2020.4.5.1)\n",
            "Collecting pathtools>=0.1.1\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Collecting graphql-core<2,>=0.5.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b0/89/00ad5e07524d8c523b14d70c685e0299a8b0de6d0727e368c41b89b7ed0b/graphql-core-1.1.tar.gz (70kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 7.0MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: promise<3,>=2.0 in /usr/local/lib/python3.6/dist-packages (from gql==0.2.0->wandb) (2.3)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0.0->wandb) (2.8)\n",
            "Collecting smmap<4,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/27/b1/e379cfb7c07bbf8faee29c4a1a2469dbea525f047c2b454c4afdefa20a30/smmap-3.0.2-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: subprocess32, watchdog, gql, pathtools, graphql-core\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp36-none-any.whl size=6489 sha256=8fa13c9cb806b85f2e6bef9aa29dee6c1f30c5507f3910436e3353f68e76c2f6\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for watchdog (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for watchdog: filename=watchdog-0.10.2-cp36-none-any.whl size=73605 sha256=dd29214106030b180ca43ff5dd88a69bda9fc3d7e7af8a36bcc50065e19e77b6\n",
            "  Stored in directory: /root/.cache/pip/wheels/bc/ed/6c/028dea90d31b359cd2a7c8b0da4db80e41d24a59614154072e\n",
            "  Building wheel for gql (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gql: filename=gql-0.2.0-cp36-none-any.whl size=7630 sha256=e89dfebca0638662e6092efb1e13240a9d8a69b9e7ab90a83e02d8ef2616667c\n",
            "  Stored in directory: /root/.cache/pip/wheels/ce/0e/7b/58a8a5268655b3ad74feef5aa97946f0addafb3cbb6bd2da23\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp36-none-any.whl size=8784 sha256=7b84a65ae7d804f900c56e81c1df4e220992cf5861d9b140b54a5b792fab5fd7\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "  Building wheel for graphql-core (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for graphql-core: filename=graphql_core-1.1-cp36-none-any.whl size=104650 sha256=9d8eb7ab5d72f843a503cdeb0fa8658b16efc8ae5669bfcbe3049461195d5fa4\n",
            "  Stored in directory: /root/.cache/pip/wheels/45/99/d7/c424029bb0fe910c63b68dbf2aa20d3283d023042521bcd7d5\n",
            "Successfully built subprocess32 watchdog gql pathtools graphql-core\n",
            "Installing collected packages: smmap, gitdb, GitPython, shortuuid, subprocess32, sentry-sdk, docker-pycreds, pathtools, watchdog, configparser, graphql-core, gql, wandb\n",
            "Successfully installed GitPython-3.1.1 configparser-5.0.0 docker-pycreds-0.4.0 gitdb-4.0.4 gql-0.2.0 graphql-core-1.1 pathtools-0.1.2 sentry-sdk-0.14.3 shortuuid-1.0.1 smmap-3.0.2 subprocess32-3.5.4 wandb-0.8.34 watchdog-0.10.2\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
            "\u001b[32mSuccessfully logged in to Weights & Biases!\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l1uRl0tiNpx5",
        "colab_type": "code",
        "outputId": "2a820af9-b177-4a53-afb3-7a41f0799d4b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "from sys import platform as sys_pf\n",
        "# if sys_pf == 'darwin':\n",
        "#     import matplotlib\n",
        "#     matplotlib.use(\"TkAgg\")\n",
        "! pip install simpleitk\n",
        "%matplotlib inline  \n",
        "import os\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "import matplotlib\n",
        "import SimpleITK as sitk\n",
        "from scipy import ndimage\n",
        "import random\n",
        "import math\n",
        "import keras\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Concatenate, Add, Average, Input, Dense, Flatten, BatchNormalization, Activation, LeakyReLU, Reshape\n",
        "from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten, Conv2D\n",
        "from tensorflow.keras import backend as K\n",
        "import tensorflow.keras.callbacks as callbacks\n",
        "import tensorflow.keras.optimizers as optimizers\n",
        "from sklearn import preprocessing as prepro\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from keras.models import *\n",
        "from keras.layers import *\n",
        "from keras.optimizers import *\n",
        "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
        "from keras import backend as keras"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting simpleitk\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f8/d8/53338c34f71020725ffb3557846c80af96c29c03bc883551a2565aa68a7c/SimpleITK-1.2.4-cp36-cp36m-manylinux1_x86_64.whl (42.5MB)\n",
            "\u001b[K     |████████████████████████████████| 42.5MB 79kB/s \n",
            "\u001b[?25hInstalling collected packages: simpleitk\n",
            "Successfully installed simpleitk-1.2.4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DMs7leBNsYp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "image_shape = (256,256,176)\n",
        "\n",
        "# Read in transformed images\n",
        "def read_file(filename):\n",
        "    data = np.loadtxt(filename, delimiter=\",\")\n",
        "    data = np.reshape(data,image_shape)\n",
        "    return data\n",
        "\n",
        "# Read in non-transformed images\n",
        "def read_original(filename):\n",
        "    reader = sitk.ImageFileReader()\n",
        "    reader.SetImageIO(\"NiftiImageIO\")\n",
        "    reader.SetFileName(filename)\n",
        "    image = reader.Execute()\n",
        "\n",
        "    # img1 = sitk.ReadImage(folder + item)  # alternative way to pull in image\n",
        "\n",
        "    # convert image into np array & perform fft\n",
        "    img = sitk.GetArrayFromImage(image)\n",
        "    # Transpose the image so the first axis is Anterior-Posterior\n",
        "    img = np.transpose(img, (2,1,0))\n",
        "    return img \n",
        "\n",
        "def visualize(orig, new):\n",
        "    plt.figure(figsize= (20,20))\n",
        "    plt.subplot(121), plt.imshow(orig, cmap='gray')\n",
        "    plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
        "    plt.subplot(122), plt.imshow(new, cmap='gray')\n",
        "    plt.title('New'), plt.xticks([]), plt.yticks([])\n",
        "    \n",
        "def visualize3(orig, blur, predict):\n",
        "    plt.figure(figsize= (30,30))\n",
        "    plt.subplot(131), plt.imshow(orig, cmap='gray')\n",
        "    plt.title('Original'), plt.xticks([]), plt.yticks([])\n",
        "    plt.subplot(132), plt.imshow(blur, cmap='gray')\n",
        "    plt.title('Blurred'), plt.xticks([]), plt.yticks([])\n",
        "    plt.subplot(133), plt.imshow(predict, cmap='gray')\n",
        "    plt.title('Model Output'), plt.xticks([]), plt.yticks([])\n",
        "\n",
        "def visualize_model(model, ind, X, y, save=False, fn=''):\n",
        "    x_input = X[ind]\n",
        "    out = model.predict(np.array(x_input[np.newaxis, ...]))\n",
        "    visualize3(y[ind][...,0], x_input[...,0], out[0,...,0])\n",
        "    if save:\n",
        "        plt.savefig(fn)\n",
        "def generate_data(t_filenames, o_filenames, min_slc = 70, max_slc = 220):\n",
        "    # Takes in the files of the transformed images and the originals to create the pairs\n",
        "    # Also sets the min and max slice numbers to take in\n",
        "    x_data = []\n",
        "    y_data = []\n",
        "    for i in range(len(t_filenames)):\n",
        "        print('Generating data for: ' + t_filenames[i])\n",
        "        x_mri = read_file(t_filenames[i])\n",
        "        y_mri = read_original(o_filenames[i])\n",
        "        for ii in np.arange(min_slc, max_slc+1):\n",
        "            x_data.append((x_mri[ii] - np.amin(x_mri[ii]))/(np.amax(x_mri[ii]) - np.amin(x_mri[ii]))) # normalizes the intensity to between 0 and 1\n",
        "            y_data.append((y_mri[ii] - np.amin(y_mri[ii]))/(np.amax(y_mri[ii]) - np.amin(y_mri[ii])))\n",
        "    # Shape of x & y will be [# slices, 256, 176, 1]\n",
        "    x_data = np.array(x_data)[..., np.newaxis]\n",
        "    y_data = np.array(y_data)[..., np.newaxis]\n",
        "    return x_data, y_data\n",
        "def generate_data_3d(t_filenames, o_filenames):\n",
        "    # Takes in the files of the transformed images and the originals to create the pairs\n",
        "    # Also sets the min and max slice numbers to take in\n",
        "    x_data = []\n",
        "    y_data = []\n",
        "    for i in range(len(t_filenames)):\n",
        "        print('Generating data for: ' + t_filenames[i])\n",
        "        x_mri = read_file(t_filenames[i])\n",
        "        y_mri = read_original(o_filenames[i])\n",
        "        x_data.append((x_mri - np.amin(x_mri))/(np.amax(x_mri) - np.amin(x_mri))) # normalizes the intensity to between 0 and 1\n",
        "        y_data.append((y_mri - np.amin(y_mri))/(np.amax(y_mri) - np.amin(y_mri)))\n",
        "    # Shape of x & y will be [# mris, 256, 256, 176, 1]\n",
        "    x_data = np.array(x_data)[..., np.newaxis]\n",
        "    y_data = np.array(y_data)[..., np.newaxis]\n",
        "    return x_data, y_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZEiTb8nN6cz",
        "colab_type": "code",
        "outputId": "119b2466-cf55-4548-9105-e8f484a86719",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        }
      },
      "source": [
        "# generate 3d training data\n",
        "trans_folder = 'gdrive/My Drive/NU_Rad/transforms/Training/'\n",
        "orig_folder = 'gdrive/My Drive/NU_Rad/mris/'\n",
        "X_filenames = []\n",
        "y_filenames = []\n",
        "for item in os.listdir(trans_folder):\n",
        "    if item.endswith(\"_trans.txt.gz\"):\n",
        "        X_filenames.append(trans_folder + item)\n",
        "        y_filenames.append(orig_folder + item[:3] + '.nii')\n",
        "\n",
        "# X, y = generate_data(['gdrive/My Drive/NU_Rad/transforms/M02_motion5_trans.txt.gz'], ['gdrive/My Drive/NU_Rad/mris/M02.nii'])\n",
        "X_train, y_train = generate_data_3d(X_filenames, y_filenames)\n",
        "\n",
        "# generate 3d validation data\n",
        "trans_folder = 'gdrive/My Drive/NU_Rad/transforms/Validation/'\n",
        "X_filenames = []\n",
        "y_filenames = []\n",
        "for item in os.listdir(trans_folder):\n",
        "    if item.endswith(\"_trans.txt.gz\"):\n",
        "        X_filenames.append(trans_folder + item)\n",
        "        y_filenames.append(orig_folder + item[:3] + '.nii')\n",
        "X_val, y_val = generate_data_3d(X_filenames, y_filenames)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M02_motion5_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M02_motion3_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M02_motion4_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M02_motion1_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M02_motion2_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M01_motion5_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M01_motion3_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M01_motion4_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M03_motion5_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M03_motion3_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M03_motion4_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M03_motion1_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M03_motion2_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M05_motion5_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M05_motion3_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M05_motion4_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M05_motion1_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Training/M01_motion1_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Validation/M06_motion5_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Validation/M06_motion3_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Validation/M06_motion4_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Validation/M06_motion1_trans.txt.gz\n",
            "Generating data for: gdrive/My Drive/NU_Rad/transforms/Validation/M06_motion2_trans.txt.gz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s3o17qTHN9Gx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Simple 3d unet with mse loss\n",
        "\n",
        "def conv_layer(filts, dim):\n",
        "    # abstracted a single conv layer out since the parameters outside of dimension were kept the same\n",
        "    return Conv3D(filts, dim, activation = 'relu', padding = 'same', kernel_initializer = 'he_normal')\n",
        "\n",
        "# u_net model\n",
        "def unet_model(lr= 1e-4, input_size = (256, 256, 176, 1), dropout_level = 0.1):\n",
        "    inputs = Input(input_size)\n",
        "\n",
        "    conv1 = conv_layer(64, 3)(inputs)\n",
        "    conv1 = conv_layer(64, 3)(conv1)\n",
        "    drop1 = Dropout(dropout_level)(conv1)\n",
        "    pool1 = MaxPooling3D(pool_size=(2, 2, 2))(drop1)\n",
        "    conv2 = conv_layer(128, 3)(pool1)\n",
        "    conv2 = conv_layer(128, 3)(conv2)\n",
        "    drop2 = Dropout(dropout_level)(conv2)\n",
        "    pool2 = MaxPooling3D(pool_size=(2, 2, 2))(drop2)\n",
        "    conv3 = conv_layer(256, 3)(pool2)\n",
        "    conv3 = conv_layer(256, 3)(conv3)\n",
        "    drop3 = Dropout(dropout_level)(conv3)\n",
        "    pool3 = MaxPooling3D(pool_size=(2, 2, 2))(drop3)\n",
        "    conv4 = conv_layer(512, 3)(pool3)\n",
        "    conv4 = conv_layer(512, 3)(conv4)\n",
        "    drop4 = Dropout(dropout_level)(conv4)\n",
        "    pool4 = MaxPooling3D(pool_size=(2, 2, 2))(drop4)\n",
        "    conv5 = conv_layer(1024, 3)(pool4)\n",
        "    conv5 = conv_layer(1024, 3)(conv5)\n",
        "    drop5 = Dropout(dropout_level)(conv5)\n",
        "\n",
        "    # Decoder\n",
        "    up6 = conv_layer(512, 2)(UpSampling3D(size = (2,2,2))(drop5))\n",
        "    merge6 = concatenate([drop4,up6], axis = 4)\n",
        "    drop6 = Dropout(dropout_level)(merge6)\n",
        "    conv6 = conv_layer(512, 3)(drop6)\n",
        "    conv6 = conv_layer(512, 3)(conv6)\n",
        "    \n",
        "    up7 = conv_layer(256, 2)(UpSampling3D(size = (2,2,2))(conv6))\n",
        "    merge7 = concatenate([conv3,up7], axis = 4)\n",
        "    drop7 = Dropout(dropout_level)(merge7)\n",
        "    conv7 = conv_layer(256, 3)(drop7)\n",
        "    conv7 = conv_layer(256, 3)(conv7)\n",
        "\n",
        "    up8 = conv_layer(128, 2)(UpSampling3D(size = (2,2,2))(conv7))\n",
        "    merge8 = concatenate([conv2,up8], axis = 4)\n",
        "    drop8 = Dropout(dropout_level)(merge8)\n",
        "    conv8 = conv_layer(128, 3)(drop8)\n",
        "    conv8 = conv_layer(128, 3)(conv8)\n",
        "\n",
        "    up9 = conv_layer(64, 2)(UpSampling3D(size = (2,2,2))(conv8))\n",
        "    merge9 = concatenate([conv1,up9], axis = 4)\n",
        "    drop9 = Dropout(dropout_level)(merge9)\n",
        "    conv9 = conv_layer(64, 3)(drop9)\n",
        "    conv9 = conv_layer(64, 3)(conv9)\n",
        "    conv9 = conv_layer(2, 3)(conv9)\n",
        "    conv10 = Conv3D(1, 1, activation = 'linear')(conv9)\n",
        "\n",
        "    model = Model(input= inputs, output= conv10)\n",
        "    model.compile(loss=\"mean_squared_error\", optimizer=\"adam\")\n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DFba3HYvQz35",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "897513e6-1a8a-41b6-8ff1-953a0534af9d"
      },
      "source": [
        "# Checkpoint\n",
        "check_path = 'gdrive/My Drive/NU_Rad/models/3d_unet_mse_model'\n",
        "checkpoint = ModelCheckpoint(check_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
        "callbacks_list = [checkpoint]\n",
        "\n",
        "model = unet_model()\n",
        "model.fit(X_train, y_train, batch_size=4, validation_split=0.1, epochs = 3, use_multiprocessing=True, callbacks=callbacks_list)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:57: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=Tensor(\"in..., outputs=Tensor(\"co...)`\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Train on 16 samples, validate on 2 samples\n",
            "Epoch 1/3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ResourceExhaustedError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-8-5783e161d8e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0munet_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[1;32m    194\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3790\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3791\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3792\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3793\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3794\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1603\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1604\u001b[0m     \"\"\"\n\u001b[0;32m-> 1605\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1607\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1643\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m   1644\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m-> 1645\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1647\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mResourceExhaustedError\u001b[0m:  OOM when allocating tensor with shape[4,256,256,176,64] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[node conv3d_25/convolution (defined at /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3009) ]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n [Op:__inference_keras_scratch_graph_16038]\n\nFunction call stack:\nkeras_scratch_graph\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pXFp_wCuX0xW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}