cw_nets_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import walk\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tqdm import tqdm\n",
    "path_to_data = \"/home/dlindenbaum/cw-tiler/cw-tiler/AOI_6_Atlanta/\"\n",
    "f = []\n",
    "for (dirpath, dirnames, filenames) in walk(path_to_data):\n",
    "    f.extend(filenames)\n",
    "    break\n",
    "#print(f)\n",
    "image_list = [os.path.join(path_to_data, filename) for filename in filenames if \"_image.tif\" in filename]\n",
    "label_list = [os.path.join(path_to_data, filename) for filename in filenames if \"_label.tif\" in filename]\n",
    "\n",
    "\n",
    "\n",
    "print(\"Number of images: {}\".format(len(image_list)))\n",
    "print(\"Number of labels: {}\".format(len(label_list)))\n",
    "image_list = np.sort(image_list)\n",
    "label_list = np.sort(label_list)\n",
    "image_list_filter = []\n",
    "label_list_filter = []\n",
    "\n",
    "import rasterio\n",
    "for image, label in tqdm(zip(image_list, label_list)):\n",
    "    with rasterio.open(image) as src:\n",
    "        data = src.read()\n",
    "        if data.max()==0:\n",
    "            #print(\"bad\")\n",
    "            pass\n",
    "            \n",
    "        else:\n",
    "            image_list_filter.append(image)\n",
    "            label_list_filter.append(label)\n",
    "    \n",
    "\n",
    "image_list_train, image_list_test, label_list_train, label_list_test = train_test_split(image_list_filter, label_list_filter, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from cw_nets.keras_tools.keras_geotiff import RasterDataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "from cw_nets.keras_tools.unet_keras import unet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.optimizers import SGD, Adam, Adagrad\n",
    "\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras import utils\n",
    "\n",
    "n_channels =8\n",
    "n_width = 800\n",
    "n_height = 800\n",
    "n_classes = 1\n",
    "batch_size = 5\n",
    "params={\"dim\": (n_width, n_height),\n",
    "       \"batch_size\": batch_size,\n",
    "        \"n_classes\":n_classes,\n",
    "        \"n_channels\":n_channels,\n",
    "        \"shuffle\":True,\n",
    "        \"max_value\": 2**11 -1, # Use Max Value of Chip or you can specify a max value to normalize by\n",
    "        \"img_norm\": \"sub_divide\", #Divide by max value and normalize to 0,1\n",
    "        \"low_ram\": True\n",
    "       }\n",
    "\n",
    "\n",
    "model = unet((n_channels, n_width, n_height), n_classes=1,\n",
    "             kernel=3\n",
    "            )\n",
    "\n",
    "training_generator = RasterDataGenerator(image_list_train, label_list_train, **params)\n",
    "validation_generator = RasterDataGenerator(image_list_test, label_list_test, **params)\n",
    "\n",
    "\n",
    " # set callbacks\n",
    "print (\"Setting callbacks...\")\n",
    "model_name = \"AOI_6_unet_model_v1\"\n",
    "early_stopping_patience = 4\n",
    "\n",
    "model_checkpoint = ModelCheckpoint(model_name, monitor='val_loss', \n",
    "                                           save_best_only=True)\n",
    "early_stopping = EarlyStopping(monitor='val_loss', \n",
    "                                       patience=early_stopping_patience, \n",
    "                                       verbose=1, mode='auto')\n",
    "print (\"Callbacks successfully set\")\n",
    "\n",
    "\n",
    "epochs=100\n",
    "\n",
    "\n",
    "\n",
    "model.fit_generator(generator=training_generator,\n",
    "                   validation_data=validation_generator,\n",
    "                   use_multiprocessing=True,\n",
    "                    workers=8,\n",
    "                   verbose=1,\n",
    "                   epochs=epochs,\n",
    "                    callbacks=[model_checkpoint, early_stopping]\n",
    "                   )\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "2**11 -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from matplotlib import pyplot as plt\n",
    "import rasterio\n",
    "\n",
    "idx = 50\n",
    "image = image_list_test[idx] \n",
    "label = label_list_test[idx]\n",
    "n_channels =8\n",
    "n_width = 800\n",
    "n_height = 800\n",
    "n_classes = 1\n",
    "batch_size = 5\n",
    "\n",
    "\n",
    "params={\"dim\": (n_width, n_height),\n",
    "       \"batch_size\": batch_size,\n",
    "        \"n_classes\":n_classes,\n",
    "        \"n_channels\":n_channels,\n",
    "        \"shuffle\":True,\n",
    "        \"max_value\": -1, # Use Max Value of Chip or you can specify a max value to normalize by\n",
    "        \"img_norm\": \"divide\", #Divide by max value and normalize to 0,1\n",
    "        \"low_ram\": True\n",
    "       }\n",
    "\n",
    "\n",
    "\n",
    "model = unet((n_channels, n_width, n_height), n_classes=1,\n",
    "             kernel=3\n",
    "            )\n",
    "\n",
    "model.load_weights('AOI_6_unet_model_v4')\n",
    "#model.compile(optimizer='adadelta')\n",
    "\n",
    "\n",
    "with rasterio.open(image) as src:\n",
    "    \n",
    "    data = src.read().astype(float)\n",
    "    plt.imshow(data[1])\n",
    "    if params['max_value']==-1:\n",
    "        max_value = data.max()\n",
    "    else:\n",
    "        max_value = params['max_value']\n",
    "        \n",
    "    if params['img_norm'] == \"sub_divide\":\n",
    "        img = np.clip((data*1.0/(max_value/2) - 1), -1, 1)\n",
    "    else:\n",
    "        img = data\n",
    "        img *= 1.0/max_value\n",
    "    \n",
    "    pr = model.predict(np.array([img]))\n",
    "    \n",
    "with rasterio.open(label) as src:\n",
    "    imga = src.read().astype(float)\n",
    "    plt.imshow(np.squeeze(imga))\n",
    "    \n",
    "print(pr.shape)\n",
    "#plt.imshow(np.squeeze(pr>0.15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.load_weights('AOI_6_unet_model_v2')\n",
    "#model.compile(optimizer='adadelta')\n",
    "\n",
    "\n",
    "with rasterio.open(image) as src:\n",
    "    \n",
    "    data = src.read().astype(float)\n",
    "    plt.imshow(data[1])\n",
    "    if params['max_value']==-1:\n",
    "        max_value = data.max()\n",
    "    else:\n",
    "        max_value = params['max_value']\n",
    "        \n",
    "    if params['img_norm'] == \"sub_divide\":\n",
    "        data = np.clip((data*1.0/(max_value/2) - 1), -1, 1)\n",
    "    else:\n",
    "        data *= 1.0/max_value\n",
    "    \n",
    "    pr = model.predict(np.array([data]))\n",
    "with rasterio.open(label) as src:\n",
    "    imga = src.read().astype(float)\n",
    "    plt.imshow(np.squeeze(imga))\n",
    "    \n",
    "print(pr.shape)\n",
    "plt.imshow(np.squeeze(pr>0.15))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:bassis]",
   "language": "python",
   "name": "conda-env-bassis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}