complete-30-gan-closs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# HiDDeN / complete-30-gan-closs.py\n",
    "# Gourav Siddhad\n",
    "# 21-Apr-2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "from numpy.random import seed\n",
    "seed(1607)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(1607)\n",
    "\n",
    "print('Importing Libraries', end='')\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy.random import seed\n",
    "import random\n",
    "import re\n",
    "import os\n",
    "import pandas as pd\n",
    "import time\n",
    "import csv\n",
    "import math\n",
    "\n",
    "from PIL import Image, ImageFile\n",
    "\n",
    "from skimage import io\n",
    "from skimage.io import imread, imshow\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "\n",
    "import tensorflow as tf\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.models import Model, Sequential, load_model\n",
    "from keras.layers import Activation, Input, Dense, Conv2D, BatchNormalization, ReLU\n",
    "from keras.layers import GlobalAveragePooling2D, Flatten, Reshape, Lambda, Add\n",
    "from keras import optimizers\n",
    "from keras.preprocessing import sequence\n",
    "from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n",
    "\n",
    "from keras.utils import to_categorical, plot_model\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger\n",
    "from keras.layers.merge import concatenate\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "print(' - Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Configuration\n",
    "H, W, C, L = 128, 128, 3, 30\n",
    "all_messages_train = []\n",
    "all_messages_validate = []\n",
    "all_messages_test = []\n",
    "\n",
    "# Training train_options\n",
    "batch_size = 12\n",
    "n_epochs = 200\n",
    "\n",
    "modelpath = 'models/'\n",
    "\n",
    "train_folder = 'dataset128/original/train/'\n",
    "validate_folder = 'dataset128/original/validate/'\n",
    "test_folder = 'dataset128/original/test/'\n",
    "    \n",
    "print('Model    - ', modelpath)\n",
    "print('Train    - ', train_folder)\n",
    "print('Validate - ', validate_folder)\n",
    "print('Test     - ', test_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Reading Image List', end='')\n",
    "train_list = os.listdir(train_folder)\n",
    "validate_list = os.listdir(validate_folder)\n",
    "test_list = os.listdir(test_folder)\n",
    "print(' - Done')\n",
    "\n",
    "print('Train    - ', len(train_list))\n",
    "print('Validate - ', len(validate_list))\n",
    "print('Test - ', len(test_list))\n",
    "print()\n",
    "\n",
    "message_train = []\n",
    "message_validate = []\n",
    "message_test = []\n",
    "\n",
    "print('Reading Messages', end='')\n",
    "with open(modelpath + str(L) + '-train-msg.csv') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file)\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count%2 is 0:\n",
    "            message_train.append(row)\n",
    "        line_count += 1\n",
    "csv_file.close()\n",
    "\n",
    "with open(modelpath + str(L) + '-validate-msg.csv') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file)\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count%2 is 0:\n",
    "            message_validate.append(row)\n",
    "        line_count += 1\n",
    "csv_file.close()\n",
    "\n",
    "with open(modelpath + str(L) + '-test-msg.csv') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file)\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count%2 is 0:\n",
    "            message_test.append(row)\n",
    "        line_count += 1\n",
    "csv_file.close()\n",
    "print(' - Done')\n",
    "\n",
    "print('Train Messages    - ', len(message_train))\n",
    "print('Validate Messages - ', len(message_validate))\n",
    "print('Test Messages     - ', len(message_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_msg_plates(message):\n",
    "    msg = []\n",
    "    for l in message:\n",
    "        if l is '1':\n",
    "            temp = np.ones((H, W), dtype='float32')\n",
    "            msg.append(temp)\n",
    "        elif l is '0':\n",
    "            temp = np.zeros((H, W), dtype='float32')\n",
    "            msg.append(temp)\n",
    "    return np.array(msg)\n",
    "\n",
    "def gen_msg(message):\n",
    "    msg = []\n",
    "    for l in message:\n",
    "        if l is '1':\n",
    "            msg.append(float(1))\n",
    "        if l is '0':\n",
    "            msg.append(float(0))\n",
    "    return np.array(msg, dtype='float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lamI = 0.7\n",
    "lamG = 0.001\n",
    "\n",
    "def encoder_loss(y_true, y_pred):\n",
    "    # Euclidean Distance\n",
    "    loss = K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1))\n",
    "    return loss/(C * H * W)\n",
    "\n",
    "def decoder_loss(y_true, y_pred):\n",
    "    # Euclidean Distance\n",
    "    loss = K.sqrt(K.sum(K.square(y_pred - y_true), axis=-1))\n",
    "    return loss/L\n",
    "\n",
    "def adversary_loss_alone(y_true, y_pred):\n",
    "    # Log Loss\n",
    "    loss = K.log(1.0 - y_true) + K.log(y_pred)\n",
    "    return loss\n",
    "\n",
    "def adversary_loss(y_true, y_pred):\n",
    "    # Log Loss\n",
    "    loss = K.log(1.0 - y_pred)\n",
    "    return loss\n",
    "\n",
    "adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Encoder Network')\n",
    "# Encoder\n",
    "def build_encoder():\n",
    "    input_e = Input(shape=(H, W, C))\n",
    "    input_m = Input(shape=(L, H, W))\n",
    "    layer_e1 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(input_e)\n",
    "    layer_e2 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_e1)\n",
    "    layer_e3 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_e2)\n",
    "    layer_e4 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_e3)\n",
    "\n",
    "    # Reshape Message\n",
    "    layer_r = Reshape((H, W, L))(input_m)\n",
    "\n",
    "    # Integrate Message to Model\n",
    "    merge = concatenate([layer_e4, input_e, layer_r])\n",
    "    layer_e5 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(merge)\n",
    "    output_e = Conv2D(filters=C, kernel_size=(1, 1), strides=(1, 1), padding='same', name='Enc')(layer_e5)\n",
    "    \n",
    "    model_e = Model(inputs=[input_e, input_m], outputs=output_e)\n",
    "    model_e.compile(loss=encoder_loss, optimizer=adam, metrics=['accuracy'])\n",
    "    return model_e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('Decoder Network')\n",
    "# Decoder\n",
    "def build_decoder():\n",
    "    input_d = Input(shape=(H, W, C))\n",
    "    layer_d1 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(input_d)\n",
    "    layer_d2 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d1)\n",
    "    layer_d3 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d2)\n",
    "    layer_d4 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d3)\n",
    "    layer_d5 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d4)\n",
    "    layer_d6 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d5)\n",
    "    layer_d7 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d6)\n",
    "    layer_d8 = Sequential([Conv2D(filters=L, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_d7)\n",
    "    layer_d9 = GlobalAveragePooling2D(data_format=None)(layer_d8)\n",
    "    output_d = Activation('linear')(layer_d9)\n",
    "    \n",
    "    model_d = Model(inputs=input_d, outputs=output_d)\n",
    "    model_d.compile(loss=decoder_loss, optimizer=adam, metrics=['binary_accuracy'])\n",
    "    # output_d = Dense(L, use_bias=False, activation='sigmoid', name='Dec')(layer_d9)\n",
    "    return model_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Adversary Network')\n",
    "# Adversary\n",
    "def build_adversary():\n",
    "    input_a = Input(shape=(H, W, C))\n",
    "    layer_a1 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(input_a)\n",
    "    layer_a2 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_a1)\n",
    "    layer_a3 = Sequential([Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='same'),\n",
    "                    BatchNormalization(axis=-1, epsilon=1e-05, momentum=0.1, center=True, scale=True),\n",
    "                    ReLU(max_value=None, negative_slope=0.0, threshold=0.0)])(layer_a2)\n",
    "    layer_a4 = GlobalAveragePooling2D(data_format=None)(layer_a3)\n",
    "    output_a = Dense(1, activation='linear', name='Adv')(layer_a4)\n",
    "#     output_a = Dense(2, use_bias=False, activation='softmax', name='Adv')(layer_a4)\n",
    "\n",
    "    model_a = Model(inputs=input_a, outputs=output_a)\n",
    "    model_a.compile(loss=adversary_loss_alone, optimizer=adam, metrics=['accuracy'])\n",
    "    \n",
    "    return model_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Combined Network')\n",
    "# Combined\n",
    "def build_combined(encoder, decoder, adversary):\n",
    "    adversary.trainable = False\n",
    "    \n",
    "    c_iinput = Input(shape=(H, W, C))\n",
    "    c_minput = Input(shape=(L, H, W))\n",
    "    \n",
    "    c_eout = encoder([c_iinput, c_minput])\n",
    "    c_dout = decoder(c_eout)\n",
    "    c_aout = adversary(c_eout)\n",
    "    \n",
    "    combined = Model(inputs=[c_iinput, c_minput], outputs=[c_eout, c_dout, c_aout])\n",
    "\n",
    "    combined.compile(loss=[encoder_loss, decoder_loss, adversary_loss], loss_weights=[lamI, 1.0, lamG],\n",
    "                     optimizer=adam, metrics=['accuracy', 'binary_accuracy'])\n",
    "    \n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Combined Model')\n",
    "encoder = build_encoder()\n",
    "decoder = build_decoder()\n",
    "adversary = build_adversary()\n",
    "combined = build_combined(encoder, decoder, adversary)\n",
    "combined.summary()\n",
    "\n",
    "print()\n",
    "print('Metrics')\n",
    "print(combined.metrics_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DataGenerator(file_list, folder_path, messages, index, batch_size=12):\n",
    "    indexes = np.arange(len(file_list))\n",
    "    \n",
    "    index = indexes[index*batch_size : (index+1)*batch_size]\n",
    "    file_list_temp = [file_list[k] for k in index]\n",
    "    message_temp = [messages[k] for k in index]\n",
    "\n",
    "    images = []\n",
    "    batch_msg = []\n",
    "    dec_msg = []\n",
    "\n",
    "    for i, ID in enumerate(file_list_temp):\n",
    "        img = io.imread(folder_path + ID)\n",
    "        img = img.reshape([H, W, C])\n",
    "        images.append(np.array(img, dtype='float32')/255)\n",
    "        \n",
    "        msg = gen_msg_plates(message_temp[i])\n",
    "        msg2 = gen_msg(message_temp[i])\n",
    "        batch_msg.append(msg)\n",
    "        dec_msg.append(msg2)\n",
    "\n",
    "    return np.array(images), np.array(batch_msg), np.array(dec_msg)\n",
    "\n",
    "def d_bit_accuracy(true, pred):\n",
    "    correct = 0\n",
    "    total = len(true)*L\n",
    "    for i in range(len(true)):\n",
    "        for j in range(len(true[i])):\n",
    "            if int(np.round(true[i][j])) == (np.round(pred[i][j])):\n",
    "                correct += 1\n",
    "    acc = correct/total    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To remove the tf warning ;)\n",
    "a = tf.to_int32(tf.Variable(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "e_loss, d_loss, a_loss = [], [], []\n",
    "e_acc, d_acc, a_acc = [], [], []\n",
    "v_e_loss, v_d_loss, v_a_loss = [], [], []\n",
    "v_e_acc, v_d_acc, v_a_acc = [], [], []\n",
    "v_db_acc = []\n",
    "train_loss, val_loss = [], []\n",
    "\n",
    "tbatches = len(train_list)/batch_size\n",
    "vbatches = len(validate_list)/batch_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print('Training Complete Network')\n",
    "for epoch in range(n_epochs):\n",
    "    print('\\nEpoch : ', epoch+1, '/', n_epochs, '\\t\\t\\t', '[', end='')\n",
    "    \n",
    "    com_score = []\n",
    "    v_score, v_dbit_acc = [], []\n",
    "\n",
    "    # Shuffle Data after each Epoch\n",
    "    train_list, message_train = shuffle(train_list, message_train)\n",
    "    \n",
    "    tstartt = time.time()\n",
    "    for nbatch in range(int(tbatches)):\n",
    "        if nbatch%12 == 0:\n",
    "            print('#', end='')\n",
    "                    \n",
    "        # Generate Images\n",
    "        r1, r2, r3 = DataGenerator(train_list, train_folder, message_train, nbatch)\n",
    "        \n",
    "        # Generate Encoder Images\n",
    "        fake_images = encoder.predict([r1, r2])\n",
    "        \n",
    "        # Create two batches\n",
    "        X = np.concatenate([r1, fake_images])\n",
    "        y_dis = np.concatenate([np.zeros((np.int64(batch_size), 1)), np.ones((np.int64(batch_size), 1))])\n",
    "        # Train discriminator\n",
    "        adversary.trainable = True\n",
    "        adversary.train_on_batch(X, y_dis)\n",
    "        \n",
    "        # Train Combined\n",
    "        y_gen = np.ones((np.int64(batch_size), 1))\n",
    "        adversary.trainable = False\n",
    "        com_score.append(combined.train_on_batch([r1, r2], [r1, r3, y_gen]))\n",
    "\n",
    "    tendt = time.time()\n",
    "    print('|', end='')\n",
    "    vstartt = time.time()\n",
    "\n",
    "    # Shuffle Data after each Epoch\n",
    "    validate_list, message_validate = shuffle(validate_list, message_validate)\n",
    "    \n",
    "    # Validate Model\n",
    "    for nbatch in range(int(vbatches)):\n",
    "        if nbatch%12 == 0:\n",
    "            print('#', end='')\n",
    "        v1, v2, v3 = DataGenerator(validate_list, validate_folder, message_validate, nbatch)\n",
    "        \n",
    "        y_gen = np.ones((np.int64(batch_size), 1))\n",
    "        v_score.append(combined.evaluate([v1, v2], [v1, v3, y_gen], verbose=0))\n",
    "#         res_v , d_out, _ = combined.predict([v1, v2])\n",
    "#         v_dbit_acc.append(d_bit_accuracy(v3, d_out))\n",
    "\n",
    "    vendt = time.time()\n",
    "    print(']')\n",
    "\n",
    "    # Calculating Metrics\n",
    "    avgl_e, avgl_d, avgl_a = [], [], []\n",
    "    avga_e, avga_d, avga_a = [], [], []\n",
    "    avgl = []\n",
    "    for com in com_score:\n",
    "        avgl.append(0 if math.isnan(com[0]) else com[0])\n",
    "        avgl.append(com[0])\n",
    "        avgl_e.append(com[1])\n",
    "        avgl_d.append(com[2])\n",
    "        avgl_a.append(0 if math.isnan(com[3]) else com[3])\n",
    "        avga_e.append(com[4])\n",
    "        avga_d.append(com[7])\n",
    "        avga_a.append(com[8])\n",
    "    train_loss.append(np.mean(avgl))\n",
    "    e_loss.append(np.mean(avgl_e))\n",
    "    d_loss.append(np.mean(avgl_d))\n",
    "    a_loss.append(np.mean(avgl_a))\n",
    "    e_acc.append(np.mean(avga_e))\n",
    "    d_acc.append(np.mean(avga_d))\n",
    "    a_acc.append(np.mean(avga_a))\n",
    "    \n",
    "    avgl_e, avgl_d, avgl_a = [], [], []\n",
    "    avga_e, avga_d, avga_a = [], [], []\n",
    "    avgl = []\n",
    "    for com in v_score:\n",
    "        avgl.append(0 if math.isnan(com[0]) else com[0])\n",
    "        avgl_e.append(com[1])\n",
    "        avgl_d.append(com[2])\n",
    "        avgl_a.append(0 if math.isnan(com[3]) else com[3])\n",
    "        avga_e.append(com[4])\n",
    "        avga_d.append(com[7])\n",
    "        avga_a.append(com[8])\n",
    "    val_loss.append(np.mean(avgl))\n",
    "    v_e_loss.append(np.mean(avgl_e))\n",
    "    v_d_loss.append(np.mean(avgl_d))\n",
    "    v_a_loss.append(np.mean(avgl_a))\n",
    "    v_e_acc.append(np.mean(avga_e))\n",
    "    v_d_acc.append(np.mean(avga_d))\n",
    "    v_a_acc.append(np.mean(avga_a))\n",
    "    \n",
    "#     v_db_acc.append(np.mean(v_dbit_acc))\n",
    "    \n",
    "    print()\n",
    "    print('Training   - ', tendt - tstartt, 'sec')\n",
    "    print('Train     | Loss  : {:10.7f}\\t |'.format(train_loss[epoch]))\n",
    "    print('Encoder   | Loss  : {:10.7f}'.format(e_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(e_acc[epoch]))\n",
    "    print('Decoder   | Loss  : {:10.7f}'.format(d_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(d_acc[epoch]))\n",
    "    print('Adversary | Loss  : {:10.7f}'.format(a_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(a_acc[epoch]))\n",
    "    print()\n",
    "    print('Validation - ', vendt - vstartt, 'sec')\n",
    "    print('Validate  | Loss  : {:10.7f}\\t |'.format(val_loss[epoch]))\n",
    "    print('Encoder   | Loss  : {:10.7f}'.format(v_e_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(v_e_acc[epoch]))\n",
    "    print('Decoder   | Loss  : {:10.7f}'.format(v_d_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(v_d_acc[epoch]))\n",
    "#     print('Bit Accuracy : {:10.7f} |'.format(v_db_acc[epoch]))\n",
    "    print('Adversary | Loss  : {:10.7f}'.format(v_a_loss[epoch]), end='\\t')\n",
    "    print(' | Accuracy : {:10.7f} |'.format(v_a_acc[epoch]))\n",
    "    \n",
    "    # Save Complete Model\n",
    "    combined.save(modelpath + 'complete-closs-' + str(L) + '-' + str(epoch) + '.h5')\n",
    "    \n",
    "    # Print Images\n",
    "    in_img = []\n",
    "    in_msg = []\n",
    "    \n",
    "    p1, p2, _ = DataGenerator(validate_list, validate_folder, message_validate, 0, 3)\n",
    "    out_img = encoder.predict([p1, p2])\n",
    "    \n",
    "    fig, ax = plt.subplots(2, 3, sharex='col', sharey='row')#, figsize=(5, 5))\n",
    "    plt.tight_layout()\n",
    "    for j in range(3):\n",
    "        ax[0, j].imshow((p1[j]*255).astype(np.uint8))\n",
    "        ax[1, j].imshow((out_img[j]*255).astype(np.uint8))\n",
    "        ax[0, j].autoscale(False)\n",
    "        ax[1, j].autoscale(False)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot Encoder Accuracy\n",
    "plt.plot(e_acc)\n",
    "plt.plot(v_e_acc)\n",
    "plt.title('Encoder Accuracy - Complete ' + str(L))\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Enc Acc', 'Val Enc Acc'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-enc-acc.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()\n",
    "\n",
    "# Plot Decoder Accuracy\n",
    "plt.plot(d_acc)\n",
    "plt.plot(v_d_acc)\n",
    "plt.plot(v_db_acc)\n",
    "plt.title('Decoder Accuracy - Complete ' + str(L))\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Dec Acc', 'Val Dec Acc', 'Val Dec Bit Acc'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-dec-acc.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()\n",
    "\n",
    "# Plot Adversary Accuracy\n",
    "plt.plot(a_acc)\n",
    "plt.plot(v_a_acc)\n",
    "plt.title('Adversary Accuracy - Complete ' + str(L))\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Adv Acc', 'Val Adv Acc'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-adv-acc.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()\n",
    "\n",
    "# Plot Encoder Loss\n",
    "plt.plot(e_loss)\n",
    "plt.plot(v_e_loss)\n",
    "plt.title('Encoder Loss - Complete ' + str(L))\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Enc Loss', 'Val Enc Loss'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-enc-loss.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()\n",
    "\n",
    "# Plot Decoder Loss\n",
    "plt.plot(d_loss)\n",
    "plt.plot(v_d_loss)\n",
    "plt.title('Decoder Loss - Complete ' + str(L))\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Dec Loss', 'Val Dec Loss'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-dec-loss.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()\n",
    "\n",
    "# Plot Adversary Loss\n",
    "plt.plot(a_loss)\n",
    "plt.plot(v_a_loss)\n",
    "plt.title('Adversary Loss - Complete ' + str(L))\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend(['Adv Loss', 'Val Adv Loss'], loc='upper left')\n",
    "plt.savefig(modelpath + 'complete-closs-' + str(L) + '-adv-loss.png', dpi=300, pad_inches=0.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the Best Accuracy of Models\n",
    "print('Maximum Validation Accuracy - ')\n",
    "e_max, d_max, a_max = 0, 0, 0\n",
    "e_min, d_min, a_min = 0, 0, 0\n",
    "\n",
    "# Encoder\n",
    "maxa, maxlen = 0, len(v_e_acc)\n",
    "maxacc = 0\n",
    "minloss = 1\n",
    "for i in range(maxlen):\n",
    "    if (v_e_acc[i])>=maxacc:\n",
    "        maxacc = v_e_acc[i]\n",
    "        e_max = i\n",
    "    if (v_e_loss[i]<=minloss):\n",
    "        minloss = v_e_loss[i]\n",
    "        e_min = i\n",
    "print('Encoder   : {:10.7f} | Index : {}'.format(maxacc, e_max))\n",
    "print('Encoder   : {:10.7f} | Index : {}'.format(minloss, e_min))\n",
    "\n",
    "# Decoder\n",
    "maxa, maxlen = 0, len(v_d_acc)\n",
    "maxacc = 0\n",
    "minloss = 1\n",
    "for i in range(maxlen):\n",
    "    if (v_d_acc[i])>=maxacc:\n",
    "        maxacc = v_d_acc[i]\n",
    "        d_max = i\n",
    "    if (v_d_loss[i]<=minloss):\n",
    "        minloss = v_d_loss[i]\n",
    "        d_min = i\n",
    "print('Decoder   : {:10.7f} | Index : {}'.format(maxacc, d_max))\n",
    "print('Decoder   : {:10.7f} | Index : {}'.format(minloss, d_min))\n",
    "\n",
    "# Adversary\n",
    "maxa, maxlen = 0, len(v_a_acc)\n",
    "maxacc = 0\n",
    "minloss = 1\n",
    "for i in range(maxlen):\n",
    "    if (v_a_acc[i])>=maxacc:\n",
    "        maxacc = v_a_acc[i]\n",
    "        a_max = i\n",
    "    if (v_a_loss[i]<=minloss):\n",
    "        minloss = v_a_loss[i]\n",
    "        a_min = i\n",
    "print('Adversary : {:10.7f} | Index : {}'.format(maxacc, a_max))\n",
    "print('Adversary : {:10.7f} | Index : {}'.format(minloss, a_min))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Load Best Model\n",
    "print('Loading Best Model', end='')\n",
    "bmodel = 'complete-com-' + str(L) + '-' + str(e_min)\n",
    "l_model = load_model(modelpath + bmodel + '.h5')#, custom_objects={'encoder_loss': encoder_loss})\n",
    "l_model.load_weights(modelpath + bmodel + '.h5')\n",
    "l_model.save(modelpath + bmodel + '.hdf5')\n",
    "l_model = load_model(modelpath + bmodel + '.hdf5')#, custom_objects={'encoder_loss': encoder_loss})\n",
    "# bmodel_d = 'complete-d-' + str(L) + '-' + str(d_min)\n",
    "# l_model_d = load_model(modelpath + bmodel_d + '.h5')#, custom_objects={'decoder_loss': decoder_loss})\n",
    "# l_model_d.load_weights(modelpath + bmodel_d + '.h5')\n",
    "# l_model_d.save(modelpath + bmodel_d + '.hdf5')\n",
    "# l_model_d = load_model(modelpath + bmodel_d + '.hdf5')#, custom_objects={'decoder_loss': decoder_loss})\n",
    "# bmodel_a = 'complete-a-' + str(L) + '-' + str(a_min)\n",
    "# l_model_a = load_model(modelpath + bmodel_a + '.h5')\n",
    "# l_model_a.load_weights(modelpath + bmodel_a + '.h5')\n",
    "# l_model_a.save(modelpath + bmodel_a + '.hdf5')\n",
    "# l_model_a = load_model(modelpath + bmodel_a + '.hdf5')\n",
    "print(' - Done')\n",
    "\n",
    "ttbatches = len(test_list)/batch_size\n",
    "fscore, lscore = [], []\n",
    "b_dbit_acc, b_dbit_acc = [], []\n",
    "\n",
    "print('Testing Models - ', end='')\n",
    "for nbatch in range(int(ttbatches)):\n",
    "    if nbatch%12 == 0:\n",
    "        print('#', end='')\n",
    "    t1, t2, t3 = DataGenerator(test_list, test_folder, message_test, nbatch)\n",
    "    \n",
    "    y_gen = np.ones((np.int64(batch_size), 1))\n",
    "    \n",
    "    fscore.append(combined.evaluate([t1, t2], [t1, t3, y_gen], verbose=0))\n",
    "    lscore.append(l_model.evaluate([t1, t2], [t1, t3, y_gen], verbose=0))\n",
    "    \n",
    "    res_v , d_out, _ = combined.predict([t1, t2])\n",
    "    b_dbit_acc.append(d_bit_accuracy(t3, d_out))\n",
    "    res_v, d_out, _ = l_model.predict([t1, t2])\n",
    "    l_dbit_acc.append(d_bit_accuracy(t3, d_out))\n",
    "    \n",
    "print(' - Done')\n",
    "\n",
    "ef_score, df_score, af_score = [], [], []\n",
    "lef_score, ldf_score, laf_score = [], [], []\n",
    "\n",
    "for sample in fscore_e:\n",
    "    ef_score.append(sample[4])\n",
    "for sample in fscore_d:\n",
    "    df_score.append(sample[5])\n",
    "for sample in fscore_a:\n",
    "    af_score.append(sample[6])\n",
    "\n",
    "for sample in lscore_e:\n",
    "    lef_score.append(sample[1])\n",
    "for sample in lscore_d:\n",
    "    ldf_score.append(sample[2])\n",
    "for sample in lscore_a:\n",
    "    laf_score.append(sample[3])\n",
    "\n",
    "print()\n",
    "print('Accuracy  |    Final   |     Best   |')\n",
    "print('-'*9, '+', '-'*10, '+', '-'*10, '|')\n",
    "print('Encoder   | {:10.8f} | {:10.8f} |'.format(np.mean(ef_score), np.mean(lef_score)))\n",
    "print('Decoder   | {:10.8f} | {:10.8f} |'.format(np.mean(df_score), np.mean(ldf_score)))\n",
    "print('Adversary | {:10.8f} | {:10.8f} |'.format(np.mean(af_score), np.mean(laf_score)))\n",
    "print('Decoder b | {:10.8f} | {:10.8f} |'.format(np.mean(b_dbit_acc), np.mean(l_dbit_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save Images\n",
    "def save_images(dfolder, rfolder, file_list, messages, bmodel):\n",
    "    for index in range(len(test_list)//batch_size):\n",
    "        print(index, end=' ')\n",
    "        file_list_temp = file_list[index*batch_size : (index+1)*batch_size]\n",
    "        message_temp = messages[index*batch_size : (index+1)*batch_size]\n",
    "\n",
    "        images = []\n",
    "        batch_msg = []\n",
    "        for i, ID in enumerate(file_list_temp):\n",
    "            img = io.imread(rfolder + ID)\n",
    "            img = img.reshape([H, W, 3])\n",
    "            msg = gen_msg_plates(message_temp[i])\n",
    "\n",
    "            images.append(np.array(img)/255)\n",
    "            batch_msg.append(msg)\n",
    "        \n",
    "        output, _, _ = bmodel.predict([np.array(images), np.array(batch_msg)])\n",
    "        \n",
    "        for i in range(len(output)):\n",
    "            img = output[i]\n",
    "            formatted = (img * 255 / np.max(img))\n",
    "            formatted = formatted.astype('uint8')\n",
    "            final = Image.fromarray(formatted)\n",
    "            ImageFile.MAXBLOCK = final.size[0] * final.size[1] * 3\n",
    "            final.save(dfolder + file_list_temp[i], \"JPEG\", quality=100, optimize=True, progressive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save Test\n",
    "# if np.mean(ef_score)>np.mean(lef_score):\n",
    "#     bmodel = model_e\n",
    "# else:\n",
    "#     bmodel = l_model_e\n",
    "bmodel = combined\n",
    "\n",
    "print('Saving Encoded Test Images - ', end='')\n",
    "save_images('dataset128/complete-' + str(L) + '-eda/test/', test_folder, test_list, message_test, bmodel)        \n",
    "print(' - Done')\n",
    "print('Saving Encoded Validate Images - ', end='')\n",
    "save_images('dataset128/complete-' + str(L) + '-eda/validate/', validate_folder, validate_list, message_validate, bmodel)        \n",
    "print(' - Done')\n",
    "print('Saving Encoded Train Images - ', end='')\n",
    "save_images('dataset128/complete-' + str(L) + '-eda/train/', train_folder, train_list, message_train, bmodel)        \n",
    "print(' - Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}