SMOTE_GAN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d7973263",
   "metadata": {},
   "source": [
    "# Combination of synthetic minority oversampling technique and generative adversarial networks for data augmentation\n",
    "\n",
    "### Ehsan Farahbakhsh<sup>1</sup>, Sabin Zahirovic<sup>1</sup>, Brent I. A. McInnes<sup>2</sup>, Sara Polanco<sup>1</sup>, Fabian Kohlmann<sup>3</sup>, Maria Seton<sup>1</sup>, R. Dietmar M&uuml;ller<sup>1</sup>\n",
    "\n",
    "<sup>1</sup>*EarthByte Group, School of Geosciences, The University of Sydney, Sydney, Australia*\n",
    "\n",
    "<sup>2</sup>*John de Laeter Centre, Faculty of Science and Engineering, Curtin University, Perth, Australia*\n",
    "\n",
    "<sup>3</sup>*Lithodat Pty. Ltd., Melbourne, Australia*\n",
    "\n",
    "This notebook enables the user to combine two methods of data agumentation including synthetic minority oversampling technique and generative adversarial networks for increasing the number of positive samples. The output of this notebook is used to create a spatio-temporal mineral prospectivity model of the Papua New Guinea and Solomon Islands region using plate motion models.\n",
    "\n",
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf4a1a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "import imageio\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from ipywidgets import interact\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "import moviepy.editor as mpy\n",
    "import numpy as np\n",
    "from numpy import genfromtxt\n",
    "from numpy import ones\n",
    "from numpy import zeros\n",
    "from numpy.random import randint\n",
    "from numpy.random import randn\n",
    "import os\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "from scipy import stats\n",
    "\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.regularizers import L1, L2, L1L2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faefd808",
   "metadata": {},
   "source": [
    "### SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5d2322f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xy_train_file = './augmentation/Xy_train_muller2019.csv'\n",
    "# Xy_train_file = './augmentation/Xy_train_muller2016.csv'\n",
    "\n",
    "Xy_train = pd.read_csv(Xy_train_file, index_col=False)\n",
    "features_label_list = Xy_train.columns.tolist()\n",
    "\n",
    "# split the training dataset based on the label column\n",
    "Xy_train_positive = Xy_train.loc[Xy_train['label']==1]\n",
    "Xy_train_unlabelled = Xy_train.loc[Xy_train['label']==0]\n",
    "print('Number of positive training samples before removing outliers:', Xy_train_positive.shape[0])\n",
    "print('Number of unlabelled training samples before removing outliers:', Xy_train_unlabelled.shape[0])\n",
    "# remove outliers\n",
    "Xy_train_positive = Xy_train_positive[(np.abs(stats.zscore(Xy_train_positive.drop(columns=['label']))) < 3).all(axis=1)]\n",
    "Xy_train_unlabelled = Xy_train_unlabelled[(np.abs(stats.zscore(Xy_train_unlabelled.drop(columns=['label']))) < 3).all(axis=1)]\n",
    "print('Number of positive training samples after removing outliers:', Xy_train_positive.shape[0])\n",
    "print('Number of unlabelled training samples after removing outliers:', Xy_train_unlabelled.shape[0])\n",
    "\n",
    "Xy_train = pd.concat([Xy_train_positive, Xy_train_unlabelled]).reset_index(drop=True)\n",
    "Xy_train = Xy_train.to_numpy()\n",
    "features = Xy_train[:, :-1]\n",
    "labels = Xy_train[:, -1]\n",
    "\n",
    "# SMOTE\n",
    "smote = SMOTE(random_state=42)\n",
    "X_sm, y_sm = smote.fit_resample(features, labels)\n",
    "smote_samples = np.concatenate((X_sm, y_sm.reshape(-1, 1)), axis=1)\n",
    "X_positive = smote_samples[np.where(smote_samples[:, -1]==1)]\n",
    "X_positive = X_positive[:, 0:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ab3903a",
   "metadata": {},
   "source": [
    "### GAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2072e02d",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-2\n",
    "\n",
    "# define the standalone discriminator model\n",
    "def define_discriminator(n_inputs):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(int(n_inputs*10), activation='LeakyReLU', activity_regularizer=L2(learning_rate), kernel_initializer='he_uniform', input_dim=n_inputs))\n",
    "    model.add(Dense(1, activation='sigmoid', activity_regularizer=L2(learning_rate)))\n",
    "    # compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "# define the standalone generator model\n",
    "def define_generator(latent_dim, n_outputs):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(int(n_outputs*5), activation='LeakyReLU', activity_regularizer=L1(learning_rate), kernel_initializer='he_uniform', input_dim=latent_dim))\n",
    "    model.add(Dense(n_outputs, activation='linear', activity_regularizer=L1(learning_rate)))\n",
    "    # model.add(Dense(n_outputs, activation='linear'))\n",
    "    return model\n",
    "\n",
    "# define the combined generator and discriminator model for updating the generator\n",
    "def define_gan(generator, discriminator):\n",
    "    # make weights in the discriminator not trainable\n",
    "    discriminator.trainable = False\n",
    "    # connect them\n",
    "    model = Sequential()\n",
    "    # add generator\n",
    "    model.add(generator)\n",
    "    # add the discriminator\n",
    "    model.add(discriminator)\n",
    "    # compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam')\n",
    "    return model\n",
    "\n",
    "n_samples = int(features.shape[0]-(2*labels.sum()))\n",
    "# sample real data\n",
    "def sample_real_data(n=n_samples):\n",
    "    X_rand = X_positive[randint(X_positive.shape[0], size=n), :]\n",
    "    y_rand = ones((n, 1))\n",
    "    return X_rand, y_rand\n",
    "\n",
    "# generate points in the latent space as input for the generator\n",
    "def generate_latent_points(latent_dim, n=n_samples):\n",
    "    # generate points in the latent space\n",
    "    x_input = randn(latent_dim * n)\n",
    "    # reshape into a batch of inputs for the network\n",
    "    x_input = x_input.reshape(n, latent_dim)\n",
    "    return x_input\n",
    "\n",
    "# use the generator to generate n fake examples with class labels\n",
    "def generate_fake_samples(generator, latent_dim, n=n_samples):\n",
    "    # generate points in the latent space\n",
    "    x_input = generate_latent_points(latent_dim, n)\n",
    "    # predict outputs\n",
    "    X_fake = generator.predict(x_input, verbose=0)\n",
    "    # create class labels\n",
    "    y_fake = zeros((n, 1))\n",
    "    return X_fake, y_fake\n",
    "\n",
    "# evaluate the discriminator\n",
    "def summarize_performance(epoch, generator, discriminator, latent_dim):\n",
    "    # sample real data\n",
    "    x_real, y_real = sample_real_data()\n",
    "    # evaluate the discriminator on real examples\n",
    "    _, acc_real = discriminator.evaluate(X_sm, y_sm, verbose=0)\n",
    "    # prepare fake examples\n",
    "    x_fake, y_fake = generate_fake_samples(generator, latent_dim)\n",
    "    # evaluate the discriminator on fake examples\n",
    "    _, acc_fake = discriminator.evaluate(x_fake, y_fake, verbose=0)\n",
    "    return x_real, x_fake, acc_real, acc_fake\n",
    "\n",
    "# train the generator and discriminator\n",
    "def train(g_model, d_model, gan_model, latent_dim, n_epochs, smote_gan_file, n_batch=128, n_eval=100):\n",
    "    # determine half the size of one batch, for updating the discriminator\n",
    "    half_batch = int(n_batch / 2)\n",
    "    acc_real_all = []\n",
    "    acc_fake_all = []\n",
    "    # manually enumerate epochs\n",
    "    for i in range(n_epochs):\n",
    "        # prepare real samples\n",
    "        x_real, y_real = sample_real_data(half_batch)\n",
    "        # prepare fake examples\n",
    "        x_fake, y_fake = generate_fake_samples(g_model, latent_dim, half_batch)\n",
    "        # update discriminator\n",
    "        d_model.train_on_batch(x_real, y_real)\n",
    "        d_model.train_on_batch(x_fake, y_fake)\n",
    "        # prepare points in the latent space as input for the generator\n",
    "        x_gan = generate_latent_points(latent_dim, n_batch)\n",
    "        # create inverted labels for the fake samples\n",
    "        y_gan = ones((n_batch, 1))\n",
    "        # update the generator via the discriminator's error\n",
    "        gan_model.train_on_batch(x_gan, y_gan)\n",
    "        \n",
    "        # plot data points\n",
    "        # ----------------\n",
    "        # evaluate the model every n_eval epochs\n",
    "        if (i+1) % n_eval == 0:\n",
    "            # summarize discriminator performance\n",
    "            x_real, x_fake, acc_real, acc_fake = summarize_performance(i, g_model, d_model, latent_dim)\n",
    "            print(i+1, acc_real, acc_fake)\n",
    "                \n",
    "        if i == n_epochs-1:\n",
    "            # remove outliers\n",
    "            x_fake = x_fake[(np.abs(stats.zscore(x_fake)) < 3).all(axis=1)]\n",
    "            x_fake_ones = np.full((x_fake.shape[0], 1), 2)\n",
    "            x_fake = np.concatenate((x_fake, x_fake_ones), axis=1)\n",
    "            smote_gan_samples = np.concatenate((Xy_train, x_fake), axis=0)\n",
    "            smote_gan_samples = pd.DataFrame(smote_gan_samples, columns=features_label_list)\n",
    "            smote_gan_samples.to_csv(smote_gan_file, index=False)\n",
    "            \n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0768a650",
   "metadata": {},
   "source": [
    "### Multiple Iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2bf9d3d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# size of the latent space\n",
    "latent_dim = int(features.shape[1]*0.5)\n",
    "\n",
    "n_iter = 10\n",
    "acc_real_all = []\n",
    "acc_fake_all = []\n",
    "\n",
    "for i in range(n_iter):\n",
    "    print('--------------------')\n",
    "    print(f'Iteration {i+1}')\n",
    "    print('--------------------')\n",
    "\n",
    "    # create the discriminator\n",
    "    discriminator = define_discriminator(features.shape[1])\n",
    "    # create the generator\n",
    "    generator = define_generator(latent_dim, features.shape[1])\n",
    "    # create the gan\n",
    "    gan_model = define_gan(generator, discriminator)    \n",
    "    # train model\n",
    "    train(generator, discriminator, gan_model, latent_dim, n_epochs=2000,\n",
    "          smote_gan_file=f'./augmentation/smote_gan_muller2019_{i+1}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39e920a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "autoenc_pip",
   "language": "python",
   "name": "autoenc_pip"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}