PEMFC_Prognostics_DL.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kar10tik/Long-Term-PEMFC-DL-Prognostics/blob/main/PEMFC_Prognostics_DL.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Author: Kartik Sahajpal\n",
        "# Licensed under MIT License"
      ],
      "metadata": {
        "id": "pmYuQCXf_XAI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Requirements"
      ],
      "metadata": {
        "id": "eZV7VkgoIAjc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i75mYRQ3zUdr"
      },
      "outputs": [],
      "source": [
        "# Install required libraries\n",
        "!pip install optuna\n",
        "!pip install tensorflow==2.9.0\n",
        "!pip install statsmodels\n",
        "!pip install plotly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uifPC5lZCs0u"
      },
      "outputs": [],
      "source": [
        "# Check Available GPUs\n",
        "!nvidia-smi\n",
        "!cat /proc/meminfo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tWLgbKqdvLBf"
      },
      "outputs": [],
      "source": [
        "# Import required libraries\n",
        "import optuna\n",
        "import pandas as pd\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import scipy\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.metrics import mean_absolute_percentage_error, mean_absolute_error, r2_score, mean_squared_error\n",
        "import tensorflow as tf\n",
        "#%tensorflow_version 2.9\n",
        "from tensorflow import keras, data\n",
        "from random import random\n",
        "# import matplotlib as mpl\n",
        "# mpl.rcParams['figure.dpi'] = 600\n",
        "# print(tf.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O1fJ48RCfXiK"
      },
      "outputs": [],
      "source": [
        "# Set Random Seeds to Get Reproducible Results\n",
        "seed_value = 0\n",
        "\n",
        "# Set `PYTHONHASHSEED` environment variable at a fixed value\n",
        "import os\n",
        "os.environ['PYTHONHASHSEED']=str(seed_value)\n",
        "\n",
        "# Set `python` built-in pseudo-random generator at a fixed value\n",
        "import random\n",
        "random.seed(seed_value)\n",
        "\n",
        "# Set `numpy` pseudo-random generator at a fixed value\n",
        "np.random.seed(seed_value)\n",
        "\n",
        "# Set the `tensorflow` pseudo-random generator at a fixed value\n",
        "tf.random.set_seed(seed_value)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PWNCyRs4_eet"
      },
      "outputs": [],
      "source": [
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set column display limit if needed\n",
        "from google.colab.data_table import DataTable\n",
        "DataTable.max_columns = 30"
      ],
      "metadata": {
        "id": "uZCWR3qtTu9H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load the IEEE 2014 PHM Data Challenge Dataset"
      ],
      "metadata": {
        "id": "Bj7lvGyqF7Z7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "38Hfi93tSBVk"
      },
      "outputs": [],
      "source": [
        "# IEEE PHM 2014 Data Challenge Dataset without ripples\n",
        "df_part1 = pd.read_csv(r'/content/drive/MyDrive/IEEE2014DataChallengeData/FC1_Without_Ripples/FC1_Ageing_part1.csv', encoding = \"ISO-8859-1\")\n",
        "df_part2 = pd.read_csv(r'/content/drive/MyDrive/IEEE2014DataChallengeData/FC1_Without_Ripples/FC1_Ageing_part2.csv', encoding = \"ISO-8859-1\")\n",
        "df_part3 = pd.read_csv(r'/content/drive/MyDrive/IEEE2014DataChallengeData/FC1_Without_Ripples/FC1_Ageing_part3.csv', encoding = \"ISO-8859-1\")\n",
        "frames = [df_part1, df_part2, df_part3]\n",
        "phm_dataset_1 = pd.concat(frames, ignore_index = True).astype('float32')\n",
        "# Elucidate statistical features\n",
        "# phm_dataset_1.describe().transpose()\n",
        "# Plot stack voltage as a function of time\n",
        "# plt.plot(list(phm_dataset['Time (h)']), list(phm_dataset['Utot (V)']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8K4I-_chlRjH"
      },
      "outputs": [],
      "source": [
        "# IEEE PHM 2014 Data Challenge Dataset with ripples\n",
        "df_part1 = pd.read_csv(r'/content/drive/MyDrive/IEEE2014DataChallengeData/Full_FC2_With_Ripples/FC2_Ageing_part1.csv', encoding = \"ISO-8859-1\")\n",
        "df_part2 = pd.read_csv(r'/content/drive/MyDrive/IEEE2014DataChallengeData/Full_FC2_With_Ripples/FC2_Ageing_part2.csv', encoding = \"ISO-8859-1\")\n",
        "phm_dataset_2 = pd.concat([df_part1, df_part2], ignore_index = True).dropna(axis=1, how='all').astype('float32')\n",
        "# Elucidate statistical features\n",
        "# phm_dataset_2.describe().transpose()\n",
        "# Plot stack voltage as a function of time\n",
        "# plt.plot(list(phm_dataset_2['Time (h)']), list(phm_dataset_2['Utot (V)']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gwM3mA1pnLxO"
      },
      "outputs": [],
      "source": [
        "def series_to_supervised(data, n_in = 1, n_out = 1, dropnan = True):\n",
        "\t\"\"\"\n",
        "  Multivariate Time Series\n",
        "  Convert series to supervised learning\n",
        "  Standard practice in time series forecasting to use lagged observations (e.g. t-1) as input variables to forecast the current time step (t)\n",
        "  Code sourced from https://machinelearningmastery.com/convert-time-series-supervised-learning-problem-python/\n",
        "\t\"\"\"\n",
        "\tn_vars = 1 if type(data) is list else data.shape[1]\n",
        "\tdf = pd.DataFrame(data)\n",
        "\tcols, names = list(), list()\n",
        "\t# input sequence (t-n, ... t-1)\n",
        "\tfor i in range(n_in, 0, -1):\n",
        "\t\tcols.append(df.shift(i))\n",
        "\t\tnames += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]\n",
        "\t# forecast sequence (t, t+1, ... t+n)\n",
        "\tfor i in range(0, n_out):\n",
        "\t\tcols.append(df.shift(-i))\n",
        "\t\tif i == 0:\n",
        "\t\t\tnames += [('var%d(t)' % (j+1)) for j in range(n_vars)]\n",
        "\t\telse:\n",
        "\t\t\tnames += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]\n",
        "\tagg = pd.concat(cols, axis = 1)\n",
        "\tagg.columns = names\n",
        "\t# drop rows with NaN values\n",
        "\tif dropnan:\n",
        "\t\tagg.dropna(inplace = True)\n",
        "\treturn agg"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ao8KRQHaHoaJ"
      },
      "source": [
        "# Model Implementation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_LYhz3F9Ho24"
      },
      "outputs": [],
      "source": [
        "BATCH_SIZE = 72\n",
        "class Neural_Networks():\n",
        "    \"\"\"\n",
        "    Implements structure, training, and validation of neural network models.\n",
        "    \"\"\"\n",
        "    def __init__(self, x_train, y_train, x_valid, y_valid, x_test, y_test, volt_scaler) -> None:\n",
        "        \"\"\"\n",
        "        volt_scaler is the MinMaxScaler object used to normalize Utot(V) in range (0,1)\n",
        "        \"\"\"\n",
        "        self.mae = keras.losses.MeanAbsoluteError()\n",
        "        self.x_train, self.y_train = x_train, y_train  # Training set\n",
        "        self.x_valid, self.y_valid = x_valid, y_valid # Validation set\n",
        "        self.x_test, self.y_test = x_test, y_test   # Test set\n",
        "        self.volt_scaler = volt_scaler\n",
        "        # Hyperparameter tuning results for static loading dataset with 50% data as training set\n",
        "        # stored as attributes for easy accessibility\n",
        "        self.opt_LSTM_units, self.opt_GRU_units = 75, 92\n",
        "        self.opt_LSTM_LR, self.opt_GRU_LR = 0.00045967464871265887, 0.0006899549137745159 #Adam\n",
        "        self.opt_LSTM_dropout, self.opt_GRU_dropout = 0.10204101575985941, 0.06017063859125542\n",
        "        self.opt_CNN_LSTM_units, self.opt_CNN_GRU_units = 108, 102\n",
        "        self.opt_CNN_LSTM_filters, self.opt_CNN_GRU_filters = 93, 97\n",
        "        self.opt_CNN_LSTM_LR, self.opt_CNN_GRU_LR = 0.00039458365054797465, 0.0005623902764462512 #Adam\n",
        "        self.opt_CNN_LSTM_dropout, self.opt_CNN_GRU_dropout = 0.21587038777591175, 0.06863870820110385\n",
        "        self.opt_CNN_BidirLSTM_units, self.opt_CNN_BidirGRU_units = 100, 71\n",
        "        self.opt_CNN_BidirLSTM_filters, self.opt_CNN_BidirGRU_filters = 79, 95\n",
        "        self.opt_CNN_BidirLSTM_LR, self.opt_CNN_BidirGRU_LR = 0.0001415542408060021, 0.00011281449394542564 #Adam\n",
        "        self.opt_CNN_BidirLSTM_dropout, self.opt_CNN_BidirGRU_dropout = 0.311649235809069, 0.22507334379858374\n",
        "\n",
        "\n",
        "    def build_LSTM_model(self, hidden_neurons, activator, drop_out, optimizer):\n",
        "        \"\"\"\n",
        "        LSTM Network.\n",
        "        Use bidir_LSTM_layer in keras.Sequential() to implement a Bidirectional LSTM.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        LSTM_layer = keras.layers.LSTM(hidden_neurons, input_shape = (self.x_train.shape[1], self.x_train.shape[2]),\n",
        "                dropout = drop_out, activation = activator)\n",
        "        # For bidirectional LSTM, move input_shape argument from LSTM layer to Bidirectional\n",
        "        # bidir_LSTM_layer = keras.layers.Bidirectional(LSTM_layer)\n",
        "        # Dropout added as validation loss increasing => DNN Overfitting\n",
        "        output_layer = keras.layers.Dense(output_neurons)\n",
        "        lstm_model = keras.Sequential(layers = (LSTM_layer, output_layer))\n",
        "        lstm_model.reset_states()\n",
        "        lstm_model.compile(optimizer = optimizer, loss = self.mae)\n",
        "        lstm_model.summary()\n",
        "        return lstm_model\n",
        "\n",
        "\n",
        "    def build_GRU_model(self, hidden_neurons, activator, drop_out, OPTIMIZER = 'adam'):\n",
        "        \"\"\"\n",
        "        GRU Network.\n",
        "        Default OPTIMIZER used when not supplied.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        GRU_layer = keras.layers.GRU(hidden_neurons, dropout = drop_out, activation = activator)\n",
        "        gru_model = keras.Sequential(layers = (GRU_layer, keras.layers.Dense(output_neurons)))\n",
        "        gru_model.reset_states()\n",
        "        gru_model.compile(optimizer = OPTIMIZER, loss = self.mae)\n",
        "        return gru_model\n",
        "\n",
        "\n",
        "    def build_CNN_LSTM_model(self, hidden_neurons, activator_CNN, drop_out, filters = 64, OPTIMIZER = 'adam', activator_LSTM = 'tanh'):\n",
        "        \"\"\"\n",
        "        LSTM-CNN Network.\n",
        "        Default values of filters, OPTIMIZER, activator_LSTM used when not supplied.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        LSTM1 = keras.layers.LSTM(hidden_neurons, dropout = drop_out, activation = activator_LSTM)\n",
        "        conv1D = keras.layers.Conv1D(filters = filters, kernel_size = 3, strides = 1, activation = activator_CNN,\n",
        "                padding = 'causal', input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        cnn_lstm_model = keras.Sequential(layers = (conv1D, LSTM1, keras.layers.Dense(output_neurons)))\n",
        "        cnn_lstm_model.reset_states()\n",
        "        cnn_lstm_model.compile(optimizer = OPTIMIZER, loss = self.mae)\n",
        "        return cnn_lstm_model\n",
        "\n",
        "\n",
        "    def build_CNN_BidirLSTM_model(self, hidden_neurons, activator_CNN, drop_out, filters = 64, OPTIMIZER = 'adam', activator_LSTM = 'tanh'):\n",
        "        \"\"\"\n",
        "        Bidirectional LSTM-CNN Network.\n",
        "        Default values of filters, OPTIMIZER, and activator_LSTM used when not supplied.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        LSTM1 = keras.layers.LSTM(hidden_neurons, dropout = drop_out, activation = activator_LSTM)\n",
        "        bidir_LSTM1 = keras.layers.Bidirectional(LSTM1, input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        conv1D = keras.layers.Conv1D(filters = filters, kernel_size = 3, strides = 1, activation = activator_CNN,\n",
        "                padding = 'causal', input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        cnn_bidirlstm_model = keras.Sequential(layers = (conv1D, bidir_LSTM1, keras.layers.Dense(output_neurons)))\n",
        "        cnn_bidirlstm_model.reset_states()\n",
        "        cnn_bidirlstm_model.compile(optimizer = OPTIMIZER, loss = self.mae)\n",
        "        return cnn_bidirlstm_model\n",
        "\n",
        "\n",
        "    def build_CNN_GRU_model(self, hidden_neurons, activator, drop_out, filters = 64, OPTIMIZER = 'adam'):\n",
        "        \"\"\"\n",
        "        1D-CNN-GRU Network.\n",
        "        Default values of filters and OPTIMIZER used when not supplied.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        GRU_layer = keras.layers.GRU(hidden_neurons, dropout = drop_out)\n",
        "        conv1D = keras.layers.Conv1D(filters = filters, kernel_size = 3, strides = 1, activation = activator,\n",
        "                padding = 'causal', input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        cnn_gru_model = keras.Sequential(layers = (conv1D, GRU_layer, keras.layers.Dense(output_neurons)))\n",
        "        cnn_gru_model.reset_states()\n",
        "        cnn_gru_model.compile(optimizer = OPTIMIZER, loss = self.mae)\n",
        "        return cnn_gru_model\n",
        "\n",
        "\n",
        "    def build_CNN_BidirGRU_model(self, hidden_neurons, activator, drop_out, filters = 64, OPTIMIZER = 'adam'):\n",
        "        \"\"\"\n",
        "        1D-CNN-Bidirectional GRU.\n",
        "        Default values of filters and OPTIMIZER used when not supplied.\n",
        "        \"\"\"\n",
        "        keras.backend.clear_session()\n",
        "        output_neurons = 1\n",
        "        GRU_layer = keras.layers.GRU(hidden_neurons, dropout = drop_out)\n",
        "        bidir_GRU = keras.layers.Bidirectional(GRU_layer, input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        conv1D = keras.layers.Conv1D(filters = filters, kernel_size = 3, strides = 1, activation = activator,\n",
        "                padding = 'causal', input_shape = (self.x_train.shape[1], self.x_train.shape[2]))\n",
        "        cnn_bidirgru_model = keras.Sequential(layers = (conv1D, bidir_GRU, keras.layers.Dense(output_neurons)))\n",
        "        cnn_bidirgru_model.reset_states()\n",
        "        cnn_bidirgru_model.compile(optimizer = OPTIMIZER, loss = self.mae)\n",
        "        return cnn_bidirgru_model\n",
        "\n",
        "\n",
        "    def train_model(self, model, epoch_size = 150, BATCH_SIZE = BATCH_SIZE):\n",
        "        \"\"\"\n",
        "        Train the model with specified epochs and batch size.\n",
        "        Default values of epoch_size and BATCH_SIZE used when not supplied.\n",
        "        \"\"\"\n",
        "        early_stop = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 30, mode = 'min')\n",
        "        history = model.fit(self.x_train, self.y_train, epochs = epoch_size, batch_size = BATCH_SIZE, callbacks = [early_stop],\n",
        "                validation_data=  (self.x_valid, self.y_valid), shuffle = False)\n",
        "        return history\n",
        "\n",
        "\n",
        "    def predict(self, model):\n",
        "        \"\"\"\n",
        "        model: DNN model built\n",
        "        volt_scaler: Scaler used to fit independent variable (stack voltage) data during initial data processing\n",
        "        \"\"\"\n",
        "        # Predict\n",
        "        yhat = model.predict(self.x_test)\n",
        "        x_test = self.x_test.reshape((self.x_test.shape[0], self.x_test.shape[2]))\n",
        "        inv_yhat = np.concatenate((yhat, x_test[:, 1:]), axis=1)\n",
        "        inv_yhat = self.volt_scaler.inverse_transform(inv_yhat)\n",
        "        inv_yhat = inv_yhat[:,0]\n",
        "        # invert scaling for actual\n",
        "        y_test = self.y_test.reshape((len(self.y_test), 1))\n",
        "        inv_y = np.concatenate((y_test, x_test[:, 1:]), axis=1)\n",
        "        inv_y = self.volt_scaler.inverse_transform(inv_y)\n",
        "        inv_y = inv_y[:,0]\n",
        "        return inv_y, inv_yhat\n",
        "\n",
        "\n",
        "    def plot_loss_curves(self, history, epoch_size):\n",
        "        \"\"\"\n",
        "        Visualize Model Results\n",
        "        \"\"\"\n",
        "        training_loss = history.history['loss']\n",
        "        validation_loss = history.history['val_loss']\n",
        "        epochs = range(1, epoch_size + 1)\n",
        "        plt.figure()\n",
        "        plt.plot(epochs, training_loss, label = 'Training Loss')\n",
        "        plt.plot(epochs, validation_loss, label = 'Validation Loss')\n",
        "        plt.legend()\n",
        "        plt.show()\n",
        "\n",
        "\n",
        "    def evaluate_metrics(self, y_true, y_pred):\n",
        "        \"\"\"\n",
        "        Return RMSE, MAPE, R2\n",
        "        \"\"\"\n",
        "        rmse = math.sqrt(mean_squared_error(y_true, y_pred))\n",
        "        mape = mean_absolute_percentage_error(y_true, y_pred)\n",
        "        r2 = r2_score(y_true, y_pred)\n",
        "        return rmse, mape, r2\n",
        "\n",
        "\n",
        "    def evaluate_loss_function(self, y_true, y_pred):\n",
        "        \"\"\"\n",
        "        RMSE is chosen as the loss function for hyperparameter optimization\n",
        "        \"\"\"\n",
        "        return math.sqrt(mean_squared_error(y_true, y_pred))\n",
        "\n",
        "\n",
        "    def loop_model(self, n_loops = 50, df_type = 2):\n",
        "        \"\"\"\n",
        "        Outputs the MAPE, RMSE, R2, and RUL Prediction RE over 50 trials as arrays\n",
        "        Loop - Min, Max, Average of RMSE, MAPE, R2, predicted RUL, and relative error of predicted RUL over number of trials, default 50\n",
        "        df_type = 1 corresponds to the static loading dataset\n",
        "        df_type = 2 corresponds to the dynamic loading dataset\n",
        "        Uncomment the line corresponding to the model you wish to evaluate\n",
        "        \"\"\"\n",
        "        arr_RMSE, arr_MAPE, arr_R2, arr_RUL, arr_RUL_RE = list(), list(), list(), list(), list()\n",
        "        for _ in range(n_loops):\n",
        "          print(\"Trial \", _+1 ,\" of\", n_loops)\n",
        "          model = self.build_LSTM_model(self.opt_LSTM_units, 'tanh', self.opt_LSTM_dropout, tf.keras.optimizers.Adam(learning_rate = self.opt_LSTM_LR))\n",
        "          # model = self.build_GRU_model(self.opt_GRU_units, 'tanh', self.opt_GRU_dropout, tf.keras.optimizers.Adam(learning_rate = self.opt_GRU_LR))\n",
        "          # model = self.build_CNN_LSTM_model(self.opt_CNN_LSTM_units, 'tanh', self.opt_CNN_LSTM_dropout, self.opt_CNN_LSTM_filters, tf.keras.optimizers.Adam(learning_rate = self.opt_CNN_LSTM_LR))\n",
        "          # model = self.build_CNN_GRU_model(self.opt_CNN_GRU_units, 'tanh', self.opt_CNN_GRU_dropout, self.opt_CNN_GRU_filters, tf.keras.optimizers.Adam(learning_rate = self.opt_CNN_GRU_LR))\n",
        "          # model = self.build_CNN_BidirLSTM_model(self.opt_CNN_BidirLSTM_units, 'tanh', self.opt_CNN_BidirLSTM_dropout, self.opt_CNN_BidirLSTM_filters, tf.keras.optimizers.Adam(learning_rate = self.opt_CNN_BidirLSTM_LR))\n",
        "          # model = self.build_CNN_BidirGRU_model(self.opt_CNN_BidirGRU_units, 'tanh', self.opt_CNN_BidirGRU_dropout, self.opt_CNN_BidirGRU_filters, tf.keras.optimizers.Adam(learning_rate = self.opt_CNN_BidirGRU_LR))\n",
        "          history = self.train_model(model, epoch_size = 150, BATCH_SIZE = BATCH_SIZE)\n",
        "          y_true, y_pred = self.predict(model)\n",
        "          rmse, mape, r2 = self.evaluate_metrics(y_true, y_pred)\n",
        "          # initial voltage for threshold chosen as first voltage in training set of each dataset\n",
        "          threshold_4perc = 3.1971 if df_type == 2 else 3.2028\n",
        "          validation_h = len(self.y_valid)\n",
        "          true_failure_h = [i for i,v in enumerate(y_true) if v < threshold_4perc][0]\n",
        "          true_RUL = validation_h + true_failure_h\n",
        "          try:\n",
        "            pred_failure_h = [i for i,v in enumerate(y_pred) if v < threshold_4perc][0]\n",
        "          except IndexError:\n",
        "            pred_failure_h = len(y_pred)\n",
        "          pred_RUL = validation_h + pred_failure_h\n",
        "          arr_RUL.append(pred_RUL) # RUL = validation set time + time to failure\n",
        "          arr_RUL_RE.append(abs(pred_RUL - true_RUL)/true_RUL)\n",
        "          print(f\"RMSE: {rmse}, MAPE: {mape}\")\n",
        "          arr_RMSE.append(rmse)\n",
        "          arr_MAPE.append(mape)\n",
        "          arr_R2.append(r2)\n",
        "          del model\n",
        "          keras.backend.clear_session()\n",
        "\n",
        "        arr_RMSE, arr_MAPE, arr_R2, arr_RUL, arr_RUL_RE = np.array(arr_RMSE), np.array(arr_MAPE), np.array(arr_R2), np.array(arr_RUL), np.array(arr_RUL_RE)\n",
        "        min_RMSE, min_MAPE, min_R2, min_RUL, min_RUL_RE = arr_RMSE.min(), arr_MAPE.min(), arr_R2.min(), arr_RUL.min(), arr_RUL_RE.min()\n",
        "        max_RMSE, max_MAPE, max_R2, max_RUL, max_RUL_RE = arr_RMSE.max(), arr_MAPE.max(), arr_R2.max(), arr_RUL.max(), arr_RUL_RE.max()\n",
        "        avg_RMSE, avg_MAPE, avg_R2, avg_RUL, avg_RUL_RE = arr_RMSE.mean(), arr_MAPE.mean(), arr_R2.mean(), arr_RUL.mean(), arr_RUL_RE.mean()\n",
        "        print(\"RMSE Array\", arr_RMSE, end = '\\n')\n",
        "        print(\"MAPE Array\", arr_MAPE, end = '\\n')\n",
        "        print(\"R2 Array\", arr_R2, end = '\\n')\n",
        "        print(\"RUL Array\", arr_RUL, end = '\\n')\n",
        "        print(\"RUL_RE Array\", arr_RUL_RE, end = '\\n')\n",
        "        print(f\"Mean RMSE: {avg_RMSE}, Min RMSE: {min_RMSE}, Max RMSE: {max_RMSE}\")\n",
        "        print(f\"Mean MAPE: {avg_MAPE}, Min MAPE: {min_MAPE}, Max MAPE: {max_MAPE}\")\n",
        "        print(f\"Mean R2: {avg_R2}, Min R2: {min_R2}, Max R2: {max_R2}\")\n",
        "        print(f\"True RUL: {true_RUL}, Min RUL: {min_RUL}, Mean RUL: {avg_RUL}, Max RUL: {max_RUL}\")\n",
        "        print(f\"Min RUL_RE: {min_RUL_RE}, Mean RUL_RE: {avg_RUL_RE}, Max RUL_RE: {max_RUL_RE}\")\n",
        "        return arr_RMSE, arr_MAPE, arr_R2, arr_RUL, arr_RUL_RE, y_pred, y_true"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Neural_Networks_2(Neural_Networks):\n",
        "  \"\"\"\n",
        "  Class for dynamic loading dataset. Methods inherited from class for static loading dataset.\n",
        "  \"\"\"\n",
        "  def __init__(self, x_train, y_train, x_valid, y_valid, x_test, y_test, volt_scaler) -> None:\n",
        "    self.mae = keras.losses.MeanAbsoluteError()\n",
        "    self.x_train, self.y_train = x_train, y_train  # Training set\n",
        "    self.x_valid, self.y_valid = x_valid, y_valid # Validation set\n",
        "    self.x_test, self.y_test = x_test, y_test   # Test set\n",
        "    self.volt_scaler = volt_scaler\n",
        "    # Hyperparameter tuning results for dynamic loading dataset\n",
        "    # stored as attributes for easy accessibility\n",
        "    self.opt_LSTM_units, self.opt_GRU_units = 82, 68\n",
        "    self.opt_LSTM_LR, self.opt_GRU_LR = 0.0003078717031343842, 0.0003398059933836796 #Adam\n",
        "    self.opt_LSTM_dropout, self.opt_GRU_dropout = 0.024500500000141777, 0.03033682440686076\n",
        "    self.opt_CNN_LSTM_units, self.opt_CNN_GRU_units = 104, 107\n",
        "    self.opt_CNN_LSTM_filters, self.opt_CNN_GRU_filters = 92, 88\n",
        "    self.opt_CNN_LSTM_LR, self.opt_CNN_GRU_LR = 0.00040525360755873315, 0.0001806959844971354 #Adam\n",
        "    self.opt_CNN_LSTM_dropout, self.opt_CNN_GRU_dropout = 0.0873443385220109, 0.010175502955337826\n",
        "    self.opt_CNN_BidirLSTM_units, self.opt_CNN_BidirGRU_units = 97, 107\n",
        "    self.opt_CNN_BidirLSTM_filters, self.opt_CNN_BidirGRU_filters = 91, 100\n",
        "    self.opt_CNN_BidirLSTM_LR, self.opt_CNN_BidirGRU_LR = 0.00017984981754231067, 0.00018921126679578512 #Adam\n",
        "    self.opt_CNN_BidirLSTM_dropout, self.opt_CNN_BidirGRU_dropout = 0.043454063827613305, 0.010853916830585342"
      ],
      "metadata": {
        "id": "-oxJlgem3Rok"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Dataset Processor Class"
      ],
      "metadata": {
        "id": "URrmxF2ZV7Ym"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3hl5bjrqeRTY"
      },
      "outputs": [],
      "source": [
        "class df_handler():\n",
        "  \"\"\"\n",
        "  Class implementing downsampling, Savitzky-Golay filtering,\n",
        "  \"\"\"\n",
        "  def __init__(self, dataset: pd.DataFrame):\n",
        "      self.fc_dataset = dataset\n",
        "\n",
        "\n",
        "  def reconstruct_dataset(self):\n",
        "      \"\"\"\n",
        "      Downsample the dataset at 1 h interval.\n",
        "      Returns the sampled time indices, stack voltage, dataset, and features\n",
        "      \"\"\"\n",
        "      time_original = self.fc_dataset['Time (h)']\n",
        "      sampled_indexes = self.fc_dataset.astype('int32').drop_duplicates(subset = ['Time (h)']).index\n",
        "      sampled_df = self.fc_dataset.iloc[sampled_indexes, :]\n",
        "      time_h = sampled_df['Time (h)']\n",
        "      volt_total = sampled_df['Utot (V)'].values.tolist()\n",
        "      features = sampled_df.copy(deep = True).drop(labels = ['Time (h)', 'Utot (V)'], axis = 1)\n",
        "      return time_h, volt_total, sampled_df, features\n",
        "\n",
        "\n",
        "  def smooth_dataset(self, sampled_df):\n",
        "      \"\"\"\n",
        "      Smooths the (sampled) dataset by Savitzky-Golay smoothing\n",
        "      The window length and polynomial order may be tweaked by the variables window_length and polynomial_order, respectively.\n",
        "      \"\"\"\n",
        "      window_length = 21\n",
        "      polynomial_order = 2\n",
        "      smoothed_dataset = pd.DataFrame()\n",
        "      for col in sampled_df.columns[1:]: #Exclude Time (h) column\n",
        "        smoothed_dataset[col] = scipy.signal.savgol_filter(sampled_df[col], window_length, polynomial_order)\n",
        "      return smoothed_dataset\n",
        "\n",
        "\n",
        "  def train_test_split(self, train_frac, smoothed_dataset):\n",
        "      \"\"\"\n",
        "      Splits dataset into train-validation-test sets after smoothing features\n",
        "      50% (520 h) train, 10% validation (115 h), 40% test (519 h) for FC1: wihout ripple dataset\n",
        "      50% (460 h) train, 10% validation (102 h), 40% test (458 h) for FC2: ripple dataset\n",
        "      train_frac: fraction of data to be used for training; of the remaining datset, 10% is for validation and the rest is the test set\n",
        "      \"\"\"\n",
        "      sup_df = series_to_supervised(smoothed_dataset)\n",
        "      num_cols = len(smoothed_dataset.columns)\n",
        "      n = len(sup_df)\n",
        "      # Drop variables not to predict at the next time step; predict only the stack's total voltage.\n",
        "      # Do not drop column 29 with output voltage\n",
        "      sup_dataset = sup_df.drop(columns = sup_df.columns[list(_ for _ in range(num_cols, 2*num_cols) if _ != 29)], axis = 1)\n",
        "      sup_values = sup_dataset.values\n",
        "      sup_train, sup_valid, sup_test = sup_values[:int(n*train_frac),:], sup_values[int(n*train_frac):int(n*(train_frac+0.1)),:], sup_values[int(n*(train_frac+0.1)):,:]\n",
        "      features_train, features_valid, features_test = sup_train[:, :-1], sup_valid[:, :-1], sup_test[:, :-1]\n",
        "      train_y, valid_y, test_y = sup_train[:, -1], sup_valid[:, -1], sup_test[:, -1]\n",
        "      return features_train, train_y, features_valid, valid_y, features_test, test_y\n",
        "\n",
        "\n",
        "  def normalize_split_sets(self, features_train, features_valid, features_test, train_y, valid_y, test_y):\n",
        "      \"\"\"\n",
        "      Normalize the dataset features in the range (0, 1)\n",
        "      \"\"\"\n",
        "      # Normalize features\n",
        "      feature_scaler = MinMaxScaler(feature_range = (0, 1))\n",
        "      normalized_train_features = feature_scaler.fit_transform(features_train)\n",
        "      normalized_valid_features = feature_scaler.transform(features_valid)\n",
        "      normalized_test_features = feature_scaler.transform(features_test)\n",
        "      normalized_train_features = normalized_train_features.reshape((normalized_train_features.shape[0], 1, normalized_train_features.shape[1]))\n",
        "      normalized_valid_features = normalized_valid_features.reshape((normalized_valid_features.shape[0], 1, normalized_valid_features.shape[1]))\n",
        "      normalized_test_features = normalized_test_features.reshape((normalized_test_features.shape[0], 1, normalized_test_features.shape[1]))\n",
        "      # Normalize output voltage\n",
        "      volt_scaler = MinMaxScaler(feature_range = (0, 1))\n",
        "      normalized_train_volt = volt_scaler.fit_transform(train_y.reshape(train_y.shape[0], 1))\n",
        "      normalized_valid_volt = volt_scaler.transform(valid_y.reshape(valid_y.shape[0], 1))\n",
        "      normalized_test_volt = volt_scaler.transform(test_y.reshape(test_y.shape[0], 1))\n",
        "      return (normalized_train_features, normalized_train_volt, normalized_valid_features, normalized_valid_volt,\n",
        "                              normalized_test_features, normalized_test_volt, volt_scaler)\n",
        "\n",
        "\n",
        "  def plot_prediction_results(self, y_train, y_valid, y_test, y_true, y_pred, true_RUL, pred_RUL):\n",
        "      \"\"\"\n",
        "      Plot the predicted voltage as a function of time\n",
        "      \"\"\"\n",
        "      training_time_arr = [_ for _ in range(len(y_train))]\n",
        "      validation_time_arr = [_ for _ in range(len(y_train), len(y_train) + len(y_valid))]\n",
        "      prediction_time_arr = [_ for _ in range(len(y_train) + len(y_valid), len(y_train) + len(y_valid) + len(y_test))]\n",
        "      plt.axvline(x = training_time_arr[0], linestyle = '--')\n",
        "      plt.axvline(x = training_time_arr[-1], linestyle = '--')\n",
        "      plt.axvline(x = validation_time_arr[-1], linestyle = '--')\n",
        "      #plt.axvline(color= 'green', x = true_RUL, linestyle = '--')\n",
        "      #plt.axvline(color = 'red', x = pred_RUL, linestyle = '--')\n",
        "      plt.ylabel('Stack Voltage (V)')\n",
        "      plt.xlabel('Time (h)')\n",
        "      plt.plot(training_time_arr, y_train, label = 'Training voltage')\n",
        "      plt.plot(validation_time_arr, y_valid, label = 'Validation voltage')\n",
        "      plt.plot(prediction_time_arr, y_true, label = 'True voltage')\n",
        "      plt.plot(prediction_time_arr, y_pred, label = 'Predicted voltage')\n",
        "      plt.legend(loc = 'upper right', prop = {'size': 8})"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Preprocess the Dataset"
      ],
      "metadata": {
        "id": "YCOAizU9V_wW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pjubn2Ildq3M"
      },
      "outputs": [],
      "source": [
        "# Reconstruct and smooth dataset 1\n",
        "fc1 = df_handler(phm_dataset_1)\n",
        "time_1, volt_1, df_1, features_1 = fc1.reconstruct_dataset()\n",
        "smoothed_df_1 = fc1.smooth_dataset(df_1)\n",
        "x_train_1, y_train_1, x_valid_1, y_valid_1, x_test_1, y_test_1 = fc1.train_test_split(0.5, smoothed_df_1)\n",
        "x_train_s1, y_train_s1, x_valid_s1, y_valid_s1, x_test_s1, y_test_s1, volt_scaler_1 = \\\n",
        "                        fc1.normalize_split_sets(x_train_1, x_valid_1, x_test_1, y_train_1, y_valid_1, y_test_1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y30_D_jviD_Q"
      },
      "outputs": [],
      "source": [
        "# Reconstruct and smooth dataset 2\n",
        "fc2 = df_handler(phm_dataset_2)\n",
        "time_2, volt_2, df_2, features_2 = fc2.reconstruct_dataset()\n",
        "smoothed_df_2 = fc2.smooth_dataset(df_2)\n",
        "x_train_2, y_train_2, x_valid_2, y_valid_2, x_test_2, y_test_2 = fc2.train_test_split(0.5, smoothed_df_2)\n",
        "x_train_s2, y_train_s2, x_valid_s2, y_valid_s2, x_test_s2, y_test_s2, volt_scaler_2 = \\\n",
        "                        fc2.normalize_split_sets(x_train_2, x_valid_2, x_test_2, y_train_2, y_valid_2, y_test_2)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## PLOT FC1 RAW, FILTERED, AND SAMPLED VOLTAGES ##\n",
        "plt.figure()\n",
        "plt.plot(phm_dataset_1['Time (h)'], phm_dataset_1['Utot (V)'], 'orange', label = 'Raw voltage')\n",
        "plt.plot(time_1, volt_1, 'tab:blue', label = 'Sampled voltage')\n",
        "plt.plot(time_1, smoothed_df_1['Utot (V)'], 'lime', label = 'Filtered voltage')\n",
        "plt.xlabel('Time (h)')\n",
        "plt.ylabel('Stack Voltage (V)')\n",
        "plt.legend(loc = 'upper right')\n",
        "plt.grid(visible=True, which='major', color = 'darkgray', lw = 0.2)\n",
        "# Download the figure\n",
        "# plt.savefig('demo.png', transparent = True)\n",
        "# from google.colab import files\n",
        "# files.download(\"demo.png\")"
      ],
      "metadata": {
        "id": "k44ksI31r-7c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9jRDvke90rAf"
      },
      "outputs": [],
      "source": [
        "## PLOT FC2 RAW, FILTERED, AND SAMPLED VOLTAGES ##\n",
        "plt.figure()\n",
        "#plt.plot(phm_dataset_2['Time (h)'], phm_dataset_2['Utot (V)'], 'orange', label = 'Raw voltage')\n",
        "plt.plot(time_2, volt_2, 'tab:blue', label = 'Sampled voltage')\n",
        "plt.plot(time_2, smoothed_df_2['Utot (V)'], 'lime', label = 'Filtered voltage')\n",
        "plt.xlabel('Time (h)')\n",
        "plt.ylabel('Stack Voltage (V)')\n",
        "plt.legend(loc = 'upper right')\n",
        "plt.grid(visible=True, which='major', color = 'darkgray', lw = 0.2)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model Evaluation"
      ],
      "metadata": {
        "id": "C1YXNtOsWFGl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vYATa2wtJMoi"
      },
      "outputs": [],
      "source": [
        "# Run to evaluate model performance\n",
        "networks = Neural_Networks(x_train_s1, y_train_s1, x_valid_s1, y_valid_s1, x_test_s1, y_test_s1, volt_scaler_1)\n",
        "# networks = Neural_Networks_2(x_train_s2, y_train_s2, x_valid_s2, y_valid_s2, x_test_s2, y_test_s2, volt_scaler_2)\n",
        "# networks = Neural_Networks(x_train_s1, y_train_s1, x_valid_s2, y_valid_s2, x_test_s2, y_test_s2, volt_scaler_2)\n",
        "# Evaluate RMSE, MAPE, R2\n",
        "arr_RMSE, arr_MAPE, arr_R2, arr_RUL, arr_RUL_RE, y_pred, y_true = networks.loop_model(1,2)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KUZAZ7eNzMjn"
      },
      "source": [
        "#Hyperparameter Optimization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iJyIBYbqzPgi"
      },
      "outputs": [],
      "source": [
        "class optuna_search():\n",
        "    \"\"\"\n",
        "    Class employing the Optuna Python package for hyperparameter optimization\n",
        "    The cuDNN GPU accelerator is not applied to LSTM/GRU layers not meeting specified criteria (https://stackoverflow.com/a/68846093)\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, x_train, y_train, x_valid, y_valid, x_test, y_test, scaler) -> None:\n",
        "        self.neural_network = Neural_Networks(x_train, y_train, x_valid, y_valid, x_test, y_test, scaler)\n",
        "        self.x_train = x_train\n",
        "        self.y_train = y_train\n",
        "        self.x_test = x_test\n",
        "        self.y_test = y_test\n",
        "        self.scaler = scaler\n",
        "\n",
        "\n",
        "    def build_deep_LSTM_model(self, trial):\n",
        "        \"\"\"\n",
        "        Function to build LSTM models with arbitrary number of hidden layers.\n",
        "        Definition changed each time a new model with arbitrary number of hidden layers is optimized.\n",
        "        Used for pure LSTM and GRU models only\n",
        "        \"\"\"\n",
        "        output_neurons = 1\n",
        "        layers = list()\n",
        "        optimizer =  self.create_optimizer(trial)\n",
        "        n_layers = trial.suggest_int(\"n_hidden_layers\", 1, 10)\n",
        "        drop_out = trial.suggest_float('dropout', 0.01, 0.5)\n",
        "        if n_layers > 1:\n",
        "          layers = [keras.layers.LSTM(trial.suggest_int(\"n_units_L{}\".format(_), 30, 100),\n",
        "                                            input_shape = (self.x_train.shape[1], self.x_train.shape[2]), return_sequences = True) for _ in range(n_layers)]\n",
        "        layers.append(keras.layers.LSTM(trial.suggest_int(\"n_units_L{}\".format(n_layers), 30, 100),\n",
        "                                        input_shape = (self.x_train.shape[1], self.x_train.shape[2]), dropout = drop_out))\n",
        "        layers.append(keras.layers.Dense(output_neurons))\n",
        "        deep_model = keras.Sequential(layers = layers)\n",
        "        deep_model.reset_states()\n",
        "        deep_model.compile(optimizer = optimizer, loss = 'mae')\n",
        "        return deep_model\n",
        "\n",
        "\n",
        "    def build_deep_GRU_model(self, trial):\n",
        "        \"\"\"\n",
        "        Function to build GRU models with arbitrary number of hidden layers.\n",
        "        \"\"\"\n",
        "        output_neurons = 1\n",
        "        layers = list()\n",
        "        optimizer =  self.create_optimizer(trial)\n",
        "        n_layers = trial.suggest_int(\"n_hidden_layers\", 1, 10)\n",
        "        drop_out = trial.suggest_float('dropout', 0.01, 0.5)\n",
        "        if n_layers > 1:\n",
        "          layers = [keras.layers.GRU(trial.suggest_int(\"n_units_L{}\".format(_), 30, 100),\n",
        "                                            input_shape = (self.x_train.shape[1], self.x_train.shape[2]), return_sequences = True) for _ in range(n_layers)]\n",
        "        layers.append(keras.layers.GRU(trial.suggest_int(\"n_units_L{}\".format(n_layers), 30, 100),\n",
        "                                        input_shape = (self.x_train.shape[1], self.x_train.shape[2]), dropout = drop_out))\n",
        "        layers.append(keras.layers.Dense(output_neurons))\n",
        "        deep_model = keras.Sequential(layers = layers)\n",
        "        deep_model.reset_states()\n",
        "        deep_model.compile(optimizer = optimizer, loss = 'mae')\n",
        "        return deep_model\n",
        "\n",
        "\n",
        "    def create_optimizer(self, trial):\n",
        "        \"\"\"\n",
        "        Create optimizer with specific ranges for hyperparameter search.\n",
        "        optimizer_selected: Optimizer type (RMSprop/Adam/SGD)\n",
        "        trial.suggest_float() locates a hyperparameter sample in the search space using TPE for each trial\n",
        "        \"\"\"\n",
        "        kwargs = {}\n",
        "        optimizer_options = ['Adam', 'SGD', 'RMSprop']\n",
        "        optimizer_selected = trial.suggest_categorical(\"optimizer\", optimizer_options)\n",
        "        if optimizer_selected == \"RMSprop\":\n",
        "            kwargs[\"learning_rate\"] = trial.suggest_float(\n",
        "                \"rmsprop_learning_rate\", 1e-5, 1e-1, log=True)\n",
        "            kwargs[\"decay\"] = trial.suggest_float(\"rmsprop_decay\", 0.85, 0.99)\n",
        "            kwargs[\"momentum\"] = trial.suggest_float(\"rmsprop_momentum\", 1e-5, 1e-1, log=True)\n",
        "        elif optimizer_selected == \"Adam\":\n",
        "            kwargs[\"learning_rate\"] = trial.suggest_float(\"adam_learning_rate\", 1e-5, 1e-1, log=True)\n",
        "        elif optimizer_selected == \"SGD\":\n",
        "            kwargs[\"learning_rate\"] = trial.suggest_float(\n",
        "                \"sgd_opt_learning_rate\", 1e-5, 1e-1, log=True)\n",
        "            kwargs[\"momentum\"] = trial.suggest_float(\"sgd_opt_momentum\", 1e-3, 99e-2, log=True)\n",
        "\n",
        "        optimizer = getattr(tf.keras.optimizers.legacy, optimizer_selected)(**kwargs)\n",
        "        return optimizer\n",
        "\n",
        "\n",
        "    def objective_function(self, trial):\n",
        "        \"\"\"\n",
        "        Returns objective function value to be optimized\n",
        "        Uncomment the line corresponding to the hyperparameters to optimize.\n",
        "        \"\"\"\n",
        "        #dropout = trial.suggest_float('dropout', 0.01, 0.5)\n",
        "        #units = trial.suggest_int('units', 30, 110) #hidden_neurons\n",
        "        #units_1 = trial.suggest_int('units_1', 30, 100) #hidden_neurons\n",
        "        #activation = trial.suggest_categorical(\"activation\", ['relu', 'tanh', 'softsign'])\n",
        "        #activation_1 = trial.suggest_categorical(\"activation\", ['relu', 'tanh', 'softsign'])\n",
        "        #optimizer =  self.create_optimizer(trial)\n",
        "        #filters = trial.suggest_int('filters', 30, 100) #filters for CNN\n",
        "        #test_model = self.build_deep_LSTM_model(trial)\n",
        "        test_model = self.build_deep_GRU_model(trial)\n",
        "        self.neural_network.train_model(test_model)\n",
        "        y_true, y_pred = self.neural_network.predict(test_model)\n",
        "        return self.neural_network.evaluate_loss_function(y_true, y_pred)\n",
        "\n",
        "\n",
        "    def optimize_study(self):\n",
        "        \"\"\"\n",
        "        Configures the hyperparameter optimization settings.\n",
        "        \"\"\"\n",
        "        from optuna.visualization import plot_optimization_history, plot_param_importances\n",
        "        from optuna.importance import get_param_importances\n",
        "        study = optuna.create_study(direction = \"minimize\", sampler = optuna.samplers.TPESampler(),\n",
        "                                    pruner = optuna.pruners.HyperbandPruner())\n",
        "        study.optimize(self.objective_function, n_trials = 500)\n",
        "        plot_optimization_history(study).show()\n",
        "        print(get_param_importances(study))\n",
        "        return study.best_params, study.best_value\n",
        "\n",
        "\n",
        "    def optimize(self):\n",
        "        \"\"\"\n",
        "        Performs the optimization process.\n",
        "        \"\"\"\n",
        "        best_params, best_values = self.optimize_study()\n",
        "        print(f\"Best params: {best_params}\\n Best value: {best_values}\")\n",
        "        return self"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DRVNHA8JMAg6"
      },
      "outputs": [],
      "source": [
        "# Run to optimize hyperparameters\n",
        "param_optimizer = optuna_search(x_train_s1, y_train_s1, x_valid_s1, y_valid_s1, x_test_s1, y_test_s1, volt_scaler_1)\n",
        "param_optimizer.optimize()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}