new_flow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyedflib\n",
    "import os\n",
    "import shutil\n",
    "import glob\n",
    "import datetime\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from numpy import mean, sqrt, square, arange\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.signal import butter, sosfilt, sosfreqz\n",
    "\n",
    "\n",
    "def butter_bandpass(lowcut, highcut, fs, order=5):\n",
    "    nyq = 0.5 * fs\n",
    "    low = lowcut / nyq\n",
    "    high = highcut / nyq\n",
    "    sos = butter(order, [low, high], analog=False, btype='band', output='sos')\n",
    "    return sos\n",
    "\n",
    "\n",
    "def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):\n",
    "    sos = butter_bandpass(lowcut, highcut, fs, order=order)\n",
    "    y = sosfilt(sos, data)\n",
    "    return y\n",
    "\n",
    "\n",
    "# apply bandpass filter\n",
    "\n",
    "fs = 25\n",
    "lowcut = 0.55\n",
    "highcut = 3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>counter</th>\n",
       "      <th>unixTimes</th>\n",
       "      <th>accelerometerX</th>\n",
       "      <th>accelerometerY</th>\n",
       "      <th>accelerometerZ</th>\n",
       "      <th>capacitorsA</th>\n",
       "      <th>capacitorsB</th>\n",
       "      <th>capacitorsC</th>\n",
       "      <th>gyroscopeX</th>\n",
       "      <th>...</th>\n",
       "      <th>sleep_state</th>\n",
       "      <th>Leg Movement</th>\n",
       "      <th>Hypopnea</th>\n",
       "      <th>Arousal</th>\n",
       "      <th>RERA</th>\n",
       "      <th>Relative Desaturation</th>\n",
       "      <th>Obstructive Apnea</th>\n",
       "      <th>Central Apnea</th>\n",
       "      <th>Leg Movement (L)</th>\n",
       "      <th>Leg Movement (R)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2568</td>\n",
       "      <td>2569</td>\n",
       "      <td>1645771253032</td>\n",
       "      <td>1353.0</td>\n",
       "      <td>-472.0</td>\n",
       "      <td>-3934.0</td>\n",
       "      <td>30734.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-149.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2569</td>\n",
       "      <td>2574</td>\n",
       "      <td>1645771253039</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2570</td>\n",
       "      <td>2570</td>\n",
       "      <td>1645771253072</td>\n",
       "      <td>1346.0</td>\n",
       "      <td>-486.0</td>\n",
       "      <td>-3924.0</td>\n",
       "      <td>30733.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-142.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2571</td>\n",
       "      <td>2575</td>\n",
       "      <td>1645771253079</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2572</td>\n",
       "      <td>2571</td>\n",
       "      <td>1645771253112</td>\n",
       "      <td>1342.0</td>\n",
       "      <td>-471.0</td>\n",
       "      <td>-3923.0</td>\n",
       "      <td>30733.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-171.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  counter      unixTimes  accelerometerX  accelerometerY  \\\n",
       "0        2568     2569  1645771253032          1353.0          -472.0   \n",
       "1        2569     2574  1645771253039             NaN             NaN   \n",
       "2        2570     2570  1645771253072          1346.0          -486.0   \n",
       "3        2571     2575  1645771253079             NaN             NaN   \n",
       "4        2572     2571  1645771253112          1342.0          -471.0   \n",
       "\n",
       "   accelerometerZ  capacitorsA  capacitorsB  capacitorsC  gyroscopeX  ...  \\\n",
       "0         -3934.0      30734.0          NaN          NaN      -149.0  ...   \n",
       "1             NaN          NaN          NaN          NaN         NaN  ...   \n",
       "2         -3924.0      30733.0          NaN          NaN      -142.0  ...   \n",
       "3             NaN          NaN          NaN          NaN         NaN  ...   \n",
       "4         -3923.0      30733.0          NaN          NaN      -171.0  ...   \n",
       "\n",
       "   sleep_state  Leg Movement  Hypopnea  Arousal  RERA  Relative Desaturation  \\\n",
       "0         -1.0             0         0        0     0                      0   \n",
       "1         -1.0             0         0        0     0                      0   \n",
       "2         -1.0             0         0        0     0                      0   \n",
       "3         -1.0             0         0        0     0                      0   \n",
       "4         -1.0             0         0        0     0                      0   \n",
       "\n",
       "   Obstructive Apnea  Central Apnea Leg Movement (L) Leg Movement (R)  \n",
       "0                  0              0                0                0  \n",
       "1                  0              0                0                0  \n",
       "2                  0              0                0                0  \n",
       "3                  0              0                0                0  \n",
       "4                  0              0                0                0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf = pd.read_csv(r'prof_data\\resteaze\\00000192-resteaze-sync.csv', sep=',')\n",
    "mdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WK    322643\n",
       "N2    140754\n",
       "N3     55779\n",
       "NS     33179\n",
       "N1     19597\n",
       "Name: sleep_stage, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf = mdf.apply(lambda x: pd.Series(x.dropna().values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = mdf[mdf.sleep_state != -1].reset_index()\n",
    "\n",
    "all_df = all_df[[\n",
    "    'unixTimes', 'accelerometerX', 'accelerometerY', 'accelerometerZ',\n",
    "    'gyroscopeX', 'gyroscopeY', 'gyroscopeZ', 'ledGreen',\n",
    "    'sleep_stage', 'sleep_state'\n",
    "]].dropna()\n",
    "\n",
    "# Binary Classification\n",
    "all_df[\"sleep_state\"] = np.where(all_df[\"sleep_state\"] == 0, 0, 1)\n",
    "\n",
    "# RMS of accelerometer\n",
    "all_df['accelerometer'] = all_df[[\n",
    "    'accelerometerX', 'accelerometerY', 'accelerometerZ'\n",
    "]].apply(lambda x: sqrt(square(x['accelerometerX']) + square(x['accelerometerY']) + square(x['accelerometerZ'])), axis=1)\n",
    "\n",
    "# RMS of gyroscope\n",
    "all_df['gyroscope'] = all_df[[\n",
    "    'gyroscopeX', 'gyroscopeY', 'gyroscopeZ'\n",
    "]].apply(lambda x: sqrt(square(x['gyroscopeX']) + square(x['gyroscopeY']) + square(x['gyroscopeZ'])), axis=1)\n",
    "\n",
    "all_df = all_df.drop(columns=['accelerometerX', 'accelerometerY', 'accelerometerZ', 'gyroscopeX', 'gyroscopeY', 'gyroscopeZ'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df['ledGreen'] = butter_bandpass_filter(all_df['ledGreen'],\n",
    "                                            lowcut,\n",
    "                                            highcut,\n",
    "                                            fs,\n",
    "                                            order=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unixTimes</th>\n",
       "      <th>ledGreen</th>\n",
       "      <th>sleep_stage</th>\n",
       "      <th>sleep_state</th>\n",
       "      <th>accelerometer</th>\n",
       "      <th>gyroscope</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1645771913034</td>\n",
       "      <td>119.082818</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4172.718418</td>\n",
       "      <td>297.546635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1645771913034</td>\n",
       "      <td>658.482111</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4176.995451</td>\n",
       "      <td>296.528245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1645771913074</td>\n",
       "      <td>1701.987486</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4182.306302</td>\n",
       "      <td>299.315887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1645771913074</td>\n",
       "      <td>2813.322086</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4182.463389</td>\n",
       "      <td>316.483807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1645771913114</td>\n",
       "      <td>3399.263287</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4171.734052</td>\n",
       "      <td>296.447297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250744</th>\n",
       "      <td>1645776902232</td>\n",
       "      <td>21.508890</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4198.440306</td>\n",
       "      <td>402.203928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250745</th>\n",
       "      <td>1645776902234</td>\n",
       "      <td>33.029222</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4199.829163</td>\n",
       "      <td>417.749925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250746</th>\n",
       "      <td>1645776902273</td>\n",
       "      <td>43.759520</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4196.294318</td>\n",
       "      <td>425.135273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250747</th>\n",
       "      <td>1645776902274</td>\n",
       "      <td>54.575285</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4205.252192</td>\n",
       "      <td>412.175933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250748</th>\n",
       "      <td>1645776902313</td>\n",
       "      <td>65.288622</td>\n",
       "      <td>WK</td>\n",
       "      <td>0</td>\n",
       "      <td>4208.247260</td>\n",
       "      <td>413.226330</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>250749 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            unixTimes     ledGreen sleep_stage  sleep_state  accelerometer  \\\n",
       "0       1645771913034   119.082818          WK            0    4172.718418   \n",
       "1       1645771913034   658.482111          WK            0    4176.995451   \n",
       "2       1645771913074  1701.987486          WK            0    4182.306302   \n",
       "3       1645771913074  2813.322086          WK            0    4182.463389   \n",
       "4       1645771913114  3399.263287          WK            0    4171.734052   \n",
       "...               ...          ...         ...          ...            ...   \n",
       "250744  1645776902232    21.508890          WK            0    4198.440306   \n",
       "250745  1645776902234    33.029222          WK            0    4199.829163   \n",
       "250746  1645776902273    43.759520          WK            0    4196.294318   \n",
       "250747  1645776902274    54.575285          WK            0    4205.252192   \n",
       "250748  1645776902313    65.288622          WK            0    4208.247260   \n",
       "\n",
       "         gyroscope  \n",
       "0       297.546635  \n",
       "1       296.528245  \n",
       "2       299.315887  \n",
       "3       316.483807  \n",
       "4       296.447297  \n",
       "...            ...  \n",
       "250744  402.203928  \n",
       "250745  417.749925  \n",
       "250746  425.135273  \n",
       "250747  412.175933  \n",
       "250748  413.226330  \n",
       "\n",
       "[250749 rows x 6 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WK    250749\n",
       "Name: sleep_stage, dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df['sleep_stage'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = all_df.drop(['unixTimes', 'sleep_stage', 'sleep_state'], axis=1)\n",
    "Y = all_df['sleep_state']\n",
    "Y = LabelEncoder().fit_transform(Y)\n",
    "X = StandardScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combine all the features\n",
    "features = np.sum(X, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(250749,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['WK'], dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df['sleep_stage'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.0 ('tf')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "7884e113244ffff6edd713e86b3c7e6f0d272676a6838c0ee96a8e2f769e2153"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}