dataWrangling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark\n",
    "from pyspark.context import SparkContext\n",
    "from pyspark.sql import SparkSession, Window\n",
    "from pyspark.sql import functions as f\n",
    "from pyspark.sql.functions import col, signum\n",
    "\n",
    "spark = SparkSession.builder.master(\"local[1]\").getOrCreate()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+--------+-------+------+--------+----------+------+------+\n",
      "|  Open|    High|    Low| Close|  Volume|      Time|Status|Ticker|\n",
      "+------+--------+-------+------+--------+----------+------+------+\n",
      "|326.65|  326.67| 326.63|326.63|791327.0|1578622260|    ok|   SPY|\n",
      "|295.99|  295.99| 295.99|295.99|   780.0|1571102520|    ok|   SPY|\n",
      "|306.67|306.6735|306.355|306.45|352821.0|1591139280|    ok|   SPY|\n",
      "|310.05|  310.09| 310.01|310.02| 56166.0|1574295780|    ok|   SPY|\n",
      "|308.88|  308.93| 308.87|308.91| 71974.0|1575415920|    ok|   SPY|\n",
      "+------+--------+-------+------+--------+----------+------+------+\n",
      "only showing top 5 rows\n",
      "\n",
      "200255\n"
     ]
    }
   ],
   "source": [
    "#Need to carry over from day to day\n",
    "#Dollar bars\n",
    "df=spark.read.parquet(\"SPYcandlestick.parquet\")\n",
    "df=df.withColumn(\"Ticker\",f.lit(\"SPY\"))\n",
    "df.show(5)\n",
    "print(df.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+\n",
      "|      DolExch|\n",
      "+-------------+\n",
      "|    526530.09|\n",
      "|1.232265061E7|\n",
      "| 6.16455684E7|\n",
      "|4.233734953E7|\n",
      "|3.878613142E7|\n",
      "+-------------+\n",
      "only showing top 5 rows\n",
      "\n",
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+\n",
      "|  Open|  High|   Low| Close| Volume|      Time|Status|Ticker|           DolExch|        CumDolExch|\n",
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+\n",
      "|297.53|297.53|297.53|297.53|  100.0|1562167500|    ok|   SPY|29752.999999999996|29752.999999999996|\n",
      "| 297.5|297.56| 297.5|297.56| 1500.0|1562167620|    ok|   SPY|          446340.0|          476093.0|\n",
      "|297.51|297.51|297.51|297.51|  100.0|1562167680|    ok|   SPY|           29751.0|          505844.0|\n",
      "| 297.5| 297.5|297.47| 297.5| 1438.0|1562167740|    ok|   SPY|          427805.0|          933649.0|\n",
      "| 297.5|297.51|297.47|297.49| 1634.0|1562167800|    ok|   SPY|486098.66000000003|1419747.6600000001|\n",
      "|297.46|297.47|297.46|297.47|  269.0|1562167860|    ok|   SPY| 80019.43000000001|        1499767.09|\n",
      "|297.47|297.47|297.46|297.47| 4819.0|1562167980|    ok|   SPY|1433507.9300000002|2933275.0200000005|\n",
      "|297.49|297.49|297.49|297.49|  155.0|1562168040|    ok|   SPY|46110.950000000004|2979385.9700000007|\n",
      "|297.49|297.53|297.49|297.53| 5945.0|1562168100|    ok|   SPY|1768815.8499999999|        4748201.82|\n",
      "|297.53|297.53|297.53|297.53|  154.0|1562168160|    ok|   SPY|45819.619999999995|        4794021.44|\n",
      "|297.51|297.51|297.48|297.51| 6500.0|1562168280|    ok|   SPY|         1933815.0|        6727836.44|\n",
      "|297.49| 297.5|297.46|297.49| 3900.0|1562168340|    ok|   SPY|         1160211.0|        7888047.44|\n",
      "|297.53|297.54|297.49|297.53|12661.0|1562168400|    ok|   SPY|3767027.3299999996|     1.165507477E7|\n",
      "|297.53|297.54| 297.5|297.54|11728.0|1562168460|    ok|   SPY|        3489549.12|     1.514462389E7|\n",
      "|297.52|297.52|297.52|297.52| 1055.0|1562168700|    ok|   SPY|          313883.6|     1.545850749E7|\n",
      "| 297.5| 297.5| 297.5| 297.5|  600.0|1562168760|    ok|   SPY|          178500.0|     1.563700749E7|\n",
      "|297.52|297.52|297.52|297.52|  262.0|1562168820|    ok|   SPY| 77950.23999999999|     1.571495773E7|\n",
      "|297.52|297.52|297.52|297.52|  900.0|1562168880|    ok|   SPY|          267768.0|     1.598272573E7|\n",
      "|297.54|297.54|297.52|297.52| 2500.0|1562168940|    ok|   SPY|          743800.0|     1.672652573E7|\n",
      "|297.46|297.46|297.46|297.46|  100.0|1562169060|    ok|   SPY|29745.999999999996|     1.675627173E7|\n",
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n",
      "+--------------------+\n",
      "|          CumDolExch|\n",
      "+--------------------+\n",
      "| 9.310278973979998E7|\n",
      "| 5.810578673321801E9|\n",
      "|1.439130926434330...|\n",
      "|1.502274517295381E10|\n",
      "|1.935903786558241...|\n",
      "+--------------------+\n",
      "only showing top 5 rows\n",
      "\n",
      "+----+\n",
      "|Mark|\n",
      "+----+\n",
      "|   1|\n",
      "|   0|\n",
      "+----+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "df=df.withColumn('DolExch',col('Close')*col('Volume'))\n",
    "df.select(\"DolExch\").distinct().show(5)\n",
    "df=df.withColumn(\"CumDolExch\",f.sum(\"DolExch\").over(Window.partitionBy('Ticker').orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "df.show()\n",
    "df.select(\"CumDolExch\").distinct().show(5)\n",
    "df=df.withColumn('DolBars', col('CumDolExch')%(1E7))#Take data pt every 10M dollars exchanged per stock\n",
    "df=df.withColumn('Mark',f.when(df.DolBars<=df.DolExch,1).otherwise(0))\n",
    "df.select(\"Mark\").distinct().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+------------------+----+-------+\n",
      "|  Open|  High|   Low| Close| Volume|      Time|Status|Ticker|           DolExch|        CumDolExch|           DolBars|Mark|CumMark|\n",
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+------------------+----+-------+\n",
      "|297.53|297.53|297.53|297.53|  100.0|1562167500|    ok|   SPY|29752.999999999996|29752.999999999996|29752.999999999996|   1|      1|\n",
      "| 297.5|297.56| 297.5|297.56| 1500.0|1562167620|    ok|   SPY|          446340.0|          476093.0|          476093.0|   0|      1|\n",
      "|297.51|297.51|297.51|297.51|  100.0|1562167680|    ok|   SPY|           29751.0|          505844.0|          505844.0|   0|      1|\n",
      "| 297.5| 297.5|297.47| 297.5| 1438.0|1562167740|    ok|   SPY|          427805.0|          933649.0|          933649.0|   0|      1|\n",
      "| 297.5|297.51|297.47|297.49| 1634.0|1562167800|    ok|   SPY|486098.66000000003|1419747.6600000001|1419747.6600000001|   0|      1|\n",
      "|297.46|297.47|297.46|297.47|  269.0|1562167860|    ok|   SPY| 80019.43000000001|        1499767.09|        1499767.09|   0|      1|\n",
      "|297.47|297.47|297.46|297.47| 4819.0|1562167980|    ok|   SPY|1433507.9300000002|2933275.0200000005|2933275.0200000005|   0|      1|\n",
      "|297.49|297.49|297.49|297.49|  155.0|1562168040|    ok|   SPY|46110.950000000004|2979385.9700000007|2979385.9700000007|   0|      1|\n",
      "|297.49|297.53|297.49|297.53| 5945.0|1562168100|    ok|   SPY|1768815.8499999999|        4748201.82|        4748201.82|   0|      1|\n",
      "|297.53|297.53|297.53|297.53|  154.0|1562168160|    ok|   SPY|45819.619999999995|        4794021.44|        4794021.44|   0|      1|\n",
      "|297.51|297.51|297.48|297.51| 6500.0|1562168280|    ok|   SPY|         1933815.0|        6727836.44|        6727836.44|   0|      1|\n",
      "|297.49| 297.5|297.46|297.49| 3900.0|1562168340|    ok|   SPY|         1160211.0|        7888047.44|        7888047.44|   0|      1|\n",
      "|297.53|297.54|297.49|297.53|12661.0|1562168400|    ok|   SPY|3767027.3299999996|     1.165507477E7|1655074.7699999996|   1|      2|\n",
      "|297.53|297.54| 297.5|297.54|11728.0|1562168460|    ok|   SPY|        3489549.12|     1.514462389E7| 5144623.890000001|   0|      2|\n",
      "|297.52|297.52|297.52|297.52| 1055.0|1562168700|    ok|   SPY|          313883.6|     1.545850749E7|        5458507.49|   0|      2|\n",
      "| 297.5| 297.5| 297.5| 297.5|  600.0|1562168760|    ok|   SPY|          178500.0|     1.563700749E7|        5637007.49|   0|      2|\n",
      "|297.52|297.52|297.52|297.52|  262.0|1562168820|    ok|   SPY| 77950.23999999999|     1.571495773E7|        5714957.73|   0|      2|\n",
      "|297.52|297.52|297.52|297.52|  900.0|1562168880|    ok|   SPY|          267768.0|     1.598272573E7|        5982725.73|   0|      2|\n",
      "|297.54|297.54|297.52|297.52| 2500.0|1562168940|    ok|   SPY|          743800.0|     1.672652573E7|        6726525.73|   0|      2|\n",
      "|297.46|297.46|297.46|297.46|  100.0|1562169060|    ok|   SPY|29745.999999999996|     1.675627173E7|        6756271.73|   0|      2|\n",
      "+------+------+------+------+-------+----------+------+------+------------------+------------------+------------------+----+-------+\n",
      "only showing top 20 rows\n",
      "\n",
      "+-------+\n",
      "|CumMark|\n",
      "+-------+\n",
      "|     26|\n",
      "|     29|\n",
      "|    474|\n",
      "|    964|\n",
      "|   1677|\n",
      "|   1697|\n",
      "|   1806|\n",
      "|   1950|\n",
      "|   2040|\n",
      "|   2214|\n",
      "+-------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df=df.withColumn('CumMark',f.sum(\"Mark\").over(Window.partitionBy('Ticker').orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "df.show(20)\n",
    "df.select(\"CumMark\").distinct().show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------+--------+--------+--------+----------+------+------+--------------------+--------------------+------------------+----+-------+\n",
      "|    Open|    High|     Low|   Close|  Volume|      Time|Status|Ticker|             DolExch|          CumDolExch|           DolBars|Mark|CumMark|\n",
      "+--------+--------+--------+--------+--------+----------+------+------+--------------------+--------------------+------------------+----+-------+\n",
      "|  297.24|  297.32|  297.23|  297.27|131122.0|1562175840|    ok|   SPY|       3.897863694E7|    9.201479921487E8|147992.14869999886|   1|     26|\n",
      "|  297.34|  297.43|  297.33| 297.415| 86366.0|1562176020|    ok|   SPY|       2.568654389E7|    9.941301698587E8| 4130169.858700037|   1|     29|\n",
      "|  297.79| 297.795|  297.74|297.7515|146296.0|1562360220|    ok|   SPY|4.3559853444000006E7|1.701835819697521...| 8358196.975212097|   1|    474|\n",
      "|  296.38|  296.43|  296.35|  296.43| 79853.0|1562624820|    ok|   SPY|       2.367082479E7|3.144494718092561...| 4947180.925613403|   1|    964|\n",
      "|  298.46|   298.5|  298.46| 298.495| 64795.0|1562790060|    ok|   SPY|      1.9340983525E7|5.772624603771107...| 6246037.711074829|   1|   1677|\n",
      "|  298.57|  298.59|  298.55|  298.57| 44822.0|1562791320|    ok|   SPY|       1.338250454E7|5.800918234991108E10| 9182349.911079407|   1|   1697|\n",
      "|298.4572|  298.52|298.4572|298.4928|104767.0|1562798580|    ok|   SPY|     3.12721951776E7|6.157188233413515...|1882334.1351547241|   1|   1806|\n",
      "|  298.49| 298.515|298.4572|  298.51|124208.0|1562798640|    ok|   SPY|          5803332.91|6.157768566704516E10| 7685667.045158386|   0|   1806|\n",
      "| 299.205|  299.26| 299.135|  299.25| 86656.0|1562866980|    ok|   SPY|         2.5931808E7|6.783055274410758E10| 552744.1075820923|   1|   1950|\n",
      "|  299.11|  299.18|   299.1| 299.145|203635.0|1562872380|    ok|   SPY|6.0916392074999996E7| 7.08576223962395E10| 7622396.239501953|   1|   2040|\n",
      "| 298.335| 298.335|  298.22|  298.25|232626.0|1562883540|    ok|   SPY|        6.93807045E7|7.708821395264009E10| 8213952.640090942|   1|   2214|\n",
      "|  298.73|  298.77|  298.71|  298.75| 59341.0|1562885700|    ok|   SPY|       1.772812375E7|7.778956607736502E10| 9566077.365020752|   1|   2250|\n",
      "|  299.71|299.7197|  299.67|  299.69| 46344.0|1562958720|    ok|   SPY|       1.388883336E7|8.498217935916705E10|2179359.1670532227|   1|   2453|\n",
      "| 299.955|  299.97| 299.885| 299.885| 41246.0|1562962260|    ok|   SPY|1.2369056709999999E7|8.609851950074261E10| 8519500.742614746|   1|   2509|\n",
      "| 299.845|  299.87|  299.83|  299.84| 27028.0|1562963520|    ok|   SPY|          8104075.52|8.647172340098793E10|1723400.9879302979|   1|   2529|\n",
      "| 300.305|  300.31|  300.19|  300.26|199329.0|1563223560|    ok|   SPY|       5.985052554E7|9.826790836646931E10| 7908366.469314575|   1|   2927|\n",
      "|  300.55|  300.62|  300.54|  300.55|223121.0|1563234960|    ok|   SPY|6.7059016550000004E7|1.019073447187982...| 7344718.798278809|   1|   3091|\n",
      "| 299.985|  299.99| 299.915|  299.93|162045.0|1563321120|    ok|   SPY|       4.860215685E7|1.136697184780471...| 9718478.047164917|   1|   3506|\n",
      "|  298.85|  298.86|  298.83|  298.83| 39188.0|1563398100|    ok|   SPY|       1.171055004E7|1.211350708031971...| 5070803.197143555|   1|   3764|\n",
      "| 298.755|  298.81|  298.73|  298.79| 33317.0|1563566400|    ok|   SPY|   9954786.430000002|1.514689114261610...|  8911426.16104126|   1|   4590|\n",
      "+--------+--------+--------+--------+--------+----------+------+------+--------------------+--------------------+------------------+----+-------+\n",
      "only showing top 20 rows\n",
      "\n",
      "Row(Open=297.24, High=297.32, Low=297.23, Close=297.27, Volume=131122.0, Time=1562175840, Status='ok', Ticker='SPY', DolExch=38978636.94, CumDolExch=920147992.1487, DolBars=147992.14869999886, Mark=1, CumMark=26)\n",
      "Row(Open=297.24, High=297.32, Low=297.23, Close=297.27, Volume=131122.0, Time=1562175840, Status='ok', Ticker='SPY', DolExch=38978636.94, CumDolExch=920147992.1487, DolBars=147992.14869999886, Mark=1, CumMark=26)\n",
      "Row(Open=297.53, High=297.53, Low=297.53, Close=297.53, Volume=100.0, Time=1562167500, Status='ok', Ticker='SPY', DolExch=29752.999999999996, CumDolExch=29752.999999999996, DolBars=29752.999999999996, Mark=1, CumMark=1, next_val=297.53, label=0.0)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Row(Ticker='SPY', Time=1562167500, Close=297.53, Volume=100.0, label=0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=df.withColumn(\"Volume\", f.sum(\"Volume\").over(Window.partitionBy(\"CumMark\").orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "\n",
    "df=df.withColumn(\"Low\",f.min(\"Low\").over(Window.partitionBy(\"CumMark\").orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "df.show()\n",
    "print(df.head())\n",
    "df=df.withColumn(\"Open\",f.first(\"Open\").over(Window.partitionBy(\"CumMark\").orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "print(df.head())\n",
    "df=df.withColumn(\"Close\",f.last(\"Close\").over(Window.partitionBy(\"CumMark\").orderBy(\"Time\").rowsBetween(-sys.maxsize,0)))\n",
    "df=df.filter(df.Mark==1)\n",
    "\n",
    "#Add target vector(2 for +change, 1 for no change 0 for -change)\n",
    "df=df.withColumn(\"next_val\", f.lead(col(\"Close\"),1).over(Window.partitionBy('Ticker').orderBy(df[\"Time\"])))#Need to add partitionBy()\n",
    "df=df.withColumn(\"label\", (col(\"next_val\")-col(\"Close\"))/col(\"Close\"))\n",
    "print(df.head())\n",
    "df=df.select(\"Ticker\",\"Time\",\"Close\",\"Volume\",\"label\")#.withColumn(\"Target\", 100*(col(\"next_val\")-col(\"Close\"))/col(\"Close\"))\n",
    "#df=df.drop(\"next_val\").withColumn(\"Target\",1+signum(col(\"Target\")))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'SPY': '1'}\n",
      "root\n",
      " |-- Ticker: string (nullable = false)\n",
      " |-- Time: long (nullable = true)\n",
      " |-- Close: double (nullable = true)\n",
      " |-- Volume: double (nullable = true)\n",
      " |-- label: double (nullable = true)\n",
      "\n",
      "root\n",
      " |-- Ticker: integer (nullable = true)\n",
      " |-- Time: long (nullable = true)\n",
      " |-- Close: double (nullable = true)\n",
      " |-- Volume: double (nullable = true)\n",
      " |-- label: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Convert Ticker value to int to include in ML algo, dictionary present (tick_dict) to convert back if needed\n",
    "tickers=df.select(\"Ticker\").distinct().rdd.flatMap(lambda x: x).collect()\n",
    "\n",
    "tick_dict = {val : str(idx + 1) for idx, val in enumerate(tickers)} \n",
    "print(tick_dict)\n",
    "\n",
    "from pyspark.sql.types import IntegerType\n",
    "df=df.replace(to_replace=tick_dict, subset=['Ticker'])\n",
    "\n",
    "df.printSchema()\n",
    "df=df.withColumn(\"Ticker\",col(\"Ticker\").cast(IntegerType()))\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118183\n",
      "+------+----------+-------+--------+--------------------+--------------------+\n",
      "|Ticker|      Time|  Close|  Volume|               label|            features|\n",
      "+------+----------+-------+--------+--------------------+--------------------+\n",
      "|     1|1562344620| 297.97| 14200.0|-3.35604255462035...|[1.0,297.97,14200.0]|\n",
      "|     1|1562365800| 298.14| 53824.0|2.180183806265436...|[1.0,298.14,53824.0]|\n",
      "|     1|1562696880| 296.26| 47518.0|3.037872139338143E-4|[1.0,296.26,47518.0]|\n",
      "|     1|1562721000| 297.08| 57480.0|1.346438669719283...|[1.0,297.08,57480.0]|\n",
      "|     1|1562781000| 299.34|238514.0|8.351707088940372E-5|[1.0,299.34,23851...|\n",
      "|     1|1562870520| 298.73| 83874.0|-3.34750443544413...|[1.0,298.73,83874.0]|\n",
      "|     1|1562961660| 299.97| 28375.0|1.000100010000090...|[1.0,299.97,28375.0]|\n",
      "|     1|1562970600| 300.18|193117.0|1.332533813046187...|[1.0,300.18,19311...|\n",
      "|     1|1563299640| 300.65| 78073.0|1.663063362714497...|[1.0,300.65,78073.0]|\n",
      "|     1|1563395400|298.895| 43447.0|-8.36414125360988...|[1.0,298.895,4344...|\n",
      "+------+----------+-------+--------+--------------------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.feature import VectorAssembler\n",
    "#Create input features\n",
    "df_cols=df.columns\n",
    "df_cols=[ elem for elem in df_cols if elem not in [\"Time\",\"label\"]]\n",
    "#Move features to a single vector\n",
    "assembler=VectorAssembler(inputCols=df_cols,outputCol=\"features\")\n",
    "df=assembler.transform(df)\n",
    "df=df.na.drop()\n",
    "df=df.dropDuplicates()\n",
    "print(df.count())\n",
    "df.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-7b30673def0d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mfeatureIndexer\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0mVectorIndexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputCol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"features\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputCol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"indexedFeatures\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m# Split the data into training and test sets (last 20% held out for testing)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/pyspark/ml/base.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, dataset, params)\u001b[0m\n\u001b[1;32m    127\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             raise ValueError(\"Params must be either a param map or a list/tuple of param maps, \"\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/pyspark/ml/wrapper.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 321\u001b[0;31m         \u001b[0mjava_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit_java\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    322\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjava_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_copyValues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/pyspark/ml/wrapper.py\u001b[0m in \u001b[0;36m_fit_java\u001b[0;34m(self, dataset)\u001b[0m\n\u001b[1;32m    316\u001b[0m         \"\"\"\n\u001b[1;32m    317\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_transfer_params_to_java\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 318\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_java_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    319\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1301\u001b[0m             \u001b[0mproto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEND_COMMAND_PART\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1303\u001b[0;31m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1304\u001b[0m         return_value = get_return_value(\n\u001b[1;32m   1305\u001b[0m             answer, self.gateway_client, self.target_id, self.name)\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36msend_command\u001b[0;34m(self, command, retry, binary)\u001b[0m\n\u001b[1;32m   1031\u001b[0m         \u001b[0mconnection\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_connection\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1032\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1033\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconnection\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1034\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbinary\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1035\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_connection_guard\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/py4j/java_gateway.py\u001b[0m in \u001b[0;36msend_command\u001b[0;34m(self, command)\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1199\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1200\u001b[0;31m             \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msmart_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1201\u001b[0m             \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Answer received: {0}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0manswer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1202\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0manswer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproto\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRETURN_MESSAGE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.7/lib/python3.7/socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    587\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.regression import RandomForestRegressor\n",
    "from pyspark.ml.feature import VectorIndexer\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "\n",
    "\n",
    "\n",
    "featureIndexer =VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\").fit(df)\n",
    "\n",
    "# Split the data into training and test sets (last 20% held out for testing)\n",
    "df=df.sort(\"Time\")\n",
    "trainingData=df.limit(int(df.count()*0.8))\n",
    "maxTime=trainingData.agg({'Time':'max'}).collect()[0]\n",
    "maxTime=maxTime[\"max(Time)\"]\n",
    "testData=df.filter(df.Time>maxTime)\n",
    "testData.show(10)\n",
    "\n",
    "\n",
    "# Train a RandomForest model.\n",
    "rf = RandomForestRegressor(featuresCol=\"indexedFeatures\")\n",
    "\n",
    "# Chain indexer and forest in a Pipeline\n",
    "pipeline = Pipeline(stages=[featureIndexer, rf])\n",
    "\n",
    "paramGrid = ParamGridBuilder().addGrid(rf.maxDepth, [3, 10, 20]) \\\n",
    "                              .addGrid(rf.numTrees, [5, 10]) \\\n",
    "                              .build()\n",
    "\n",
    "crossval = CrossValidator(estimator=pipeline,\n",
    "                          estimatorParamMaps=paramGrid,\n",
    "                          evaluator=RegressionEvaluator(),\n",
    "                          numFolds=5)\n",
    "\n",
    "# Train model.  This also runs the indexer.\n",
    "model = crossval.fit(trainingData)\n",
    "\n",
    "# Make predictions.\n",
    "predictions = model.transform(testData)\n",
    "\n",
    "# Select example rows to display.\n",
    "predictions.select(\"prediction\", \"label\", \"features\").show(5)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = RegressionEvaluator(\n",
    "    labelCol=\"label\", predictionCol=\"prediction\", metricName=\"r2\")\n",
    "r2 = evaluator.evaluate(predictions)\n",
    "print(\"R Squared on test data = %g\" % r2)\n",
    "\n",
    "#rfModel = model.stages[1]\n",
    "#print(rfModel)  # summary only\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#r2=0.000213893 for SPY_candlestick random split 80/20%\n",
    "#r2=-0.000156503 for SPY_candlestick with first 80% being train data, last 20% is test\n",
    "#r2=9.93879e-05 for \"\" with first 80% being train data, use 5-fold CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#RMSE test> RMSE train -> likely overfitting the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranFor=RandomForestRegressor(labelCol=\"label\", featuresCol=\"features\")\n",
    "modelRf=ranFor.fit(trainingData)\n",
    "print(modelRf.featureImportances) #[Ticker, Close, Volume]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions=predictions.repartition(100)\n",
    "predictions.write.parquet('SPYpred.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(dir(rf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df.agg({\"Time\":'max'}).collect()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int(df.count()*0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testData.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions.filter(predictions.prediction<0).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#IDEA: Add length of bar as a feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ticker</th>\n",
       "      <th>pred_times</th>\n",
       "      <th>Close</th>\n",
       "      <th>Volume</th>\n",
       "      <th>label</th>\n",
       "      <th>features</th>\n",
       "      <th>eval_times</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1562344620</td>\n",
       "      <td>297.97</td>\n",
       "      <td>14200.0</td>\n",
       "      <td>-0.000336</td>\n",
       "      <td>[1.0, 297.97, 14200.0]</td>\n",
       "      <td>1562344680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1562365800</td>\n",
       "      <td>298.14</td>\n",
       "      <td>53824.0</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>[1.0, 298.14, 53824.0]</td>\n",
       "      <td>1562365860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1562696880</td>\n",
       "      <td>296.26</td>\n",
       "      <td>47518.0</td>\n",
       "      <td>0.000304</td>\n",
       "      <td>[1.0, 296.26, 47518.0]</td>\n",
       "      <td>1562696940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1562721000</td>\n",
       "      <td>297.08</td>\n",
       "      <td>57480.0</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>[1.0, 297.08, 57480.0]</td>\n",
       "      <td>1562721060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1562781000</td>\n",
       "      <td>299.34</td>\n",
       "      <td>238514.0</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>[1.0, 299.34, 238514.0]</td>\n",
       "      <td>1562781060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118178</th>\n",
       "      <td>1</td>\n",
       "      <td>1592504040</td>\n",
       "      <td>311.65</td>\n",
       "      <td>156630.0</td>\n",
       "      <td>-0.000417</td>\n",
       "      <td>[1.0, 311.65, 156630.0]</td>\n",
       "      <td>1592504100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118179</th>\n",
       "      <td>1</td>\n",
       "      <td>1592513280</td>\n",
       "      <td>311.21</td>\n",
       "      <td>99791.0</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>[1.0, 311.21, 99791.0]</td>\n",
       "      <td>1592513340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118180</th>\n",
       "      <td>1</td>\n",
       "      <td>1592587680</td>\n",
       "      <td>314.03</td>\n",
       "      <td>209341.0</td>\n",
       "      <td>0.000446</td>\n",
       "      <td>[1.0, 314.03, 209341.0]</td>\n",
       "      <td>1592587740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118181</th>\n",
       "      <td>1</td>\n",
       "      <td>1592857380</td>\n",
       "      <td>310.17</td>\n",
       "      <td>70228.0</td>\n",
       "      <td>0.000161</td>\n",
       "      <td>[1.0, 310.17, 70228.0]</td>\n",
       "      <td>1592857440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118182</th>\n",
       "      <td>1</td>\n",
       "      <td>1592866920</td>\n",
       "      <td>310.83</td>\n",
       "      <td>103716.0</td>\n",
       "      <td>0.000273</td>\n",
       "      <td>[1.0, 310.83, 103716.0]</td>\n",
       "      <td>1592866980</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>118183 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Ticker  pred_times   Close    Volume     label  \\\n",
       "0            1  1562344620  297.97   14200.0 -0.000336   \n",
       "1            1  1562365800  298.14   53824.0  0.000218   \n",
       "2            1  1562696880  296.26   47518.0  0.000304   \n",
       "3            1  1562721000  297.08   57480.0  0.000135   \n",
       "4            1  1562781000  299.34  238514.0  0.000084   \n",
       "...        ...         ...     ...       ...       ...   \n",
       "118178       1  1592504040  311.65  156630.0 -0.000417   \n",
       "118179       1  1592513280  311.21   99791.0  0.000161   \n",
       "118180       1  1592587680  314.03  209341.0  0.000446   \n",
       "118181       1  1592857380  310.17   70228.0  0.000161   \n",
       "118182       1  1592866920  310.83  103716.0  0.000273   \n",
       "\n",
       "                       features  eval_times  \n",
       "0        [1.0, 297.97, 14200.0]  1562344680  \n",
       "1        [1.0, 298.14, 53824.0]  1562365860  \n",
       "2        [1.0, 296.26, 47518.0]  1562696940  \n",
       "3        [1.0, 297.08, 57480.0]  1562721060  \n",
       "4       [1.0, 299.34, 238514.0]  1562781060  \n",
       "...                         ...         ...  \n",
       "118178  [1.0, 311.65, 156630.0]  1592504100  \n",
       "118179   [1.0, 311.21, 99791.0]  1592513340  \n",
       "118180  [1.0, 314.03, 209341.0]  1592587740  \n",
       "118181   [1.0, 310.17, 70228.0]  1592857440  \n",
       "118182  [1.0, 310.83, 103716.0]  1592866980  \n",
       "\n",
       "[118183 rows x 7 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Pred pandas df for timeseriescv\n",
    "pdDf=df.toPandas()\n",
    "pdDf=pdDf.rename(columns={\"Time\": \"pred_times\"})\n",
    "pdDf['eval_times']=pdDf['pred_times']+60;\n",
    "display(pdDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([    0,     1,     2, ..., 98829, 98830, 98831]), array([98832, 98833, 98834, 98835, 98836, 98837, 98838, 98839, 98840,\n",
      "       98841, 98842, 98843, 98844, 98845, 98846, 98847, 98848, 98849,\n",
      "       98850, 98851, 98852, 98853, 98854, 98855, 98856, 98857, 98858,\n",
      "       98859, 98860, 98861, 98862, 98863, 98864, 98865, 98866, 98867,\n",
      "       98868, 98869, 98870, 98871, 98872, 98873, 98874, 98875, 98876,\n",
      "       98877, 98878, 98879, 98880, 98881, 98882, 98883, 98884, 98885,\n",
      "       98886, 98887, 98888, 98889, 98890, 98891, 98892, 98893, 98894,\n",
      "       98895, 98896, 98897, 98898, 98899, 98900, 98901, 98902, 98903,\n",
      "       98904, 98905, 98906, 98907, 98908, 98909, 98910, 98911, 98912,\n",
      "       98913, 98914, 98915, 98916, 98917, 98918, 98919, 98920, 98921,\n",
      "       98922, 98923, 98924, 98925, 98926, 98927, 98928, 98929, 98930,\n",
      "       98931, 98932, 98933, 98934, 98935, 98936, 98937, 98938, 98939,\n",
      "       98940, 98941, 98942, 98943, 98944, 98945, 98946, 98947, 98948,\n",
      "       98949, 98950, 98951, 98952, 98953, 98954, 98955, 98956, 98957,\n",
      "       98958, 98959, 98960, 98961, 98962, 98963, 98964, 98965, 98966,\n",
      "       98967, 98968, 98969, 98970, 98971, 98972, 98973, 98974, 98975,\n",
      "       98976, 98977, 98978, 98979, 98980, 98981, 98982, 98983, 98984,\n",
      "       98985, 98986, 98987, 98988, 98989, 98990, 98991, 98992, 98993,\n",
      "       98994, 98995, 98996, 98997, 98998, 98999, 99000, 99001, 99002,\n",
      "       99003, 99004, 99005, 99006, 99007, 99008, 99009, 99010, 99011,\n",
      "       99012, 99013, 99014, 99015, 99016, 99017, 99018, 99019, 99020,\n",
      "       99021, 99022, 99023, 99024, 99025, 99026, 99027, 99028, 99029,\n",
      "       99030, 99031, 99032, 99033, 99034, 99035, 99036, 99037, 99038,\n",
      "       99039, 99040, 99041, 99042, 99043, 99044, 99045, 99046, 99047,\n",
      "       99048, 99049, 99050, 99051, 99052, 99053, 99054, 99055, 99056,\n",
      "       99057, 99058, 99059, 99060, 99061, 99062, 99063, 99064, 99065,\n",
      "       99066, 99067, 99068, 99069, 99070, 99071, 99072, 99073, 99074,\n",
      "       99075, 99076, 99077, 99078, 99079, 99080, 99081, 99082, 99083,\n",
      "       99084, 99085, 99086, 99087, 99088, 99089, 99090, 99091, 99092,\n",
      "       99093, 99094, 99095, 99096, 99097, 99098, 99099, 99100, 99101,\n",
      "       99102, 99103, 99104, 99105, 99106, 99107, 99108, 99109, 99110,\n",
      "       99111, 99112, 99113, 99114, 99115, 99116, 99117, 99118, 99119,\n",
      "       99120, 99121, 99122, 99123, 99124, 99125, 99126, 99127, 99128,\n",
      "       99129, 99130, 99131, 99132, 99133, 99134, 99135, 99136, 99137,\n",
      "       99138, 99139, 99140, 99141, 99142, 99143, 99144, 99145, 99146,\n",
      "       99147, 99148, 99149, 99150, 99151, 99152, 99153, 99154, 99155,\n",
      "       99156, 99157, 99158, 99159, 99160, 99161, 99162, 99163, 99164,\n",
      "       99165, 99166, 99167, 99168, 99169, 99170, 99171, 99172, 99173,\n",
      "       99174, 99175, 99176, 99177, 99178, 99179, 99180, 99181, 99182,\n",
      "       99183, 99184, 99185, 99186, 99187, 99188, 99189, 99190, 99191,\n",
      "       99192, 99193, 99194, 99195, 99196, 99197, 99198, 99199, 99200,\n",
      "       99201, 99202, 99203, 99204, 99205, 99206, 99207, 99208, 99209,\n",
      "       99210, 99211, 99212, 99213, 99214, 99215, 99216, 99217, 99218,\n",
      "       99219, 99220, 99221, 99222, 99223, 99224, 99225, 99226, 99227,\n",
      "       99228, 99229, 99230, 99231, 99232, 99233, 99234, 99235, 99236,\n",
      "       99237, 99238, 99239, 99240, 99241, 99242, 99243, 99244, 99245,\n",
      "       99246, 99247, 99248, 99249, 99250, 99251, 99252, 99253, 99254,\n",
      "       99255, 99256, 99257, 99258, 99259, 99260, 99261, 99262, 99263,\n",
      "       99264, 99265, 99266, 99267, 99268, 99269, 99270, 99271, 99272,\n",
      "       99273, 99274, 99275, 99276, 99277, 99278, 99279, 99280, 99281,\n",
      "       99282, 99283, 99284, 99285, 99286, 99287, 99288, 99289, 99290,\n",
      "       99291, 99292, 99293, 99294, 99295, 99296, 99297, 99298, 99299,\n",
      "       99300, 99301, 99302, 99303, 99304, 99305, 99306, 99307, 99308,\n",
      "       99309, 99310, 99311, 99312, 99313, 99314, 99315, 99316, 99317,\n",
      "       99318, 99319, 99320, 99321, 99322, 99323, 99324, 99325, 99326,\n",
      "       99327, 99328, 99329, 99330, 99331, 99332, 99333, 99334, 99335,\n",
      "       99336, 99337, 99338, 99339, 99340, 99341, 99342, 99343, 99344,\n",
      "       99345, 99346, 99347, 99348, 99349, 99350, 99351, 99352, 99353,\n",
      "       99354, 99355, 99356, 99357, 99358, 99359, 99360, 99361, 99362,\n",
      "       99363, 99364, 99365, 99366, 99367, 99368, 99369, 99370, 99371,\n",
      "       99372, 99373, 99374, 99375, 99376, 99377, 99378, 99379, 99380,\n",
      "       99381, 99382, 99383, 99384, 99385, 99386, 99387, 99388, 99389,\n",
      "       99390, 99391, 99392, 99393, 99394, 99395, 99396, 99397, 99398,\n",
      "       99399, 99400, 99401, 99402, 99403, 99404, 99405, 99406, 99407,\n",
      "       99408, 99409, 99410, 99411, 99412, 99413, 99414, 99415, 99416,\n",
      "       99417, 99418, 99419, 99420, 99421, 99422, 99423, 99424, 99425,\n",
      "       99426, 99427, 99428, 99429, 99430, 99431, 99432, 99433, 99434,\n",
      "       99435, 99436, 99437, 99438, 99439, 99440, 99441, 99442, 99443,\n",
      "       99444, 99445]))\n"
     ]
    }
   ],
   "source": [
    "#USING MLFLOW library instead\n",
    "from timeseriescv import cross_validation as cv\n",
    "crossval=cv.PurgedWalkForwardCV() #Default settings: n_splits=10, n_test_splits=1, min_train_splits=2, max_train_splits=None\n",
    "crossval=crossval.split(pdDf, pdDf['label'],pdDf['pred_times'],pdDf['eval_times'],True)\n",
    "print(next(crossval))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': [200, 400, 600, 800, 1000, 1200, 1400, 1600, 1800, 2000], 'max_features': ['auto', 'sqrt'], 'max_depth': [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, None], 'min_samples_split': [2, 5, 10], 'min_samples_leaf': [1, 2, 4], 'bootstrap': [True, False]}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "import numpy as np\n",
    "# Number of trees in random forest\n",
    "n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 10)]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "# Maximum number of levels in tree\n",
    "max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth.append(None)\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap}\n",
    "print(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_rows=df.select(\"features\").collect()\n",
    "feat_list=[row.features for row in feat_rows]\n",
    "label_rows=df.select(\"label\").collect()\n",
    "label_list=[row.label for row in label_rows]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "No fits were performed. Was the CV iterator empty? Were there no candidates?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-24af4b98823c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mrfr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mRandomForestRegressor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mrf_cv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mGridSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrfr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_grid\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrandom_grid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcrossval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mrf_cv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeat_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    734\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 736\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1187\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1188\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/share/virtualenvs/grantjensen-LUZRbGIG/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    716\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    717\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 718\u001b[0;31m                     raise ValueError('No fits were performed. '\n\u001b[0m\u001b[1;32m    719\u001b[0m                                      \u001b[0;34m'Was the CV iterator empty? '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    720\u001b[0m                                      'Were there no candidates?')\n",
      "\u001b[0;31mValueError\u001b[0m: No fits were performed. Was the CV iterator empty? Were there no candidates?"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "rfr=RandomForestRegressor()\n",
    "rf_cv=GridSearchCV(estimator=rfr, param_grid=random_grid, cv=crossval)\n",
    "rf_cv.fit(feat_list, label_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "ename": "StopIteration",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mStopIteration\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-78-e6e0e1e461ec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcrossval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mStopIteration\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print(next(crossval))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0003037872139338143\n"
     ]
    }
   ],
   "source": [
    "print(label_list[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.write.parquet(\"CleanedSpy.parquet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}