Recommendation-system-spark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.recommendation import ALS,ALSModel\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import col,UserDefinedFunction\n",
    "from pyspark.sql.types import *\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import warnings\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sc = SparkSession.builder.appName('als-recommender').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"movie recommendation\") \\\n",
    "    .config(\"spark.driver.maxResultSize\", \"100g\") \\\n",
    "    .config(\"spark.driver.memory\", \"100g\") \\\n",
    "    .config(\"spark.executor.memory\", \"10g\") \\\n",
    "    .config(\"spark.master\", \"local[16]\") \\\n",
    "    .config(\"spark.local.dir\",\"/home/bh_parijat/spark_temp/\")\\\n",
    "    .getOrCreate()\n",
    "# get spark context\n",
    "#sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_rdd  = sc.sparkContext.textFile('ml-1m/ratings.dat') \\\n",
    "                .map(lambda line: line.split(\"::\")) \\\n",
    "                .map(lambda x: (int(x[0]),int(x[1]),int(x[2])))\\\n",
    "                .cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_rdd = sc.sparkContext.textFile('ml-1m/movies.dat')\\\n",
    "                .map(lambda line : line.split(\"::\"))\\\n",
    "                .map(lambda x:(int(x[0]),x[1]))\\\n",
    "                .cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 'Toy Story (1995)'), (2, 'Jumanji (1995)')]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies_rdd.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies_schema = StructType([StructField('movieId',LongType(),True),StructField('name',StringType(),True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = sc.createDataFrame(movies_rdd,schema = schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_names = lambda x: re.sub(r'\\(\\d{4}\\)',\"\",x).rstrip().lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'toy'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "udf('Toy(1995)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#movies = movies.rdd.map(lambda x:(x[0],x[1])) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 1193, 5), (1, 661, 3)]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_rdd.take(2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ratings_rdd = ratings_rdd.sample(False,0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "schema = StructType([StructField('userId',LongType(),True),StructField('movieId',LongType(),True),StructField('rating',LongType(),True)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "ratings = sc.createDataFrame(ratings_rdd,schema=schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(userId=1, movieId=1193, rating=5),\n",
       " Row(userId=1, movieId=661, rating=3),\n",
       " Row(userId=1, movieId=914, rating=3)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.take(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spilt data for training, validation and testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, validation_data, test_data = ratings.randomSplit([0.6,0.2,0.2],seed=90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def model_train(train_data, validation_data, num_iters, ranks, regularizations):\n",
    "    \n",
    "    min_error = np.float('inf')\n",
    "    \n",
    "    best_rank = None\n",
    "    \n",
    "    best_model = None\n",
    "    \n",
    "    best_regularization = None\n",
    "    \n",
    "    for rank in ranks:\n",
    "        \n",
    "        for param in regularizations:\n",
    "            \n",
    "            model = ALS(rank=rank,maxIter=num_iters,regParam=param,userCol='userId',itemCol='movieId',nonnegative = True).fit(train_data)\n",
    "            \n",
    "            train =  train_data.select(['userId','movieId'])\n",
    "            \n",
    "            valid = validation_data.select(['userId','movieId'])\n",
    "            \n",
    "            \n",
    "            \n",
    "            predictions = model.transform(valid).fillna(0)\n",
    "            \n",
    "            \n",
    "            cond = [predictions.userId == validation_data.userId, predictions.movieId == validation_data.movieId]\n",
    "            temp = validation_data.join(predictions,cond).select(validation_data.rating, predictions.prediction)\n",
    "            \n",
    "            #print(temp.take(2))\n",
    "            error = math.sqrt(temp.rdd.map(lambda x: (x[-1]-x[-2])**2).mean())\n",
    "            \n",
    "            if error<min_error:\n",
    "                min_error = error\n",
    "                best_rank = rank\n",
    "                best_regularization = param\n",
    "                best_model = model\n",
    "            \n",
    "            #avg_error = error.select(\"select avg()\")\n",
    "            #print(error.take(2),error.mean(),error.sum(),type(error),error.count())\n",
    "            #print(predictions.take(2))\n",
    "            \n",
    "    return best_model,best_rank,best_regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model,rank,regular_param = model_train(train_data, validation_data, 10,range(8,22,2),[0.1,1,2,4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rank is 20 and regularization hyper-parameter is 0.1\n"
     ]
    }
   ],
   "source": [
    "model.write().overwrite().save(str(pathlib.Path().absolute()/'model'))\n",
    "print(\"rank is {} and regularization hyper-parameter is {}\".format(rank,regular_param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the learning curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_training_curve(train,rank,regular_param, iterations):\n",
    "    \n",
    "    errors = []\n",
    "    \n",
    "    for num_iters in range(1,iterations):\n",
    "        model = ALS(rank = rank, maxIter = num_iters, regParam = regular_param, userCol = 'userId', itemCol = 'movieId', nonnegative = True ).fit(train)\n",
    "\n",
    "        predictions = model.transform(train).fillna(0)\n",
    "\n",
    "        condition = [train.userId == predictions.userId,  train.movieId == predictions.movieId]\n",
    "\n",
    "        error = predictions.join(train,condition).select(predictions.prediction,train.rating).rdd.map(lambda x: (x[0]-x[1])**2).mean()\n",
    "\n",
    "        errors.append(math.sqrt(error))\n",
    "        \n",
    "        #print(predictions.schema)\n",
    "        \n",
    "        predictions.select(predictions.userId,predictions.movieId,predictions.prediction).withColumnRenamed(\"prediction\",\"rating\")\n",
    "        \n",
    "    return errors\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = get_training_curve(train_data,rank=20,regular_param=0.1,iterations=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f30ee161350>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGsNJREFUeJzt3XuMXOd53/HvM5fd5V5mxL2I3FneJJmUyB1Ztb115KhOlMhJSKWwUKBFq7Q2ItgVAiiuUxio06CNi+aPtsgFSZDECqGoalpDRmELqWpIstPEidD6ElG2LO1yZYm6kUvukksutffLXJ7+cWa5F+7uDMlZnpkzvw+w2DnnvDvn0Uj6nTPve857zN0REZFoiYVdgIiIVJ/CXUQkghTuIiIRpHAXEYkghbuISAQp3EVEIkjhLiISQQp3EZEIUriLiERQIqwdd3d3+4EDB8LavYhIXXr55ZcvuntPuXahhfuBAwc4ceJEWLsXEalLZvZeJe3ULSMiEkEKdxGRCFK4i4hEkMJdRCSCFO4iIhGkcBcRiaCy4W5me83s22Y2bGZDZvb5DdqYmf2hmZ0ys1fN7MPbU66IiFSikjP3PPAFdz8M3As8ZmZH1rU5Bhws/TwKfLmqVa7y47Fp/tNzw8wu5rdrFyIida9suLv7qLv/oPR6GhgG+tY1ewj4cw98D7jFzHqrXi0wcnmOP33xbYZHp7bj7UVEIuGa+tzN7ADwIeD76zb1AWdWLY9w9QEAM3vUzE6Y2Ynx8fFrq7SkP5MGYPDs5HX9vYhII6g43M2sHfg68Gvuvv602Tb4E79qhftxdx9w94GenrJTI2xoV6qZ7vYmBs/pzF1EZDMVhbuZJQmC/Svu/swGTUaAvauW9wDnbry8DWuhP5NmSOEuIrKpSq6WMeDPgGF3/71Nmj0LfLp01cy9wKS7j1axzjWyfSnePD/NQq6wXbsQEalrlcwKeR/wKeA1M3ultO43gH0A7v448BzwIHAKmAMeqX6pK/ozafJF543z03xwzy3buSsRkbpUNtzd/f+ycZ/66jYOPFatosrJXhlUnVK4i4hsoC7vUN3buYOOlgRD53TFjIjIRuoy3M2MbCatK2ZERDZRl+EO0J9JMTw6Ra5QDLsUEZGaU7fhnu1Ls5Qv8tb4TNiliIjUnDoO9xQAQ2fVNSMisl7dhvtt3e3sSMYZ1KCqiMhV6jbc4zHjcG+HztxFRDZQt+EOQb/70LlJisWrprEREWlo9R3umTSzSwXem5gLuxQRkZpS1+HeXxpU1fS/IiJr1XW4H7y1g2TcNKgqIrJOXYd7UyLGnbs1qCoisl5dhzsE/e5D5yYJ5i4TERGIQLj396W5PJfj3ORC2KWIiNSM+g/3jAZVRUTWq/twP7w7RcxgSOEuInJF3Yf7jqY4H7i1Xc9UFRFZpe7DHSjN7a4zdxGRZZEI9yOZFOenFrkwrUFVERGISLhn+4JnqqprRkQkEIlwP5JZnttdXTMiIhCRcE+1JDnQ1aozdxGRkkiEO0C/BlVFRK4oG+5m9qSZXTCzwU22p83sf5vZj8xsyMweqX6Z5fX3pTgzMc/kXC6M3YuI1JRKztyfAo5usf0x4KS73wPcD/yumTXdeGnXJptZHlTV2buISNlwd/cXgYmtmgAdZmZAe6ltvjrlVW55GgL1u4uIQKIK7/FHwLPAOaAD+KfuXqzC+16TrvZmMukW9buLiFCdAdVfAF4BMsDfA/7IzFIbNTSzR83shJmdGB8fr8Ku1zqSSWsCMRERqhPujwDPeOAU8A5w10YN3f24uw+4+0BPT08Vdr1Wti/F2xdnmV286b1CIiI1pRrhfhp4AMDMdgF3Am9X4X2vWTaTxh1eH1O/u4g0trJ97mb2NMFVMN1mNgJ8CUgCuPvjwG8BT5nZa4ABX3T3i9tW8RaWpyEYPDvFR/Z3hlGCiEhNKBvu7v5wme3ngJ+vWkU3YFeqma62JvW7i0jDi8wdqgBmRn9fmkFdDikiDS5S4Q6QzaR48/w0i/lC2KWIiIQmeuHelyZfdN4Ymwm7FBGR0EQu3K88MFs3M4lIA4tcuO/rbKWjJaFBVRFpaJELdzOjP5PSHDMi0tAiF+4Q3Mw0PDpFvnDTp7gREakJkQz3/r4Ui/kib43Phl2KiEgoIhnuy3O7q99dRBpVJMP99p52WpIx9buLSMOKZLjHY8aR3pQuhxSRhhXJcIfggdknz01RLHrYpYiI3HSRDfdsX4qZxTzvTcyFXYqIyE0X2XDv1wOzRaSBRTbcD+3qIBk3Bs9qUFVEGk9kw70pEePQrg6duYtIQ4psuENwvfvg2UncNagqIo0l2uHel+LyXI7RyYWwSxERuakiHe79fbpTVUQaU6TD/fDuFDFDj90TkYYT6XDf0RTnjp52hnTmLiINJtLhDsFj9zTHjIg0msiHe38mxdjUAuPTi2GXIiJy0zRAuOtOVRFpPGXD3cyeNLMLZja4RZv7zewVMxsys7+tbok35kjpgdnqmhGRRlLJmftTwNHNNprZLcCfAJ90937gn1SntOpI70iyv6tVZ+4i0lDKhru7vwhMbNHkl4Bn3P10qf2FKtVWNcGdqjpzF5HGUY0+90PATjP7GzN72cw+vVlDM3vUzE6Y2Ynx8fEq7LoyRzIpTk/MMTmXu2n7FBEJUzXCPQF8BPhF4BeAf29mhzZq6O7H3X3A3Qd6enqqsOvKZEt3qg6NqmtGRBpDNcJ9BHjB3Wfd/SLwInBPFd63avpLg6onNagqIg2iGuH+v4CPm1nCzFqBnwCGq/C+VdPd3kxvukVzzIhIw0iUa2BmTwP3A91mNgJ8CUgCuPvj7j5sZi8ArwJF4Al33/SyybD0Z9KaY0ZEGkbZcHf3hyto89vAb1elom3Sn0nxV6+fZ24pT2tT2X9sEZG6Fvk7VJdl+9K4w/DodNiliIhsuwYK9+U7VdXvLiLR1zDhvjvVQldbkwZVRaQhNEy4mxlHMindqSoiDaFhwh2Cfvc3L0yzmC+EXYqIyLZqrHDPpMkVnDfPz4RdiojItmqscC8NqqrfXUSirqHCfe/OVjqaEwzqihkRibiGCvdYLBhU1YM7RCTqGircIRhUHR6dIl8ohl2KiMi2acBwT7GQK/L2xdmwSxER2TYNF+7LD8zWoKqIRFnDhfvt3W20JGPqdxeRSGu4cE/EYxzuTenMXUQireHCHYKbmU6em6JY9LBLERHZFg0Z7v2ZFNOLeU5PzIVdiojItmjIcL/ywGz1u4tIRDVkuB/c1U4ybrpTVUQiqyHDvTkR59CuDg2qikhkNWS4Q9DvPnRuCncNqopI9DRsuGf70kzMLjE2tRB2KSIiVdew4b5yp6oGVUUkeho23A/3dhAzTUMgItFUNtzN7Ekzu2Bmg2Xa/X0zK5jZP65eeduntSnB7T3tDOmKGRGJoErO3J8Cjm7VwMziwH8BvlmFmm6arOZ2F5GIKhvu7v4iMFGm2eeArwMXqlHUzZLtSzM6ucDFmcWwSxERqaob7nM3sz7gHwGP33g5N9fyoKrO3kUkaqoxoPr7wBfdvVCuoZk9amYnzOzE+Ph4FXZ9Y45k9MBsEYmmRBXeYwD4qpkBdAMPmlne3f9ifUN3Pw4cBxgYGAj97qH0jiT7Ols5qTN3EYmYGw53d79t+bWZPQV8Y6Ngr1XZvpTmmBGRyKnkUsinge8Cd5rZiJl9xsx+xcx+ZfvL2379mTTvXZpjcj4XdikiIlVT9szd3R+u9M3c/ZdvqJoQ9Jf63U+em+Jjd3SFXI2ISHU07B2qy1aumFHXjIhER8OHe09HM7tTLbocUkQipeHDHUqDqrocUkQiROFO0DXz1vgMc0v5sEsREakKhTvBoGrRYXh0OuxSRESqQuHOygOzT2pQVUQiQuEO9KZb6Gxr0oM7RCQyFO6AmdGf0Z2qIhIdCveS/kyaN85Ps5Qvhl2KiMgNU7iXZPtS5ArOG+c1qCoi9U/hXpLVnaoiEiEK95J9na10NCc0qCoikaBwL4nFjMMaVBWRiFC4r5LNpBkenaJQDP05IiIiN0Thvkq2L8VCrsjb4zNhlyIickMU7qss36mqrhkRqXcK91Vu726jORHToKqI1D2F+yqJeIzDvSldDikidU/hvk62L8XQ2SmKGlQVkTqmcF8nm0kzvZjnzOW5sEsREbluCvd1lp+pqn53EalnCvd1Du1uJxEz9buLSF1TuK/TnIhzaFcHg3pgtojUMYX7BoJB1UncNagqIvWpbLib2ZNmdsHMBjfZ/s/N7NXSz3fM7J7ql3lz9WfSXJpdYmxqIexSRESuSyVn7k8BR7fY/g7w0+7+QeC3gONVqCtU2b4UAEMaVBWROlU23N39RWBii+3fcffLpcXvAXuqVFtoDvemMNM0BCJSv6rd5/4Z4PnNNprZo2Z2wsxOjI+PV3nX1dPalOCOnnZdDikidatq4W5mP0MQ7l/crI27H3f3AXcf6Onpqdaut0V/RtMQiEj9qkq4m9kHgSeAh9z9UjXeM2zZTJrRyQUuzSyGXYqIyDW74XA3s33AM8Cn3P2NGy+pNvQvD6rqencRqUOJcg3M7GngfqDbzEaALwFJAHd/HPhNoAv4EzMDyLv7wHYVfLNcmYbg3CQ/dai2u5BERNYrG+7u/nCZ7Z8FPlu1impEekeSvZ07dDmkiNQl3aG6hWwmrUFVEalLCvctZPvSvHtpjqmFXNiliIhcE4X7FvozwaDqSQ2qikidUbhvYWVud3XNiEh9UbhvoaejmV2pZp25i0jdUbiXkc2kNceMiNQdhXsZ/X1pTl2YYX6pEHYpIiIVU7iXkc2kKDoMj6lrRkTqh8K9jP6+YFBV0xCISD1RuJeRSbewszXJkK6YEZE6onAvw8zI9mlQVUTqi8K9Av2ZND8em2YpXwy7FBGRiijcK9CfSZErOG9emA67FBGRiijcK5BdHlTVDJEiUicU7hXY39lKe3NC/e4iUjcU7hWIxYwjmZTmmBGRuqFwr1B/JsXw6DSFooddiohIWQr3CmUzaeZzBd65OBN2KSIiZSncK7Q8qDqoQVURqQMK9wrd0dNGcyKmfncRqQsK9wol4jHu6k1pjhkRqQsK92uQzaQYPDeJuwZVRaS2KdyvQbYvzfRCnjMT82GXIiKypbLhbmZPmtkFMxvcZLuZ2R+a2Skze9XMPlz9MmtDdvmZqrqZSURqXCVn7k8BR7fYfgw4WPp5FPjyjZdVmw7tbicRM4YU7iJS48qGu7u/CExs0eQh4M898D3gFjPrrVaBtaQ5Eefgrg5dDikiNa8afe59wJlVyyOldZGULU1DoEFVEall1Qh322DdhslnZo+a2QkzOzE+Pl6FXd982b40l2aXOD+1GHYpIiKbqka4jwB7Vy3vAc5t1NDdj7v7gLsP9PT0VGHXN19/JgWgfncRqWnVCPdngU+Xrpq5F5h099EqvG9NOtybwkzTEIhIbUuUa2BmTwP3A91mNgJ8CUgCuPvjwHPAg8ApYA54ZLuKrQVtzQlu727T5ZAiUtPKhru7P1xmuwOPVa2iOpDtS/PSO1tdQCQiEi7doXod+jMpzk0uMDG7FHYpIiIbUrhfh+U7VTWoKiK1SuF+HfozmttdRGqbwv06pFuT7O3coUFVEalZCvfr1N+b5pXT7/PuxdmwSxERuUrZq2VkY8fu3s0LQ2Pc/zt/w+HeFMeyuzmW3c3BXR1hlyYigoU1R8rAwICfOHEilH1Xy8jlOV4YHOOFwTFePn0Zd/jAre0cy+7maHY3R3pTmG00O4OIyPUxs5fdfaBsO4V7dZyfWuCbQ2M8/9oY33/nEkWH/V2tHM3u5li2l3v2pBX0InLDFO4hujSzyF+ePM9zg2N859RF8kUnk27haLaXY3fv5iP7dhKLKehF5Nop3GvE5FyO/zN8nucHx3jxzXGW8kV6Opo52h/00X/0tk4ScY1ri0hlFO41aGYxz1+/foEXBkf59uvjzOcKdLY18fNHdnE0u5ufvKObpoSCXkQ2p3CvcfNLBf72jQs8PzjGXw1fYGYxT6olwScO7+LY3b18/GA3Lcl42GWKSI1RuNeRhVyB/3fqIs8PjvGXJ88zOZ+jrSnOz9x1Kw/e3cv9d/bQ2qSrVkWk8nBXYtSAlmScBw7v4oHDu8gVinz3rUs8PzjGt4bG+Maro7QkY/z0oR4evLuXn73rVjpakmGXLCI1TmfuNaxQdP7unQleGBzlhaExzk8t0hSP8Q8OdnMsu5ufO7KLW1qbwi5TRG4idctETLHo/PDM+zz/2ijPD45x9v15EjHjY3d0cd8HujnQ1cq+zjb2dbXS3qwvZCJRpXCPMHfntbOTPF+6O/addfPbdLY1sa+zdeWna+X17lSLrrEXqWMK9wYyOZ/jzMQcpyfmeO9S8PvMxBzvTcxy7v0FCsWVf8dN8Rh7Onewr7OV/Z2t7C2F/v6uNvZ27tDArUiN04BqA0nvSJLuS5PtS1+1LVcoMvr+Au9NzHK6dAA4XToAvPzuZaYX82vad7c3s790pr+3dABYPvO/taNZUyiI1AmFe8Ql47EgnLtar9rm7rw/l1sJ/VXB/3fvTPAXr5xl9Re7lmSMvTuv7urZ39XKnp2tui5fpIYo3BuYmbGzrYmdbU3cs/eWq7Yv5YucfX+e9y7NXtXt8923LzG3VFjTfmdrks62Jrramulsa6KzvYmutqbg9ar1Xe1N7Gxt0t24IttI4S6bakrEuK27jdu6267a5u5cml1a6d+/NMeF6eCh4ZdmlnhrfIaX3l3i8twSxU2GdTpaEqvCvzl4vcEBoas9eK1vBiKVU7jLdTEzutub6W5v5sP7dm7arlB0JudzTMwucmlmiYnZJS7OLjExsxSsmw3WjVye40cj73N5don8JkeDtqY4ne2rDgRt6w4EpW2plgTtLQk6mpO0JGMaJ5CGpHCXbRWP2ZXw/cCt5du7O1PzeS7NLgbfAkrhv/yNYHn92OQCJ89NMTG7xFKhuOX+25sTtDcn6GgJfre3rFtuTpYOBivb2ta1b2tKENclpFJHKgp3MzsK/AEQB55w9/+8bvs+4L8Bt5Ta/Lq7P1flWqUBmBnp1iTp1iS395Rv7+7MLOZXDgQzS0wv5phZLDCzkGdmMcfMQp7pxXxpOWh7+tLclXXzuUL5HRF8c2i/EvjJ4GCw0cGi9HtHMk5LMs6Opvia1y2JWOl3XPccyLYpG+5mFgf+GPg5YAR4ycyedfeTq5r9O+B/uvuXzewI8BxwYBvqFVnDzOhoSdLRkmR/19VjA5XIF4rMLhZKB4X8VQeD5eXZVeuC7TkuTC+stF/Mc623jTQnYkHoLwd/Mk5LMhYsJ+O0lA4CO5pW1jWvar980LjyN2vWBeubEjGa4uqeajSVnLl/FDjl7m8DmNlXgYeA1eHuQKr0Og2cq2aRItspEY+Rbo2Rbr2xCdncnbmlQhD+C3kWcgUWcgXmcwXml4Lfi7lisFxat5AvsFDaNp8rrvzNUoHJ+VxpubjmPa5XUyJGc7wU9qWf5sRK+Afr4lfWrW670i5+1d83r/n7GM2J+Jr3XP7bRMxIJmIkYzGScSMeMx1wtlEl4d4HnFm1PAL8xLo2/wH4lpl9DmgDPrHRG5nZo8CjAPv27bvWWkVqmpnRVuqv35Uq3/56uDuL+eKag8Zy+C+sOmAE64M2S/kiS/kii4XilddL+SJLhSKLueD38rrJ+VzQNl9Y02759WaD3dcrGTeS8SD4gwNAjGTCSMZiJJa3xWM0xY1EaV1TfGVb8GMk4jGSsXXt46varzqwxGNGonRwScSC943Hg9fx0nssb9toebmOjZZr6WBVSbhvVO36f8MPA0+5+++a2ceA/25mWXdfM9Ll7seB4xBMP3A9BYs0MjO70uVy9Z0J269Q9FUHi8KWB4rFqw4MKweIXL5IrujkC0VyhSK5gpMrFMmXfudKbfLFtdvmc4VgXd7JFVe1v9Km9LpYvOYusmqIGcHBonQACQ4AwcFl9QHh4Y/u47Mfv31ba6kk3EeAvauW93B1t8tngKMA7v5dM2sBuoEL1ShSRGpDPGZBv35THKjd5wq4O4WiBweS1eFfdAoFJ18MXucLy+2uXi4Undy65XwheM9Cqf1Km82X86WD2Orl7vbmbf8MKgn3l4CDZnYbcBb4Z8AvrWtzGngAeMrMDgMtwHg1CxURqZRZ6Uw5TsPe/Fb2/m93zwO/CnwTGCa4KmbIzP6jmX2y1OwLwL80sx8BTwO/7GFNNykiIpVd5166Zv25det+c9Xrk8B91S1NRESul2ZuEhGJIIW7iEgEKdxFRCJI4S4iEkEKdxGRCFK4i4hEkIV1ObqZjQPvXeefdwMXq1hOvdPnsZY+jxX6LNaKwuex393LTogdWrjfCDM74e4DYddRK/R5rKXPY4U+i7Ua6fNQt4yISAQp3EVEIqhew/142AXUGH0ea+nzWKHPYq2G+Tzqss9dRES2Vq9n7iIisoW6C3czO2pmPzazU2b262HXEyYz22tm3zazYTMbMrPPh11T2MwsbmY/NLNvhF1L2MzsFjP7mpm9Xvpv5GNh1xQWM/vXpf9HBs3s6dIDhSKtrsLdzOLAHwPHgCPAw2Z2JNyqQpUHvuDuh4F7gcca/PMA+DzBcwcE/gB4wd3vAu6hQT8XM+sD/hUw4O5ZIE7w0KFIq6twBz4KnHL3t919Cfgq8FDINYXG3Ufd/Qel19ME//P2hVtVeMxsD/CLwBNh1xI2M0sBPwX8GYC7L7n7++FWFaoEsMPMEkArVz8qNHLqLdz7gDOrlkdo4DBbzcwOAB8Cvh9uJaH6feDfAMVyDRvA7QSPuvyvpW6qJ8ysLeyiwuDuZ4HfIXgc6Cgw6e7fCreq7Vdv4W4brGv4y33MrB34OvBr7j4Vdj1hMLN/CFxw95fDrqVGJIAPA1929w8Bs0BDjlGZ2U6Cb/i3ARmgzcz+RbhVbb96C/cRYO+q5T00wNerrZhZkiDYv+Luz4RdT4juAz5pZu8SdNf9rJn9j3BLCtUIMOLuy9/kvkYQ9o3oE8A77j7u7jngGeAnQ65p29VbuL8EHDSz28ysiWBQ5NmQawqNmRlBn+qwu/9e2PWEyd3/rbvvcfcDBP9d/LW7R/7sbDPuPgacMbM7S6seAE6GWFKYTgP3mllr6f+ZB2iAweWKHpBdK9w9b2a/CnyTYMT7SXcfCrmsMN0HfAp4zcxeKa37jdIDzUU+B3yldCL0NvBIyPWEwt2/b2ZfA35AcIXZD2mAO1V1h6qISATVW7eMiIhUQOEuIhJBCncRkQhSuIuIRJDCXUQkghTuIiIRpHAXEYkghbuISAT9f7RV8n6H4vM/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The minimum training error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7909596187552549"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min(errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_predictions = ALSModel.read().load((str(pathlib.Path().absolute()/'model'))).transform(test_data).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_error = math.sqrt(test_predictions.rdd.map(lambda x: (x.rating-x.prediction)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8744198541233533"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ids(movies,fav_movies_names):\n",
    "    \n",
    "    movie_ids = movies.filter(lambda x :)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}